From 1b36c45bfaef2840deea8e2803c1e0327955b65f Mon Sep 17 00:00:00 2001 From: Terence Date: Tue, 22 Dec 2020 14:02:59 +0800 Subject: [PATCH 1/4] Add serving histogram metrics Signed-off-by: Terence --- .../service/OnlineServingServiceV2.java | 42 ++++++++++++++++++- .../main/java/feast/serving/util/Metrics.java | 27 ++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index a78a94ab00..94f74f2a93 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -16,6 +16,8 @@ */ package feast.serving.service; +import static feast.common.models.FeatureTable.getFeatureTableStringRef; + import com.google.protobuf.Duration; import feast.common.models.FeatureV2; import feast.proto.core.FeatureProto; @@ -159,8 +161,21 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re populateCountMetrics(statusMap, projectName); } } - entityValuesMap.get(entityRow).putAll(allValueMaps); - entityStatusesMap.get(entityRow).putAll(allStatusMaps); + populateHistogramMetrics(entityRows, featureReferences, projectName); + + // Build response field values from entityValuesMap and entityStatusesMap + // Response field values should be in the same order as the entityRows provided by the user. + List fieldValuesList = + entityRows.stream() + .map( + entityRow -> { + return GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(entityValuesMap.get(entityRow)) + .putAllStatuses(entityStatusesMap.get(entityRow)) + .build(); + }) + .collect(Collectors.toList()); + return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); } // Build response field values from entityValuesMap and entityStatusesMap @@ -295,6 +310,29 @@ private static boolean checkOutsideMaxAge( return timeDifference > maxAge.getSeconds(); } + /** + * Populate histogram metrics that can be used for analysing online retrieval calls + * + * @param entityRows entity rows provided in request + * @param featureReferences feature references provided in request + * @param project project name provided in request + */ + private void populateHistogramMetrics( + List entityRows, + List featureReferences, + String project) { + Metrics.requestEntityCount.labels(project).observe(Double.valueOf(entityRows.size())); + Metrics.requestFeatureCount.labels(project).observe(Double.valueOf(featureReferences.size())); + + Set featureTableRefs = + featureReferences.stream() + .map(featureReference -> getFeatureTableStringRef(project, featureReference)) + .collect(Collectors.toSet()); + Metrics.requestFeatureTableCount + .labels(project) + .observe(Double.valueOf(featureTableRefs.size())); + } + /** * Populate count metrics that can be used for analysing online retrieval calls * diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java index 13cbe0e1cc..8f3ed5a76c 100644 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ b/serving/src/main/java/feast/serving/util/Metrics.java @@ -37,6 +37,33 @@ public class Metrics { .labelNames("project", "feature_name") .register(); + public static final Histogram requestEntityCount = + Histogram.build() + .buckets(1, 2, 5, 10) + .name("request_entity_count") + .subsystem("feast_serving") + .help("Number of entity rows per request") + .labelNames("project") + .register(); + + public static final Histogram requestFeatureCount = + Histogram.build() + .buckets(1, 2, 5, 10, 15, 20) + .name("request_feature_count") + .subsystem("feast_serving") + .help("Number of feature rows per request") + .labelNames("project") + .register(); + + public static final Histogram requestFeatureTableCount = + Histogram.build() + .buckets(1, 2, 5, 10) + .name("request_feature_table_count") + .subsystem("feast_serving") + .help("Number of feature tables per request") + .labelNames("project") + .register(); + public static final Counter notFoundKeyCount = Counter.build() .name("not_found_feature_count") From 0303d04bae484dd14de1504655397f8e2a385a7d Mon Sep 17 00:00:00 2001 From: Terence Date: Wed, 23 Dec 2020 09:00:07 +0800 Subject: [PATCH 2/4] Clean ups Signed-off-by: Terence --- .../feast/serving/service/OnlineServingServiceV2.java | 7 ++++--- serving/src/main/java/feast/serving/util/Metrics.java | 8 -------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index 94f74f2a93..42040f1ccf 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -324,13 +324,14 @@ private void populateHistogramMetrics( Metrics.requestEntityCount.labels(project).observe(Double.valueOf(entityRows.size())); Metrics.requestFeatureCount.labels(project).observe(Double.valueOf(featureReferences.size())); - Set featureTableRefs = + long countDistinctFeatureTables = featureReferences.stream() .map(featureReference -> getFeatureTableStringRef(project, featureReference)) - .collect(Collectors.toSet()); + .distinct() + .count(); Metrics.requestFeatureTableCount .labels(project) - .observe(Double.valueOf(featureTableRefs.size())); + .observe(Double.valueOf(countDistinctFeatureTables)); } /** diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java index 8f3ed5a76c..8c9aa3eb4e 100644 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ b/serving/src/main/java/feast/serving/util/Metrics.java @@ -29,14 +29,6 @@ public class Metrics { .labelNames("method") .register(); - public static final Counter requestCount = - Counter.build() - .name("request_feature_count") - .subsystem("feast_serving") - .help("number of feature rows requested") - .labelNames("project", "feature_name") - .register(); - public static final Histogram requestEntityCount = Histogram.build() .buckets(1, 2, 5, 10) From b1f0e5c1d77dee5b0793beb903b027f8bd513f14 Mon Sep 17 00:00:00 2001 From: Terence Date: Wed, 23 Dec 2020 12:03:18 +0800 Subject: [PATCH 3/4] Add more buckets Signed-off-by: Terence --- serving/src/main/java/feast/serving/util/Metrics.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java index 8c9aa3eb4e..8879531840 100644 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ b/serving/src/main/java/feast/serving/util/Metrics.java @@ -31,7 +31,7 @@ public class Metrics { public static final Histogram requestEntityCount = Histogram.build() - .buckets(1, 2, 5, 10) + .buckets(1, 2, 5, 10, 20, 50, 100, 200) .name("request_entity_count") .subsystem("feast_serving") .help("Number of entity rows per request") @@ -40,7 +40,7 @@ public class Metrics { public static final Histogram requestFeatureCount = Histogram.build() - .buckets(1, 2, 5, 10, 15, 20) + .buckets(1, 2, 5, 10, 15, 20, 30, 50) .name("request_feature_count") .subsystem("feast_serving") .help("Number of feature rows per request") @@ -49,7 +49,7 @@ public class Metrics { public static final Histogram requestFeatureTableCount = Histogram.build() - .buckets(1, 2, 5, 10) + .buckets(1, 2, 5, 10, 20) .name("request_feature_table_count") .subsystem("feast_serving") .help("Number of feature tables per request") From 130e58d74b747ad763b4fea3ed89b317ae116e05 Mon Sep 17 00:00:00 2001 From: Terence Date: Wed, 23 Dec 2020 13:30:04 +0800 Subject: [PATCH 4/4] Fix conflict Signed-off-by: Terence --- .../service/OnlineServingServiceV2.java | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index 42040f1ccf..748b587e8f 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -161,22 +161,10 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re populateCountMetrics(statusMap, projectName); } } - populateHistogramMetrics(entityRows, featureReferences, projectName); - - // Build response field values from entityValuesMap and entityStatusesMap - // Response field values should be in the same order as the entityRows provided by the user. - List fieldValuesList = - entityRows.stream() - .map( - entityRow -> { - return GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(entityValuesMap.get(entityRow)) - .putAllStatuses(entityStatusesMap.get(entityRow)) - .build(); - }) - .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); + entityValuesMap.get(entityRow).putAll(allValueMaps); + entityStatusesMap.get(entityRow).putAll(allStatusMaps); } + populateHistogramMetrics(entityRows, featureReferences, projectName); // Build response field values from entityValuesMap and entityStatusesMap // Response field values should be in the same order as the entityRows provided by the user.