Merge branch 'master' into fix/model_load

examples/pt2/README.md

@@ -23,8 +23,8 @@ As an example let's expand our getting started guide with the only difference be
 
 ```
 mkdir model_store
-torch-model-archiver --model-name densenet161 --version 1.0 --model-file ./serve/examples/image_classifier/densenet_161/model.py --export-path model_store --extra-files ./serve/examples/image_classifier/index_to_name.json --handler image_classifier
-torchserve --start --ncs --model-store model_store --models densenet161.mar --config-file model_config.yaml
+torch-model-archiver --model-name densenet161 --version 1.0 --model-file ./serve/examples/image_classifier/densenet_161/model.py --export-path model_store --extra-files ./serve/examples/image_classifier/index_to_name.json --handler image_classifier --config-file model_config.yaml
+torchserve --start --ncs --model-store model_store --models densenet161.mar
 ```
 
 The exact same approach works with any other model, what's going on is the below

frontend/server/src/main/java/org/pytorch/serve/metrics/configuration/MetricConfiguration.java

@@ -0,0 +1,91 @@
+package org.pytorch.serve.metrics.configuration;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.composer.ComposerException;
+import org.yaml.snakeyaml.constructor.Constructor;
+
+public class MetricConfiguration {
+    private static final Logger logger = LoggerFactory.getLogger(MetricConfiguration.class);
+    private List<String> dimensions;
+
+    @SuppressWarnings("checkstyle:MemberName")
+    private MetricTypes ts_metrics;
+
+    @SuppressWarnings("checkstyle:MemberName")
+    private MetricTypes model_metrics;
+
+    public void setDimensions(List<String> dimensions) {
+        this.dimensions = dimensions;
+    }
+
+    public List<String> getDimensions() {
+        return this.dimensions;
+    }
+
+    @SuppressWarnings("checkstyle:MethodName")
+    public void setTs_metrics(MetricTypes tsMetrics) {
+        this.ts_metrics = tsMetrics;
+    }
+
+    @SuppressWarnings("checkstyle:MethodName")
+    public MetricTypes getTs_metrics() {
+        return this.ts_metrics;
+    }
+
+    @SuppressWarnings("checkstyle:MethodName")
+    public void setModel_metrics(MetricTypes modelMetrics) {
+        // The Hostname dimension is included by default for model metrics
+        modelMetrics.setCounter(this.addHostnameDimensionToMetrics(modelMetrics.getCounter()));
+        modelMetrics.setGauge(this.addHostnameDimensionToMetrics(modelMetrics.getGauge()));
+        modelMetrics.setHistogram(this.addHostnameDimensionToMetrics(modelMetrics.getHistogram()));
+        this.model_metrics = modelMetrics;
+    }
+
+    @SuppressWarnings("checkstyle:MethodName")
+    public MetricTypes getModel_metrics() {
+        return this.model_metrics;
+    }
+
+    public void validate() {
+        if (this.ts_metrics != null) {
+            ts_metrics.validate();
+        }
+
+        if (this.model_metrics != null) {
+            model_metrics.validate();
+        }
+    }
+
+    public static MetricConfiguration loadConfiguration(String configFilePath)
+            throws FileNotFoundException, ComposerException, RuntimeException {
+        Constructor constructor = new Constructor(MetricConfiguration.class);
+        Yaml yaml = new Yaml(constructor);
+        FileInputStream inputStream = new FileInputStream(new File(configFilePath));
+        MetricConfiguration config = yaml.load(inputStream);
+        config.validate();
+        logger.info("Successfully loaded metrics configuration from {}", configFilePath);
+
+        return config;
+    }
+
+    private List<MetricSpecification> addHostnameDimensionToMetrics(
+            List<MetricSpecification> metricsSpec) {
+        if (metricsSpec == null) {
+            return metricsSpec;
+        }
+
+        for (MetricSpecification spec : metricsSpec) {
+            List<String> dimensions = spec.getDimensions();
+            dimensions.add("Hostname");
+            spec.setDimensions(dimensions);
+        }
+
+        return metricsSpec;
+    }
+}

frontend/server/src/main/java/org/pytorch/serve/metrics/configuration/MetricSpecification.java

@@ -0,0 +1,48 @@
+package org.pytorch.serve.metrics.configuration;
+
+import java.util.List;
+
+public class MetricSpecification {
+    private String name;
+    private String unit;
+    private List<String> dimensions;
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return this.name;
+    }
+
+    public void setUnit(String unit) {
+        this.unit = unit;
+    }
+
+    public String getUnit() {
+        return this.unit;
+    }
+
+    public void setDimensions(List<String> dimensions) {
+        this.dimensions = dimensions;
+    }
+
+    public List<String> getDimensions() {
+        return this.dimensions;
+    }
+
+    @Override
+    public String toString() {
+        return "name: " + this.name + ", unit: " + this.unit + ", dimensions: " + this.dimensions;
+    }
+
+    public void validate() {
+        if (this.name == null || this.name.isEmpty()) {
+            throw new RuntimeException("Metric name cannot be empty. " + this);
+        }
+
+        if (this.unit == null || this.unit.isEmpty()) {
+            throw new RuntimeException("Metric unit cannot be empty. " + this);
+        }
+    }
+}

frontend/server/src/main/java/org/pytorch/serve/metrics/configuration/MetricTypes.java

@@ -0,0 +1,53 @@
+package org.pytorch.serve.metrics.configuration;
+
+import java.util.List;
+
+public class MetricTypes {
+    private List<MetricSpecification> counter;
+    private List<MetricSpecification> gauge;
+    private List<MetricSpecification> histogram;
+
+    public void setCounter(List<MetricSpecification> counter) {
+        this.counter = counter;
+    }
+
+    public List<MetricSpecification> getCounter() {
+        return this.counter;
+    }
+
+    public void setGauge(List<MetricSpecification> gauge) {
+        this.gauge = gauge;
+    }
+
+    public List<MetricSpecification> getGauge() {
+        return this.gauge;
+    }
+
+    public void setHistogram(List<MetricSpecification> histogram) {
+        this.histogram = histogram;
+    }
+
+    public List<MetricSpecification> getHistogram() {
+        return this.histogram;
+    }
+
+    public void validate() {
+        if (this.counter != null) {
+            for (MetricSpecification spec : this.counter) {
+                spec.validate();
+            }
+        }
+
+        if (this.gauge != null) {
+            for (MetricSpecification spec : this.gauge) {
+                spec.validate();
+            }
+        }
+
+        if (this.histogram != null) {
+            for (MetricSpecification spec : this.histogram) {
+                spec.validate();
+            }
+        }
+    }
+}

frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java

@@ -66,6 +66,7 @@ public final class ConfigManager {
     private static final String TS_JOB_QUEUE_SIZE = "job_queue_size";
     private static final String TS_NUMBER_OF_GPU = "number_of_gpu";
     private static final String TS_METRICS_CONFIG = "metrics_config";
+    private static final String TS_METRICS_MODE = "metrics_mode";
     private static final String TS_DISABLE_SYSTEM_METRICS = "disable_system_metrics";
 
     // IPEX config option that can be set at config.properties
@@ -387,6 +388,10 @@ public String getTorchRunLogDir() {
         return torchrunLogDir;
     }
 
+    public String getMetricsMode() {
+        return getProperty(TS_METRICS_MODE, "log");
+    }
+
     public boolean isSystemMetricsDisabled() {
         return Boolean.parseBoolean(getProperty(TS_DISABLE_SYSTEM_METRICS, "false"));
     }
@@ -660,6 +665,8 @@ public String dumpConfigurations() {
                 + prop.getProperty(TS_METRICS_FORMAT, METRIC_FORMAT_PROMETHEUS)
                 + "\nEnable metrics API: "
                 + prop.getProperty(TS_ENABLE_METRICS_API, "true")
+                + "\nMetrics mode: "
+                + getMetricsMode()
                 + "\nDisable system metrics: "
                 + isSystemMetricsDisabled()
                 + "\nWorkflow Store: "

frontend/server/src/test/java/org/pytorch/serve/metrics/MetricConfigurationTest.java

@@ -0,0 +1,136 @@
+package org.pytorch.serve.metrics.configuration;
+
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import org.pytorch.serve.util.ConfigManager;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+import org.yaml.snakeyaml.composer.ComposerException;
+
+public class MetricConfigurationTest {
+    @Test
+    public void testLoadValidConfiguration()
+            throws FileNotFoundException, ComposerException, RuntimeException {
+        MetricConfiguration config =
+                MetricConfiguration.loadConfiguration(
+                        "src/test/resources/metrics/valid_configuration.yaml");
+
+        Assert.assertEquals(
+                config.getDimensions(),
+                new ArrayList<String>(
+                        Arrays.asList("ModelName", "ModelVersion", "Level", "Hostname")));
+
+        Assert.assertEquals(config.getTs_metrics().getCounter().size(), 2);
+
+        MetricSpecification spec = config.getTs_metrics().getCounter().get(0);
+        Assert.assertEquals(spec.getName(), "Requests2XX");
+        Assert.assertEquals(spec.getUnit(), "Count");
+        Assert.assertEquals(
+                spec.getDimensions(), new ArrayList<String>(Arrays.asList("Level", "Hostname")));
+
+        spec = config.getTs_metrics().getCounter().get(1);
+        Assert.assertEquals(spec.getName(), "InferenceRequestsTotal");
+        Assert.assertEquals(spec.getUnit(), "Count");
+        Assert.assertEquals(
+                spec.getDimensions(),
+                new ArrayList<String>(Arrays.asList("ModelName", "ModelVersion", "Hostname")));
+
+        Assert.assertEquals(config.getTs_metrics().getGauge().size(), 2);
+
+        spec = config.getTs_metrics().getGauge().get(0);
+        Assert.assertEquals(spec.getName(), "QueueTime");
+        Assert.assertEquals(spec.getUnit(), "Milliseconds");
+        Assert.assertEquals(
+                spec.getDimensions(), new ArrayList<String>(Arrays.asList("Level", "Hostname")));
+
+        spec = config.getTs_metrics().getGauge().get(1);
+        Assert.assertEquals(spec.getName(), "WorkerThreadTime");
+        Assert.assertEquals(spec.getUnit(), "Milliseconds");
+        Assert.assertEquals(
+                spec.getDimensions(), new ArrayList<String>(Arrays.asList("Level", "Hostname")));
+
+        Assert.assertEquals(config.getTs_metrics().getHistogram(), null);
+
+        Assert.assertEquals(config.getModel_metrics().getCounter(), null);
+
+        Assert.assertEquals(config.getModel_metrics().getGauge().size(), 2);
+
+        spec = config.getModel_metrics().getGauge().get(0);
+        Assert.assertEquals(spec.getName(), "HandlerTime");
+        Assert.assertEquals(spec.getUnit(), "ms");
+        Assert.assertEquals(
+                spec.getDimensions(),
+                new ArrayList<String>(Arrays.asList("ModelName", "Level", "Hostname")));
+
+        spec = config.getModel_metrics().getGauge().get(1);
+        Assert.assertEquals(spec.getName(), "PredictionTime");
+        Assert.assertEquals(spec.getUnit(), "ms");
+        Assert.assertEquals(
+                spec.getDimensions(),
+                new ArrayList<String>(Arrays.asList("ModelName", "Level", "Hostname")));
+
+        Assert.assertEquals(config.getModel_metrics().getHistogram(), null);
+    }
+
+    @Test
+    public void testLoadValidConfigurationEmptyMetricDimensions()
+            throws FileNotFoundException, ComposerException, RuntimeException {
+        MetricConfiguration config =
+                MetricConfiguration.loadConfiguration(
+                        "src/test/resources/metrics/valid_configuration_empty_metric_dimensions.yaml");
+
+        Assert.assertEquals(config.getDimensions(), null);
+
+        Assert.assertEquals(config.getTs_metrics().getCounter().size(), 1);
+
+        MetricSpecification spec = config.getTs_metrics().getCounter().get(0);
+        Assert.assertEquals(spec.getName(), "InferenceRequestsTotal");
+        Assert.assertEquals(spec.getUnit(), "Count");
+        Assert.assertEquals(spec.getDimensions(), null);
+
+        Assert.assertEquals(config.getTs_metrics().getGauge(), null);
+
+        Assert.assertEquals(config.getTs_metrics().getHistogram(), null);
+
+        Assert.assertEquals(config.getModel_metrics(), null);
+    }
+
+    @Test
+    public void testLoadInvalidConfigurationMissingDimension() {
+        Assert.assertThrows(
+                ComposerException.class,
+                () ->
+                        MetricConfiguration.loadConfiguration(
+                                "src/test/resources/metrics/invalid_configuration_missing_dimension.yaml"));
+    }
+
+    @Test
+    public void testLoadInvalidConfigurationMissingMetricName() {
+        Assert.assertThrows(
+                RuntimeException.class,
+                () ->
+                        MetricConfiguration.loadConfiguration(
+                                "src/test/resources/metrics/invalid_configuration_missing_metric_name.yaml"));
+    }
+
+    @Test
+    public void testLoadInvalidConfigurationMissingMetricUnit() {
+        Assert.assertThrows(
+                RuntimeException.class,
+                () ->
+                        MetricConfiguration.loadConfiguration(
+                                "src/test/resources/metrics/invalid_configuration_missing_metric_unit.yaml"));
+    }
+
+    @Test
+    public void testMetricsModeConfiguration() {
+        ConfigManager configManager = ConfigManager.getInstance();
+        String existingMetricsModeConfiguration = configManager.getMetricsMode();
+        Assert.assertEquals(existingMetricsModeConfiguration, "log");
+        configManager.setProperty("metrics_mode", "test_metrics_mode");
+        Assert.assertEquals(configManager.getMetricsMode(), "test_metrics_mode");
+        // Restore original metrics mode configuration
+        configManager.setProperty("metrics_mode", existingMetricsModeConfiguration);
+    }
+}

frontend/server/src/test/resources/metrics/invalid_configuration_missing_dimension.yaml

@@ -0,0 +1,9 @@
+dimensions:
+  - &model_name "ModelName"
+  - &model_version "ModelVersion"
+
+ts_metrics:
+  counter:
+    - name: InferenceRequestsTotal
+      unit: Count
+      dimensions: [*model_name, *model_version, *hostname]

frontend/server/src/test/resources/metrics/invalid_configuration_missing_metric_name.yaml

@@ -0,0 +1,9 @@
+dimensions:
+  - &model_name "ModelName"
+  - &model_version "ModelVersion"
+  - &hostname "Hostname"
+
+ts_metrics:
+  counter:
+    - unit: Count
+      dimensions: [*model_name, *model_version, *hostname]