[RFC]: Metrics Refactoring #1492 Draft PR

examples/Huggingface_Transformers/Transformer_handler_generalized.py

@@ -1,21 +1,22 @@
-from abc import ABC
+import ast
 import json
 import logging
 import os
-import ast
+from abc import ABC
+
 import torch
 import transformers
+from captum.attr import LayerIntegratedGradients
 from transformers import (
-    AutoModelForSequenceClassification,
-    AutoTokenizer,
+    AutoModelForCausalLM,
     AutoModelForQuestionAnswering,
+    AutoModelForSequenceClassification,
     AutoModelForTokenClassification,
-    AutoModelForCausalLM,
+    AutoTokenizer,
+    GPT2TokenizerFast,
 )
-from transformers import GPT2TokenizerFast
 
 from ts.torch_handler.base_handler import BaseHandler
-from captum.attr import LayerIntegratedGradients
 
 logger = logging.getLogger(__name__)
 logger.info("Transformers version %s", transformers.__version__)

frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java

@@ -5,6 +5,7 @@
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.util.SelfSignedCertificate;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -39,6 +40,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
+
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
@@ -65,6 +67,7 @@ public final class ConfigManager {
     private static final String TS_NETTY_CLIENT_THREADS = "netty_client_threads";
     private static final String TS_JOB_QUEUE_SIZE = "job_queue_size";
     private static final String TS_NUMBER_OF_GPU = "number_of_gpu";
+    private static final String TS_METRICS_CONFIG = "metrics_config";
 
     // IPEX config option that can be set at config.properties
     private static final String TS_IPEX_ENABLE = "ipex_enable";
@@ -363,6 +366,40 @@ public int getNumberOfGpu() {
         return getIntProperty(TS_NUMBER_OF_GPU, 0);
     }
 
+    public String getMetricsConfig() {
+        // Getting absolute path of working directory, and finding the metrics_default yaml.
+        // Assumptions are that users are running torchserve within the serve directory, and that the name of the
+        // serve directory is still 'serve' (i.e. users did not rename serve)
+
+        String basePath = System.getProperty("user.dir");
+        String yamlModulePath = "/frontend/server/src/test/resources/metrics_default.yaml";
+
+        String[] basePathList = basePath.split("/");
+        List<String> serveBasePath = new ArrayList<String>();
+        int serveIndex = basePathList.length;
+
+        for (int i = 0; i < basePathList.length; i++) {
+            String directory = basePathList[i];
+            if ("serve".equals(directory)) {
+                serveIndex = i;
+            }
+        }
+
+        if (serveIndex == basePathList.length) {
+            serveIndex--;
+        }
+
+        for (int j = 0; j < serveIndex + 1; j++) {
+            String directory = basePathList[j];
+            serveBasePath.add(directory);
+        }
+
+        String serveBasePathString = String.join("/", serveBasePath);
+        String yamlAbsolutePath = serveBasePathString + yamlModulePath;
+
+        return getProperty(TS_METRICS_CONFIG, yamlAbsolutePath);
+    }
+
     public String getTsDefaultServiceHandler() {
         return getProperty(TS_DEFAULT_SERVICE_HANDLER, null);
     }
@@ -510,11 +547,11 @@ public SslContext getSslContext() throws IOException, GeneralSecurityException {
         } else {
             SelfSignedCertificate ssc = new SelfSignedCertificate();
             privateKey = ssc.key();
-            chain = new X509Certificate[] {ssc.cert()};
+            chain = new X509Certificate[]{ssc.cert()};
         }
 
         return SslContextBuilder.forServer(privateKey, chain)
-                .protocols(new String[] {"TLSv1.2"})
+                .protocols(new String[]{"TLSv1.2"})
                 .ciphers(supportedCiphers)
                 .build();
     }
@@ -793,7 +830,8 @@ public void setInitialWorkerPort(int initialPort) {
 
     private void setModelConfig() {
         String modelConfigStr = prop.getProperty(MODEL_CONFIG, null);
-        Type type = new TypeToken<Map<String, Map<String, JsonObject>>>() {}.getType();
+        Type type = new TypeToken<Map<String, Map<String, JsonObject>>>() {
+        }.getType();
 
         if (modelConfigStr != null) {
             this.modelConfig = JsonUtils.GSON.fromJson(modelConfigStr, type);
@@ -835,7 +873,8 @@ public static final class Arguments {
         private boolean snapshotDisabled;
         private String workflowStore;
 
-        public Arguments() {}
+        public Arguments() {
+        }
 
         public Arguments(CommandLine cmd) {
             tsConfigFile = cmd.getOptionValue("ts-config-file");

frontend/server/src/main/java/org/pytorch/serve/wlm/WorkerLifeCycle.java

@@ -124,6 +124,9 @@ public void startWorker(int port) throws WorkerInitializationException, Interrup
         argl.add(connector.isUds() ? "--sock-name" : "--port");
         argl.add(connector.getSocketPath());
 
+        argl.add("--metrics-config");
+        argl.add(configManager.getMetricsConfig());
+
         String[] envp =
                 EnvironmentUtils.getEnvString(
                         workingDir.getAbsolutePath(),

frontend/server/src/test/resources/config.properties

@@ -46,3 +46,4 @@ models={\
 # install_py_dep_per_model=false
 # enable_metrics_api=false
 workflow_store=../archive/src/test/resources/workflows
+# metrics_config=../../../ts/tests/unit_tests/metrics_yaml_testing/metrics.yaml

frontend/server/src/test/resources/metrics_default.yaml

@@ -0,0 +1,35 @@
+dimensions:
+  ModelName: "my_tc"  # customer should edit based on their own models that are being used
+  Level: "Model"
+
+# not yet implemented in frontend
+ts_metrics:
+  counter:
+    - name:
+        unit: ms
+        dimensions: [ModelName, Level]
+  gauge:
+    - name:
+        unit: ms
+        dimensions: [ModelName, Level]
+  histogram:
+    - name:
+        unit: ms
+        dimensions: [ModelName, Level]
+
+model_metrics:
+  counter:
+    - InferenceTimeInMS:
+        unit: ms
+        dimensions: [ModelName, Level]
+    - NumberOfMetrics:
+        unit: count
+        dimensions: [ModelName, Level]
+  gauge:
+    - GaugeModelMetricNameExample:
+        unit: ms
+        dimensions: [ModelName, Level]
+  histogram:
+    - HistogramModelMetricNameExample:
+        unit: ms
+        dimensions: [ModelName, Level]

ts/arg_parser.py

@@ -1,5 +1,3 @@
-
-
 """
 This module parses the arguments given through the torchserve command-line. This is used by model-server
 at runtime.
@@ -14,48 +12,71 @@ class ArgParser(object):
     Argument parser for torchserve and torchserve-export commands
     TODO : Add readme url
     """
+
     @staticmethod
     def ts_parser():
         """
         Argument parser for torchserve start service
         """
-        parser = argparse.ArgumentParser(prog='torchserve', description='Torchserve')
+        parser = argparse.ArgumentParser(prog="torchserve", description="Torchserve")
 
         sub_parse = parser.add_mutually_exclusive_group(required=False)
-        sub_parse.add_argument('-v', '--version', action='store_true', help='Return TorchServe Version')
-        sub_parse.add_argument('--start', action='store_true', help='Start the model-server')
-        sub_parse.add_argument('--stop', action='store_true', help='Stop the model-server')
+        sub_parse.add_argument(
+            "-v", "--version", action="store_true", help="Return TorchServe Version"
+        )
+        sub_parse.add_argument(
+            "--start", action="store_true", help="Start the model-server"
+        )
+        sub_parse.add_argument(
+            "--stop", action="store_true", help="Stop the model-server"
+        )
 
-        parser.add_argument('--ts-config',
-                            dest='ts_config',
-                            help='Configuration file for model server')
-        parser.add_argument('--model-store',
-                            required=False,
-                            dest='model_store',
-                            help='Model store location from where local or default models can be loaded')
-        parser.add_argument('--workflow-store',
-                            required=False,
-                            dest='workflow_store',
-                            help='Workflow store location from where local or default workflows can be loaded')
-        parser.add_argument('--models',
-                            metavar='MODEL_PATH1 MODEL_NAME=MODEL_PATH2...',
-                            nargs='+',
-                            help='Models to be loaded using [model_name=]model_location format. '
-                                 'Location can be a HTTP URL or a model archive file  in MODEL_STORE.')
-        parser.add_argument('--log-config',
-                            dest='log_config',
-                            help='Log4j configuration file for model server')
-        parser.add_argument('--foreground',
-                            help='Run the model server in foreground. If this option is disabled, the model server'
-                                 ' will run in the background.',
-                            action='store_true')
-        parser.add_argument('--no-config-snapshots', '--ncs',
-                            dest='no_config_snapshots',
-                            help='Prevents to server from storing config snapshot files.',
-                            action='store_true')
-        parser.add_argument('--plugins-path', '--ppath',
-                            dest='plugins_path',
-                            help='plugin jars to be included in torchserve class path')
+        parser.add_argument(
+            "--ts-config", dest="ts_config", help="Configuration file for model server"
+        )
+        parser.add_argument(
+            "--model-store",
+            required=False,
+            dest="model_store",
+            help="Model store location from where local or default models can be loaded",
+        )
+        parser.add_argument(
+            "--workflow-store",
+            required=False,
+            dest="workflow_store",
+            help="Workflow store location from where local or default workflows can be loaded",
+        )
+        parser.add_argument(
+            "--models",
+            metavar="MODEL_PATH1 MODEL_NAME=MODEL_PATH2...",
+            nargs="+",
+            help="Models to be loaded using [model_name=]model_location format. "
+            "Location can be a HTTP URL or a model archive file  in MODEL_STORE.",
+        )
+        parser.add_argument(
+            "--log-config",
+            dest="log_config",
+            help="Log4j configuration file for model server",
+        )
+        parser.add_argument(
+            "--foreground",
+            help="Run the model server in foreground. If this option is disabled, the model server"
+            " will run in the background.",
+            action="store_true",
+        )
+        parser.add_argument(
+            "--no-config-snapshots",
+            "--ncs",
+            dest="no_config_snapshots",
+            help="Prevents to server from storing config snapshot files.",
+            action="store_true",
+        )
+        parser.add_argument(
+            "--plugins-path",
+            "--ppath",
+            dest="plugins_path",
+            help="plugin jars to be included in torchserve class path",
+        )
 
         return parser
 
@@ -65,30 +86,47 @@ def model_service_worker_args():
         ArgParser for backend worker. Takes the socket name and socket type.
         :return:
         """
-        parser = argparse.ArgumentParser(prog='model-server-worker', description='Model Server Worker')
-        parser.add_argument('--sock-type',
-                            required=True,
-                            dest="sock_type",
-                            type=str,
-                            choices=["unix", "tcp"],
-                            help='Socket type the model service worker would use. The options are\n'
-                                 'unix: The model worker expects to unix domain-socket\n'
-                                 'tcp: The model worker expects a host-name and port-number')
+        parser = argparse.ArgumentParser(
+            prog="model-server-worker", description="Model Server Worker"
+        )
+        parser.add_argument(
+            "--sock-type",
+            required=True,
+            dest="sock_type",
+            type=str,
+            choices=["unix", "tcp"],
+            help="Socket type the model service worker would use. The options are\n"
+            "unix: The model worker expects to unix domain-socket\n"
+            "tcp: The model worker expects a host-name and port-number",
+        )
+
+        parser.add_argument(
+            "--sock-name",
+            required=False,
+            dest="sock_name",
+            type=str,
+            help="If 'sock-type' is 'unix', sock-name is expected to be a string. "
+            'Eg: --sock-name "test_sock"',
+        )
 
-        parser.add_argument('--sock-name',
-                            required=False,
-                            dest="sock_name",
-                            type=str,
-                            help='If \'sock-type\' is \'unix\', sock-name is expected to be a string. '
-                                 'Eg: --sock-name \"test_sock\"')
+        parser.add_argument(
+            "--host",
+            type=str,
+            help="If 'sock-type' is 'tcp' this is expected to have a host IP address",
+        )
 
-        parser.add_argument('--host',
-                            type=str,
-                            help='If \'sock-type\' is \'tcp\' this is expected to have a host IP address')
+        parser.add_argument(
+            "--port",
+            type=str,
+            help="If 'sock-type' is 'tcp' this is expected to have the host port to bind on",
+        )
 
-        parser.add_argument('--port',
-                            type=str,
-                            help='If \'sock-type\' is \'tcp\' this is expected to have the host port to bind on')
+        parser.add_argument(
+            "--metrics-config",
+            dest="metrics_config",
+            type=str,
+            help="(Yaml) File to be passed for metrics configuration.",
+        )
 
         return parser