From 5b9adf7a54c21332d7552f980641a4763638e2d7 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Wed, 27 Mar 2024 12:24:09 -0700 Subject: [PATCH 01/16] adding mps support to base handler and regression test --- test/regression_tests.py | 16 +++++++++++++--- test/resources/model-config.yaml | 1 + ts/torch_handler/base_handler.py | 2 ++ ts_scripts/marsgen.py | 15 ++++++++++++--- 4 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 test/resources/model-config.yaml diff --git a/test/regression_tests.py b/test/regression_tests.py index afb29748da..cf12939970 100644 --- a/test/regression_tests.py +++ b/test/regression_tests.py @@ -15,7 +15,7 @@ from ts_scripts.utils import check_python_version, try_and_handle -def regression_tests(binaries, pypi, conda, nightly): +def regression_tests(binaries, pypi, conda, nightly, mps): now = datetime.datetime.now() print("Current date and time : " + now.strftime("%Y-%m-%d %H:%M:%S")) @@ -39,7 +39,10 @@ def regression_tests(binaries, pypi, conda, nightly): install_from_src() # Generate mar file - mg.generate_mars() + if mps: + mg.generate_mars(mps=True) + else: + mg.generate_mars() # Run newman api tests test_api( @@ -82,7 +85,14 @@ def regression_tests(binaries, pypi, conda, nightly): required=False, help="Run regression tests using conda", ) + + parser.add_argument( + "--mps", + action="store_true", + required=False, + help="Run regression tests using mps", + ) args = parser.parse_args() - regression_tests(args.binaries, args.pypi, args.conda, args.nightly) + regression_tests(args.binaries, args.pypi, args.conda, args.nightly, args.mps) diff --git a/test/resources/model-config.yaml b/test/resources/model-config.yaml new file mode 100644 index 0000000000..9be205017d --- /dev/null +++ b/test/resources/model-config.yaml @@ -0,0 +1 @@ +mps: "enable" diff --git a/ts/torch_handler/base_handler.py b/ts/torch_handler/base_handler.py index 68c330c854..d370263505 100644 --- a/ts/torch_handler/base_handler.py +++ b/ts/torch_handler/base_handler.py @@ -149,6 +149,8 @@ def initialize(self, context): self.device = torch.device( self.map_location + ":" + str(properties.get("gpu_id")) ) + elif hasattr(self, "model_yaml_config") and "mps" in self.model_yaml_config and self.model_yaml_config["mps"] == "enable": + self.device = torch.device("mps") elif XLA_AVAILABLE: self.device = xm.xla_device() else: diff --git a/ts_scripts/marsgen.py b/ts_scripts/marsgen.py index 4c7fb4a8a8..23009fe5ba 100644 --- a/ts_scripts/marsgen.py +++ b/ts_scripts/marsgen.py @@ -43,7 +43,7 @@ def gen_mar(model_store=None): print(f"## Symlink {src}, {dst} successfully.") -def generate_model(model, model_store_dir): +def generate_model(model, model_store_dir, mps): serialized_file_path = None if model.get("serialized_file_remote", None): if model.get("gen_scripted_file_path", None): @@ -74,6 +74,10 @@ def generate_model(model, model_store_dir): export_path = model.get("export_path", model_store_dir) + config_file = None + if mps: + config_file = os.path.join(REPO_ROOT, "test","resources","model-config.yaml") + cmd = model_archiver_command_builder( model["model_name"], model["version"], @@ -85,6 +89,7 @@ def generate_model(model, model_store_dir): archive_format, requirements_file, export_path, + config_file=config_file, ) print(f"## In directory: {os.getcwd()} | Executing command: {cmd}\n") try: @@ -101,7 +106,7 @@ def generate_model(model, model_store_dir): os.remove(serialized_file_path) -def generate_mars(mar_config=MAR_CONFIG_FILE_PATH, model_store_dir=MODEL_STORE_DIR): +def generate_mars(mar_config=MAR_CONFIG_FILE_PATH, model_store_dir=MODEL_STORE_DIR, mps=False): """ By default generate_mars reads ts_scripts/mar_config.json and outputs mar files in dir model_store_gen - mar_config.json defines a list of models' mar file parameters. They are: @@ -125,7 +130,7 @@ def generate_mars(mar_config=MAR_CONFIG_FILE_PATH, model_store_dir=MODEL_STORE_D models = json.loads(f.read()) for model in models: - generate_model(model, model_store_dir) + generate_model(model, model_store_dir, mps) os.chdir(cwd) @@ -141,6 +146,7 @@ def model_archiver_command_builder( requirements_file=None, export_path=None, force=True, + config_file=None, ): cmd = "torch-model-archiver" @@ -176,6 +182,9 @@ def model_archiver_command_builder( if force: cmd += " --force" + + if config_file: + cmd += " --config-file {0}".format(config_file) return cmd From c3f060a83abafdf7feee5335716c9c38e70dd219 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Wed, 27 Mar 2024 13:02:47 -0700 Subject: [PATCH 02/16] fixed method --- ts_scripts/marsgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts_scripts/marsgen.py b/ts_scripts/marsgen.py index 23009fe5ba..31d8dacd98 100644 --- a/ts_scripts/marsgen.py +++ b/ts_scripts/marsgen.py @@ -43,7 +43,7 @@ def gen_mar(model_store=None): print(f"## Symlink {src}, {dst} successfully.") -def generate_model(model, model_store_dir, mps): +def generate_model(model, model_store_dir, mps=False): serialized_file_path = None if model.get("serialized_file_remote", None): if model.get("gen_scripted_file_path", None): From 31c093c473b38b83672ed0ca39a286573db1df7b Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 29 Mar 2024 09:38:36 -0700 Subject: [PATCH 03/16] mps support --- .../org/pytorch/serve/util/ConfigManager.java | 20 ++++++++++++++++++- test/regression_tests.py | 15 +++----------- ts/torch_handler/base_handler.py | 4 ++-- ts_scripts/marsgen.py | 14 +++---------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java index bc1e60265c..85eeb662fd 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java @@ -8,6 +8,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.reflect.Field; import java.lang.reflect.Type; import java.net.InetAddress; @@ -835,7 +836,24 @@ private static int getAvailableGpu() { for (String id : ids) { gpuIds.add(Integer.parseInt(id)); } - } else { + } else if (System.getProperty("os.name").startsWith("Mac")){ + Process process = + Runtime.getRuntime().exec("system_profiler SPDisplaysDataType"); + int ret = process.waitFor(); + if (ret != 0) { + return 0; + } + List list = + IOUtils.readLines(process.getInputStream(), StandardCharsets.UTF_8); + if (list.isEmpty() ) { + throw new AssertionError("Unexpected response."); + } + String input = list.get(7); + String[] parts = input.split(":"); + String numberString = parts[1].trim(); + return (Integer.parseInt(numberString)); + } + else { Process process = Runtime.getRuntime().exec("nvidia-smi --query-gpu=index --format=csv"); int ret = process.waitFor(); diff --git a/test/regression_tests.py b/test/regression_tests.py index cf12939970..4ea141f782 100644 --- a/test/regression_tests.py +++ b/test/regression_tests.py @@ -15,7 +15,7 @@ from ts_scripts.utils import check_python_version, try_and_handle -def regression_tests(binaries, pypi, conda, nightly, mps): +def regression_tests(binaries, pypi, conda, nightly): now = datetime.datetime.now() print("Current date and time : " + now.strftime("%Y-%m-%d %H:%M:%S")) @@ -39,10 +39,7 @@ def regression_tests(binaries, pypi, conda, nightly, mps): install_from_src() # Generate mar file - if mps: - mg.generate_mars(mps=True) - else: - mg.generate_mars() + mg.generate_mars() # Run newman api tests test_api( @@ -86,13 +83,7 @@ def regression_tests(binaries, pypi, conda, nightly, mps): help="Run regression tests using conda", ) - parser.add_argument( - "--mps", - action="store_true", - required=False, - help="Run regression tests using mps", - ) args = parser.parse_args() - regression_tests(args.binaries, args.pypi, args.conda, args.nightly, args.mps) + regression_tests(args.binaries, args.pypi, args.conda, args.nightly) diff --git a/ts/torch_handler/base_handler.py b/ts/torch_handler/base_handler.py index d370263505..cc9cbb9d11 100644 --- a/ts/torch_handler/base_handler.py +++ b/ts/torch_handler/base_handler.py @@ -149,8 +149,8 @@ def initialize(self, context): self.device = torch.device( self.map_location + ":" + str(properties.get("gpu_id")) ) - elif hasattr(self, "model_yaml_config") and "mps" in self.model_yaml_config and self.model_yaml_config["mps"] == "enable": - self.device = torch.device("mps") + elif torch.backends.mps.is_available(): + self.device = torch.device("mps") elif XLA_AVAILABLE: self.device = xm.xla_device() else: diff --git a/ts_scripts/marsgen.py b/ts_scripts/marsgen.py index 31d8dacd98..95a80a4865 100644 --- a/ts_scripts/marsgen.py +++ b/ts_scripts/marsgen.py @@ -43,7 +43,7 @@ def gen_mar(model_store=None): print(f"## Symlink {src}, {dst} successfully.") -def generate_model(model, model_store_dir, mps=False): +def generate_model(model, model_store_dir): serialized_file_path = None if model.get("serialized_file_remote", None): if model.get("gen_scripted_file_path", None): @@ -74,9 +74,6 @@ def generate_model(model, model_store_dir, mps=False): export_path = model.get("export_path", model_store_dir) - config_file = None - if mps: - config_file = os.path.join(REPO_ROOT, "test","resources","model-config.yaml") cmd = model_archiver_command_builder( model["model_name"], @@ -89,7 +86,6 @@ def generate_model(model, model_store_dir, mps=False): archive_format, requirements_file, export_path, - config_file=config_file, ) print(f"## In directory: {os.getcwd()} | Executing command: {cmd}\n") try: @@ -106,7 +102,7 @@ def generate_model(model, model_store_dir, mps=False): os.remove(serialized_file_path) -def generate_mars(mar_config=MAR_CONFIG_FILE_PATH, model_store_dir=MODEL_STORE_DIR, mps=False): +def generate_mars(mar_config=MAR_CONFIG_FILE_PATH, model_store_dir=MODEL_STORE_DIR): """ By default generate_mars reads ts_scripts/mar_config.json and outputs mar files in dir model_store_gen - mar_config.json defines a list of models' mar file parameters. They are: @@ -130,7 +126,7 @@ def generate_mars(mar_config=MAR_CONFIG_FILE_PATH, model_store_dir=MODEL_STORE_D models = json.loads(f.read()) for model in models: - generate_model(model, model_store_dir, mps) + generate_model(model, model_store_dir) os.chdir(cwd) @@ -146,7 +142,6 @@ def model_archiver_command_builder( requirements_file=None, export_path=None, force=True, - config_file=None, ): cmd = "torch-model-archiver" @@ -182,9 +177,6 @@ def model_archiver_command_builder( if force: cmd += " --force" - - if config_file: - cmd += " --config-file {0}".format(config_file) return cmd From a1022880f21bdf83eab7a2d7ac6e73e1295a1d9a Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 29 Mar 2024 09:47:37 -0700 Subject: [PATCH 04/16] fix format --- .../java/org/pytorch/serve/util/ConfigManager.java | 11 ++++------- test/regression_tests.py | 1 - ts_scripts/marsgen.py | 1 - 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java index 85eeb662fd..a7d3fb1af1 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java @@ -8,7 +8,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.lang.reflect.Field; import java.lang.reflect.Type; import java.net.InetAddress; @@ -836,24 +835,22 @@ private static int getAvailableGpu() { for (String id : ids) { gpuIds.add(Integer.parseInt(id)); } - } else if (System.getProperty("os.name").startsWith("Mac")){ - Process process = - Runtime.getRuntime().exec("system_profiler SPDisplaysDataType"); + } else if (System.getProperty("os.name").startsWith("Mac")) { + Process process = Runtime.getRuntime().exec("system_profiler SPDisplaysDataType"); int ret = process.waitFor(); if (ret != 0) { return 0; } List list = IOUtils.readLines(process.getInputStream(), StandardCharsets.UTF_8); - if (list.isEmpty() ) { + if (list.isEmpty()) { throw new AssertionError("Unexpected response."); } String input = list.get(7); String[] parts = input.split(":"); String numberString = parts[1].trim(); return (Integer.parseInt(numberString)); - } - else { + } else { Process process = Runtime.getRuntime().exec("nvidia-smi --query-gpu=index --format=csv"); int ret = process.waitFor(); diff --git a/test/regression_tests.py b/test/regression_tests.py index 4ea141f782..afb29748da 100644 --- a/test/regression_tests.py +++ b/test/regression_tests.py @@ -82,7 +82,6 @@ def regression_tests(binaries, pypi, conda, nightly): required=False, help="Run regression tests using conda", ) - args = parser.parse_args() diff --git a/ts_scripts/marsgen.py b/ts_scripts/marsgen.py index 95a80a4865..4c7fb4a8a8 100644 --- a/ts_scripts/marsgen.py +++ b/ts_scripts/marsgen.py @@ -74,7 +74,6 @@ def generate_model(model, model_store_dir): export_path = model.get("export_path", model_store_dir) - cmd = model_archiver_command_builder( model["model_name"], model["version"], From 1eff31afaad9ebf48494fb22c3c361713b0ea504 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 29 Mar 2024 10:11:49 -0700 Subject: [PATCH 05/16] changes to detection --- ts/torch_handler/base_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ts/torch_handler/base_handler.py b/ts/torch_handler/base_handler.py index cc9cbb9d11..9367eb57c1 100644 --- a/ts/torch_handler/base_handler.py +++ b/ts/torch_handler/base_handler.py @@ -149,8 +149,11 @@ def initialize(self, context): self.device = torch.device( self.map_location + ":" + str(properties.get("gpu_id")) ) - elif torch.backends.mps.is_available(): - self.device = torch.device("mps") + elif torch.backends.mps.is_available() and properties.get("gpu_id") is not None: + self.map_location = "mps" + self.device = torch.device( + self.map_location + ":" + str(properties.get("gpu_id")) + ) elif XLA_AVAILABLE: self.device = xm.xla_device() else: From 827fa6d0703734abe3cd2e7c8f8bf21ff924d0c0 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Mon, 1 Apr 2024 12:06:24 -0700 Subject: [PATCH 06/16] testing x86 --- .../org/pytorch/serve/util/ConfigManager.java | 34 ++++++++++++++----- ts/torch_handler/base_handler.py | 6 ++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java index a7d3fb1af1..2812319559 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java @@ -5,9 +5,12 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.SelfSignedCertificate; + +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.reflect.Field; import java.lang.reflect.Type; import java.net.InetAddress; @@ -841,15 +844,30 @@ private static int getAvailableGpu() { if (ret != 0) { return 0; } - List list = - IOUtils.readLines(process.getInputStream(), StandardCharsets.UTF_8); - if (list.isEmpty()) { - throw new AssertionError("Unexpected response."); + // List list = + // IOUtils.readLines(process.getInputStream(), StandardCharsets.UTF_8); + // if (list.isEmpty()) { + // throw new AssertionError("Unexpected response."); + // } + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + if (line.contains("Total Number of Cores:")) { + String[] parts = line.split(":"); + if (parts.length >= 2) { + return (Integer.parseInt(parts[1].trim())); + } + } } - String input = list.get(7); - String[] parts = input.split(":"); - String numberString = parts[1].trim(); - return (Integer.parseInt(numberString)); + return 0; + // throw new AssertionError("Unexpected response."); + + + // String input = list.get(7); + // String[] parts = input.split(":"); + // String numberString = parts[1].trim(); + // return (Integer.parseInt(numberString)); } else { Process process = Runtime.getRuntime().exec("nvidia-smi --query-gpu=index --format=csv"); diff --git a/ts/torch_handler/base_handler.py b/ts/torch_handler/base_handler.py index 9367eb57c1..93e6049a59 100644 --- a/ts/torch_handler/base_handler.py +++ b/ts/torch_handler/base_handler.py @@ -144,6 +144,11 @@ def initialize(self, context): self.model_yaml_config = context.model_yaml_config properties = context.system_properties + print("--------TEST 1------") + obj_dict = context.__dict__ + print(obj_dict) + print("--------TEST 2------") + print(properties) if torch.cuda.is_available() and properties.get("gpu_id") is not None: self.map_location = "cuda" self.device = torch.device( @@ -154,6 +159,7 @@ def initialize(self, context): self.device = torch.device( self.map_location + ":" + str(properties.get("gpu_id")) ) + # self.device = torch.device("mps") elif XLA_AVAILABLE: self.device = xm.xla_device() else: From 1d9975e8d412ce5d82071c837b460b42f1b95cd8 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Mon, 1 Apr 2024 13:13:39 -0700 Subject: [PATCH 07/16] adding m1 check --- .../org/pytorch/serve/util/ConfigManager.java | 23 ++++++------------- ts/torch_handler/base_handler.py | 6 +---- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java index 2812319559..4978644adb 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java @@ -5,7 +5,6 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.SelfSignedCertificate; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -844,15 +843,14 @@ private static int getAvailableGpu() { if (ret != 0) { return 0; } - // List list = - // IOUtils.readLines(process.getInputStream(), StandardCharsets.UTF_8); - // if (list.isEmpty()) { - // throw new AssertionError("Unexpected response."); - // } - - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + + BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream())); String line; while ((line = reader.readLine()) != null) { + if (line.contains("Chipset Model:") && !line.contains("Apple M1")) { + return 0; + } if (line.contains("Total Number of Cores:")) { String[] parts = line.split(":"); if (parts.length >= 2) { @@ -860,14 +858,7 @@ private static int getAvailableGpu() { } } } - return 0; - // throw new AssertionError("Unexpected response."); - - - // String input = list.get(7); - // String[] parts = input.split(":"); - // String numberString = parts[1].trim(); - // return (Integer.parseInt(numberString)); + throw new AssertionError("Unexpected response."); } else { Process process = Runtime.getRuntime().exec("nvidia-smi --query-gpu=index --format=csv"); diff --git a/ts/torch_handler/base_handler.py b/ts/torch_handler/base_handler.py index 93e6049a59..616e76d1d5 100644 --- a/ts/torch_handler/base_handler.py +++ b/ts/torch_handler/base_handler.py @@ -144,11 +144,7 @@ def initialize(self, context): self.model_yaml_config = context.model_yaml_config properties = context.system_properties - print("--------TEST 1------") - obj_dict = context.__dict__ - print(obj_dict) - print("--------TEST 2------") - print(properties) + if torch.cuda.is_available() and properties.get("gpu_id") is not None: self.map_location = "cuda" self.device = torch.device( From 29b388eaab0759d77b0381971f233bda4e5da01e Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 5 Apr 2024 10:03:43 -0700 Subject: [PATCH 08/16] adding test cases --- .../pytorch/serve/util/ConfigManagerTest.java | 14 ++ test/pytest/test_device_config.py | 181 ++++++++++++++++++ ts/torch_handler/base_handler.py | 17 +- 3 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 test/pytest/test_device_config.py diff --git a/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java b/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java index 9cb04debf1..81a088d2e5 100644 --- a/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java +++ b/frontend/server/src/test/java/org/pytorch/serve/util/ConfigManagerTest.java @@ -105,4 +105,18 @@ public void testNoWorkflowState() throws ReflectiveOperationException, IOExcepti workingDir + "/frontend/archive/src/test/resources/models", configManager.getWorkflowStore()); } + + @Test + public void testNumGpuM1() throws ReflectiveOperationException, IOException { + System.setProperty("tsConfigFile", "src/test/resources/config_test_env.properties"); + ConfigManager.Arguments args = new ConfigManager.Arguments(); + args.setModels(new String[] {"noop_v0.1"}); + args.setSnapshotDisabled(true); + ConfigManager.init(args); + ConfigManager configManager = ConfigManager.getInstance(); + String arch = System.getProperty("os.arch"); + if (arch.equals("aarch64")) { + Assert.assertTrue(configManager.getNumberOfGpu() > 0); + } + } } diff --git a/test/pytest/test_device_config.py b/test/pytest/test_device_config.py new file mode 100644 index 0000000000..139d234d4e --- /dev/null +++ b/test/pytest/test_device_config.py @@ -0,0 +1,181 @@ +import shutil +from pathlib import Path +from unittest.mock import patch +import tempfile + +import pytest +import test_utils +import requests +import os +import platform +from model_archiver import ModelArchiverConfig + + + + +CURR_FILE_PATH = Path(__file__).parent +REPO_ROOT_DIR = CURR_FILE_PATH.parent.parent +ROOT_DIR = os.path.join(tempfile.gettempdir(), "workspace") +REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../") +data_file_zero = os.path.join(REPO_ROOT, "test/pytest/test_data/0.png") +config_file = os.path.join(REPO_ROOT, "test/resources/config_token.properties") +mnist_scriptes_py = os.path.join(REPO_ROOT,"examples/image_classifier/mnist/mnist.py") + +HANDLER_PY = """ +from ts.torch_handler.base_handler import BaseHandler + +class deviceHandler(BaseHandler): + + def initialize(self, context): + super().initialize(context) + assert self.get_device().type == "mps" +""" + +MODEL_CONFIG_YAML = """ + #frontend settings + # TorchServe frontend parameters + minWorkers: 1 + batchSize: 4 + maxWorkers: 4 + """ + +MODEL_CONFIG_YAML_GPU = """ + #frontend settings + # TorchServe frontend parameters + minWorkers: 1 + batchSize: 4 + maxWorkers: 4 + deviceType: "gpu" + """ + +MODEL_CONFIG_YAML_CPU = """ + #frontend settings + # TorchServe frontend parameters + minWorkers: 1 + batchSize: 4 + maxWorkers: 4 + deviceType: "cpu" + """ + + +@pytest.fixture(scope="module") +def model_name(): + yield "mnist" + +@pytest.fixture(scope="module") +def work_dir(tmp_path_factory, model_name): + return Path(tmp_path_factory.mktemp(model_name)) + +@pytest.fixture(scope="module") +def model_config_name(request): + def get_config(param): + if param == "cpu": + return MODEL_CONFIG_YAML_CPU + elif param == "gpu": + return MODEL_CONFIG_YAML_GPU + else: + return MODEL_CONFIG_YAML + + return get_config(request.param) + +@pytest.fixture(scope="module", name="mar_file_path") +def create_mar_file(work_dir, model_archiver, model_name, model_config_name): + + + mar_file_path = work_dir.joinpath(model_name + ".mar") + + model_config_yaml_file = work_dir / "model_config.yaml" + model_config_yaml_file.write_text(model_config_name) + + model_py_file = work_dir / "model.py" + + model_py_file.write_text(mnist_scriptes_py) + + handler_py_file = work_dir / "handler.py" + handler_py_file.write_text(HANDLER_PY) + + config = ModelArchiverConfig( + model_name=model_name, + version="1.0", + serialized_file=None, + model_file=mnist_scriptes_py, #model_py_file.as_posix(), + handler=handler_py_file.as_posix(), + extra_files=None, + export_path=work_dir, + requirements_file=None, + runtime="python", + force=False, + archive_format="default", + config_file=model_config_yaml_file.as_posix(), + ) + + with patch("archiver.ArgParser.export_model_args_parser", return_value=config): + model_archiver.generate_model_archive() + + assert mar_file_path.exists() + + yield mar_file_path.as_posix() + + # Clean up files + + mar_file_path.unlink(missing_ok=True) + + # Clean up files + +@pytest.fixture(scope="module", name="model_name") +def register_model(mar_file_path, model_store, torchserve): + """ + Register the model in torchserve + """ + shutil.copy(mar_file_path, model_store) + + file_name = Path(mar_file_path).name + + model_name = Path(file_name).stem + + params = ( + ("model_name", model_name), + ("url", file_name), + ("initial_workers", "1"), + ("synchronous", "true"), + ("batch_size", "1"), + ) + + test_utils.reg_resp = test_utils.register_model_with_params(params) + + yield model_name + + test_utils.unregister_model(model_name) + + +@pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") +@pytest.mark.parametrize("model_config_name", ["gpu"], indirect=True) +def test_m1_device(model_name, model_config_name): + + response = requests.get(f"http://localhost:8081/models/{model_name}") + + print("-----TEST-----") + print(response.content) + assert response.status_code == 200, "Describe worked" + + +@pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") +@pytest.mark.parametrize("model_config_name", ["cpu"], indirect=True) +def test_m1_device_cpu(model_name, model_config_name): + + response = requests.get(f"http://localhost:8081/models/{model_name}") + + print("-----TEST-----") + print(response.content) + assert response.status_code == 404, "Describe worked" + + +@pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") +@pytest.mark.parametrize("model_config_name", ["default"], indirect=True) +def test_m1_device_default(model_name, model_config_name): + + response = requests.get(f"http://localhost:8081/models/{model_name}") + + print("-----TEST-----") + print(response.content) + assert response.status_code == 200, "Describe worked" diff --git a/ts/torch_handler/base_handler.py b/ts/torch_handler/base_handler.py index 616e76d1d5..24004126c9 100644 --- a/ts/torch_handler/base_handler.py +++ b/ts/torch_handler/base_handler.py @@ -113,7 +113,7 @@ class BaseHandler(abc.ABC): Base default handler to load torchscript or eager mode [state_dict] models Also, provides handle method per torch serve custom model specification """ - + def __init__(self): self.model = None self.mapping = None @@ -144,7 +144,7 @@ def initialize(self, context): self.model_yaml_config = context.model_yaml_config properties = context.system_properties - + if torch.cuda.is_available() and properties.get("gpu_id") is not None: self.map_location = "cuda" self.device = torch.device( @@ -152,10 +152,7 @@ def initialize(self, context): ) elif torch.backends.mps.is_available() and properties.get("gpu_id") is not None: self.map_location = "mps" - self.device = torch.device( - self.map_location + ":" + str(properties.get("gpu_id")) - ) - # self.device = torch.device("mps") + self.device = torch.device("mps") elif XLA_AVAILABLE: self.device = xm.xla_device() else: @@ -531,3 +528,11 @@ def describe_handle(self): # pylint: disable=unnecessary-pass pass # pylint: enable=unnecessary-pass + + def get_device(self): + """Get device + + Returns: + string : self device + """ + return self.device \ No newline at end of file From 5d45c220bd0a785d2343ce64a17c5450486d9b29 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 5 Apr 2024 11:22:16 -0700 Subject: [PATCH 09/16] adding test workflow --- .github/workflows/regression_tests_gpu_m1.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/regression_tests_gpu_m1.yml diff --git a/.github/workflows/regression_tests_gpu_m1.yml b/.github/workflows/regression_tests_gpu_m1.yml new file mode 100644 index 0000000000..9f9201281a --- /dev/null +++ b/.github/workflows/regression_tests_gpu_m1.yml @@ -0,0 +1,50 @@ +name: Run Regression Tests on GPU + +on: + workflow_dispatch: + push: + branches: + - mps_m1 + + +concurrency: + group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} + cancel-in-progress: true + +jobs: + regression-gpu: + # creates workflows on self hosted runner + runs-on: [self-hosted, regression-test-m1] + steps: + - name: Clean up previous run + run: | + echo "Cleaning up previous run" + ls -la ./ + sudo rm -rf ./* || true + sudo rm -rf ./.??* || true + ls -la ./ + - name: Update git + run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt-get update && sudo apt-get install git -y + - name: Check git version + run: git --version + - name: Setup Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + architecture: x64 + - name: Setup Java 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '17' + - name: Checkout TorchServe + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Install dependencies + run: | + python ts_scripts/install_dependencies.py --environment=dev --cuda=cu121 + - name: Torchserve Regression Tests + run: | + export TS_RUN_IN_DOCKER=False + python test/regression_tests.py From 09fb2010cc2adbcdaaba0e0453c23de06514ff0f Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 5 Apr 2024 11:29:07 -0700 Subject: [PATCH 10/16] modifiying tests --- .github/workflows/regression_tests_gpu_m1.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/regression_tests_gpu_m1.yml b/.github/workflows/regression_tests_gpu_m1.yml index 9f9201281a..b674334631 100644 --- a/.github/workflows/regression_tests_gpu_m1.yml +++ b/.github/workflows/regression_tests_gpu_m1.yml @@ -16,17 +16,17 @@ jobs: # creates workflows on self hosted runner runs-on: [self-hosted, regression-test-m1] steps: - - name: Clean up previous run - run: | - echo "Cleaning up previous run" - ls -la ./ - sudo rm -rf ./* || true - sudo rm -rf ./.??* || true - ls -la ./ - - name: Update git - run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt-get update && sudo apt-get install git -y - - name: Check git version - run: git --version + # - name: Clean up previous run + # run: | + # echo "Cleaning up previous run" + # ls -la ./ + # sudo rm -rf ./* || true + # sudo rm -rf ./.??* || true + # ls -la ./ + # - name: Update git + # run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt-get update && sudo apt-get install git -y + # - name: Check git version + # run: git --version - name: Setup Python 3.9 uses: actions/setup-python@v5 with: From 1096ab7b3fe1cdc24f0db0e69818c36eeca01498 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 5 Apr 2024 11:33:39 -0700 Subject: [PATCH 11/16] removing python tests --- .github/workflows/regression_tests_gpu_m1.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/regression_tests_gpu_m1.yml b/.github/workflows/regression_tests_gpu_m1.yml index b674334631..23196c8ab4 100644 --- a/.github/workflows/regression_tests_gpu_m1.yml +++ b/.github/workflows/regression_tests_gpu_m1.yml @@ -27,16 +27,16 @@ jobs: # run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt-get update && sudo apt-get install git -y # - name: Check git version # run: git --version - - name: Setup Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - architecture: x64 - - name: Setup Java 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' + # - name: Setup Python 3.9 + # uses: actions/setup-python@v5 + # with: + # python-version: 3.9 + # architecture: x64 + # - name: Setup Java 17 + # uses: actions/setup-java@v3 + # with: + # distribution: 'zulu' + # java-version: '17' - name: Checkout TorchServe uses: actions/checkout@v3 with: From 5d2879bdfbc6bf2590dffa3699e43f21498895a6 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Fri, 5 Apr 2024 11:48:14 -0700 Subject: [PATCH 12/16] remove workflow --- .github/workflows/regression_tests_gpu_m1.yml | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/regression_tests_gpu_m1.yml diff --git a/.github/workflows/regression_tests_gpu_m1.yml b/.github/workflows/regression_tests_gpu_m1.yml deleted file mode 100644 index 23196c8ab4..0000000000 --- a/.github/workflows/regression_tests_gpu_m1.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Run Regression Tests on GPU - -on: - workflow_dispatch: - push: - branches: - - mps_m1 - - -concurrency: - group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} - cancel-in-progress: true - -jobs: - regression-gpu: - # creates workflows on self hosted runner - runs-on: [self-hosted, regression-test-m1] - steps: - # - name: Clean up previous run - # run: | - # echo "Cleaning up previous run" - # ls -la ./ - # sudo rm -rf ./* || true - # sudo rm -rf ./.??* || true - # ls -la ./ - # - name: Update git - # run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt-get update && sudo apt-get install git -y - # - name: Check git version - # run: git --version - # - name: Setup Python 3.9 - # uses: actions/setup-python@v5 - # with: - # python-version: 3.9 - # architecture: x64 - # - name: Setup Java 17 - # uses: actions/setup-java@v3 - # with: - # distribution: 'zulu' - # java-version: '17' - - name: Checkout TorchServe - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Install dependencies - run: | - python ts_scripts/install_dependencies.py --environment=dev --cuda=cu121 - - name: Torchserve Regression Tests - run: | - export TS_RUN_IN_DOCKER=False - python test/regression_tests.py From 5d7f39d319a3a519955bfdba3868c4cfd1330d15 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Mon, 8 Apr 2024 09:28:07 -0700 Subject: [PATCH 13/16] removing test config file --- test/resources/model-config.yaml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test/resources/model-config.yaml diff --git a/test/resources/model-config.yaml b/test/resources/model-config.yaml deleted file mode 100644 index 9be205017d..0000000000 --- a/test/resources/model-config.yaml +++ /dev/null @@ -1 +0,0 @@ -mps: "enable" From 325688af56ec03f001c66f275e482dc3de720cca Mon Sep 17 00:00:00 2001 From: udaij12 Date: Mon, 8 Apr 2024 15:11:23 -0700 Subject: [PATCH 14/16] adding docs --- docs/apple_silicon_support.md | 129 ++++++++++++++++++++++++++++++ test/pytest/test_device_config.py | 6 +- 2 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 docs/apple_silicon_support.md diff --git a/docs/apple_silicon_support.md b/docs/apple_silicon_support.md new file mode 100644 index 0000000000..facd8a7f28 --- /dev/null +++ b/docs/apple_silicon_support.md @@ -0,0 +1,129 @@ +# Apple Silicon Support + +## What is supported +* TorchServe CI jobs now include M1 hardware in order to ensure support, [documentation](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories) on github M1 hardware. + - [Regression Tests](https://github.com/pytorch/serve/blob/master/.github/workflows/regression_tests_cpu.yml) + - [Regression binaries Test](https://github.com/pytorch/serve/blob/master/.github/workflows/regression_tests_cpu_binaries.yml) +* For [Docker](https://docs.docker.com/desktop/install/mac-install/) ensure Docker for Apple silicon is installed then follow [setup steps](https://github.com/pytorch/serve/tree/master/docker) + +## Experimental Support + +* For GPU jobs on Apple Silicon, [MPS](https://pytorch.org/docs/master/notes/mps.html) is now auto detected and enabled. To prevent TorchServe from using MPS, users have to set `deviceType: "cpu"` in model-config.yaml. + * This is an experimental feature and NOT ALL models are guaranteed to work. +* Number of GPUs now reports GPUs on Apple Silicon + +### Testing +* [Pytests](https://github.com/pytorch/serve/tree/master/test/pytest/test_device_config.py) that checks for MPS on MacOS M1 devices +* Models that have been tested and work: Resnet-18, Densenet161, Alexnet +* Models that have been tested and DO NOT work: MNIST + + +#### Example Resnet-18 Using MPS On Mac M1 Pro +``` +serve % torchserve --start --model-store model_store_gen --models resnet-18=resnet-18.mar --ncs + +Torchserve version: 0.10.0 +Number of GPUs: 16 +Number of CPUs: 10 +Max heap size: 8192 M +Python executable: /Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 +Config file: N/A +Inference address: http://127.0.0.1:8080 +Management address: http://127.0.0.1:8081 +Metrics address: http://127.0.0.1:8082 +Model Store: +Initial Models: resnet-18=resnet-18.mar +Log dir: +Metrics dir: +Netty threads: 0 +Netty client threads: 0 +Default workers per model: 16 +Blacklist Regex: N/A +Maximum Response Size: 6553500 +Maximum Request Size: 6553500 +Limit Maximum Image Pixels: true +Prefer direct buffer: false +Allowed Urls: [file://.*|http(s)?://.*] +Custom python dependency for model allowed: false +Enable metrics API: true +Metrics mode: LOG +Disable system metrics: false +Workflow Store: +CPP log config: N/A +Model config: N/A +024-04-08T14:18:02,380 [INFO ] main org.pytorch.serve.servingsdk.impl.PluginsManager - Loading snapshot serializer plugin... +2024-04-08T14:18:02,391 [INFO ] main org.pytorch.serve.ModelServer - Loading initial models: resnet-18.mar +2024-04-08T14:18:02,699 [DEBUG] main org.pytorch.serve.wlm.ModelVersionedRefs - Adding new version 1.0 for model resnet-18 +2024-04-08T14:18:02,699 [INFO ] main org.pytorch.serve.wlm.ModelManager - Model resnet-18 loaded. +2024-04-08T14:18:02,699 [DEBUG] main org.pytorch.serve.wlm.ModelManager - updateModel: resnet-18, count: 16 +... +... +serve % curl http://127.0.0.1:8080/predictions/resnet-18 -T ./examples/image_classifier/kitten.jpg +... +{ + "tabby": 0.40966302156448364, + "tiger_cat": 0.3467046618461609, + "Egyptian_cat": 0.1300288736820221, + "lynx": 0.02391958422958851, + "bucket": 0.011532187461853027 +} +... +``` +#### Conda Example + +``` +(myenv) serve % pip list | grep torch +torch 2.2.1 +torchaudio 2.2.1 +torchdata 0.7.1 +torchtext 0.17.1 +torchvision 0.17.1 +(myenv3) serve % conda install -c pytorch-nightly torchserve torch-model-archiver torch-workflow-archiver +(myenv3) serve % pip list | grep torch +torch 2.2.1 +torch-model-archiver 0.10.0b20240312 +torch-workflow-archiver 0.2.12b20240312 +torchaudio 2.2.1 +torchdata 0.7.1 +torchserve 0.10.0b20240312 +torchtext 0.17.1 +torchvision 0.17.1 +(myenv3) serve % torchserve --start --ncs --models densenet161.mar --model-store ./model_store_gen/ +Torchserve version: 0.10.0 +Number of GPUs: 0 +Number of CPUs: 10 +Max heap size: 8192 M +Config file: N/A +Inference address: http://127.0.0.1:8080 +Management address: http://127.0.0.1:8081 +Metrics address: http://127.0.0.1:8082 +Initial Models: densenet161.mar +Netty threads: 0 +Netty client threads: 0 +Default workers per model: 10 +Blacklist Regex: N/A +Maximum Response Size: 6553500 +Maximum Request Size: 6553500 +Limit Maximum Image Pixels: true +Prefer direct buffer: false +Allowed Urls: [file://.*|http(s)?://.*] +Custom python dependency for model allowed: false +Enable metrics API: true +Metrics mode: LOG +Disable system metrics: false +CPP log config: N/A +Model config: N/A +System metrics command: default +... +2024-03-12T15:58:54,702 [INFO ] main org.pytorch.serve.wlm.ModelManager - Model densenet161 loaded. +2024-03-12T15:58:54,702 [DEBUG] main org.pytorch.serve.wlm.ModelManager - updateModel: densenet161, count: 10 +Model server started. +... +(myenv3) serve % curl http://127.0.0.1:8080/predictions/densenet161 -T examples/image_classifier/kitten.jpg +{ + "tabby": 0.46661922335624695, + "tiger_cat": 0.46449029445648193, + "Egyptian_cat": 0.0661405548453331, + "lynx": 0.001292439759708941, + "plastic_bag": 0.00022909720428287983 +} \ No newline at end of file diff --git a/test/pytest/test_device_config.py b/test/pytest/test_device_config.py index 139d234d4e..374ac78949 100644 --- a/test/pytest/test_device_config.py +++ b/test/pytest/test_device_config.py @@ -156,7 +156,7 @@ def test_m1_device(model_name, model_config_name): print("-----TEST-----") print(response.content) - assert response.status_code == 200, "Describe worked" + assert response.status_code == 200, "Describe Failed" @pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") @@ -167,7 +167,7 @@ def test_m1_device_cpu(model_name, model_config_name): print("-----TEST-----") print(response.content) - assert response.status_code == 404, "Describe worked" + assert response.status_code == 404, "Describe Worked" @pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") @@ -178,4 +178,4 @@ def test_m1_device_default(model_name, model_config_name): print("-----TEST-----") print(response.content) - assert response.status_code == 200, "Describe worked" + assert response.status_code == 200, "Describe Failed" From 5575f93d42a8277efa384f702c4f5d5cc92acf90 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Mon, 8 Apr 2024 15:31:02 -0700 Subject: [PATCH 15/16] fixing spell check --- test/pytest/test_device_config.py | 35 ++++++++----------------- ts_scripts/spellcheck_conf/wordlist.txt | 2 ++ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/test/pytest/test_device_config.py b/test/pytest/test_device_config.py index 374ac78949..213162ce5d 100644 --- a/test/pytest/test_device_config.py +++ b/test/pytest/test_device_config.py @@ -1,30 +1,27 @@ +import os +import platform import shutil +import tempfile from pathlib import Path from unittest.mock import patch -import tempfile import pytest -import test_utils import requests -import os -import platform +import test_utils from model_archiver import ModelArchiverConfig - - - CURR_FILE_PATH = Path(__file__).parent REPO_ROOT_DIR = CURR_FILE_PATH.parent.parent ROOT_DIR = os.path.join(tempfile.gettempdir(), "workspace") REPO_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../") data_file_zero = os.path.join(REPO_ROOT, "test/pytest/test_data/0.png") config_file = os.path.join(REPO_ROOT, "test/resources/config_token.properties") -mnist_scriptes_py = os.path.join(REPO_ROOT,"examples/image_classifier/mnist/mnist.py") +mnist_scriptes_py = os.path.join(REPO_ROOT, "examples/image_classifier/mnist/mnist.py") HANDLER_PY = """ from ts.torch_handler.base_handler import BaseHandler -class deviceHandler(BaseHandler): +class deviceHandler(BaseHandler): def initialize(self, context): super().initialize(context) @@ -62,10 +59,12 @@ def initialize(self, context): def model_name(): yield "mnist" + @pytest.fixture(scope="module") def work_dir(tmp_path_factory, model_name): return Path(tmp_path_factory.mktemp(model_name)) + @pytest.fixture(scope="module") def model_config_name(request): def get_config(param): @@ -78,10 +77,9 @@ def get_config(param): return get_config(request.param) + @pytest.fixture(scope="module", name="mar_file_path") def create_mar_file(work_dir, model_archiver, model_name, model_config_name): - - mar_file_path = work_dir.joinpath(model_name + ".mar") model_config_yaml_file = work_dir / "model_config.yaml" @@ -98,7 +96,7 @@ def create_mar_file(work_dir, model_archiver, model_name, model_config_name): model_name=model_name, version="1.0", serialized_file=None, - model_file=mnist_scriptes_py, #model_py_file.as_posix(), + model_file=mnist_scriptes_py, # model_py_file.as_posix(), handler=handler_py_file.as_posix(), extra_files=None, export_path=work_dir, @@ -122,6 +120,7 @@ def create_mar_file(work_dir, model_archiver, model_name, model_config_name): # Clean up files + @pytest.fixture(scope="module", name="model_name") def register_model(mar_file_path, model_store, torchserve): """ @@ -151,31 +150,19 @@ def register_model(mar_file_path, model_store, torchserve): @pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") @pytest.mark.parametrize("model_config_name", ["gpu"], indirect=True) def test_m1_device(model_name, model_config_name): - response = requests.get(f"http://localhost:8081/models/{model_name}") - - print("-----TEST-----") - print(response.content) assert response.status_code == 200, "Describe Failed" @pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") @pytest.mark.parametrize("model_config_name", ["cpu"], indirect=True) def test_m1_device_cpu(model_name, model_config_name): - response = requests.get(f"http://localhost:8081/models/{model_name}") - - print("-----TEST-----") - print(response.content) assert response.status_code == 404, "Describe Worked" @pytest.mark.skipif(platform.machine() != "arm64", reason="Skip on Mac M1") @pytest.mark.parametrize("model_config_name", ["default"], indirect=True) def test_m1_device_default(model_name, model_config_name): - response = requests.get(f"http://localhost:8081/models/{model_name}") - - print("-----TEST-----") - print(response.content) assert response.status_code == 200, "Describe Failed" diff --git a/ts_scripts/spellcheck_conf/wordlist.txt b/ts_scripts/spellcheck_conf/wordlist.txt index 8df3e2852e..049fccdfc9 100644 --- a/ts_scripts/spellcheck_conf/wordlist.txt +++ b/ts_scripts/spellcheck_conf/wordlist.txt @@ -1216,3 +1216,5 @@ libomp rpath venv TorchInductor +Pytests +deviceType From 1ead54acb476a4f741a05009124acbb45b19e474 Mon Sep 17 00:00:00 2001 From: udaij12 Date: Mon, 8 Apr 2024 15:34:09 -0700 Subject: [PATCH 16/16] lint fix --- ts/torch_handler/base_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ts/torch_handler/base_handler.py b/ts/torch_handler/base_handler.py index 24004126c9..ad33de7c48 100644 --- a/ts/torch_handler/base_handler.py +++ b/ts/torch_handler/base_handler.py @@ -113,7 +113,7 @@ class BaseHandler(abc.ABC): Base default handler to load torchscript or eager mode [state_dict] models Also, provides handle method per torch serve custom model specification """ - + def __init__(self): self.model = None self.mapping = None @@ -528,11 +528,11 @@ def describe_handle(self): # pylint: disable=unnecessary-pass pass # pylint: enable=unnecessary-pass - + def get_device(self): - """Get device + """Get device Returns: string : self device """ - return self.device \ No newline at end of file + return self.device