From dc0834417f7e4450964f63e29b5466bb6cbac7e3 Mon Sep 17 00:00:00 2001 From: udaij12 <32673964+udaij12@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:31:20 -0700 Subject: [PATCH] Security documentation update (#3183) * adding security update * adding security update * doc updates * changes to model mode * adding more update * adding token auth paragraph * doc changes and logging addition * doc changes and logging addition * fix formatting * changing flag name and adding env for model api * flag fixes * fixing doc * changing misaligned name * change to variable name * change name * changing config name * spellcheck test * testing docker change * test * fixing test_util * changes to llm * adding model api flag * fixes to llm update * launcher fix * testing token * testing token * fixing docker * change branch name' * change branch name' * final changes * Doc changes Co-authored-by: Naman Nandan * adding key name --------- Co-authored-by: Matthias Reso <13337103+mreso@users.noreply.github.com> Co-authored-by: Naman Nandan --- .github/workflows/docker-ci.yaml | 2 +- README.md | 4 ++ benchmarks/utils/system_under_test.py | 4 +- docker/README.md | 3 ++ docker/dockerd-entrypoint.sh | 3 +- docker/test_container_model_prediction.sh | 2 +- docs/README.md | 3 ++ docs/inference_api.md | 2 + docs/management_api.md | 27 +++++++++++++ ...l_control_mode.md => model_api_control.md} | 31 ++++++++------- docs/token_authorization_api.md | 5 ++- examples/README.md | 3 ++ .../serve/http/TokenAuthorizationHandler.java | 11 +++++- .../org/pytorch/serve/util/ConfigManager.java | 16 ++++---- .../src/test/resources/config.properties | 2 +- .../test/resources/config_snapshot.properties | 2 +- .../test/resources/config_test_env.properties | 2 +- .../resources/config_test_workflow.properties | 2 +- .../test/resources/snapshots/snapshot1.cfg | 2 +- .../test/resources/snapshots/snapshot2.cfg | 2 +- .../test/resources/snapshots/snapshot3.cfg | 2 +- .../test/resources/snapshots/snapshot4.cfg | 2 +- .../test/resources/snapshots/snapshot5.cfg | 2 +- .../test/resources/snapshots/snapshot6.cfg | 2 +- .../test/resources/snapshots/snapshot7.cfg | 2 +- .../test/resources/snapshots/snapshot8.cfg | 2 +- .../test/resources/snapshots/snapshot9.cfg | 2 +- kubernetes/README.md | 3 ++ .../kserve/tests/configs/mnist_v1_cpu.yaml | 4 +- .../kserve/tests/configs/mnist_v2_cpu.yaml | 4 +- kubernetes/tests/scripts/test_mnist.sh | 2 +- test/pytest/test_model_control_mode.py | 38 ++++++++++++++++++- test/pytest/test_torch_compile.py | 2 +- test/pytest/test_utils.py | 2 +- test/resources/config.properties | 2 +- test/resources/config_kf.properties | 2 +- test/resources/config_kfv2.properties | 2 +- test/resources/config_model_mode.properties | 3 +- ts/arg_parser.py | 4 +- ts/launcher.py | 8 ++-- ts/llm_launcher.py | 2 +- ts/model_server.py | 4 +- ts_scripts/api_utils.py | 14 +++---- ts_scripts/tsutils.py | 8 ++-- 44 files changed, 171 insertions(+), 75 deletions(-) rename docs/{model_control_mode.md => model_api_control.md} (53%) diff --git a/.github/workflows/docker-ci.yaml b/.github/workflows/docker-ci.yaml index 053cbbf8d5..6ad10a8713 100644 --- a/.github/workflows/docker-ci.yaml +++ b/.github/workflows/docker-ci.yaml @@ -29,7 +29,7 @@ jobs: working-directory: docker run: | IMAGE_TAG=test-image-${{ matrix.python-version }} - ./build_image.sh -py "${{ matrix.python-version }}" -t "${IMAGE_TAG}" -s -r + ./build_image.sh -py "${{ matrix.python-version }}" -t "${IMAGE_TAG}" -b ${GITHUB_HEAD_REF} -s echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT - name: Container Healthcheck diff --git a/README.md b/README.md index 26e0ea01e7..c797383b2d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ +# ❗ANNOUNCEMENT: Security Changes❗ +TorchServe now enforces token authorization enabled and model API control disabled by default. These security features are intended to address the concern of unauthorized API calls and to prevent potential malicious code from being introduced to the model server. Refer the following documentation for more information: [Token Authorization](https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md), [Model API control](https://github.com/pytorch/serve/blob/master/docs/model_api_control.md) + # TorchServe + ![Nightly build](https://github.com/pytorch/serve/actions/workflows/torchserve-nightly-build.yml/badge.svg) ![Docker Nightly build](https://github.com/pytorch/serve/actions/workflows/docker-nightly-build.yml/badge.svg) ![Benchmark Nightly](https://github.com/pytorch/serve/actions/workflows/benchmark_nightly.yml/badge.svg) diff --git a/benchmarks/utils/system_under_test.py b/benchmarks/utils/system_under_test.py index 70c1014feb..3f20f8b0d2 100644 --- a/benchmarks/utils/system_under_test.py +++ b/benchmarks/utils/system_under_test.py @@ -116,7 +116,7 @@ def start(self): click.secho("*Starting local Torchserve instance...", fg="green") ts_cmd = ( - f"torchserve --start --model-store {self.execution_params['tmp_dir']}/model_store --model-api-enabled --disable-token " + f"torchserve --start --model-store {self.execution_params['tmp_dir']}/model_store --enable-model-api --disable-token-auth " f"--workflow-store {self.execution_params['tmp_dir']}/wf_store " f"--ts-config {self.execution_params['tmp_dir']}/benchmark/conf/{self.execution_params['config_properties_name']} " f" > {self.execution_params['tmp_dir']}/benchmark/logs/model_metrics.log" @@ -195,7 +195,7 @@ def start(self): f"docker run {self.execution_params['docker_runtime']} {backend_profiling} --name ts --user root -p " f"127.0.0.1:{inference_port}:{inference_port} -p 127.0.0.1:{management_port}:{management_port} " f"-v {self.execution_params['tmp_dir']}:/tmp {enable_gpu} -itd {docker_image} " - f'"torchserve --start --model-store /home/model-server/model-store --model-api-enabled --disable-token ' + f'"torchserve --start --model-store /home/model-server/model-store --enable-model-api --disable-token-auth ' f"\--workflow-store /home/model-server/wf-store " f"--ts-config /tmp/benchmark/conf/{self.execution_params['config_properties_name']} > " f'/tmp/benchmark/logs/model_metrics.log"' diff --git a/docker/README.md b/docker/README.md index 54d125e34e..64a22ca90c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,3 +1,6 @@ +## Security Changes +TorchServe now enforces token authorization enabled and model API control disabled by default. Refer the following documentation for more information: [Token Authorization](https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md), [Model API control](https://github.com/pytorch/serve/blob/master/docs/model_api_control.md) + ### Deprecation notice: [Dockerfile.neuron.dev](https://github.com/pytorch/serve/blob/master/docker/Dockerfile.neuron.dev) has been deprecated. Please refer to [deep learning containers](https://github.com/aws/deep-learning-containers/blob/master/available_images.md) repository for neuron torchserve containers. diff --git a/docker/dockerd-entrypoint.sh b/docker/dockerd-entrypoint.sh index 2534938b25..06ecc39f27 100755 --- a/docker/dockerd-entrypoint.sh +++ b/docker/dockerd-entrypoint.sh @@ -1,9 +1,10 @@ #!/bin/bash set -e + if [[ "$1" = "serve" ]]; then shift 1 - torchserve --start --ts-config /home/model-server/config.properties --disable-token + torchserve --start --ts-config /home/model-server/config.properties --disable-token-auth else eval "$@" fi diff --git a/docker/test_container_model_prediction.sh b/docker/test_container_model_prediction.sh index 58cc15e89f..ffe67ead8b 100755 --- a/docker/test_container_model_prediction.sh +++ b/docker/test_container_model_prediction.sh @@ -19,7 +19,7 @@ torch-model-archiver \ --handler=/home/model-server/mnist_handler.py \ --export-path=/home/model-server/model-store -torchserve --start --ts-config=/home/model-server/config.properties --models mnist=mnist.mar --disable-token +torchserve --start --ts-config=/home/model-server/config.properties --models mnist=mnist.mar --disable-token-auth EOF echo "Starting container ${CONTAINER}" diff --git a/docs/README.md b/docs/README.md index e03410b0dd..25baa813fe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,6 @@ +# ❗ANNOUNCEMENT: Security Changes❗ +TorchServe now enforces token authorization enabled and model API control disabled by default. These security features are intended to address the concern of unauthorized API calls and to prevent potential malicious code from being introduced to the model server. Refer the following documentation for more information: [Token Authorization](https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md), [Model API control](https://github.com/pytorch/serve/blob/master/docs/model_api_control.md) + # TorchServe TorchServe is a performant, flexible and easy to use tool for serving PyTorch eager mode and torchscripted models. diff --git a/docs/inference_api.md b/docs/inference_api.md index d661087465..2ac0250745 100644 --- a/docs/inference_api.md +++ b/docs/inference_api.md @@ -2,6 +2,8 @@ Inference API is listening on port 8080 and only accessible from localhost by default. To change the default setting, see [TorchServe Configuration](configuration.md). +For all Inference API requests, TorchServe requires the correct Inference token to be included or token authorization must be disable. For more details see [token authorization documentation](./token_authorization_api.md) + The TorchServe server supports the following APIs: * [API Description](#api-description) - Gets a list of available APIs and options diff --git a/docs/management_api.md b/docs/management_api.md index 37feb7a4f4..6d2392492b 100644 --- a/docs/management_api.md +++ b/docs/management_api.md @@ -8,9 +8,14 @@ TorchServe provides the following APIs that allows you to manage models at runti 4. [Unregister a model](#unregister-a-model) 5. [List registered models](#list-models) 6. [Set default version of a model](#set-default-version) +7. [Refresh tokens for token authorization](#token-authorization-api) The Management API listens on port 8081 and is only accessible from localhost by default. To change the default setting, see [TorchServe Configuration](./configuration.md). +Management API for registering and deleting models is disabled by default. Add `--enable-model-api` to command line when running TorchServe to enable the use of these APIs. For more details and ways to enable see [Model API control](https://github.com/pytorch/serve/blob/master/docs/model_api_control.md) + +For all Management API requests, TorchServe requires the correct Management token to be included or token authorization must be disabled. For more details see [token authorization documentation](./token_authorization_api.md) + Similar to the [Inference API](inference_api.md), the Management API provides a [API description](#api-description) to describe management APIs with the OpenAPI 3.0 specification. Alternatively, if you want to use KServe, TorchServe supports both v1 and v2 API. For more details please look into this [kserve documentation](https://github.com/pytorch/serve/tree/master/kubernetes/kserve) @@ -19,6 +24,8 @@ Alternatively, if you want to use KServe, TorchServe supports both v1 and v2 API This API follows the [ManagementAPIsService.RegisterModel](https://github.com/pytorch/serve/blob/master/frontend/server/src/main/resources/proto/management.proto) gRPC API. +To use this API after TorchServe starts, model API control has to be enabled. Add `--enable-model-api` to command line when starting TorchServe to enable the use of this API. For more details see [model API control](./model_api_control.md) + `POST /models` * `url` - Model archive download url. Supports the following locations: @@ -441,6 +448,8 @@ print(customizedMetadata) This API follows the [ManagementAPIsService.UnregisterModel](https://github.com/pytorch/serve/blob/master/frontend/server/src/main/resources/proto/management.proto) gRPC API. It returns the status of a model in the ModelServer. +To use this API after TorchServe starts, model API control has to be enabled. Add `--enable-model-api` to command line when starting TorchServe to enable the use of this API. For more details see [model API control](./model_api_control.md) + `DELETE /models/{model_name}/{version}` Use the Unregister Model API to free up system resources by unregistering specific version of a model from TorchServe: @@ -522,3 +531,21 @@ curl -v -X PUT http://localhost:8081/models/noop/2.0/set-default ``` The out is OpenAPI 3.0.1 json format. You use it to generate client code, see [swagger codegen](https://swagger.io/swagger-codegen/) for detail. + +## Token Authorization API + +TorchServe now enforces token authorization by default. Check the following documentation for more information: [Token Authorization](https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md). + +This API is used in order to generate a new key to replace either the management or inference key. + +Management Example: +``` +curl localhost:8081/token?type=management -H "Authorization: Bearer {API Token}" +``` +will replace the current management key in the key_file with a new one and will update the expiration time. + +Inference example: +``` +curl localhost:8081/token?type=inference -H "Authorization: Bearer {API Token}" +``` +will replace the current inference key in the key_file with a new one and will update the expiration time. diff --git a/docs/model_control_mode.md b/docs/model_api_control.md similarity index 53% rename from docs/model_control_mode.md rename to docs/model_api_control.md index 309a3ed137..8406a100cb 100644 --- a/docs/model_control_mode.md +++ b/docs/model_api_control.md @@ -1,28 +1,31 @@ -# Model Control Mode +# Model API Control -TorchServe now supports model control mode with two settings "none"(default) and "enabled" +TorchServe now disables the use of model API (specifically registering and deleting models) by default. The use of these APIs can be enabled through command line or config.properties file. -## Two ways to set Model Control -1. Add `--model-api-enabled` to command line when running TorchServe to switch from none to enabled mode. Command line cannot be used to set mode to none, can only be used to set to enabled -2. Add `model_api_enabled=false` or `model_api_enabled=true` to config.properties file - * `model_api_enabled=false` is default and prevents users from registering or deleting models once TorchServe is running - * `model_api_enabled=true` is not default and allows users to register and delete models using the TorchServe model load APIs +TorchServe disables the ability to register and delete models using API calls by default once TorchServe is running. This is a security feature which addresses the concern of unintended registration and deletion of models once TorchServe has started. This is applicable in the scenario where a user may upload malicious code to the model server in the form of a model or where a user may delete a model that is being used. The default behavior prevents users from registering or deleting models once TorchServe is running. Model API control can be enabled to allow users to register and delete models using the TorchServe model load and delete APIs. -Priority between cmd and config file follows the following [TorchServer standard](https://github.com/pytorch/serve/blob/c74a29e8144bc12b84196775076b0e8cf3c5a6fc/docs/configuration.md#advanced-configuration) +## Three ways to set Model API Control +1. Environment variable: use `TS_ENABLE_MODEL_API` and set to `true` to enable and `false` to disable model API use. Note that `enable_envvars_config=true` must be set in config.properties to use environment variables configuration +2. Add `--enable-model-api` to command line when starting TorchServe to switch from disabled to enabled. Command line cannot be used to disable, can only be used to enable +3. Add `enable_model_api=false` or `enable_model_api=true` to config.properties file + * `enable_model_api=false` is default and prevents users from registering or deleting models once TorchServe is running + * `enable_model_api=true` is not default and allows users to register and delete models using the TorchServe model APIs + +Priority follows the following [TorchServe standard](https://github.com/pytorch/serve/blob/c74a29e8144bc12b84196775076b0e8cf3c5a6fc/docs/configuration.md#advanced-configuration) * Example 1: - * Config file: `model_api_enabled=false` + * Config file: `enable_model_api=false` - cmd line: `torchserve --start --ncs --model-store model_store --model-api-enabled` + cmd line: `torchserve --start --ncs --model-store model_store --enable-model-api` Result: Model api mode enabled * Example 2: - * Config file: `model_api_enabled=true` + * Config file: `enable_model_api=true` cmd line: `torchserve --start --ncs --model-store model_store` Result: Mode is enabled (no way to disable api mode through cmd) -## Model Control Mode Default +## Model API Control Default At startup TorchServe loads only those models specified explicitly with the `--models` command-line option. After startup users will be unable to register or delete models in this mode. ### Example default @@ -40,11 +43,11 @@ ubuntu@ip-172-31-11-32:~/serve$ curl -X POST "http://localhost:8081/models?url= ``` ## Model Control API Enabled -Setting model control to `enabled` allows users to load and unload models using the model load APIs. +Setting model API to `enabled` allows users to load and unload models using the model load APIs. ### Example using cmd line to set mode to enabled ``` -ubuntu@ip-172-31-11-32:~/serve$ torchserve --start --ncs --model-store model_store --models resnet-18=resnet-18.mar --ts-config config.properties --model-api-enabled +ubuntu@ip-172-31-11-32:~/serve$ torchserve --start --ncs --model-store model_store --models resnet-18=resnet-18.mar --ts-config config.properties --enable-model-api ubuntu@ip-172-31-11-32:~/serve$ curl -X POST "http://localhost:8081/models?url=https://torchserve.pytorch.org/mar_files/squeezenet1_1.mar" { diff --git a/docs/token_authorization_api.md b/docs/token_authorization_api.md index e1614c54be..fbed95cc38 100644 --- a/docs/token_authorization_api.md +++ b/docs/token_authorization_api.md @@ -2,10 +2,11 @@ TorchServe now enforces token authorization by default +TorchServe enforces token authorization by default which requires the correct token to be provided when calling an API. This is a security feature which addresses the concern of unauthorized API calls. This is applicable in the scenario where an unauthorized user may try to access a running TorchServe instance. The default behavior is to enable this feature which creates a key file with the appropriate tokens to be used for API calls. Users can disable this feature to prevent token authorization from being required for API calls ([how to disable](#how-to-set-and-disable-token-authorization)), however users are warned that this will open up TorchServe to potential unauthorized API calls. ## How to set and disable Token Authorization * Global environment variable: use `TS_DISABLE_TOKEN_AUTHORIZATION` and set to `true` to disable and `false` to enable token authorization. Note that `enable_envvars_config=true` must be set in config.properties for global environment variables to be used -* Command line: Command line can only be used to disable token authorization by adding the `--disable-token` flag. +* Command line: Command line can only be used to disable token authorization by adding the `--disable-token-auth` flag. * Config properties file: use `disable_token_authorization` and set to `true` to disable and `false` to enable token authorization. Priority between env variables, cmd, and config file follows the following [TorchServer standard](https://github.com/pytorch/serve/blob/master/docs/configuration.md) @@ -13,7 +14,7 @@ Priority between env variables, cmd, and config file follows the following [Torc * Example 1: * Config file: `disable_token_authorization=false` - cmd line: `torchserve --start --ncs --model-store model_store --disable-token` + cmd line: `torchserve --start --ncs --model-store model_store --disable-token-auth` Result: Token authorization disabled through command line but enabled through config file, resulting in token authorization being disabled. Command line takes precedence * Example 2: diff --git a/examples/README.md b/examples/README.md index 2ba50fa9ea..de42112514 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,8 @@ # [Examples showcasing TorchServe Features and Integrations](#torchserve-internals) +## Security Changes +TorchServe now enforces token authorization and model API control by default. This change will impact the current examples so please check the following documentation for more information: [Token Authorization](https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md), [Model API control](https://github.com/pytorch/serve/blob/master/docs/model_api_control.md) + ## TorchServe Internals * [Creating mar file for an eager mode model](#creating-mar-file-for-eager-mode-model) diff --git a/frontend/server/src/main/java/org/pytorch/serve/http/TokenAuthorizationHandler.java b/frontend/server/src/main/java/org/pytorch/serve/http/TokenAuthorizationHandler.java index caaea20ed9..065ba80762 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/http/TokenAuthorizationHandler.java +++ b/frontend/server/src/main/java/org/pytorch/serve/http/TokenAuthorizationHandler.java @@ -85,7 +85,16 @@ public static void setupToken() { try { token = new Token(); if (token.generateKeyFile("token")) { - logger.info("Token Authorization Enabled"); + String loggingMessage = + "\n######\n" + + "TorchServe now enforces token authorization by default.\n" + + "This requires the correct token to be provided when calling an API.\n" + + "Key file located at " + + ConfigManager.getInstance().getModelServerHome() + + "/key_file.json\n" + + "Check token authorization documenation for information: https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md \n" + + "######\n"; + logger.info(loggingMessage); } } catch (IOException e) { e.printStackTrace(); diff --git a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java index cbf9daa74a..d31103e1b9 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java @@ -125,6 +125,7 @@ public final class ConfigManager { private static final String TS_HEADER_KEY_SEQUENCE_START = "ts_header_key_sequence_start"; private static final String TS_HEADER_KEY_SEQUENCE_END = "ts_header_key_sequence_end"; private static final String TS_DISABLE_TOKEN_AUTHORIZATION = "disable_token_authorization"; + private static final String TS_ENABLE_MODEL_API = "enable_model_api"; // Configuration which are not documented or enabled through environment variables private static final String USE_NATIVE_IO = "use_native_io"; @@ -135,7 +136,6 @@ public final class ConfigManager { private static final String MODEL_CONFIG = "models"; private static final String VERSION = "version"; private static final String SYSTEM_METRICS_CMD = "system_metrics_cmd"; - private static final String MODEL_CONTROL_MODE = "model_api_enabled"; // Configuration default values private static final String DEFAULT_TS_ALLOWED_URLS = "file://.*|http(s)?://.*"; @@ -258,7 +258,7 @@ private ConfigManager(Arguments args) throws IOException { } if (args.isModelEnabled().equals("true")) { - prop.setProperty(MODEL_CONTROL_MODE, args.isModelEnabled()); + prop.setProperty(TS_ENABLE_MODEL_API, args.isModelEnabled()); } String tokenDisabled = args.isTokenDisabled(); @@ -499,7 +499,7 @@ public int getNumberOfGpu() { } public boolean getModelControlMode() { - return Boolean.parseBoolean(getProperty(MODEL_CONTROL_MODE, "false")); + return Boolean.parseBoolean(getProperty(TS_ENABLE_MODEL_API, "false")); } public String getMetricsConfigPath() { @@ -1179,8 +1179,8 @@ public Arguments(CommandLine cmd) { snapshotDisabled = cmd.hasOption("no-config-snapshot"); workflowStore = cmd.getOptionValue("workflow-store"); cppLogConfigFile = cmd.getOptionValue("cpp-log-config"); - tokenAuthEnabled = cmd.hasOption("disable-token"); - modelApiEnabled = cmd.hasOption("model-api-enabled"); + tokenAuthEnabled = cmd.hasOption("disable-token-auth"); + modelApiEnabled = cmd.hasOption("enable-model-api"); } public static Options getOptions() { @@ -1235,14 +1235,14 @@ public static Options getOptions() { .build()); options.addOption( Option.builder("dt") - .longOpt("disable-token") + .longOpt("disable-token-auth") .argName("TOKEN") .desc("disables token authorization") .build()); options.addOption( Option.builder("mapi") - .longOpt("model-api-enabled") - .argName("MODEL-API-ENABLED") + .longOpt("enable-model-api") + .argName("ENABLE-MODEL-API") .desc("sets model apis to enabled") .build()); return options; diff --git a/frontend/server/src/test/resources/config.properties b/frontend/server/src/test/resources/config.properties index 72adf62af7..6c9f9458fa 100644 --- a/frontend/server/src/test/resources/config.properties +++ b/frontend/server/src/test/resources/config.properties @@ -47,4 +47,4 @@ models={\ # enable_metrics_api=false workflow_store=../archive/src/test/resources/workflows disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/config_snapshot.properties b/frontend/server/src/test/resources/config_snapshot.properties index 0a7de0d96b..92b8e991fa 100644 --- a/frontend/server/src/test/resources/config_snapshot.properties +++ b/frontend/server/src/test/resources/config_snapshot.properties @@ -34,4 +34,4 @@ enable_envvars_config=true workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/config_test_env.properties b/frontend/server/src/test/resources/config_test_env.properties index 07ac2f4cff..5f38ff5347 100644 --- a/frontend/server/src/test/resources/config_test_env.properties +++ b/frontend/server/src/test/resources/config_test_env.properties @@ -53,4 +53,4 @@ models={\ }\ } metrics_config=src/test/resources/metrics_default.yaml -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/config_test_workflow.properties b/frontend/server/src/test/resources/config_test_workflow.properties index df5d8f7c19..8c75f65146 100644 --- a/frontend/server/src/test/resources/config_test_workflow.properties +++ b/frontend/server/src/test/resources/config_test_workflow.properties @@ -53,4 +53,4 @@ models={\ }\ } metrics_config=src/test/resources/metrics_default.yaml -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot1.cfg b/frontend/server/src/test/resources/snapshots/snapshot1.cfg index 5789da62ae..bbd4eae8a2 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot1.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot1.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot2.cfg b/frontend/server/src/test/resources/snapshots/snapshot2.cfg index 3fca05db8c..1217a2ac38 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot2.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot2.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot3.cfg b/frontend/server/src/test/resources/snapshots/snapshot3.cfg index 16f1af1d20..d1e23e021e 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot3.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot3.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot4.cfg b/frontend/server/src/test/resources/snapshots/snapshot4.cfg index 64ee923b3d..7479f088ac 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot4.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot4.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot5.cfg b/frontend/server/src/test/resources/snapshots/snapshot5.cfg index 59a32a8bd1..bfd6cd7069 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot5.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot5.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot6.cfg b/frontend/server/src/test/resources/snapshots/snapshot6.cfg index 922b77f70e..c62d659597 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot6.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot6.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot7.cfg b/frontend/server/src/test/resources/snapshots/snapshot7.cfg index aed143e875..cc38f325cf 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot7.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot7.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot8.cfg b/frontend/server/src/test/resources/snapshots/snapshot8.cfg index 2a84a8a9c4..bd5c5b3d64 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot8.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot8.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/frontend/server/src/test/resources/snapshots/snapshot9.cfg b/frontend/server/src/test/resources/snapshots/snapshot9.cfg index 9241e14c69..7b34b649b3 100644 --- a/frontend/server/src/test/resources/snapshots/snapshot9.cfg +++ b/frontend/server/src/test/resources/snapshots/snapshot9.cfg @@ -20,4 +20,4 @@ metrics_address=https\://127.0.0.1\:8445 workflow_store=../archive/src/test/resources/workflows metrics_config=src/test/resources/metrics_default.yaml disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/kubernetes/README.md b/kubernetes/README.md index 6e5bd6678c..796ae4613d 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -1,5 +1,8 @@ # Torchserve on Kubernetes +## Security Changes +TorchServe now enforces token authorization and model API control by default. Refer the following documentation for more information: [Token Authorization](https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md), [Model API control](https://github.com/pytorch/serve/blob/master/docs/model_api_control.md) + ## Overview This page demonstrates a Torchserve deployment in Kubernetes using Helm Charts. It uses the DockerHub Torchserve Image for the pods and a PersistentVolume for storing config / model files. diff --git a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml index 73082c53ab..559a27b96a 100644 --- a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml @@ -15,5 +15,5 @@ spec: cpu: "100m" memory: 256Mi args: - - --disable-token - - --model-api-enabled + - --disable-token-auth + - --enable-model-api diff --git a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml index a6718567ea..851b69a1e5 100644 --- a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml @@ -16,5 +16,5 @@ spec: cpu: "100m" memory: 256Mi args: - - --disable-token - - --model-api-enabled + - --disable-token-auth + - --enable-model-api diff --git a/kubernetes/tests/scripts/test_mnist.sh b/kubernetes/tests/scripts/test_mnist.sh index f01ed2fda3..4b33e7c630 100755 --- a/kubernetes/tests/scripts/test_mnist.sh +++ b/kubernetes/tests/scripts/test_mnist.sh @@ -22,7 +22,7 @@ function start_minikube_cluster() { function build_docker_image() { eval $(minikube docker-env) - echo "model_api_enabled=true" >> $ROOT_DIR/$EXAMPLE_DIR/../docker/config.properties + echo "enable_model_api=true" >> $ROOT_DIR/$EXAMPLE_DIR/../docker/config.properties echo "disable_token_authorization=true" >> $ROOT_DIR/$EXAMPLE_DIR/../docker/config.properties docker system prune -f docker build -t $DOCKER_IMAGE --file $ROOT_DIR/$EXAMPLE_DIR/../docker/Dockerfile --build-arg EXAMPLE_DIR="${EXAMPLE_DIR}" . diff --git a/test/pytest/test_model_control_mode.py b/test/pytest/test_model_control_mode.py index c55689caa3..fa5a44d6e9 100644 --- a/test/pytest/test_model_control_mode.py +++ b/test/pytest/test_model_control_mode.py @@ -23,7 +23,7 @@ def setup_torchserve(): Path(test_utils.MODEL_STORE).mkdir(parents=True, exist_ok=True) test_utils.start_torchserve( - no_config_snapshots=True, models="mnist=mnist.mar", model_api_enabled=False + no_config_snapshots=True, models="mnist=mnist.mar", enable_model_api=False ) yield "test" @@ -122,3 +122,39 @@ def test_priority(): test_utils.stop_torchserve() assert response.status_code == 200, "model control check failed" + + +# Test priority between env variable, config.properties, and cmd +# Env sets enable_model_api to true +# config sets enable_model_api to false +# cmd sets enable_model_api to false +# Priority falls to env hence enable_model_api is true +def test_priority_env(monkeypatch): + test_var_name = "TS_ENABLE_MODEL_API" + test_var_value = "true" + monkeypatch.setenv(test_var_name, test_var_value) + + MODEL_STORE = os.path.join(ROOT_DIR, "model_store/") + PLUGIN_STORE = os.path.join(ROOT_DIR, "plugins-path") + + Path(test_utils.MODEL_STORE).mkdir(parents=True, exist_ok=True) + config_file_priority = os.path.join( + REPO_ROOT, "../resources/config_model_mode.properties" + ) + test_utils.start_torchserve( + snapshot_file=config_file_priority, + no_config_snapshots=True, + enable_model_api=False, + ) + + params = ( + ("model_name", "resnet-18"), + ("url", "resnet-18.mar"), + ("initial_workers", "1"), + ("synchronous", "true"), + ) + response = requests.post("http://localhost:8081/models", params=params) + + test_utils.stop_torchserve() + + assert response.status_code == 200, "model control check failed" diff --git a/test/pytest/test_torch_compile.py b/test/pytest/test_torch_compile.py index 3607cf2a2a..22d981d8eb 100644 --- a/test/pytest/test_torch_compile.py +++ b/test/pytest/test_torch_compile.py @@ -98,7 +98,7 @@ def test_archive_model_artifacts(self): ) def test_start_torchserve(self): - cmd = f"torchserve --start --ncs --models {MODEL_NAME}_str.mar,{MODEL_NAME}_dict.mar --model-store {MODEL_STORE_DIR} --model-api-enabled --disable-token" + cmd = f"torchserve --start --ncs --models {MODEL_NAME}_str.mar,{MODEL_NAME}_dict.mar --model-store {MODEL_STORE_DIR} --enable-model-api --disable-token-auth" subprocess.run( cmd, shell=True, diff --git a/test/pytest/test_utils.py b/test/pytest/test_utils.py index 3429f247a2..059a2c903d 100644 --- a/test/pytest/test_utils.py +++ b/test/pytest/test_utils.py @@ -47,7 +47,7 @@ def start_torchserve(*args, **kwargs): if "gen_mar" in kwargs: del kwargs["gen_mar"] kwargs.update({"disable_token": kwargs.get("disable_token", True)}) - kwargs.update({"model_api_enabled": kwargs.get("model_api_enabled", True)}) + kwargs.update({"enable_model_api": kwargs.get("enable_model_api", True)}) return start(*args, **kwargs) diff --git a/test/resources/config.properties b/test/resources/config.properties index 6762dd6616..cfe013eb2e 100644 --- a/test/resources/config.properties +++ b/test/resources/config.properties @@ -5,4 +5,4 @@ private_key_file=resources/key.pem certificate_file=resources/certs.pem install_py_dep_per_model=true disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/test/resources/config_kf.properties b/test/resources/config_kf.properties index 417ed5ad6a..c76eb566a3 100644 --- a/test/resources/config_kf.properties +++ b/test/resources/config_kf.properties @@ -5,4 +5,4 @@ private_key_file=resources/key.pem certificate_file=resources/certs.pem service_envelope=kserve disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/test/resources/config_kfv2.properties b/test/resources/config_kfv2.properties index 8b2e415db0..4cb4ec2bea 100644 --- a/test/resources/config_kfv2.properties +++ b/test/resources/config_kfv2.properties @@ -5,4 +5,4 @@ private_key_file=resources/key.pem certificate_file=resources/certs.pem service_envelope=kservev2 disable_token_authorization=true -model_api_enabled=true +enable_model_api=true diff --git a/test/resources/config_model_mode.properties b/test/resources/config_model_mode.properties index a60893dda0..b4c04ad3d9 100644 --- a/test/resources/config_model_mode.properties +++ b/test/resources/config_model_mode.properties @@ -1 +1,2 @@ -model_api_enabled=false +enable_model_api=false +enable_envvars_config=true diff --git a/ts/arg_parser.py b/ts/arg_parser.py index 657919d73f..6c0bd50ea3 100644 --- a/ts/arg_parser.py +++ b/ts/arg_parser.py @@ -83,14 +83,14 @@ def ts_parser(): help="plugin jars to be included in torchserve class path", ) parser.add_argument( - "--disable-token", + "--disable-token-auth", "--dt", dest="token_auth", help="if this option is set then token authorization is disabled", action="store_true", ) parser.add_argument( - "--model-api-enabled", + "--enable-model-api", dest="model_mode", help="enables model control apis", action="store_true", diff --git a/ts/launcher.py b/ts/launcher.py index 349332dcd9..5bdc807102 100644 --- a/ts/launcher.py +++ b/ts/launcher.py @@ -73,7 +73,7 @@ def start( plugin_folder=None, disable_token=False, models=None, - model_api_enabled=False, + enable_model_api=False, ): stop() cmd = ["torchserve", "--start"] @@ -85,11 +85,11 @@ def start( if no_config_snapshots: cmd.extend(["--no-config-snapshots"]) if disable_token: - cmd.append("--disable-token") + cmd.append("--disable-token-auth") if models: cmd.extend(["--models", models]) - if model_api_enabled: - cmd.extend(["--model-api-enabled"]) + if enable_model_api: + cmd.extend(["--enable-model-api"]) p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT) for line in p.stdout: print(line.decode("utf8").strip()) diff --git a/ts/llm_launcher.py b/ts/llm_launcher.py index 1e43888827..66a29a5194 100644 --- a/ts/llm_launcher.py +++ b/ts/llm_launcher.py @@ -134,7 +134,7 @@ def main(args): ) parser.add_argument( - "--disable_token", + "--disable_token-auth", action="store_true", help="Disable token authentication", ) diff --git a/ts/model_server.py b/ts/model_server.py index b19402c5cb..c8d70e2b30 100644 --- a/ts/model_server.py +++ b/ts/model_server.py @@ -184,10 +184,10 @@ def start() -> None: cmd.append("-ncs") if args.token_auth: - cmd.append("--disable-token") + cmd.append("--disable-token-auth") if args.model_mode: - cmd.append("--model-api-enabled") + cmd.append("--enable-model-api") if args.models: cmd.append("-m") diff --git a/ts_scripts/api_utils.py b/ts_scripts/api_utils.py index 33e45039e2..02e1fa4bc3 100755 --- a/ts_scripts/api_utils.py +++ b/ts_scripts/api_utils.py @@ -118,7 +118,7 @@ def trigger_management_tests(): """Return exit code of newman execution of management collection""" config_file = open("config.properties", "w") config_file.write("disable_token_authorization=true\n") - config_file.write("model_api_enabled=true") + config_file.write("enable_model_api=true") config_file.close() ts.start_torchserve( @@ -141,7 +141,7 @@ def trigger_inference_tests(): config_file = open("config.properties", "w") config_file.write("metrics_mode=prometheus\n") config_file.write("disable_token_authorization=true\n") - config_file.write("model_api_enabled=true") + config_file.write("enable_model_api=true") config_file.close() ts.start_torchserve( @@ -209,7 +209,7 @@ def trigger_explanation_tests(): config_file = open("config.properties", "w") config_file.write("metrics_mode=prometheus\n") config_file.write("disable_token_authorization=true\n") - config_file.write("model_api_enabled=true") + config_file.write("enable_model_api=true") config_file.close() ts.start_torchserve( @@ -236,7 +236,7 @@ def trigger_incr_timeout_inference_tests(): config_file.write("default_response_timeout=300\n") config_file.write("metrics_mode=prometheus\n") config_file.write("disable_token_authorization=true\n") - config_file.write("model_api_enabled=true") + config_file.write("enable_model_api=true") config_file.close() ts.start_torchserve( @@ -279,7 +279,7 @@ def trigger_management_tests_kf(): config_file = open("config.properties", "w") config_file.write("disable_token_authorization=true\n") - config_file.write("model_api_enabled=true\n") + config_file.write("enable_model_api=true\n") config_file.write("service_envelope=kserve") config_file.close() @@ -306,7 +306,7 @@ def trigger_inference_tests_kf(): config_file.write("service_envelope=kserve\n") config_file.write("metrics_mode=prometheus\n") config_file.write("disable_token_authorization=true\n") - config_file.write("model_api_enabled=true\n") + config_file.write("enable_model_api=true\n") config_file.close() ts.start_torchserve( @@ -349,7 +349,7 @@ def trigger_inference_tests_kfv2(): config_file.write("service_envelope=kservev2\n") config_file.write("metrics_mode=prometheus\n") config_file.write("disable_token_authorization=true\n") - config_file.write("model_api_enabled=true\n") + config_file.write("enable_model_api=true\n") config_file.close() ts.start_torchserve( diff --git a/ts_scripts/tsutils.py b/ts_scripts/tsutils.py index b3de639721..ef7b407819 100644 --- a/ts_scripts/tsutils.py +++ b/ts_scripts/tsutils.py @@ -49,7 +49,7 @@ def start_torchserve( log_file="", gen_mar=True, disable_token=True, - model_api_enabled=True, + enable_model_api=True, ): if gen_mar: mg.gen_mar(model_store) @@ -64,11 +64,11 @@ def start_torchserve( if ncs: cmd.append("--ncs") if disable_token: - cmd.append("--disable-token") + cmd.append("--disable-token-auth") if config_file: cmd.append(f"--ts-config={config_file}") - if model_api_enabled: - cmd.extend(["--model-api-enabled"]) + if enable_model_api: + cmd.extend(["--enable-model-api"]) if log_file: print(f"## Console logs redirected to file: {log_file}") print(f"## In directory: {os.getcwd()} | Executing command: {' '.join(cmd)}")