diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000000..c48822ba7a --- /dev/null +++ b/test/README.md @@ -0,0 +1,55 @@ +# TorchServe Regression Tests + +This folder contains nightly regression tests execututed against TorchServe master.These tests use [POSTMAN](https://www.postman.com/downloads/) for exercising all the Management & Inference APIs. + +### Latest Test Run Status + +![Build Status](https://codebuild.us-east-1.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoiS1QvY3lIUEdUb3hZVWNnbmJ2SEZCdExRNmNkNW9EVk1ZaFNldEk4Q0h3TU1qemwzQ29GNW0xMGFhZkxpOFpiMjUrZVVRVDUrSkh2ZDhBeFprdW5iNjRRPSIsIml2UGFyYW1ldGVyU3BlYyI6IjlvcjRqSTNMTmNhcExZbUwiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=staging_0_1_1) + +[Test Logs](https://torchserve-regression-test.s3.amazonaws.com/torch-serve-regression-test/tmp/test_exec.log) + +### Running the test manually. + +Clone Torch Serve Repo & Build the Docker Image for the execition env. + +``` +git clone https://github.com/pytorch/serve +cd serve +./build_image.sh +``` + +This would build a docker Image with a tag torchserve:1.0 in which we would run our Regression Tests. + +``` +docker run -it —user root torchserve:1.0 /bin/bash +``` + +In the Docker CLI execute the following tests. + +``` +apt-get update +apt-get install -y git wget +git clone https://github.com/pytorch/serve +cd serve +./test/regression_tests.sh +``` + +You can view the logs for Test execution & the Torch serve in the /tmp dir. + +``` +cat /tmp/test_exec.log +cat /tmp/ts.log +``` + +### Adding tests + +To add to the tests, import a collection (in /postman) to Postman and add new requests. Specifically to test for inference against a new model +* Open inference_api_test_collection.json in POST man. +* Clone an Register Model / Inferece Request combination. +* Update URL & Model Payload. + + +![POSTMAN UI](screenshot/postman.png) + +Afterwards, export the collection as a v2.1 collection and replace the existing exported collection. +To add a new suite of tests, add a new collection to /postman and update regression_tests.sh to run the new collection and buldsepc.yml to keep track of the report. diff --git a/test/buildspec.yml b/test/buildspec.yml new file mode 100644 index 0000000000..eec7baf905 --- /dev/null +++ b/test/buildspec.yml @@ -0,0 +1,18 @@ +# Build Spec for AWS CodeBuild CI + +version: 0.2 + +phases: + + build: + commands: + - ./test/regression_tests.sh + +artifacts: + files: + - /workspace/report/management_report.html + - /workspace/report/inference_report.html + - /workspace/report/snapshot_report.html + - /tmp/ts.log + - /tmp/test_exec.log + name: TS-NIGHTLY-REGRESSION-$(date +%Y-%m-%d-%H:%M) diff --git a/test/postman/environment.json b/test/postman/environment.json new file mode 100644 index 0000000000..355577a11d --- /dev/null +++ b/test/postman/environment.json @@ -0,0 +1,34 @@ +{ + "id": "68c53442-d6fa-4d54-bc03-e24f6ed313bf", + "name": "TorchServe", + "values": [ + { + "key": "hostname", + "value": "localhost", + "enabled": true + }, + { + "key": "protocol", + "value": "http", + "enabled": true + }, + { + "key": "mgmt-port", + "value": "8081", + "enabled": true + }, + { + "key": "model-name", + "value": "densenet161", + "enabled": true + }, + { + "key": "pred-port", + "value": "8080", + "enabled": true + } + ], + "_postman_variable_scope": "environment", + "_postman_exported_at": "2020-04-27T17:41:47.711Z", + "_postman_exported_using": "Postman/7.23.0" +} diff --git a/test/postman/inference_api_test_collection.json b/test/postman/inference_api_test_collection.json new file mode 100644 index 0000000000..e1702ca4c2 --- /dev/null +++ b/test/postman/inference_api_test_collection.json @@ -0,0 +1,376 @@ +{ + "info": { + "_postman_id": "a22a0432-717b-49c1-a3e1-53bf7541c487", + "name": "torchserve_regression_inference", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "Model Zoo - Register Text Classification", + "event": [ + { + "listen": "test", + "script": { + "id": "1c2662e6-d670-4a18-b6c6-2830eb28827f", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/my_text_classifier.mar&model_name=my_text_classifier&initial_workers=1&synchronous=true", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/my_text_classifier.mar" + }, + { + "key": "model_name", + "value": "my_text_classifier" + }, + { + "key": "initial_workers", + "value": "1" + }, + { + "key": "synchronous", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Model Zoo - Inference Text Classification", + "event": [ + { + "listen": "test", + "script": { + "id": "c4c913b4-7a3d-4e1e-975c-35f0d5004655", + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "protocolProfileBehavior": { + "disabledSystemHeaders": { + "content-type": true + } + }, + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "file", + "file": { + "src": "../examples/text_classification/sample_text.txt" + }, + "options": { + "raw": { + "language": "text" + } + } + }, + "url": { + "raw": "{{protocol}}://{{hostname}}:{{pred-port}}/predictions/my_text_classifier", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{pred-port}}", + "path": [ + "predictions", + "my_text_classifier" + ] + } + }, + "response": [] + }, + { + "name": "Model Zoo - Register DenseNet", + "event": [ + { + "listen": "test", + "script": { + "id": "1c2662e6-d670-4a18-b6c6-2830eb28827f", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/densenet161.mar&model_name=densenet161&initial_workers=1&synchronous=true", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/densenet161.mar" + }, + { + "key": "model_name", + "value": "densenet161" + }, + { + "key": "initial_workers", + "value": "1" + }, + { + "key": "synchronous", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Model Zoo - Inference - DenseNet", + "event": [ + { + "listen": "test", + "script": { + "id": "c4c913b4-7a3d-4e1e-975c-35f0d5004655", + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "protocolProfileBehavior": { + "disabledSystemHeaders": { + "content-type": true + } + }, + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "file", + "file": { + "src": "../examples/image_classifier/kitten.jpg" + }, + "options": { + "raw": { + "language": "text" + } + } + }, + "url": { + "raw": "{{protocol}}://{{hostname}}:{{pred-port}}/predictions/densenet161", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{pred-port}}", + "path": [ + "predictions", + "densenet161" + ] + } + }, + "response": [] + }, + { + "name": "Model Zoo - Register Model - SqueezeNet w Batch", + "event": [ + { + "listen": "test", + "script": { + "id": "1c2662e6-d670-4a18-b6c6-2830eb28827f", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar&model_name=squeezenet1_1&initial_workers=1&synchronous=true&batch_size=2&max_batch_delay=200", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar" + }, + { + "key": "model_name", + "value": "squeezenet1_1" + }, + { + "key": "initial_workers", + "value": "1" + }, + { + "key": "synchronous", + "value": "true" + }, + { + "key": "batch_size", + "value": "2" + }, + { + "key": "max_batch_delay", + "value": "200" + } + ] + } + }, + "response": [] + }, + { + "name": "Model Zoo - Inference - SqueezeNet", + "event": [ + { + "listen": "test", + "script": { + "id": "c4c913b4-7a3d-4e1e-975c-35f0d5004655", + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "protocolProfileBehavior": { + "disabledSystemHeaders": { + "content-type": true + } + }, + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "file", + "file": { + "src": "../examples/image_classifier/kitten.jpg" + }, + "options": { + "raw": { + "language": "text" + } + } + }, + "url": { + "raw": "{{protocol}}://{{hostname}}:{{pred-port}}/predictions/squeezenet1_1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{pred-port}}", + "path": [ + "predictions", + "squeezenet1_1" + ] + } + }, + "response": [] + }, + { + "name": "Inference Endpoint - Ping", + "event": [ + { + "listen": "test", + "script": { + "id": "adab6847-8c88-4b41-9a43-aaa234cd45c7", + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{pred-port}}/ping", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{pred-port}}", + "path": [ + "ping" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "id": "3d2732d6-14cf-4a16-809b-68777156679f", + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "id": "90c609d2-601a-4035-bea7-23e0aa964cf7", + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ], + "protocolProfileBehavior": {} +} diff --git a/test/postman/management_api_test_collection.json b/test/postman/management_api_test_collection.json new file mode 100644 index 0000000000..28686cdd63 --- /dev/null +++ b/test/postman/management_api_test_collection.json @@ -0,0 +1,1246 @@ +{ + "info": { + "_postman_id": "4d8b1327-d0b4-4e37-9544-06b57d9b480f", + "name": "torchserve_regression_management", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "Register Model", + "event": [ + { + "listen": "test", + "script": { + "id": "1c2662e6-d670-4a18-b6c6-2830eb28827f", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar&model_name=squeezenet1_1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/squeezenet1_1.mar" + }, + { + "key": "model_name", + "value": "squeezenet1_1" + } + ] + } + }, + "response": [] + }, + { + "name": "Get Valid Model", + "event": [ + { + "listen": "test", + "script": { + "id": "ecfbb15c-bc6d-4a32-8f9e-ff65db81ab0b", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1" + ] + } + }, + "response": [] + }, + { + "name": "Get Model with Version", + "event": [ + { + "listen": "test", + "script": { + "id": "72072739-e588-4214-b4f7-3761c3861317", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1/1.0", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1", + "1.0" + ] + } + }, + "response": [] + }, + { + "name": "Get Model - All Versions", + "event": [ + { + "listen": "test", + "script": { + "id": "cc0baaca-46eb-4cdc-a756-2d430cc71cdc", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1/all", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1", + "all" + ] + } + }, + "response": [] + }, + { + "name": "List Models", + "event": [ + { + "listen": "test", + "script": { + "id": "fd3e359f-fe11-4075-8977-f8a169d16fb4", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ] + } + }, + "response": [] + }, + { + "name": "Scale Min Workers - Asynchronous", + "event": [ + { + "listen": "test", + "script": { + "id": "aefbd490-f4f7-4d8a-9e8e-ad9489ab5784", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1?min_worker=3", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1" + ], + "query": [ + { + "key": "min_worker", + "value": "3" + } + ] + } + }, + "response": [] + }, + { + "name": "Scale Min Workers - Synchronous", + "event": [ + { + "listen": "test", + "script": { + "id": "762ffec5-7c7e-40f0-ad45-6f1c1983b754", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1?min_worker=4&synchronous=true", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1" + ], + "query": [ + { + "key": "min_worker", + "value": "4" + }, + { + "key": "synchronous", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Scale Min Workers for a Version", + "event": [ + { + "listen": "test", + "script": { + "id": "ce1c4c1c-761f-4e5d-9f04-f29247a88c34", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1/1.0?min_worker=5&synchronous=true", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1", + "1.0" + ], + "query": [ + { + "key": "min_worker", + "value": "5" + }, + { + "key": "synchronous", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Scale Min Workers with GPU", + "event": [ + { + "listen": "test", + "script": { + "id": "84b164c5-e5f6-4226-aeab-483fd8aab514", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1?min_worker=6&number_gpu=1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1" + ], + "query": [ + { + "key": "min_worker", + "value": "6" + }, + { + "key": "number_gpu", + "value": "1" + } + ] + } + }, + "response": [] + }, + { + "name": "Set Default Model Version", + "event": [ + { + "listen": "test", + "script": { + "id": "2bc3625e-0800-49fb-bf12-aa848b9ba356", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1/1.0/set-default", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1", + "1.0", + "set-default" + ] + } + }, + "response": [] + }, + { + "name": "UnRegister Model", + "event": [ + { + "listen": "test", + "script": { + "id": "dd998033-3eea-4bb5-9353-18d6d1f885d8", + "exec": [ + "pm.test(\"Successful DELETE request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/squeezenet1_1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "squeezenet1_1" + ] + } + }, + "response": [] + }, + { + "name": "Register Model with Additional Params", + "event": [ + { + "listen": "test", + "script": { + "id": "942b750b-088a-4355-89a8-89b0d263e1fc", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=squeezenet1_1.mar&model_name=squeezenet1_1&handler=serve/ts/torch_handler/image_classifier.py:handle", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "squeezenet1_1.mar" + }, + { + "key": "model_name", + "value": "squeezenet1_1" + }, + { + "key": "handler", + "value": "serve/ts/torch_handler/image_classifier.py:handle" + } + ] + } + }, + "response": [] + }, + { + "name": "Register Model with Batch Size", + "event": [ + { + "listen": "test", + "script": { + "id": "8a56636d-24b1-439c-b28e-1029db0a141c", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/resnet-152-batch.mar&model_name=resnet152&batch_size=2", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/resnet-152-batch.mar" + }, + { + "key": "model_name", + "value": "resnet152" + }, + { + "key": "batch_size", + "value": "2" + } + ] + } + }, + "response": [] + }, + { + "name": "Register Model Intial Workers 0", + "event": [ + { + "listen": "test", + "script": { + "id": "f6781541-2e92-4837-b6f9-26c61e28de8e", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/vgg11.mar&model_name=vgg11&initial_workers=0", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/vgg11.mar" + }, + { + "key": "model_name", + "value": "vgg11" + }, + { + "key": "initial_workers", + "value": "0" + } + ] + } + }, + "response": [] + }, + { + "name": "Register Model Synchronous", + "event": [ + { + "listen": "test", + "script": { + "id": "073bc446-c12f-4a2d-8fa1-044914642de2", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/resnet-18.mar&model_name=resnet18&synchronous=true", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/resnet-18.mar" + }, + { + "key": "model_name", + "value": "resnet18" + }, + { + "key": "synchronous", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "UnRegister Model", + "event": [ + { + "listen": "test", + "script": { + "id": "aa6f207e-d9ce-40cb-ab08-d7ca03ba2927", + "exec": [ + "pm.test(\"Successful DELETE request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/resnet18", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "resnet18" + ] + } + }, + "response": [] + }, + { + "name": "Register Resnet Model", + "event": [ + { + "listen": "test", + "script": { + "id": "9e195654-9864-46d4-896d-39926d89ffb3", + "exec": [ + "pm.test(\"Successful POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=resnet-18.mar&model_name=resnet18&synchronous=false", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "resnet-18.mar" + }, + { + "key": "model_name", + "value": "resnet18" + }, + { + "key": "synchronous", + "value": "false" + } + ] + } + }, + "response": [] + }, + { + "name": "List with Limit", + "event": [ + { + "listen": "test", + "script": { + "id": "76830b62-cb97-4a8e-8380-6862a6743c8b", + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?limit=1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "limit", + "value": "1" + } + ] + } + }, + "response": [] + }, + { + "name": "List with Pagination", + "event": [ + { + "listen": "test", + "script": { + "id": "e6bb5722-8b6f-4440-a74e-728c9b2f7efd", + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?limit=1&next_page_token=1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "limit", + "value": "1" + }, + { + "key": "next_page_token", + "value": "1" + } + ] + } + }, + "response": [] + }, + { + "name": "Update GPU Count", + "event": [ + { + "listen": "test", + "script": { + "id": "b678f1b6-26aa-4fe9-b2af-382f5dbf7874", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/resnet18?number_gpu=10", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "resnet18" + ], + "query": [ + { + "key": "number_gpu", + "value": "10" + } + ] + } + }, + "response": [] + }, + { + "name": "Scale up Workers - Synchronous", + "event": [ + { + "listen": "test", + "script": { + "id": "da09183e-d2d3-4100-bbb6-01395b56c0e0", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/resnet18?min_worker=5&max_worker=5&synchronous=true", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "resnet18" + ], + "query": [ + { + "key": "min_worker", + "value": "5" + }, + { + "key": "max_worker", + "value": "5" + }, + { + "key": "synchronous", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Scale up Workers - Asynchronous", + "event": [ + { + "listen": "test", + "script": { + "id": "9e1c800a-84fa-4181-ac5c-083c5071b993", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/resnet18?min_worker=6&max_worker=6&synchronous=false", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "resnet18" + ], + "query": [ + { + "key": "min_worker", + "value": "6" + }, + { + "key": "max_worker", + "value": "6" + }, + { + "key": "synchronous", + "value": "false" + } + ] + } + }, + "response": [] + }, + { + "name": "Update Timeout to -1", + "event": [ + { + "listen": "test", + "script": { + "id": "9b5509db-0260-4e36-a304-586b006e86ee", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/resnet18?timeout=-1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "resnet18" + ], + "query": [ + { + "key": "timeout", + "value": "-1" + } + ] + } + }, + "response": [] + }, + { + "name": "Update Timeout to 0", + "event": [ + { + "listen": "test", + "script": { + "id": "13a0ea39-854a-4551-adf3-d1e1aac269d6", + "exec": [ + "pm.test(\"Successful PUT request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/resnet18?timeout=0", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "resnet18" + ], + "query": [ + { + "key": "timeout", + "value": "0" + } + ] + } + }, + "response": [] + }, + { + "name": "Register Model - Invalid URL", + "event": [ + { + "listen": "test", + "script": { + "id": "66f910d7-cc2f-4b60-8f94-df8304dd399f", + "exec": [ + "pm.test(\"Invalid URL POST request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([400]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?url=https://torchserve.s3.amazonaws.com/mar_files/invalid-resnet-18.mar&model_name=invalid-resnet18", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "url", + "value": "https://torchserve.s3.amazonaws.com/mar_files/invalid-resnet-18.mar" + }, + { + "key": "model_name", + "value": "invalid-resnet18" + } + ] + } + }, + "response": [] + }, + { + "name": "Get Model - Invalid Model", + "event": [ + { + "listen": "test", + "script": { + "id": "207f7c2b-1d69-4292-bb4f-31a3147fc43b", + "exec": [ + "pm.test(\"Valid ERROR message\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([404]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/invalid_squeezenet1_1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "invalid_squeezenet1_1" + ] + } + }, + "response": [] + }, + { + "name": "List Models - Invalid Next Page Token", + "event": [ + { + "listen": "test", + "script": { + "id": "a97a22d6-1945-4992-81d0-e7508c5affa4", + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models?next_page_token=12", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models" + ], + "query": [ + { + "key": "next_page_token", + "value": "12" + } + ] + } + }, + "response": [] + }, + { + "name": "Update Worker with Invalid Worker Count", + "event": [ + { + "listen": "test", + "script": { + "id": "45736133-bb05-4dd7-80e9-f0d9f657a17f", + "exec": [ + "pm.test(\"Valid ERROR message\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([400]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/resnet18?min_worker=10&max_worker=9", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "resnet18" + ], + "query": [ + { + "key": "min_worker", + "value": "10" + }, + { + "key": "max_worker", + "value": "9" + } + ] + } + }, + "response": [] + }, + { + "name": "Update Version - Invalid Model", + "event": [ + { + "listen": "test", + "script": { + "id": "7a7e5033-0d32-4b11-a53f-b6e2e4f7ba61", + "exec": [ + "pm.test(\"Valid ERROR message\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([404]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/invalid_squeezenet1_1/1.0/set-default", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "invalid_squeezenet1_1", + "1.0", + "set-default" + ] + } + }, + "response": [] + }, + { + "name": "UnRegister Invalid Model Name", + "event": [ + { + "listen": "test", + "script": { + "id": "b0a0fa12-2de1-474e-8869-33a7c5a0895d", + "exec": [ + "pm.test(\"Valid ERROR message\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([404]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{protocol}}://{{hostname}}:{{mgmt-port}}/models/invalid_squeezenet1_1", + "protocol": "{{protocol}}", + "host": [ + "{{hostname}}" + ], + "port": "{{mgmt-port}}", + "path": [ + "models", + "invalid_squeezenet1_1" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "id": "35867f37-7b99-42a5-823e-1bde5ff3b207", + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "id": "85a997ec-639c-4e3e-b685-a44799ed33d6", + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ], + "protocolProfileBehavior": {} +} \ No newline at end of file diff --git a/test/pytest/test_snapshot.py b/test/pytest/test_snapshot.py new file mode 100644 index 0000000000..09d7b850a0 --- /dev/null +++ b/test/pytest/test_snapshot.py @@ -0,0 +1,90 @@ +import subprocess +import time +import os +import glob +import requests +import json + +def start_torchserve(model_store=None, snapshot_file=None, no_config_snapshots=False): + stop_torchserve() + cmd = ["torchserve","--start"] + model_store = model_store if (model_store != None) else "/workspace/model_store/" + cmd.extend(["--model-store", "/workspace/model_store/"]) + if(snapshot_file != None): + cmd.extend(["--ts-config", snapshot_file]) + if(no_config_snapshots): + cmd.extend(["--no-config-snapshots"]) + subprocess.run(cmd) + time.sleep(10) + + +def stop_torchserve(): + subprocess.run(["torchserve", "--stop"]) + time.sleep(5) + + +def delete_all_snapshots(): + for f in glob.glob('logs/config/*'): + os.remove(f) + assert len(glob.glob('logs/config/*')) == 0 + + +def test_snapshot_created_on_start_and_stop(): + ''' + Validates that startup.cfg & shutdown.cfg are created upon start & stop. + ''' + delete_all_snapshots() + start_torchserve() + stop_torchserve() + assert len(glob.glob('logs/config/*startup.cfg')) == 1 + assert len(glob.glob('logs/config/*shutdown.cfg')) == 1 + + +def test_snapshot_created_on_management_api_invoke(): + ''' + Validates that snapshot.cfg is created when management apis are invoked. + ''' + delete_all_snapshots() + start_torchserve() + requests.post('http://127.0.0.1:8081/models?url=/workspace/model_store/densenet161.mar') + time.sleep(10) + stop_torchserve() + assert len(glob.glob('logs/config/*snap*.cfg')) == 1 + +def test_start_from_snapshot(): + ''' + Validates if we can restore state from snapshot. + ''' + snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0] + start_torchserve(snapshot_file=snapshot_cfg) + response = requests.get('http://127.0.0.1:8081/models/') + assert json.loads(response.content)['models'][0]['modelName'] == "densenet161" + stop_torchserve() + + +def test_start_from_latest(): + ''' + Validates if latest snapshot file is picked if we dont pass snapshot arg explicitly. + ''' + start_torchserve() + response = requests.get('http://127.0.0.1:8081/models/') + assert json.loads(response.content)['models'][0]['modelName'] == "densenet161" + stop_torchserve() + +def test_no_config_snapshots_cli_option(): + ''' + Validates that --no-config-snapshots works as expected. + ''' + delete_all_snapshots() + start_torchserve(no_config_snapshots=True) + stop_torchserve() + assert len(glob.glob('logs/config/*.cfg')) == 0 + +def test_start_from_default(): + ''' + Validates that Default config is used if we dont use a config explicitly. + ''' + delete_all_snapshots() + start_torchserve() + response = requests.get('http://127.0.0.1:8081/models/') + assert len(json.loads(response.content)['models']) == 0 diff --git a/test/regression_tests.sh b/test/regression_tests.sh new file mode 100755 index 0000000000..a95193124b --- /dev/null +++ b/test/regression_tests.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +set -x +set -e + +TS_REPO="https://github.com/pytorch/serve" +BRANCH=${1:-master} +ROOT_DIR="/workspace/" +CODEBUILD_WD=$(pwd) +MODEL_STORE=$ROOT_DIR"/model_store" +TS_LOG_FILE="/tmp/ts.log" +TEST_EXECUTION_LOG_FILE="/tmp/test_exec.log" + + +install_torchserve_from_source() { + echo "Cloning & Building Torchserve Repo from " $1 + + # Install dependencies & test dependencies + pip install torch torchtext torchvision sentencepiece psutil future + sudo apt-get install npm + npm install -g newman newman-reporter-html + + # Clone & Build TorchServe + git clone -b $2 $1 + cd serve + pip install . + cd - + + # Build Model Archiver + cd serve/model-archiver + pip install . + cd - + echo "Torchserve Succesfully installed" + +} + + +generate_densenet_test_model_archive() { + + mkdir $1 && cd $1 + + # Download & create DenseNet Model Archive + wget https://download.pytorch.org/models/densenet161-8d451a50.pth + torch-model-archiver --model-name densenet161 \ + --version 1.0 --model-file $ROOT_DIR/serve/examples/image_classifier/densenet_161/model.py \ + --serialized-file $1/densenet161-8d451a50.pth \ + --extra-files $ROOT_DIR/serve/examples/image_classifier/index_to_name.json \ + --handler image_classifier + rm densenet161-8d451a50.pth + cd - + +} + + +start_torchserve() { + + # Start Torchserve with Model Store + torchserve --start --model-store $1 --models $1/densenet161.mar &> $2 + sleep 10 + curl http://127.0.0.1:8081/models + +} + + +stop_torch_serve() { + torchserve --stop +} + + +delete_model_store_snapshots() { + rm -f $MODEL_STORE/* + rm -rf logs/ +} + + +run_postman_test() { + # Run Postman Scripts + mkdir $ROOT_DIR/report/ + cd $CODEBUILD_WD/test/ + set +e + # Run Management API Tests + stop_torch_serve + start_torchserve $MODEL_STORE $TS_LOG_FILE + newman run -e postman/environment.json --bail --verbose postman/management_api_test_collection.json \ + -r cli,html --reporter-html-export $ROOT_DIR/report/management_report.html >>$1 2>&1 + + + # Run Inference API Tests after Restart + stop_torch_serve + delete_model_store_snapshots + start_torchserve $MODEL_STORE $TS_LOG_FILE + newman run -e postman/environment.json --bail --verbose postman/inference_api_test_collection.json \ + -r cli,html --reporter-html-export $ROOT_DIR/report/inference_report.html >>$1 2>&1 + set -e + cd - +} + + +run_pytest() { + + mkdir -p $ROOT_DIR/report/ + cd $CODEBUILD_WD/test/pytest + stop_torch_serve + set +e + pytest . -v >>$1 2>&1 + set -e + cd - + +} + +rm -rf $ROOT_DIR && mkdir $ROOT_DIR && cd $ROOT_DIR + +echo "** Execuing TorchServe Regression Test Suite executon for " $TS_REPO " **" + +install_torchserve_from_source $TS_REPO $BRANCH +generate_densenet_test_model_archive $MODEL_STORE +run_postman_test $TEST_EXECUTION_LOG_FILE +run_pytest $TEST_EXECUTION_LOG_FILE + +echo "** Tests Complete ** " +exit 0 diff --git a/test/screenshot/postman.png b/test/screenshot/postman.png new file mode 100644 index 0000000000..8c3d01ba75 Binary files /dev/null and b/test/screenshot/postman.png differ