diff --git a/ci/buildspec-PRT.yml b/ci/buildspec-PRT.yml new file mode 100644 index 000000000..41d50a6aa --- /dev/null +++ b/ci/buildspec-PRT.yml @@ -0,0 +1,20 @@ +# Build Spec for AWS CodeBuild CI - Performance Regression Suite + +version: 0.2 + +phases: + build: + commands: + # create and install build + - ci/scripts/linux_build.sh + - pip install dist/* + # run PRT + - ci/scripts/linux_test_perf_regression.sh + post_build: + commands: + - mkdir -p prt_artifacts/run_artifacts; + - mv tests/performance/run_artifacts/* prt_artifacts/run_artifacts + +artifacts: + files: + - prt_artifacts/**/* diff --git a/ci/buildspec-nightly.yml b/ci/buildspec-nightly.yml new file mode 100644 index 000000000..50a2a1ce4 --- /dev/null +++ b/ci/buildspec-nightly.yml @@ -0,0 +1,34 @@ +# Build Spec for AWS CodeBuild CI - Nightly + +version: 0.2 + +phases: + build: + commands: + # create and install build + - ci/scripts/linux_build.sh + - pip install dist/* + # modelarchiver tests + - ci/scripts/linux_test_modelarchiver.sh + # run python tests + - ci/scripts/linux_test_python.sh + # run api tests + - ci/scripts/linux_test_api.sh management + - ci/scripts/linux_test_api.sh inference + - ci/scripts/linux_test_api.sh https + + post_build: + commands: + - mkdir -p nightly_artifacts/build_artifacts; mkdir -p nightly_artifacts/model_archiver_tests; + - mkdir -p nightly_artifacts/mms_python_units; mkdir -p nightly_artifacts/api_tests + - mv frontend/server/build/reports/tests/test/* nightly_artifacts/build_artifacts + - mv model-archiver/result_units/* nightly_artifacts/model_archiver_tests + - mv result_units/* nightly_artifacts/mms_python_units + - mv tests/api/artifacts/* nightly_artifacts/api_tests + + +artifacts: + files: + - '**/*' + name: MMS-NIGHTLY-$(date +%Y-%m-%d) + base-directory: nightly_artifacts diff --git a/ci/buildspec-smoke.yml b/ci/buildspec-smoke.yml new file mode 100644 index 000000000..1a4467c7c --- /dev/null +++ b/ci/buildspec-smoke.yml @@ -0,0 +1,25 @@ +# Build Spec for AWS CodeBuild CI - SMOKE + +version: 0.2 + +phases: + build: + commands: + # create and install build + - ci/scripts/linux_build.sh + - pip install dist/* + # modelarchiver tests + - ci/scripts/linux_test_modelarchiver.sh + # run python tests + - ci/scripts/linux_test_python.sh + post_build: + commands: + - mkdir -p smoke_artifacts/build_artifacts; mkdir -p smoke_artifacts/model_archiver_tests; + - mkdir -p smoke_artifacts/mms_python_units; + - mv frontend/server/build/reports/tests/test/* smoke_artifacts/build_artifacts + - mv model-archiver/result_units/* smoke_artifacts/model_archiver_tests + - mv result_units/* smoke_artifacts/mms_python_units + +artifacts: + files: + - smoke_artifacts/**/* diff --git a/ci/codebuild_images/Dockerfile.python2.7 b/ci/codebuild_images/Dockerfile.python2.7 new file mode 100644 index 000000000..a7bf0d2cd --- /dev/null +++ b/ci/codebuild_images/Dockerfile.python2.7 @@ -0,0 +1,61 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# http://www.apache.org/licenses/LICENSE-2.0 +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +FROM awsdeeplearningteam/mms-build:python2.7@sha256:2b743d6724dead806873cce1330f7b8a0197399a35af47dfd7035251fdade122 + +# 2020 - Updated Build and Test dependencies + +# Python packages for MMS Server +RUN pip install psutil \ + && pip install future \ + && pip install Pillow \ + && pip install wheel \ + && pip install twine \ + && pip install requests \ + && pip install mock \ + && pip install numpy \ + && pip install Image \ + && pip install mxnet==1.5.0 \ + && pip install enum34 + +# Python packages for pytests +RUN pip install pytest==4.0.0 \ + && pip install pytest-cov \ + && pip install pytest-mock + +# Python packages for benchmark +RUN pip install pandas + +# Install NodeJS and packages for API tests +RUN curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - \ + && sudo apt-get install -y nodejs \ + && sudo npm install -g newman newman-reporter-html + +# Install jmeter for benchmark +# ToDo: Remove --no-check-certificate; temporarily added to bypass jmeter-plugins.org's expired certificate +RUN cd /opt \ + && wget https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.3.tgz \ + && tar -xzf apache-jmeter-5.3.tgz \ + && cd apache-jmeter-5.3 \ + && ln -s /opt/apache-jmeter-5.3/bin/jmeter /usr/local/bin/jmeter \ + && wget --no-check-certificate https://jmeter-plugins.org/get/ -O lib/ext/jmeter-plugins-manager-1.4.jar \ + && wget http://search.maven.org/remotecontent?filepath=kg/apc/cmdrunner/2.2/cmdrunner-2.2.jar -O lib/cmdrunner-2.2.jar \ + && java -cp lib/ext/jmeter-plugins-manager-1.4.jar org.jmeterplugins.repository.PluginManagerCMDInstaller \ + && bin/PluginsManagerCMD.sh install jpgc-synthesis=2.1,jpgc-filterresults=2.1,jpgc-mergeresults=2.1,jpgc-cmd=2.1,jpgc-perfmon=2.1 + +# bzt is used for performance regression test suite +# bzt requires python 3.6 runtime. +# Download pyenv, use pyenv to download python 3.6.5. +# The downloaded python 3.6.5 is isolated and doesn't interfere with default python(2.7) +# Only before starting the performance regression suite, py 3.6.5 is local installed(pyenv local 3.6.5) in test dir +# !! MMS server will continue using Python 2.7 !! +RUN curl https://pyenv.run | bash \ + && $HOME/.pyenv/bin/pyenv install 3.6.5 \ No newline at end of file diff --git a/ci/codebuild_images/Dockerfile.python3.6 b/ci/codebuild_images/Dockerfile.python3.6 new file mode 100644 index 000000000..bcd3b980d --- /dev/null +++ b/ci/codebuild_images/Dockerfile.python3.6 @@ -0,0 +1,51 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# http://www.apache.org/licenses/LICENSE-2.0 +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +FROM awsdeeplearningteam/mms-build:python3.6@sha256:2c1afa8834907ceec641d254dffbf4bcc659ca2d00fd6f2872d7521f32c9fa2e + +# 2020 - Updated Build and Test dependencies + +# Python packages for MMS Server +RUN pip install psutil \ + && pip install future \ + && pip install Pillow \ + && pip install wheel \ + && pip install twine \ + && pip install requests \ + && pip install mock \ + && pip install numpy \ + && pip install Image \ + && pip install mxnet==1.5.0 + +# Python packages for pytests +RUN pip install pytest==4.0.0 \ + && pip install pytest-cov \ + && pip install pytest-mock + +# Python packages for benchmark +RUN pip install pandas + +# Install NodeJS and packages for API tests +RUN curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - \ + && sudo apt-get install -y nodejs \ + && sudo npm install -g newman newman-reporter-html + +# Install jmeter for benchmark +# ToDo: Remove --no-check-certificate; temporarily added to bypass jmeter-plugins.org's expired certificate +RUN cd /opt \ + && wget https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.3.tgz \ + && tar -xzf apache-jmeter-5.3.tgz \ + && cd apache-jmeter-5.3 \ + && ln -s /opt/apache-jmeter-5.3/bin/jmeter /usr/local/bin/jmeter \ + && wget --no-check-certificate https://jmeter-plugins.org/get/ -O lib/ext/jmeter-plugins-manager-1.4.jar \ + && wget http://search.maven.org/remotecontent?filepath=kg/apc/cmdrunner/2.2/cmdrunner-2.2.jar -O lib/cmdrunner-2.2.jar \ + && java -cp lib/ext/jmeter-plugins-manager-1.4.jar org.jmeterplugins.repository.PluginManagerCMDInstaller \ + && bin/PluginsManagerCMD.sh install jpgc-synthesis=2.1,jpgc-filterresults=2.1,jpgc-mergeresults=2.1,jpgc-cmd=2.1,jpgc-perfmon=2.1 \ No newline at end of file diff --git a/ci/scripts/linux_build.sh b/ci/scripts/linux_build.sh new file mode 100755 index 000000000..019e629f9 --- /dev/null +++ b/ci/scripts/linux_build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python setup.py bdist_wheel --universal \ No newline at end of file diff --git a/ci/scripts/linux_test_api.sh b/ci/scripts/linux_test_api.sh new file mode 100755 index 000000000..d8d128a02 --- /dev/null +++ b/ci/scripts/linux_test_api.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +BASE_DIR="tests/api" +MODEL_STORE_DIR="model_store" + +ARTIFACTS_BASE_DIR="artifacts" +ARTIFACTS_MANAGEMENT_DIR="$ARTIFACTS_BASE_DIR/management" +ARTIFACTS_INFERENCE_DIR="$ARTIFACTS_BASE_DIR/inference" +ARTIFACTS_HTTPS_DIR="$ARTIFACTS_BASE_DIR/https" + +MMS_CONSOLE_LOG_FILE="mms_console.log" +MMS_CONFIG_FILE_HTTPS="resources/config.properties" + +POSTMAN_ENV_FILE="postman/environment.json" +POSTMAN_COLLECTION_MANAGEMENT="postman/management_api_test_collection.json" +POSTMAN_COLLECTION_INFERENCE="postman/inference_api_test_collection.json" +POSTMAN_COLLECTION_HTTPS="postman/https_test_collection.json" +POSTMAN_DATA_FILE_INFERENCE="postman/inference_data.json" + +REPORT_FILE="report.html" + +start_mms_server() { + multi-model-server --start --model-store $1 >> $2 2>&1 + sleep 10 +} + +start_mms_secure_server() { + multi-model-server --start --mms-config $MMS_CONFIG_FILE_HTTPS --model-store $1 >> $2 2>&1 + sleep 10 +} + +stop_mms_server() { + multi-model-server --stop + sleep 10 +} + +cleanup_model_store(){ + rm -rf $MODEL_STORE_DIR/* +} + +move_logs(){ + mv $1 logs/ + mv logs/ $2 +} + +trigger_management_tests(){ + start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE + newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_MANAGEMENT \ + -r cli,html --reporter-html-export $ARTIFACTS_MANAGEMENT_DIR/$REPORT_FILE --verbose + local EXIT_CODE=$? + stop_mms_server + move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_MANAGEMENT_DIR + cleanup_model_store + return $EXIT_CODE +} + +trigger_inference_tests(){ + start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE + newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_INFERENCE -d $POSTMAN_DATA_FILE_INFERENCE \ + -r cli,html --reporter-html-export $ARTIFACTS_INFERENCE_DIR/$REPORT_FILE --verbose + local EXIT_CODE=$? + stop_mms_server + move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_INFERENCE_DIR + cleanup_model_store + return $EXIT_CODE +} + +trigger_https_tests(){ + start_mms_secure_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE + newman run --insecure -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_HTTPS \ + -r cli,html --reporter-html-export $ARTIFACTS_HTTPS_DIR/$REPORT_FILE --verbose + local EXIT_CODE=$? + stop_mms_server + move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_HTTPS_DIR + cleanup_model_store + return $EXIT_CODE +} + +cd $BASE_DIR +mkdir -p $MODEL_STORE_DIR $ARTIFACTS_MANAGEMENT_DIR $ARTIFACTS_INFERENCE_DIR $ARTIFACTS_HTTPS_DIR + +case $1 in + 'management') + trigger_management_tests + exit $? + ;; + 'inference') + trigger_inference_tests + exit $? + ;; + 'https') + trigger_https_tests + exit $? + ;; + 'ALL') + trigger_management_tests + MGMT_EXIT_CODE=$? + trigger_inference_tests + INFR_EXIT_CODE=$? + trigger_https_tests + HTTPS_EXIT_CODE=$? + # If any one of the tests fail, exit with error + if [ "$MGMT_EXIT_CODE" -ne 0 ] || [ "$INFR_EXIT_CODE" -ne 0 ] || [ "$HTTPS_EXIT_CODE" -ne 0 ] + then exit 1 + fi + ;; + *) + echo $1 'Invalid' + echo 'Please specify any one of - management | inference | https | ALL' + exit 1 + ;; +esac \ No newline at end of file diff --git a/ci/scripts/linux_test_modelarchiver.sh b/ci/scripts/linux_test_modelarchiver.sh new file mode 100755 index 000000000..d08ad476d --- /dev/null +++ b/ci/scripts/linux_test_modelarchiver.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +cd model-archiver/ + +# Lint test +pylint -rn --rcfile=./model_archiver/tests/pylintrc model_archiver/. +PY_LINT_EXIT_CODE=$? + +# Execute python unit tests +python -m pytest --cov-report html:result_units --cov=./ model_archiver/tests/unit_tests +PY_UNITS_EXIT_CODE=$? + + +# Install model archiver module +python setup.py bdist_wheel --universal && \ +pip install dist/*.whl +MODL_ARCHVR_EXIT_CODE=$? + +# Execute integration tests +python -m pytest model_archiver/tests/integ_tests # ToDo - Report for Integration tests ? +PY_INTEG_EXIT_CODE=$? + +# If any one of the steps fail, exit with error +if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PY_UNITS_EXIT_CODE" -ne 0 ] || [ "$MODL_ARCHVR_EXIT_CODE" -ne 0 ] || [ "$PY_INTEG_EXIT_CODE" -ne 0 ] +then exit 1 +fi \ No newline at end of file diff --git a/ci/scripts/linux_test_perf_regression.sh b/ci/scripts/linux_test_perf_regression.sh new file mode 100755 index 000000000..5c86bc4d8 --- /dev/null +++ b/ci/scripts/linux_test_perf_regression.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +JMETER_PATH='/opt/apache-jmeter-5.3/bin/jmeter' + +cd tests/performance + +# Only on a python 2 environment - +PY_MAJOR_VER=$(python -c 'import sys; major = sys.version_info.major; print(major);') +if [ $PY_MAJOR_VER -eq 2 ]; then + # Hack to use python 3.6.5 for bzt installation and execution + # While MMS continues to use system python which is at 2.7.x + export PATH="/root/.pyenv/bin:/root/.pyenv/shims:$PATH" + pyenv local 3.6.5 system +fi + +# Install dependencies +pip install -r requirements.txt +pip install bzt + +# Execute performance test suite and store exit code +./run_performance_suite.py -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit +EXIT_CODE=$? + +# Exit with the same error code as that of test execution +exit $EXIT_CODE \ No newline at end of file diff --git a/ci/scripts/linux_test_python.sh b/ci/scripts/linux_test_python.sh new file mode 100755 index 000000000..5bcedce95 --- /dev/null +++ b/ci/scripts/linux_test_python.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Lint Test +pylint -rn --rcfile=./mms/tests/pylintrc mms/. +PY_LINT_EXIT_CODE=$? + +# Execute python tests +python -m pytest --cov-report html:result_units --cov=mms/ mms/tests/unit_tests/ +PYTEST_EXIT_CODE=$? + +# If any one of the tests fail, exit with error +if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PYTEST_EXIT_CODE" -ne 0 ] +then exit 1 +fi \ No newline at end of file