Codebuils setup buildspec changes.

ci/buildspec-PRT.yml

@@ -0,0 +1,20 @@
+# Build Spec for AWS CodeBuild CI - Performance Regression Suite
+
+version: 0.2
+
+phases:
+  build:
+    commands:
+      # create and install build
+      - ci/scripts/linux_build.sh
+      - pip install dist/*
+      # run PRT
+      - ci/scripts/linux_test_perf_regression.sh
+  post_build:
+    commands:
+      - mkdir -p prt_artifacts/run_artifacts;
+      - mv tests/performance/run_artifacts/* prt_artifacts/run_artifacts
+
+artifacts:
+    files:
+      - prt_artifacts/**/*

ci/buildspec-nightly.yml

@@ -0,0 +1,34 @@
+# Build Spec for AWS CodeBuild CI - Nightly
+
+version: 0.2
+
+phases:
+  build:
+    commands:
+      # create and install build
+      - ci/scripts/linux_build.sh
+      - pip install dist/*
+      # modelarchiver tests
+      - ci/scripts/linux_test_modelarchiver.sh
+      # run python tests
+      - ci/scripts/linux_test_python.sh
+      # run api tests
+      - ci/scripts/linux_test_api.sh management
+      - ci/scripts/linux_test_api.sh inference
+      - ci/scripts/linux_test_api.sh https
+
+  post_build:
+    commands:
+      - mkdir -p nightly_artifacts/build_artifacts; mkdir -p nightly_artifacts/model_archiver_tests;
+      - mkdir -p nightly_artifacts/mms_python_units;   mkdir -p nightly_artifacts/api_tests
+      - mv frontend/server/build/reports/tests/test/* nightly_artifacts/build_artifacts
+      - mv model-archiver/result_units/* nightly_artifacts/model_archiver_tests
+      - mv result_units/* nightly_artifacts/mms_python_units
+      - mv tests/api/artifacts/* nightly_artifacts/api_tests
+
+
+artifacts:
+    files:
+      - '**/*'
+    name: MMS-NIGHTLY-$(date +%Y-%m-%d)
+    base-directory: nightly_artifacts

ci/buildspec-smoke.yml

@@ -0,0 +1,25 @@
+# Build Spec for AWS CodeBuild CI - SMOKE
+
+version: 0.2
+
+phases:
+  build:
+    commands:
+      # create and install build
+      - ci/scripts/linux_build.sh
+      - pip install dist/*
+      # modelarchiver tests
+      - ci/scripts/linux_test_modelarchiver.sh
+      # run python tests
+      - ci/scripts/linux_test_python.sh
+  post_build:
+    commands:
+      - mkdir -p smoke_artifacts/build_artifacts; mkdir -p smoke_artifacts/model_archiver_tests;
+      - mkdir -p smoke_artifacts/mms_python_units;
+      - mv frontend/server/build/reports/tests/test/* smoke_artifacts/build_artifacts
+      - mv model-archiver/result_units/* smoke_artifacts/model_archiver_tests
+      - mv result_units/* smoke_artifacts/mms_python_units
+
+artifacts:
+    files:
+      - smoke_artifacts/**/*

ci/codebuild_images/Dockerfile.python2.7

@@ -0,0 +1,61 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+FROM awsdeeplearningteam/mms-build:python2.7@sha256:2b743d6724dead806873cce1330f7b8a0197399a35af47dfd7035251fdade122
+
+# 2020 - Updated Build and Test dependencies
+
+# Python packages for MMS Server
+RUN pip install psutil \
+    && pip install future \
+    && pip install Pillow \
+    && pip install wheel \
+    && pip install twine \
+    && pip install requests \
+    && pip install mock \
+    && pip install numpy \
+    && pip install Image \
+    && pip install mxnet==1.5.0 \
+    && pip install enum34
+
+# Python packages for pytests
+RUN pip install pytest==4.0.0 \
+    && pip install pytest-cov \
+    && pip install pytest-mock
+
+# Python packages for benchmark
+RUN pip install pandas
+
+# Install NodeJS and packages for API tests
+RUN curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - \
+    && sudo apt-get install -y nodejs \
+    && sudo npm install -g newman newman-reporter-html
+
+# Install jmeter for benchmark
+# ToDo: Remove --no-check-certificate; temporarily added to bypass jmeter-plugins.org's expired certificate
+RUN cd /opt \
+    && wget https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.3.tgz \
+    && tar -xzf apache-jmeter-5.3.tgz \
+    && cd apache-jmeter-5.3 \
+    && ln -s /opt/apache-jmeter-5.3/bin/jmeter /usr/local/bin/jmeter \
+    && wget --no-check-certificate https://jmeter-plugins.org/get/ -O lib/ext/jmeter-plugins-manager-1.4.jar \
+    && wget http://search.maven.org/remotecontent?filepath=kg/apc/cmdrunner/2.2/cmdrunner-2.2.jar -O lib/cmdrunner-2.2.jar \
+    && java -cp lib/ext/jmeter-plugins-manager-1.4.jar org.jmeterplugins.repository.PluginManagerCMDInstaller \
+    && bin/PluginsManagerCMD.sh install jpgc-synthesis=2.1,jpgc-filterresults=2.1,jpgc-mergeresults=2.1,jpgc-cmd=2.1,jpgc-perfmon=2.1
+
+# bzt is used for performance regression test suite
+# bzt requires python 3.6 runtime.
+# Download pyenv, use pyenv to download python 3.6.5.
+# The downloaded python 3.6.5 is isolated and doesn't interfere with default python(2.7)
+# Only before starting the performance regression suite, py 3.6.5 is local installed(pyenv local 3.6.5) in test dir
+# !! MMS server will continue using Python 2.7 !!
+RUN curl https://pyenv.run | bash \
+    && $HOME/.pyenv/bin/pyenv install 3.6.5

ci/codebuild_images/Dockerfile.python3.6

@@ -0,0 +1,51 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+FROM awsdeeplearningteam/mms-build:python3.6@sha256:2c1afa8834907ceec641d254dffbf4bcc659ca2d00fd6f2872d7521f32c9fa2e
+
+# 2020 - Updated Build and Test dependencies
+
+# Python packages for MMS Server
+RUN pip install psutil \
+    && pip install future \
+    && pip install Pillow \
+    && pip install wheel \
+    && pip install twine \
+    && pip install requests \
+    && pip install mock \
+    && pip install numpy \
+    && pip install Image \
+    && pip install mxnet==1.5.0
+
+# Python packages for pytests
+RUN pip install pytest==4.0.0 \
+    && pip install pytest-cov \
+    && pip install pytest-mock
+
+# Python packages for benchmark
+RUN pip install pandas
+
+# Install NodeJS and packages for API tests
+RUN curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - \
+    && sudo apt-get install -y nodejs \
+    && sudo npm install -g newman newman-reporter-html
+
+# Install jmeter for benchmark
+# ToDo: Remove --no-check-certificate; temporarily added to bypass jmeter-plugins.org's expired certificate
+RUN cd /opt \
+    && wget https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.3.tgz \
+    && tar -xzf apache-jmeter-5.3.tgz \
+    && cd apache-jmeter-5.3 \
+    && ln -s /opt/apache-jmeter-5.3/bin/jmeter /usr/local/bin/jmeter \
+    && wget --no-check-certificate https://jmeter-plugins.org/get/ -O lib/ext/jmeter-plugins-manager-1.4.jar \
+    && wget http://search.maven.org/remotecontent?filepath=kg/apc/cmdrunner/2.2/cmdrunner-2.2.jar -O lib/cmdrunner-2.2.jar \
+    && java -cp lib/ext/jmeter-plugins-manager-1.4.jar org.jmeterplugins.repository.PluginManagerCMDInstaller \
+    && bin/PluginsManagerCMD.sh install jpgc-synthesis=2.1,jpgc-filterresults=2.1,jpgc-mergeresults=2.1,jpgc-cmd=2.1,jpgc-perfmon=2.1

ci/scripts/linux_build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python setup.py bdist_wheel --universal

ci/scripts/linux_test_api.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+BASE_DIR="tests/api"
+MODEL_STORE_DIR="model_store"
+
+ARTIFACTS_BASE_DIR="artifacts"
+ARTIFACTS_MANAGEMENT_DIR="$ARTIFACTS_BASE_DIR/management"
+ARTIFACTS_INFERENCE_DIR="$ARTIFACTS_BASE_DIR/inference"
+ARTIFACTS_HTTPS_DIR="$ARTIFACTS_BASE_DIR/https"
+
+MMS_CONSOLE_LOG_FILE="mms_console.log"
+MMS_CONFIG_FILE_HTTPS="resources/config.properties"
+
+POSTMAN_ENV_FILE="postman/environment.json"
+POSTMAN_COLLECTION_MANAGEMENT="postman/management_api_test_collection.json"
+POSTMAN_COLLECTION_INFERENCE="postman/inference_api_test_collection.json"
+POSTMAN_COLLECTION_HTTPS="postman/https_test_collection.json"
+POSTMAN_DATA_FILE_INFERENCE="postman/inference_data.json"
+
+REPORT_FILE="report.html"
+
+start_mms_server() {
+  multi-model-server --start --model-store $1 >> $2 2>&1
+  sleep 10
+}
+
+start_mms_secure_server() {
+  multi-model-server --start --mms-config $MMS_CONFIG_FILE_HTTPS --model-store $1 >> $2 2>&1
+  sleep 10
+}
+
+stop_mms_server() {
+  multi-model-server --stop
+  sleep 10
+}
+
+cleanup_model_store(){
+  rm -rf $MODEL_STORE_DIR/*
+}
+
+move_logs(){
+  mv $1 logs/
+  mv logs/ $2
+}
+
+trigger_management_tests(){
+  start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE
+  newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_MANAGEMENT \
+             -r cli,html --reporter-html-export $ARTIFACTS_MANAGEMENT_DIR/$REPORT_FILE --verbose
+  local EXIT_CODE=$?
+  stop_mms_server
+  move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_MANAGEMENT_DIR
+  cleanup_model_store
+  return $EXIT_CODE
+}
+
+trigger_inference_tests(){
+  start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE
+  newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_INFERENCE -d $POSTMAN_DATA_FILE_INFERENCE \
+             -r cli,html --reporter-html-export $ARTIFACTS_INFERENCE_DIR/$REPORT_FILE --verbose
+  local EXIT_CODE=$?
+  stop_mms_server
+  move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_INFERENCE_DIR
+  cleanup_model_store
+  return $EXIT_CODE
+}
+
+trigger_https_tests(){
+  start_mms_secure_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE
+  newman run --insecure -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_HTTPS \
+             -r cli,html --reporter-html-export $ARTIFACTS_HTTPS_DIR/$REPORT_FILE --verbose
+  local EXIT_CODE=$?
+  stop_mms_server
+  move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_HTTPS_DIR
+  cleanup_model_store
+  return $EXIT_CODE
+}
+
+cd $BASE_DIR
+mkdir -p $MODEL_STORE_DIR $ARTIFACTS_MANAGEMENT_DIR $ARTIFACTS_INFERENCE_DIR $ARTIFACTS_HTTPS_DIR
+
+case $1 in
+   'management')
+      trigger_management_tests
+      exit $?
+      ;;
+   'inference')
+      trigger_inference_tests
+      exit $?
+      ;;
+   'https')
+      trigger_https_tests
+      exit $?
+      ;;
+   'ALL')
+      trigger_management_tests
+      MGMT_EXIT_CODE=$?
+      trigger_inference_tests
+      INFR_EXIT_CODE=$?
+      trigger_https_tests
+      HTTPS_EXIT_CODE=$?
+      # If any one of the tests fail, exit with error
+      if [ "$MGMT_EXIT_CODE" -ne 0 ] || [ "$INFR_EXIT_CODE" -ne 0 ] || [ "$HTTPS_EXIT_CODE" -ne 0 ]
+      then exit 1
+      fi
+      ;;
+   *)
+     echo $1 'Invalid'
+     echo 'Please specify any one of - management | inference | https | ALL'
+     exit 1
+     ;;
+esac

ci/scripts/linux_test_modelarchiver.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+cd model-archiver/
+
+# Lint test
+pylint -rn --rcfile=./model_archiver/tests/pylintrc model_archiver/.
+PY_LINT_EXIT_CODE=$?
+
+# Execute python unit tests
+python -m pytest --cov-report html:result_units --cov=./ model_archiver/tests/unit_tests
+PY_UNITS_EXIT_CODE=$?
+
+
+# Install model archiver module
+python setup.py bdist_wheel --universal && \
+pip install dist/*.whl
+MODL_ARCHVR_EXIT_CODE=$?
+
+# Execute integration tests
+python -m pytest model_archiver/tests/integ_tests # ToDo - Report for Integration tests ?
+PY_INTEG_EXIT_CODE=$?
+
+# If any one of the steps fail, exit with error
+if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PY_UNITS_EXIT_CODE" -ne 0 ] || [ "$MODL_ARCHVR_EXIT_CODE" -ne 0 ] || [ "$PY_INTEG_EXIT_CODE" -ne 0 ]
+then exit 1
+fi

ci/scripts/linux_test_perf_regression.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+JMETER_PATH='/opt/apache-jmeter-5.3/bin/jmeter'
+
+cd tests/performance
+
+# Only on a python 2 environment -
+PY_MAJOR_VER=$(python -c 'import sys; major = sys.version_info.major; print(major);')
+if [ $PY_MAJOR_VER -eq 2 ]; then
+  # Hack to use python 3.6.5 for bzt installation and execution
+  # While MMS continues to use system python which is at 2.7.x
+  export PATH="/root/.pyenv/bin:/root/.pyenv/shims:$PATH"
+  pyenv local 3.6.5 system
+fi
+
+# Install dependencies
+pip install -r requirements.txt
+pip install bzt
+
+# Execute performance test suite and store exit code
+./run_performance_suite.py  -j $JMETER_PATH -e xlarge -x example* --no-compare-local --no-monit
+EXIT_CODE=$?
+
+# Exit with the same error code as that of test execution
+exit $EXIT_CODE

ci/scripts/linux_test_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Lint Test
+pylint -rn --rcfile=./mms/tests/pylintrc mms/.
+PY_LINT_EXIT_CODE=$?
+
+# Execute python tests
+python -m pytest --cov-report html:result_units --cov=mms/ mms/tests/unit_tests/
+PYTEST_EXIT_CODE=$?
+
+# If any one of the tests fail, exit with error
+if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PYTEST_EXIT_CODE" -ne 0 ]
+then exit 1
+fi