Manage AML environments from conda YAML files

.gitignore

@@ -93,6 +93,7 @@ ENV/
 env.bak/
 venv.bak/
 *.vscode
+condaenv.*
 
 # Spyder project settings
 .spyderproject

.pipelines/diabetes_regression-ci-build-train.yml

@@ -105,26 +105,15 @@ stages:
     timeoutInMinutes: 0
     steps:
     - template: diabetes_regression-template-get-model-version.yml
-    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
-      displayName: 'Azure ML Model Deploy'
-      inputs:
-        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
-        modelSourceType: manualSpec
-        modelName: '$(MODEL_NAME)'
-        modelVersion: $(MODEL_VERSION)
-        inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
-        deploymentTarget: ACI
-        deploymentName: $(ACI_DEPLOYMENT_NAME)
-        deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aci.yml'
-        overwriteExistingDeployment: true
     - task: AzureCLI@1
-      displayName: 'Smoke test'
+      displayName: Azure ML Model Deploy and smoke test
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.pipelines.deploy_web_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
           python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
 
 - stage: 'Deploy_AKS'
@@ -138,27 +127,15 @@ stages:
     timeoutInMinutes: 0
     steps:
     - template: diabetes_regression-template-get-model-version.yml
-    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
-      displayName: 'Azure ML Model Deploy'
-      inputs:
-        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
-        modelSourceType: manualSpec
-        modelName: '$(MODEL_NAME)'
-        modelVersion: $(MODEL_VERSION)
-        inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
-        deploymentTarget: AKS
-        aksCluster: $(AKS_COMPUTE_NAME)
-        deploymentName: $(AKS_DEPLOYMENT_NAME)
-        deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aks.yml'
-        overwriteExistingDeployment: true
     - task: AzureCLI@1
-      displayName: 'Smoke test'
+      displayName: Azure ML Model Deploy and smoke test
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python -m ml_service.pipelines.deploy_web_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)" --compute_target "$(AKS_COMPUTE_NAME)"
           python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
 
 - stage: 'Deploy_Webapp'

diabetes_regression/ci_dependencies.yml

@@ -0,0 +1,28 @@
+name: mlopspython_ci
+
+dependencies:
+
+  # The python interpreter version.
+- python=3.7.5
+
+- r=3.6.0
+- r-essentials=3.6.0
+- numpy=1.18.1
+- pandas=1.0.0
+- scikit-learn=0.22.1
+
+- pip=20.0.2
+- pip:
+
+  # dependencies shared with other environment .yml files.
+  - azureml-sdk==1.0.79
+
+  # Additional pip dependencies for the CI environment.
+  - pytest==5.3.1
+  - pytest-cov==2.8.1
+  - requests==2.22.0
+  - python-dotenv==0.10.3
+  - flake8==3.7.9
+  - flake8_formatter_junit_xml==0.0.6
+  - azure-cli==2.0.77
+  - tox==3.14.3

diabetes_regression/scoring/inference_config.yml

@@ -1,9 +1,9 @@
 entryScript: score.py
 runtime: python
-condaFile: conda_dependencies.yml
+condaFile: ../scoring_dependencies.yml
 extraDockerfileSteps:
 schemaFile:
 sourceDirectory:
 enableGpu: False
 baseImage:
-baseImageRegistry:
+baseImageRegistry:

diabetes_regression/scoring_dependencies.yml

@@ -0,0 +1,23 @@
+# Details about the Conda environment file format:
+# https://conda.io/docs/using/envs.html#create-environment-file-by-hand
+
+name: diabetes_scoring
+
+dependencies:
+
+  # The python interpreter version.
+- python=3.7.5
+
+  # Required by azureml-defaults, installed separately through Conda to
+  # get a prebuilt version and not require build tools for the install.
+- psutil=5.6 #latest
+
+- numpy=1.18.1
+- pandas=1.0.0
+- scikit-learn=0.22.1
+
+- pip=20.0.2
+- pip:
+  # You must list azureml-defaults as a pip dependency
+  - azureml-defaults==1.0.85
+  - inference-schema[numpy-support]==1.0.1

diabetes_regression/training_dependencies.yml

@@ -0,0 +1,18 @@
+name: diabetes_training
+
+dependencies:
+
+  # The python interpreter version.
+- python=3.7.5
+
+- numpy=1.18.1
+- pandas=1.0.0
+- scikit-learn=0.22.1
+#- r
+#- r-essentials
+#- tensorflow
+#- keras
+
+- pip=20.0.2
+- pip:
+  - azureml-core==1.0.79

docs/code_description.md

@@ -2,9 +2,9 @@
 
 ### Environment Setup
 
-- `environment_setup/requirements.txt` : It consists of a list of python packages which are needed by the train.py to run successfully on host agent (locally).
+- `environment_setup/ci_environment.yml` : Conda environment definition for the CI environment.
 
-- `environment_setup/install_requirements.sh` : This script prepares the python environment i.e. install the Azure ML SDK and the packages specified in requirements.txt
+- `environment_setup/install_requirements.sh` : This script prepares a local conda environment i.e. install the Azure ML SDK and the packages specified in environment definitions.
 
 - `environment_setup/iac-*.yml, arm-templates` : Infrastructure as Code piplines to create and delete required resources along with corresponding arm-templates.
 
@@ -32,12 +32,13 @@
 - `diabetes_regression/training/train.py` : a training step of an ML training pipeline.
 - `diabetes_regression/evaluate/evaluate_model.py` : an evaluating step of an ML training pipeline which registers a new trained model if evaluation shows the new model is more performant than the previous one.
 - `diabetes_regression/evaluate/register_model.py` : (LEGACY) registers a new trained model if evaluation shows the new model is more performant than the previous one.
+- `diabetes_regression/training/training_dependencies.yml` : contains a list of dependencies required by train.py to be installed in a deployable Docker Image
 - `diabetes_regression/training/R/r_train.r` : training a model with R basing on a sample dataset (weight_data.csv).
 - `diabetes_regression/training/R/train_with_r.py` : a python wrapper (ML Pipeline Step) invoking R training script on ML Compute 
 - `diabetes_regression/training/R/train_with_r_on_databricks.py` : a python wrapper (ML Pipeline Step) invoking R training script on Databricks Compute
 - `diabetes_regression/training/R/weight_data.csv` : a sample dataset used by R script (r_train.r) to train a model
 
 ### Scoring
 - `diabetes_regression/scoring/score.py` : a scoring script which is about to be packed into a Docker Image along with a model while being deployed to QA/Prod environment.
-- `diabetes_regression/scoring/conda_dependencies.yml` : contains a list of dependencies required by score.py to be installed in a deployable Docker Image 
+- `diabetes_regression/scoring/scoring_dependencies.yml` : contains a list of dependencies required by score.py to be installed in a deployable Docker Image 
 - `diabetes_regression/scoring/inference_config.yml`, deployment_config_aci.yml, deployment_config_aks.yml : configuration files for the [AML Model Deploy](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.private-vss-services-azureml&ssr=false#overview) pipeline task for ACI and AKS deployment targets.

environment_setup/Dockerfile

@@ -4,11 +4,16 @@ LABEL org.label-schema.vendor = "Microsoft" \
       org.label-schema.url = "https://hub.docker.com/r/microsoft/mlopspython" \
       org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython"
 
-       
+COPY diabetes_regression/ci_dependencies.yml /setup/
 
-COPY environment_setup/requirements.txt  /setup/
-
-RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt && \ 
-    conda install -c r r-essentials
+RUN conda env create -f /setup/ci_dependencies.yml
 
-CMD ["python"]
+# activate environment
+ENV PATH /usr/local/envs/mlopspython_ci/bin:$PATH
+RUN /bin/bash -c "source activate mlopspython_ci"
+
+# Verify conda installation.
+# This serves as workaround for https://github.com/conda/conda/issues/8537 (conda env create doesn't fail
+# if pip installation fails, for example due to a wrong package version).
+# The `az` command is not available if pip has not run (and installed azure-cli).
+RUN az --version

environment_setup/install_requirements.sh

@@ -24,6 +24,8 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
+set -eux
 
-python --version
-pip install -r requirements.txt
+conda env create -f diabetes_regression/ci_dependencies.yml
+
+conda activate mlopspython_ci

ml_service/pipelines/deploy_web_service.py

@@ -0,0 +1,102 @@
+import argparse
+import os
+from azureml.core import Workspace
+from azureml.core.webservice import AciWebservice, AksWebservice
+from azureml.core.model import InferenceConfig, Model
+from ml_service.util.env_variables import Env
+from ml_service.util.manage_environment import get_environment
+
+
+def main():
+    parser = argparse.ArgumentParser("deploy_web_service.py")
+
+    parser.add_argument(
+        "--type",
+        type=str,
+        choices=["AKS", "ACI"],
+        required=True,
+        help="type of service"
+    )
+    parser.add_argument(
+        "--service",
+        type=str,
+        required=True,
+        help="Name of the service to deploy"
+    )
+    parser.add_argument(
+        "--compute_target",
+        type=str,
+        help="Name of the compute target. Only applicable if type = AKS"
+    )
+    args = parser.parse_args()
+
+    if args.type == "AKS" and args.compute_target is None:
+        raise ValueError("--compute_target is required")
+
+    e = Env()
+    # Get Azure machine learning workspace
+    aml_workspace = Workspace.get(
+        name=e.workspace_name,
+        subscription_id=e.subscription_id,
+        resource_group=e.resource_group
+    )
+    print("get_workspace:")
+    print(aml_workspace)
+
+    # Create a reusable scoring environment
+    environment = get_environment(
+       aml_workspace, "diabetes_scoring",
+       "diabetes_regression/scoring_dependencies.yml")
+
+    inference_config = InferenceConfig(
+        entry_script='score.py',
+        source_directory=os.path.join(e.sources_directory_train, "scoring"),
+        environment=environment,
+    )
+
+    service_description = f'Scoring model version {e.model_version}'
+
+    if args.type == "AKS":
+
+        deployment_config = AksWebservice.deploy_configuration(
+            description=service_description,
+            tags={"BuildId": e.build_id},
+            compute_target_name=args.compute_target,
+            autoscale_enabled=True,
+            autoscale_min_replicas=1,
+            autoscale_max_replicas=3,
+            autoscale_refresh_seconds=10,
+            autoscale_target_utilization=70,
+            auth_enabled=True,
+            cpu_cores=1,
+            memory_gb=4,
+            scoring_timeout_ms=5000,
+            replica_max_concurrent_requests=2,
+            max_request_wait_time=5000,
+        )
+
+    else:
+
+        deployment_config = AciWebservice.deploy_configuration(
+            description=service_description,
+            tags={"BuildId": e.build_id},
+            cpu_cores=1,
+            memory_gb=4,
+        )
+
+    model = Model(aml_workspace, name=e.model_name, version=e.model_version)
+
+    print(f'Deploying model {model} as service {args.service}')
+    service = Model.deploy(
+        workspace=aml_workspace,
+        name=args.service,
+        models=[model],
+        inference_config=inference_config,
+        deployment_config=deployment_config,
+        overwrite=True,
+    )
+    service.wait_for_deployment(show_output=True)
+
+
+if __name__ == '__main__':
+    main()