Remove sys.path from everywhere to enable IntelliSense (#162)

.pipelines/azdo-abtest-pipeline.yml

@@ -40,7 +40,7 @@ stages:
         inlineScript: |
           set -e
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt          
+          python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt          
       displayName: 'Create Scoring Image'
       name: 'buildscoringimage'
 

.pipelines/azdo-base-pipeline.yml

@@ -10,7 +10,7 @@ steps:
   enabled: 'true'
 
 - script: |
-   pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit
+   python -m pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit
   displayName: 'Run unit tests'
   enabled: 'true'
   env:

.pipelines/diabetes_regression-ci-build-train.yml

@@ -31,11 +31,12 @@ stages:
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
           # Invoke the Python building and publishing a training pipeline
-          python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }}
+          python -m ml_service.pipelines.diabetes_regression_build_train_pipeline
       displayName: 'Publish Azure Machine Learning Pipeline'
 
 - stage: 'Trigger_AML_Pipeline'
@@ -53,10 +54,11 @@ stages:
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
+        workingDirectory: $(Build.SourcesDirectory)
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution
+          python -m ml_service.pipelines.run_train_pipeline --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution
           # Set AMLPIPELINEID variable for next AML Pipeline task in next job
           AMLPIPELINEID="$(cat pipeline_id.txt)"
           echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID"
@@ -95,7 +97,7 @@ stages:
     container: mcr.microsoft.com/mlops/python:latest
     timeoutInMinutes: 0
     steps:
-    - template: azdo-template-get-model-version.yml
+    - template: diabetes_regression-template-get-model-version.yml
 - stage: 'Deploy_ACI'
   displayName: 'Deploy to ACI'
   dependsOn: Trigger_AML_Pipeline
@@ -108,7 +110,7 @@ stages:
     container: mcr.microsoft.com/mlops/python:latest
     timeoutInMinutes: 0
     steps:
-    - template: azdo-template-get-model-version.yml
+    - template: diabetes_regression-template-get-model-version.yml
     - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
       displayName: 'Azure ML Model Deploy'
       inputs:
@@ -129,7 +131,7 @@ stages:
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python ml_service/util/smoke_test_scoring_service.py --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
+          python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
 
 - stage: 'Deploy_AKS'
   displayName: 'Deploy to AKS'
@@ -143,7 +145,7 @@ stages:
     container: mcr.microsoft.com/mlops/python:latest
     timeoutInMinutes: 0
     steps:
-    - template: azdo-template-get-model-version.yml
+    - template: diabetes_regression-template-get-model-version.yml
     - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
       displayName: 'Azure ML Model Deploy'
       inputs:
@@ -165,7 +167,7 @@ stages:
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python ml_service/util/smoke_test_scoring_service.py --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
+          python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
 
 - stage: 'Deploy_Webapp'
   displayName: 'Deploy to Webapp'
@@ -179,7 +181,7 @@ stages:
     container: mcr.microsoft.com/mlops/python:latest
     timeoutInMinutes: 0
     steps:
-    - template: azdo-template-get-model-version.yml
+    - template: diabetes_regression-template-get-model-version.yml
     - task: AzureCLI@1
       displayName: 'Create scoring image and set IMAGE_LOCATION variable'
       inputs:
@@ -188,7 +190,7 @@ stages:
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt
+          python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt
           # Output image location to Azure DevOps job
           IMAGE_LOCATION="$(cat image_location.txt)"
           echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
@@ -207,4 +209,4 @@ stages:
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python ml_service/util/smoke_test_scoring_service.py --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"
+          python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"

.pipelines/diabetes_regression-ci-image.yml

@@ -28,9 +28,10 @@ steps:
   inputs:
     azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
     scriptLocation: inlineScript
+    workingDirectory: $(Build.SourcesDirectory)
     inlineScript: |
       set -e
       export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-      python3 $(Build.SourcesDirectory)/ml_service/util/create_scoring_image.py   
+      python3 -m ml_service.util.create_scoring_image   
   displayName: 'Create Scoring Image'
 

.pipelines/azdo-template-get-model-version.yml → .pipelines/diabetes_regression-template-get-model-version.yml

@@ -6,7 +6,7 @@ steps:
     inlineScript: |
       set -e # fail on error
       export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-      python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt"
+      python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline --build_id $(Build.BuildId) --output_model_version_file "model_version.txt"
       # Output model version to Azure DevOps job
       MODEL_VERSION="$(cat model_version.txt)"
       echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION"

diabetes_regression/evaluate/evaluate_model.py

@@ -24,47 +24,54 @@
 POSSIBILITY OF SUCH DAMAGE.
 """
 import os
-import sys
-from azureml.core import Run, Workspace, Experiment
+from azureml.core import Run
 import argparse
 import traceback
+from util.model_helper import get_model_by_tag
 
 run = Run.get_context()
-if (run.id.startswith('OfflineRun')):
-    from dotenv import load_dotenv
-    # For local development, set values in this section
-    load_dotenv()
-    sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
-    if (sources_dir is None):
-        sources_dir = 'diabetes_regression'
-    path_to_util = os.path.join(".", sources_dir, "util")
-    sys.path.append(os.path.abspath(path_to_util))  # NOQA: E402
-    from model_helper import get_model_by_tag
-    workspace_name = os.environ.get("WORKSPACE_NAME")
-    experiment_name = os.environ.get("EXPERIMENT_NAME")
-    resource_group = os.environ.get("RESOURCE_GROUP")
-    subscription_id = os.environ.get("SUBSCRIPTION_ID")
-    tenant_id = os.environ.get("TENANT_ID")
-    model_name = os.environ.get("MODEL_NAME")
-    app_id = os.environ.get('SP_APP_ID')
-    app_secret = os.environ.get('SP_APP_SECRET')
-    build_id = os.environ.get('BUILD_BUILDID')
-    # run_id useful to query previous runs
-    run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70"
 
-    aml_workspace = Workspace.get(
-        name=workspace_name,
-        subscription_id=subscription_id,
-        resource_group=resource_group
-    )
-    ws = aml_workspace
-    exp = Experiment(ws, experiment_name)
-else:
-    sys.path.append(os.path.abspath("./util"))  # NOQA: E402
-    from model_helper import get_model_by_tag
-    exp = run.experiment
-    ws = run.experiment.workspace
-    run_id = 'amlcompute'
+# if you would like to run this script on a local computer
+# the following code is a good starting point for you
+# use
+# python -m evaluate.evaluate_model
+# in diabetes_regression folder context
+
+# if (run.id.startswith('OfflineRun')):
+#     from dotenv import load_dotenv
+#     # For local development, set values in this section
+#     load_dotenv()
+#     sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
+#     if (sources_dir is None):
+#         sources_dir = 'diabetes_regression'
+#     path_to_util = os.path.join(".", sources_dir, "util")
+#     sys.path.append(os.path.abspath(path_to_util))  # NOQA: E402
+#     from model_helper import get_model_by_tag
+#     workspace_name = os.environ.get("WORKSPACE_NAME")
+#     experiment_name = os.environ.get("EXPERIMENT_NAME")
+#     resource_group = os.environ.get("RESOURCE_GROUP")
+#     subscription_id = os.environ.get("SUBSCRIPTION_ID")
+#     tenant_id = os.environ.get("TENANT_ID")
+#     model_name = os.environ.get("MODEL_NAME")
+#     app_id = os.environ.get('SP_APP_ID')
+#     app_secret = os.environ.get('SP_APP_SECRET')
+#     build_id = os.environ.get('BUILD_BUILDID')
+#     # run_id useful to query previous runs
+#     run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70"
+
+#     aml_workspace = Workspace.get(
+#         name=workspace_name,
+#         subscription_id=subscription_id,
+#         resource_group=resource_group
+#     )
+#     ws = aml_workspace
+#     exp = Experiment(ws, experiment_name)
+
+# comment the following three lines
+# if you would like to use Offline mode
+exp = run.experiment
+ws = run.experiment.workspace
+run_id = 'amlcompute'
 
 parser = argparse.ArgumentParser("evaluate")
 parser.add_argument(

docs/code_description.md

@@ -24,7 +24,7 @@
 - `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py` : builds and publishes an ML training pipeline. It uses R on ML Compute.
 - `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py` : builds and publishes an ML training pipeline. It uses R on Databricks Compute.
 - `ml_service/pipelines/run_train_pipeline.py` : invokes a published ML training pipeline (Python on ML Compute) via REST API.
-- `ml_service/pipelines/verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model.
+- `ml_service/pipelines/diabetes_regression_verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model.
 - `ml_service/util` : contains common utility functions used to build and publish an ML training pipeline.
 
 ### Code

ml_service/pipelines/diabetes_regression_build_train_pipeline.py

@@ -4,11 +4,8 @@
 from azureml.core import Workspace
 from azureml.core.runconfig import RunConfiguration, CondaDependencies
 from azureml.core import Dataset, Datastore
-import os
-import sys
-sys.path.append(os.path.abspath("./ml_service/util"))  # NOQA: E402
-from attach_compute import get_compute
-from env_variables import Env
+from ml_service.util.attach_compute import get_compute
+from ml_service.util.env_variables import Env
 
 
 def main():

ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py

@@ -2,11 +2,8 @@
 from azureml.pipeline.core import Pipeline
 from azureml.core import Workspace
 from azureml.core.runconfig import RunConfiguration, CondaDependencies
-import os
-import sys
-sys.path.append(os.path.abspath("./ml_service/util"))  # NOQA: E402
-from attach_compute import get_compute
-from env_variables import Env
+from ml_service.util.attach_compute import get_compute
+from ml_service.util.env_variables import Env
 
 
 def main():

ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py

@@ -1,11 +1,8 @@
 from azureml.pipeline.core import Pipeline
 from azureml.core import Workspace
-import os
-import sys
-sys.path.append(os.path.abspath("./ml_service/util"))  # NOQA: E402
-from attach_compute import get_compute
+from ml_service.util.attach_compute import get_compute
 from azureml.pipeline.steps import DatabricksStep
-from env_variables import Env
+from ml_service.util.env_variables import Env
 
 
 def main():

ml_service/pipelines/verify_train_pipeline.py → ml_service/pipelines/diabetes_regression_verify_train_pipeline.py

@@ -1,24 +1,21 @@
-import os
-import sys
 import argparse
+import sys
+import os
 from azureml.core import Run, Experiment, Workspace
-sys.path.append(os.path.abspath("./ml_service/util"))  # NOQA: E402
-from env_variables import Env
+from ml_service.util.env_variables import Env
+from diabetes_regression.util.model_helper import get_model_by_tag
 
 
 def main():
 
     run = Run.get_context()
+
     if (run.id.startswith('OfflineRun')):
         from dotenv import load_dotenv
-        # For local development, set values in this section
         load_dotenv()
         sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
         if (sources_dir is None):
             sources_dir = 'diabetes_regression'
-        path_to_util = os.path.join(".", sources_dir, "util")
-        sys.path.append(os.path.abspath(path_to_util))  # NOQA: E402
-        from model_helper import get_model_by_tag
         workspace_name = os.environ.get("WORKSPACE_NAME")
         experiment_name = os.environ.get("EXPERIMENT_NAME")
         resource_group = os.environ.get("RESOURCE_GROUP")
@@ -32,9 +29,6 @@ def main():
         ws = aml_workspace
         exp = Experiment(ws, experiment_name)
     else:
-        sys.path.append(os.path.abspath("./util"))  # NOQA: E402
-        from model_helper import get_model_by_tag
-        ws = run.experiment.workspace
         exp = run.experiment
 
     e = Env()

ml_service/pipelines/run_train_pipeline.py

@@ -1,10 +1,7 @@
 from azureml.pipeline.core import PublishedPipeline
 from azureml.core import Workspace
-import os
-import sys
 import argparse
-sys.path.append(os.path.abspath("./ml_service/util"))  # NOQA: E402
-from env_variables import Env
+from ml_service.util.env_variables import Env
 
 
 def main():

ml_service/util/attach_compute.py

@@ -2,7 +2,7 @@
 from azureml.core.compute import AmlCompute
 from azureml.core.compute import ComputeTarget
 from azureml.exceptions import ComputeTargetException
-from env_variables import Env
+from ml_service.util.env_variables import Env
 
 
 def get_compute(

ml_service/util/create_scoring_image.py

@@ -1,11 +1,9 @@
 import os
-import sys
 import argparse
 from azureml.core import Workspace
 from azureml.core.image import ContainerImage, Image
 from azureml.core.model import Model
-sys.path.append(os.path.abspath("./ml_service/util"))  # NOQA: E402
-from env_variables import Env
+from ml_service.util.env_variables import Env
 
 e = Env()
 

ml_service/util/smoke_test_scoring_service.py

@@ -1,12 +1,9 @@
-import os
-import sys
 import argparse
 import requests
 import time
 from azureml.core import Workspace
 from azureml.core.webservice import AksWebservice, AciWebservice
-sys.path.append(os.path.abspath("./ml_service/util"))  # NOQA: E402
-from env_variables import Env
+from ml_service.util.env_variables import Env
 import secrets
 
 

tests/unit/code_test.py

@@ -1,11 +1,7 @@
-import sys
-import os
 import numpy as np
 from azureml.core.run import Run
 from unittest.mock import Mock
-sys.path.append(os.path.abspath(
-        "./diabetes_regression/training"))  # NOQA: E402
-from train import train_model
+from diabetes_regression.training.train import train_model
 
 
 def test_train_model():