From 8556d629968ddb12f68c9fc1e7607e0d2a38cc3d Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Fri, 24 Jan 2020 22:39:37 +0100 Subject: [PATCH 1/9] . --- .gitignore | 2 + .pipelines/azdo-base-pipeline.yml | 30 +++++-------- .pipelines/azdo-ci-build-train.yml | 24 +++------- .pipelines/azdo-pr-build-train.yml | 4 +- .../azdo-template-get-model-version.yml | 2 +- README.md | 2 +- code/scoring/score.py | 2 +- code/training/R/train_with_r_on_databricks.py | 4 +- docs/development_setup.md | 44 +++++++++++++++++++ docs/getting_started.md | 5 +-- environment_setup/install_requirements.sh | 2 - ml_service/util/smoke_test_scoring_service.py | 2 +- tests/unit/code_test.py | 4 +- tests/unit/data_test.py | 8 ++-- tox.ini | 20 +++++++++ 15 files changed, 99 insertions(+), 56 deletions(-) create mode 100644 docs/development_setup.md create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 7bac8768..e0a583ec 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,8 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +*-testresults.xml +test-output.xml # Translations *.mo diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml index 926b404f..35dab17e 100644 --- a/.pipelines/azdo-base-pipeline.yml +++ b/.pipelines/azdo-base-pipeline.yml @@ -1,26 +1,20 @@ -# this pipeline should be ignored for now -parameters: - pipelineType: 'training' - steps: - script: | - flake8 --output-file=$(Build.BinariesDirectory)/lint-testresults.xml --format junit-xml - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'Run code quality tests' - enabled: 'true' - -- script: | - pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit - displayName: 'Run unit tests' - enabled: 'true' - env: - SP_APP_SECRET: '$(SP_APP_SECRET)' + tox + displayName: 'Linting & unit tests' - task: PublishTestResults@2 condition: succeededOrFailed() inputs: - testResultsFiles: '$(Build.BinariesDirectory)/*-testresults.xml' + testResultsFiles: '*-testresults.xml' testRunTitle: 'Linting & Unit tests' failTaskOnFailedTests: true - displayName: 'Publish linting and unit test results' - enabled: 'true' + displayName: 'Publish test results' + +- task: PublishCodeCoverageResults@1 + displayName: 'Publish coverage report' + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: 'coverage.xml' + failIfCoverageEmpty: true diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index e1ed0dc6..3f5549ca 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -7,22 +7,22 @@ trigger: exclude: - docs/ - environment_setup/ - - ml_service/util/create_scoring_image.* + - ml_service/util/create_scoring_image.py - ml_service/util/smoke_test_scoring_service.py variables: - template: azdo-variables.yml - group: devopsforai-aml-vg +pool: + vmImage: ubuntu-latest stages: - stage: 'Model_CI' displayName: 'Model CI' jobs: - - job: "Model_CI_Pipeline" - displayName: "Model CI Pipeline" - pool: - vmImage: 'ubuntu-latest' + - job: "Publish_Pipeline" + displayName: "Publish AML Pipeline" container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: @@ -35,7 +35,7 @@ stages: set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) # Invoke the Python building and publishing a training pipeline - python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }} + python ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }} displayName: 'Publish Azure Machine Learning Pipeline' - stage: 'Trigger_AML_Pipeline' @@ -44,8 +44,6 @@ stages: - job: "Get_Pipeline_ID" condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) displayName: "Get Pipeline ID for execution" - pool: - vmImage: 'ubuntu-latest' container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: @@ -56,7 +54,7 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution + python ml_service/pipelines/run_train_pipeline.py --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution # Set AMLPIPELINEID variable for next AML Pipeline task in next job AMLPIPELINEID="$(cat pipeline_id.txt)" echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID" @@ -89,8 +87,6 @@ stages: - job: "Training_Run_Report" dependsOn: "Run_ML_Pipeline" displayName: "Determine if evaluation succeeded and new model is registered" - pool: - vmImage: 'ubuntu-latest' container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: @@ -102,8 +98,6 @@ stages: jobs: - job: "Deploy_ACI" displayName: "Deploy to ACI" - pool: - vmImage: 'ubuntu-latest' container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: @@ -137,8 +131,6 @@ stages: jobs: - job: "Deploy_AKS" displayName: "Deploy to AKS" - pool: - vmImage: 'ubuntu-latest' container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: @@ -173,8 +165,6 @@ stages: jobs: - job: "Deploy_Webapp" displayName: "Deploy to Webapp" - pool: - vmImage: 'ubuntu-latest' container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: diff --git a/.pipelines/azdo-pr-build-train.yml b/.pipelines/azdo-pr-build-train.yml index 24231b2a..870a3c82 100644 --- a/.pipelines/azdo-pr-build-train.yml +++ b/.pipelines/azdo-pr-build-train.yml @@ -4,16 +4,14 @@ pr: include: - master -pool: +pool: vmImage: 'ubuntu-latest' container: mcr.microsoft.com/mlops/python:latest - variables: - template: azdo-variables.yml - group: devopsforai-aml-vg - steps: - template: azdo-base-pipeline.yml \ No newline at end of file diff --git a/.pipelines/azdo-template-get-model-version.yml b/.pipelines/azdo-template-get-model-version.yml index f69f3366..4a7dd39d 100644 --- a/.pipelines/azdo-template-get-model-version.yml +++ b/.pipelines/azdo-template-get-model-version.yml @@ -6,7 +6,7 @@ steps: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt" + python ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt" # Output model version to Azure DevOps job MODEL_VERSION="$(cat model_version.txt)" echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION" diff --git a/README.md b/README.md index e622ba75..3ce71c3f 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ This reference architecture shows how to implement continuous integration (CI), Once you have registered your ML model, you can use Azure ML + Azure DevOps to deploy it. -[Azure DevOps release pipeline](https://docs.microsoft.com/en-us/azure/devops/pipelines/release/?view=azure-devops) packages the new model along with the scoring file and its python dependencies into a [docker image](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#image) and pushes it to [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro). This image is used to deploy the model as [web service](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#web-service) across QA and Prod environments. The QA environment is running on top of [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/) and the Prod environment is built with [Azure Kubernetes Service (AKS)](https://docs.microsoft.com/en-us/azure/aks/intro-kubernetes). +The [Azure DevOps multi-stage pipeline](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/stages?view=azure-devops&tabs=yaml) packages the new model along with the scoring file and its python dependencies into a [docker image](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#image) and pushes it to [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro). This image is used to deploy the model as [web service](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#web-service) across QA and Prod environments. The QA environment is running on top of [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/) and the Prod environment is built with [Azure Kubernetes Service (AKS)](https://docs.microsoft.com/en-us/azure/aks/intro-kubernetes). ### Repo Details diff --git a/code/scoring/score.py b/code/scoring/score.py index b78a435c..4a1c6152 100644 --- a/code/scoring/score.py +++ b/code/scoring/score.py @@ -57,7 +57,7 @@ def run(raw_data, request_headers): request_headers.get("X-Ms-Request-Id", ""), request_headers.get("Traceparent", ""), len(result) - )) + )) return {"result": result.tolist()} diff --git a/code/training/R/train_with_r_on_databricks.py b/code/training/R/train_with_r_on_databricks.py index 1a120bd0..c571d609 100644 --- a/code/training/R/train_with_r_on_databricks.py +++ b/code/training/R/train_with_r_on_databricks.py @@ -11,5 +11,5 @@ args, unknown = parser.parse_known_args() folder = args.AZUREML_SCRIPT_DIRECTORY_NAME -os.system("cd " + "/dbfs/" + folder + - " && Rscript r_train.r && ls -ltr model.rds") +os.system("cd " + "/dbfs/" + folder + + " && Rscript r_train.r && ls -ltr model.rds") diff --git a/docs/development_setup.md b/docs/development_setup.md new file mode 100644 index 00000000..e625e7d8 --- /dev/null +++ b/docs/development_setup.md @@ -0,0 +1,44 @@ +## Development environment setup + +### Setup + +Please be aware that the local environment also needs access to the Azure subscription so you have to have Contributor access on the Azure ML Workspace. + +In order to configure the project locally, create a copy of `.env.example` in the root directory and name it `.env`. Fill out all missing values and adjust the existing ones to suit your requirements. + +### Installation + +[Install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli). The Azure CLI will be used to log you in interactively. + +Create a virtual environment using [venv](https://docs.python.org/3/library/venv.html), [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) or [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv). + +Here is an example for setting up and activating a `venv` environment with Python 3: + +``` +python3 -mvenv .venv +source .venv/bin/activate +``` + +Install the required Python modules in your virtual environment. + +``` +pip install -r environment_setup/build-image/requirements.txt +``` + +### Running local code + +To run your local ML pipeline code on Azure ML, run a command such as the following (in bash, all on one line): + +``` +export BUILD_BUILDID=$(uuidgen); python ml_service/pipelines/build_train_pipeline.py && python ml_service/pipelines/run_train_pipeline.py +``` + +BUILD_BUILDID is a variable used to uniquely identify the ML pipeline between the +`build_train_pipeline.py` and `run_train_pipeline.py` scripts. In Azure DevOps it is +set to the current build number. In a local environment, we can use a command such as +`uuidgen` so set a different random identifier on each run, ensuring there are +no collisions. + +### Local testing + +Before committing, run `tox` to execute linter and unit test checks. diff --git a/docs/getting_started.md b/docs/getting_started.md index a46d5304..775132b8 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -80,10 +80,7 @@ There are more variables used in the project. They're defined in two places, one ### Local configuration -In order to configure the project locally, create a copy of `.env.example` in the root directory and name it `.env`. Fill out all missing values and adjust the existing ones to suit your requirements. - -For local development, you will also need to [install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli). The Azure CLI will be used to log you in interactively. -Please be aware that the local environment also needs access to the Azure subscription so you have to have Contributor access on the Azure ML Workspace. +For instructions on how to set up a local development environment, refer to the [Development environment setup instructions](development_setup.md). ### Azure DevOps configuration diff --git a/environment_setup/install_requirements.sh b/environment_setup/install_requirements.sh index 1bdd081d..930514a6 100644 --- a/environment_setup/install_requirements.sh +++ b/environment_setup/install_requirements.sh @@ -26,6 +26,4 @@ python --version -pip install azure-cli==2.0.46 -pip install --upgrade azureml-sdk[cli] pip install -r requirements.txt \ No newline at end of file diff --git a/ml_service/util/smoke_test_scoring_service.py b/ml_service/util/smoke_test_scoring_service.py index 753ef23e..136656f6 100644 --- a/ml_service/util/smoke_test_scoring_service.py +++ b/ml_service/util/smoke_test_scoring_service.py @@ -52,7 +52,7 @@ def call_web_app(url, headers): response.raise_for_status() return response.json() except requests.exceptions.HTTPError as e: - if i == retries-1: + if i == retries - 1: raise e print(e) print("Retrying...") diff --git a/tests/unit/code_test.py b/tests/unit/code_test.py index 06654b2f..1703d54b 100644 --- a/tests/unit/code_test.py +++ b/tests/unit/code_test.py @@ -8,9 +8,9 @@ def test_train_model(): - X_train = np.array([1, 2, 3, 4, 5, 6]).reshape(-1, 1) + X_train = np.array([1, 2, 3, 4, 5, 6]).reshape(-1, 1) y_train = np.array([10, 9, 8, 8, 6, 5]) - X_test = np.array([3, 4]).reshape(-1, 1) + X_test = np.array([3, 4]).reshape(-1, 1) y_test = np.array([8, 7]) data = {"train": {"X": X_train, "y": y_train}, "test": {"X": X_test, "y": y_test}} diff --git a/tests/unit/data_test.py b/tests/unit/data_test.py index 8b40b8bc..4148f029 100644 --- a/tests/unit/data_test.py +++ b/tests/unit/data_test.py @@ -120,8 +120,8 @@ def test_check_distribution(): mean = np.mean(dataset.values, axis=0) std = np.mean(dataset.values, axis=0) assert ( - np.sum(abs(mean - historical_mean) > - shift_tolerance * abs(historical_mean)) - or np.sum(abs(std - historical_std) > - shift_tolerance * abs(historical_std)) > 0 + np.sum(abs(mean - historical_mean) + > shift_tolerance * abs(historical_mean)) + or np.sum(abs(std - historical_std) + > shift_tolerance * abs(historical_std)) > 0 ) diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..17faeda5 --- /dev/null +++ b/tox.ini @@ -0,0 +1,20 @@ +[tox] +minversion = 3.14.3 +skipsdist = True + +[flake8] +# ignore obsolete warning +ignore = W503 +exclude = .git,__pycache__,.venv,.tox,**/site-packages/**/*.py,**/lib/**.py,**/bin/**.py + +[pytest] +junit_family = legacy + +[testenv] +whitelist_externals = + flake8 + pytest +passenv = HOME +commands = + flake8 --output-file=lint-testresults.xml --format junit-xml + pytest tests/unit --cov=code --cov-report=xml --junitxml=unit-testresults.xml From 979d846a26b07ccb3dae95947882c1b182f70473 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Fri, 24 Jan 2020 22:42:10 +0100 Subject: [PATCH 2/9] . --- environment_setup/Dockerfile | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile index 5e7b7581..0779e0c6 100644 --- a/environment_setup/Dockerfile +++ b/environment_setup/Dockerfile @@ -4,11 +4,20 @@ LABEL org.label-schema.vendor = "Microsoft" \ org.label-schema.url = "https://hub.docker.com/r/microsoft/mlopspython" \ org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython" - +# Utilities for package installation +RUN apt-get update && apt-get install -y \ + curl \ + apt-transport-https \ + gcc -COPY environment_setup/requirements.txt /setup/ - -RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt && \ - conda install -c r r-essentials +# Install dotnet runtime used to generate code coverage report +RUN curl -O https://packages.microsoft.com/config/ubuntu/19.04/packages-microsoft-prod.deb +RUN dpkg -i packages-microsoft-prod.deb +RUN apt-get update && apt-get install -y dotnet-runtime-3.1 -CMD ["python"] \ No newline at end of file +# Packages for running R +RUN conda install -c r r-essentials + +# Install Python modules +COPY requirements.txt /setup/ +RUN pip install --upgrade -r /setup/requirements.txt From 08e76e3d44b42fd8f15c143082d126fea9a6c9e7 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Fri, 24 Jan 2020 23:26:27 +0100 Subject: [PATCH 3/9] Update requirements.txt --- environment_setup/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/environment_setup/requirements.txt b/environment_setup/requirements.txt index f99e7f4b..babb1ddc 100644 --- a/environment_setup/requirements.txt +++ b/environment_setup/requirements.txt @@ -1,4 +1,5 @@ pytest>=5.3 +pytest-cov>=2.8.1 requests>=2.22 numpy>=1.17 pandas>=0.25 @@ -6,5 +7,6 @@ scikit-learn>=0.21.3 azureml-sdk>=1.0 python-dotenv>=0.10.3 flake8>=3.7 -flake8_formatter_junit_xml +flake8_formatter_junit_xml>=0.0.6 +tox>=3.14.3 azure-cli==2.0.76 From f25590ae7ff087942bcecdee16619055daad3c3f Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Sat, 25 Jan 2020 07:08:43 +0100 Subject: [PATCH 4/9] Update azdo-ci-build-train.yml --- .pipelines/azdo-ci-build-train.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 3f5549ca..8e49c6a6 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -21,8 +21,8 @@ stages: - stage: 'Model_CI' displayName: 'Model CI' jobs: - - job: "Publish_Pipeline" - displayName: "Publish AML Pipeline" + - job: "Model_CI_Pipeline" + displayName: "Model CI Pipeline" container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: From 2b7707f7a58a85c533cc58c596fae8aad95276ff Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 30 Jan 2020 05:51:46 +0100 Subject: [PATCH 5/9] Update Dockerfile --- environment_setup/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile index 0779e0c6..7e1ec070 100644 --- a/environment_setup/Dockerfile +++ b/environment_setup/Dockerfile @@ -11,13 +11,13 @@ RUN apt-get update && apt-get install -y \ gcc # Install dotnet runtime used to generate code coverage report -RUN curl -O https://packages.microsoft.com/config/ubuntu/19.04/packages-microsoft-prod.deb -RUN dpkg -i packages-microsoft-prod.deb -RUN apt-get update && apt-get install -y dotnet-runtime-3.1 +RUN curl -O https://packages.microsoft.com/config/ubuntu/19.04/packages-microsoft-prod.deb \ + && dpkg -i packages-microsoft-prod.deb \ + && apt-get update && apt-get install -y dotnet-runtime-3.1 # Packages for running R RUN conda install -c r r-essentials # Install Python modules -COPY requirements.txt /setup/ +COPY environment_setup/requirements.txt /setup/ RUN pip install --upgrade -r /setup/requirements.txt From 26e1a0148e4c20f0352bb98a9fb3f9d2791db5e5 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 30 Jan 2020 06:23:50 +0100 Subject: [PATCH 6/9] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 17faeda5..9f49a364 100644 --- a/tox.ini +++ b/tox.ini @@ -17,4 +17,4 @@ whitelist_externals = passenv = HOME commands = flake8 --output-file=lint-testresults.xml --format junit-xml - pytest tests/unit --cov=code --cov-report=xml --junitxml=unit-testresults.xml + pytest tests/unit --cov=diabetes_regression --cov-report=xml --junitxml=unit-testresults.xml From ef14bc8685a69616a0303a0c21b6f2661ee48d8f Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 30 Jan 2020 18:56:23 +0100 Subject: [PATCH 7/9] Update development_setup.md --- docs/development_setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development_setup.md b/docs/development_setup.md index e625e7d8..8565ff04 100644 --- a/docs/development_setup.md +++ b/docs/development_setup.md @@ -22,7 +22,7 @@ source .venv/bin/activate Install the required Python modules in your virtual environment. ``` -pip install -r environment_setup/build-image/requirements.txt +pip install -r environment_setup/requirements.txt ``` ### Running local code From b4268afca7e24eae848b493b23f0efa08c9360f0 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Thu, 30 Jan 2020 20:24:31 +0100 Subject: [PATCH 8/9] Reverted Dockerfile --- .pipelines/azdo-base-pipeline.yml | 1 + environment_setup/Dockerfile | 21 ++++++--------------- tox.ini | 2 +- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml index 35dab17e..50c13c1c 100644 --- a/.pipelines/azdo-base-pipeline.yml +++ b/.pipelines/azdo-base-pipeline.yml @@ -17,4 +17,5 @@ steps: inputs: codeCoverageTool: Cobertura summaryFileLocation: 'coverage.xml' + reportDirectory: 'htmlcov' failIfCoverageEmpty: true diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile index 7e1ec070..5e7b7581 100644 --- a/environment_setup/Dockerfile +++ b/environment_setup/Dockerfile @@ -4,20 +4,11 @@ LABEL org.label-schema.vendor = "Microsoft" \ org.label-schema.url = "https://hub.docker.com/r/microsoft/mlopspython" \ org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython" -# Utilities for package installation -RUN apt-get update && apt-get install -y \ - curl \ - apt-transport-https \ - gcc + -# Install dotnet runtime used to generate code coverage report -RUN curl -O https://packages.microsoft.com/config/ubuntu/19.04/packages-microsoft-prod.deb \ - && dpkg -i packages-microsoft-prod.deb \ - && apt-get update && apt-get install -y dotnet-runtime-3.1 - -# Packages for running R -RUN conda install -c r r-essentials - -# Install Python modules COPY environment_setup/requirements.txt /setup/ -RUN pip install --upgrade -r /setup/requirements.txt + +RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt && \ + conda install -c r r-essentials + +CMD ["python"] \ No newline at end of file diff --git a/tox.ini b/tox.ini index 9f49a364..a0c318c4 100644 --- a/tox.ini +++ b/tox.ini @@ -17,4 +17,4 @@ whitelist_externals = passenv = HOME commands = flake8 --output-file=lint-testresults.xml --format junit-xml - pytest tests/unit --cov=diabetes_regression --cov-report=xml --junitxml=unit-testresults.xml + pytest tests/unit --cov=diabetes_regression --cov-report=html --cov-report=xml --junitxml=unit-testresults.xml From dc001442ccd22c339facf898d21e8b68d52334a7 Mon Sep 17 00:00:00 2001 From: Alexandre Gattiker Date: Fri, 31 Jan 2020 07:22:53 +0100 Subject: [PATCH 9/9] Do not use tox virtualenv --- .pipelines/azdo-base-pipeline.yml | 2 +- lint-and-test.sh | 4 ++++ tox.ini | 13 ------------- 3 files changed, 5 insertions(+), 14 deletions(-) create mode 100755 lint-and-test.sh diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml index 50c13c1c..ad0e4ad8 100644 --- a/.pipelines/azdo-base-pipeline.yml +++ b/.pipelines/azdo-base-pipeline.yml @@ -1,6 +1,6 @@ steps: - script: | - tox + ./lint-and-test.sh displayName: 'Linting & unit tests' - task: PublishTestResults@2 diff --git a/lint-and-test.sh b/lint-and-test.sh new file mode 100755 index 00000000..77c646ba --- /dev/null +++ b/lint-and-test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eux +flake8 --output-file=lint-testresults.xml --format junit-xml +python -m pytest tests/unit --cov=diabetes_regression --cov-report=html --cov-report=xml --junitxml=unit-testresults.xml diff --git a/tox.ini b/tox.ini index a0c318c4..e24d8ae1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,3 @@ -[tox] -minversion = 3.14.3 -skipsdist = True - [flake8] # ignore obsolete warning ignore = W503 @@ -9,12 +5,3 @@ exclude = .git,__pycache__,.venv,.tox,**/site-packages/**/*.py,**/lib/**.py,**/b [pytest] junit_family = legacy - -[testenv] -whitelist_externals = - flake8 - pytest -passenv = HOME -commands = - flake8 --output-file=lint-testresults.xml --format junit-xml - pytest tests/unit --cov=diabetes_regression --cov-report=html --cov-report=xml --junitxml=unit-testresults.xml