From 88dad1a68050145a96770e2027b8bf37489d7b79 Mon Sep 17 00:00:00 2001 From: David Tesar Date: Tue, 11 Feb 2020 16:55:23 -0800 Subject: [PATCH 1/6] move replaceprojectname outside class --- bootstrap/bootstrap.py | 73 +++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index d796f6df..11d85bc0 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -3,7 +3,6 @@ import argparse # from git import Repo - class Helper: def __init__(self, project_directory, project_name): @@ -57,38 +56,6 @@ def deletedir(self): os.system( 'rmdir /S /Q "{}"'.format(os.path.join(self._project_directory, dir))) # NOQA: E501 - def replaceprojectname(self): - # Replace instances of diabetes_regression within files - dirs = [r".env.example", - r".pipelines\azdo-base-pipeline.yml", - r".pipelines\azdo-pr-build-train.yml", - r".pipelines\diabetes_regression-ci-build-train.yml", - r".pipelines\diabetes_regression-ci-image.yml", - r".pipelines\diabetes_regression-template-get-model-version.yml", # NOQA: E501 - r".pipelines\diabetes_regression-variables.yml", - r"environment_setup\Dockerfile", - r"environment_setup\install_requirements.sh", - r"ml_service\pipelines\diabetes_regression_build_train_pipeline_with_r_on_dbricks.py", # NOQA: E501 - r"ml_service\pipelines\diabetes_regression_build_train_pipeline_with_r.py", # NOQA: E501 - r"ml_service\pipelines\diabetes_regression_build_train_pipeline.py", # NOQA: E501 - r"ml_service\pipelines\diabetes_regression_verify_train_pipeline.py", # NOQA: E501 - r"ml_service\util\create_scoring_image.py", - r"diabetes_regression\azureml_environment.json", - r"diabetes_regression\conda_dependencies.yml", - r"diabetes_regression\evaluate\evaluate_model.py", - r"diabetes_regression\training\test_train.py"] # NOQA: E501 - - for file in dirs: - fin = open(os.path.join(self._project_directory, file), - "rt", encoding="utf8") - data = fin.read() - data = data.replace("diabetes_regression", self.project_name) - fin.close() - fin = open(os.path.join(self._project_directory, file), - "wt", encoding="utf8") - fin.write(data) - fin.close() - def cleandir(self): # Clean up directories dirs = ["data", "experimentation"] @@ -107,6 +74,38 @@ def validateargs(self): if(len(self._project_name) < 3 or len(self._project_name) > 15): raise Exception("Project name should be 3 to 15 chars long") +def replaceprojectname(project_dir, project_name, rename_name): + # Replace instances of rename_name within files with project_name + dirs = [r".env.example", + r".pipelines\azdo-base-pipeline.yml", + r".pipelines\azdo-pr-build-train.yml", + r".pipelines\diabetes_regression-ci-build-train.yml", + r".pipelines\diabetes_regression-ci-image.yml", + r".pipelines\diabetes_regression-template-get-model-version.yml", # NOQA: E501 + r".pipelines\diabetes_regression-variables.yml", + r"environment_setup\Dockerfile", + r"environment_setup\install_requirements.sh", + r"ml_service\pipelines\diabetes_regression_build_train_pipeline_with_r_on_dbricks.py", # NOQA: E501 + r"ml_service\pipelines\diabetes_regression_build_train_pipeline_with_r.py", # NOQA: E501 + r"ml_service\pipelines\diabetes_regression_build_train_pipeline.py", # NOQA: E501 + r"ml_service\pipelines\diabetes_regression_verify_train_pipeline.py", # NOQA: E501 + r"ml_service\util\create_scoring_image.py", + r"diabetes_regression\azureml_environment.json", + r"diabetes_regression\conda_dependencies.yml", + r"diabetes_regression\evaluate\evaluate_model.py", + r"diabetes_regression\training\test_train.py"] # NOQA: E501 + + for file in dirs: + fin = open(os.path.join(project_dir, file), + "rt", encoding="utf8") + data = fin.read() + data = data.replace(rename_name, project_name) + fin.close() + fin = open(os.path.join(project_dir, file), + "wt", encoding="utf8") + fin.write(data) + fin.close() + def main(args): parser = argparse.ArgumentParser(description='New Template') @@ -122,10 +121,10 @@ def main(args): helper.validateargs() # helper.clonerepo() helper.cleandir() - helper.replaceprojectname() - helper.deletedir() - helper.renamefiles() - helper.renamedir() + replaceprojectname(project_directory,project_name,"diabetes_regression") + # helper.deletedir() + # helper.renamefiles() + # helper.renamedir() except Exception as e: print(e) return 0 From 91ab08324810aa848b4bfa9b8951ef4999ad1e84 Mon Sep 17 00:00:00 2001 From: David Tesar Date: Wed, 12 Feb 2020 08:37:20 -0800 Subject: [PATCH 2/6] change tag to diabetes_regression --- diabetes_regression/register/register_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diabetes_regression/register/register_model.py b/diabetes_regression/register/register_model.py index b2d454bb..95b9e897 100644 --- a/diabetes_regression/register/register_model.py +++ b/diabetes_regression/register/register_model.py @@ -119,7 +119,7 @@ def register_aml_model( if (build_id != 'none'): model_already_registered(model_name, exp, run_id) run = Run(experiment=exp, run_id=run_id) - tagsValue = {"area": "diabetes", "type": "regression", + tagsValue = {"area": "diabetes_regression", "BuildId": build_id, "run_id": run_id, "experiment_name": exp.name} if (build_uri is not None): @@ -127,7 +127,7 @@ def register_aml_model( else: run = Run(experiment=exp, run_id=run_id) if (run is not None): - tagsValue = {"area": "diabetes", "type": "regression", + tagsValue = {"area": "diabetes_regression", "run_id": run_id, "experiment_name": exp.name} else: print("A model run for experiment", exp.name, From 15be5a81213ef4bd406d8dbe2c1457ab7b2c2eb4 Mon Sep 17 00:00:00 2001 From: David Tesar Date: Wed, 12 Feb 2020 08:39:51 -0800 Subject: [PATCH 3/6] tag to diabetes_regression --- ml_service/util/create_scoring_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_service/util/create_scoring_image.py b/ml_service/util/create_scoring_image.py index cc6fc2ee..41ff635c 100644 --- a/ml_service/util/create_scoring_image.py +++ b/ml_service/util/create_scoring_image.py @@ -39,7 +39,7 @@ runtime="python", conda_file="conda_dependencies.yml", description="Image with ridge regression model", - tags={"area": "diabetes", "type": "regression"}, + tags={"area": "diabetes_regression"}, ) image = Image.create( From dd70b3a78cc2d26fb74939059da88bcf7ecadcd6 Mon Sep 17 00:00:00 2001 From: David Tesar Date: Wed, 12 Feb 2020 08:43:38 -0800 Subject: [PATCH 4/6] lint fixes --- bootstrap/bootstrap.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 11d85bc0..99ab78bc 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -3,6 +3,7 @@ import argparse # from git import Repo + class Helper: def __init__(self, project_directory, project_name): @@ -74,6 +75,7 @@ def validateargs(self): if(len(self._project_name) < 3 or len(self._project_name) > 15): raise Exception("Project name should be 3 to 15 chars long") + def replaceprojectname(project_dir, project_name, rename_name): # Replace instances of rename_name within files with project_name dirs = [r".env.example", @@ -97,12 +99,12 @@ def replaceprojectname(project_dir, project_name, rename_name): for file in dirs: fin = open(os.path.join(project_dir, file), - "rt", encoding="utf8") + "rt", encoding="utf8") data = fin.read() data = data.replace(rename_name, project_name) fin.close() fin = open(os.path.join(project_dir, file), - "wt", encoding="utf8") + "wt", encoding="utf8") fin.write(data) fin.close() @@ -121,10 +123,12 @@ def main(args): helper.validateargs() # helper.clonerepo() helper.cleandir() - replaceprojectname(project_directory,project_name,"diabetes_regression") - # helper.deletedir() - # helper.renamefiles() - # helper.renamedir() + replaceprojectname(project_directory, project_name, + "diabetes_regression") + replaceprojectname(project_directory, project_name, "diabetes") + helper.deletedir() + helper.renamefiles() + helper.renamedir() except Exception as e: print(e) return 0 From 9edfe349469bc8d7a8b2280702d80c26039418e2 Mon Sep 17 00:00:00 2001 From: David Tesar Date: Wed, 12 Feb 2020 08:45:46 -0800 Subject: [PATCH 5/6] add register_model --- bootstrap/bootstrap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 99ab78bc..0aa7c2e6 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -95,6 +95,7 @@ def replaceprojectname(project_dir, project_name, rename_name): r"diabetes_regression\azureml_environment.json", r"diabetes_regression\conda_dependencies.yml", r"diabetes_regression\evaluate\evaluate_model.py", + r"diabetes_regression\register\register_model.py", r"diabetes_regression\training\test_train.py"] # NOQA: E501 for file in dirs: From 4c160feaffd67122527c1a24d7f5ce2872a0783d Mon Sep 17 00:00:00 2001 From: David Tesar Date: Wed, 12 Feb 2020 08:59:21 -0800 Subject: [PATCH 6/6] enhance readme --- bootstrap/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bootstrap/README.md b/bootstrap/README.md index bbbbff09..1a4551a9 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -2,8 +2,10 @@ To use this existing project structure and scripts for your new ML project, you can quickly get started from the existing repository, bootstrap and create a template that works for your ML project. Bootstraping will prepare a similar directory structure for your project which includes renaming files and folders, deleting and cleaning up some directories and fixing imports and absolute path based on your project name. This will enable reusing various resources like pre-built pipelines and scripts for your new project. -To bootstrap from the existing MLOpsPython repository clone this repository and run bootstrap.py script as below +To bootstrap from the existing MLOpsPython repository clone this repository, ensure Python is installed locally, and run bootstrap.py script as below ->python bootstrap.py --d [dirpath] --n [projectname] +`python bootstrap.py --d [dirpath] --n [projectname]` -Where [dirpath] is the absolute path to the root of your directory where MLOps repo is cloned and [projectname] is the name of your ML project +Where `[dirpath]` is the absolute path to the root of your directory where MLOps repo is cloned and `[projectname]` is the name of your ML project. + +[This article](https://docs.microsoft.com/azure/machine-learning/tutorial-convert-ml-experiment-to-production#use-your-own-model-with-mlopspython-code-template) will also assist to use this code template for your own ML project.