Canary deployment (#125)

microsoft · Dec 12, 2019 · 8d1be26 · 8d1be26
1 parent 2a22e75
commit 8d1be26
Show file tree

Hide file tree

Showing 20 changed files with 505 additions and 2 deletions.
diff --git a/.pipelines/azdo-abtest-pipeline.yml b/.pipelines/azdo-abtest-pipeline.yml
@@ -0,0 +1,162 @@
+pr: none
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    exclude:
+    - docs/
+    - environment_setup/
+    - ml_service/util/create_scoring_image.*
+    - ml_service/util/smoke_test_scoring_service.py
+
+variables:
+- group: 'devopsforai-aml-vg'
+- name: 'helmVersion'
+  value: 'v3.0.1'
+- name: 'helmDownloadURL'
+  value: 'https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz'
+- name: 'blueReleaseName'
+  value: 'model-blue'
+- name: 'greenReleaseName'
+  value: 'model-green'
+- name: 'SCORE_SCRIPT'
+  value: 'scoreA.py'
+
+
+stages:
+- stage: 'Building'
+  jobs:
+  - job: "Build_Scoring_image"
+    timeoutInMinutes: 0
+    pool:
+      vmImage: 'ubuntu-latest'
+    container: mcr.microsoft.com/mlops/python:latest
+    steps:
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt          
+      displayName: 'Create Scoring Image'
+      name: 'buildscoringimage'
+
+    - publish: image_location.txt
+      artifact: image_location
+
+    - publish: $(System.DefaultWorkingDirectory)/charts
+      artifact: allcharts
+
+
+- stage: 'Blue_Staging'
+  jobs:
+  - deployment: "Deploy_to_Staging"
+    timeoutInMinutes: 0
+    environment: abtestenv
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - script: |
+              IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)"
+              echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
+            displayName: 'Get Image Location'
+          - template: azdo-helm-upgrade.yml
+            parameters:
+              chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model'
+              releaseName: $(blueReleaseName) 
+              overrideValues: 'deployment.name=$(blueReleaseName),deployment.bluegreen=blue,deployment.image.name=$(IMAGE_LOCATION)'
+
+- stage: 'Blue_50'
+  jobs:
+  - job: 'Blue_Rollout_50'
+    displayName: 50 50 rollout to blue environment
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-helm-upgrade.yml
+      parameters:
+        chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio'
+        releaseName: 'abtest-istio' 
+        overrideValues: 'weight.blue=50,weight.green=50'
+
+- stage: 'Blue_100'
+  jobs:
+  - deployment: 'blue_Rollout_100'
+    timeoutInMinutes: 0
+    environment: abtestenv
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - template: azdo-helm-upgrade.yml
+            parameters:
+              chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio'
+              releaseName: 'abtest-istio' 
+              overrideValues: 'weight.blue=100,weight.green=0'
+
+- stage: 'Rollback'
+  dependsOn: 'Blue_100'
+  condition: failed()
+  jobs:
+  - deployment: 'Roll_Back'
+    displayName: 'Roll Back after failure'
+    environment: abtestenv
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - template: azdo-helm-upgrade.yml
+            parameters:
+              chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio'
+              releaseName: 'abtest-istio' 
+              overrideValues: 'weight.blue=0,weight.green=100'
+
+- stage: 'Set_Production_Tag'
+  dependsOn: 'Blue_100'
+  condition: succeeded()
+  jobs:
+  - deployment: 'green_blue_tagging'
+    timeoutInMinutes: 0
+    environment: abtestenv
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - script: |
+              IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)"
+              echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
+            displayName: 'Get Image Location'
+          - template: azdo-helm-upgrade.yml
+            parameters:
+              chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model'
+              releaseName: $(greenReleaseName) 
+              overrideValues: 'deployment.name=$(greenReleaseName),deployment.bluegreen=green,deployment.image.name=$(IMAGE_LOCATION)'
+
+- stage: 'Green_100'
+  jobs:
+  - job: 'Prod_Rollout_100'
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-helm-upgrade.yml
+      parameters:
+        chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio'
+        releaseName: 'abtest-istio' 
+        overrideValues: 'weight.blue=0,weight.green=100'
+
+- stage: 'Disable_blue'
+  condition: always()
+  jobs:
+  - job: 'blue_disable'
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-helm-install.yml
+    - task: HelmDeploy@0
+      displayName: 'helm uninstall blue'
+      inputs:
+        connectionType: 'Kubernetes Service Connection'
+        kubernetesServiceConnection: $(K8S_AB_SERVICE_CONNECTION)
+        command: delete
+        arguments: $(blueReleaseName) --namespace $(K8S_AB_NAMESPACE)
diff --git a/.pipelines/azdo-ci-image.yml b/.pipelines/azdo-ci-image.yml
@@ -0,0 +1,36 @@
+pr: none
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - ml_service/util/create_scoring_image.py
+    - ml_service/util/Dockerfile
+    - code/scoring/
+    exclude:
+    - code/scoring/deployment_config_aci.yml
+    - code/scoring/deployment_config_aks.yml    
+
+pool: 
+  vmImage: 'ubuntu-latest'
+
+container: mcr.microsoft.com/mlops/python:latest
+
+variables:
+- group: devopsforai-aml-vg
+- name: 'SCORE_SCRIPT'
+  value: 'scoreB.py'
+
+steps:
+
+- task: AzureCLI@1
+  inputs:
+    azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+    scriptLocation: inlineScript
+    inlineScript: |
+      set -e
+      export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+      python3 $(Build.SourcesDirectory)/ml_service/util/create_scoring_image.py   
+  displayName: 'Create Scoring Image'
+
diff --git a/.pipelines/azdo-helm-install.yml b/.pipelines/azdo-helm-install.yml
@@ -0,0 +1,9 @@
+steps:
+- task: Bash@3
+  displayName: 'Install Helm $(helmVersion)'
+  inputs:
+    targetType: inline
+    script:  wget -q $(helmDownloadURL) -O /tmp/$FILENAME && tar -zxvf /tmp/$FILENAME -C /tmp && sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm
+  env:
+    HELM_VERSION: $(helmVersion)
+    FILENAME: helm-$(helmVersion)-linux-amd64.tar.gz
diff --git a/.pipelines/azdo-helm-upgrade.yml b/.pipelines/azdo-helm-upgrade.yml
@@ -0,0 +1,19 @@
+parameters:
+  chartPath: ''
+  releaseName: '' 
+  overrideValues: ''
+
+steps:
+- template: azdo-helm-install.yml
+- task: HelmDeploy@0
+  displayName: 'helm upgrade'
+  inputs:
+    connectionType: 'Kubernetes Service Connection'
+    kubernetesServiceConnection: $(K8S_AB_SERVICE_CONNECTION)
+    command: upgrade
+    chartType: FilePath
+    chartPath: ${{ parameters.chartPath }}
+    releaseName: ${{ parameters.releaseName }}  
+    overrideValues: ${{ parameters.overrideValues }}
+    install: true
+    arguments: --namespace $(K8S_AB_NAMESPACE)
diff --git a/.pipelines/azdo-variables.yml b/.pipelines/azdo-variables.yml
@@ -37,4 +37,6 @@ variables:
   value: 'mltrained'
   # Optional. Used by a training pipeline with R on Databricks
 - name: DB_CLUSTER_ID
-  value: ''
+  value: ''
+- name: SCORE_SCRIPT
+  value: score.py
diff --git a/charts/abtest-istio/Chart.yaml b/charts/abtest-istio/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: abtest-istio 
+version: 0.1.0
diff --git a/charts/abtest-istio/templates/istio-canary.yaml b/charts/abtest-istio/templates/istio-canary.yaml
@@ -0,0 +1,60 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: mlmodel-gateway
+  namespace: abtesting
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+  - port:
+      number: {{ .Values.ingress.port }}
+      name: http
+      protocol: HTTP
+    hosts:
+    - "*"
+---      
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: mlmodel-virtualservice
+  namespace: abtesting
+spec:
+  gateways:
+    - mlmodel-gateway 
+  hosts:
+    - '*'    
+  http:
+  - match:
+    - uri:
+        prefix: /score
+      headers:
+        x-api-version:
+          exact: 'blue'
+    route:
+    - destination:
+        host: {{ .Values.svc.name }}-blue.abtesting.svc.cluster.local  
+        port:
+          number: {{ .Values.svc.port }}
+  - match:
+    - uri:
+        prefix: /score
+      headers:
+        x-api-version:
+          exact: 'green'
+    route:
+    - destination:
+        host: {{ .Values.svc.name }}-green.abtesting.svc.cluster.local  
+        port:
+          number: {{ .Values.svc.port }}
+  - route:
+    - destination:
+        host: {{ .Values.svc.name }}-green.abtesting.svc.cluster.local  
+        port:
+          number: {{ .Values.svc.port }}
+      weight: {{ .Values.weight.green }}
+    - destination:
+        host: {{ .Values.svc.name }}-blue.abtesting.svc.cluster.local  
+        port:
+          number: {{ .Values.svc.port }}
+      weight: {{ .Values.weight.blue }}
diff --git a/charts/abtest-istio/values.yaml b/charts/abtest-istio/values.yaml
@@ -0,0 +1,15 @@
+ingress:
+  port: 80
+
+svc:
+  port: 5001
+  name: model-svc
+
+
+weight:
+  green: 50
+  blue: 50
+
+uri:
+  prefix: /score
+
diff --git a/charts/abtest-model/Chart.yaml b/charts/abtest-model/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: abtest-model
+version: 0.1.0
diff --git a/charts/abtest-model/templates/deployment.yaml b/charts/abtest-model/templates/deployment.yaml
@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.deployment.name }}
+  namespace: {{ .Values.namespace }}
+  labels:
+    app: {{ .Values.appname }}
+    model_version: {{ .Values.deployment.bluegreen }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.appname }}
+      model_version: {{ .Values.deployment.bluegreen }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.appname }}
+        model_version: {{ .Values.deployment.bluegreen }}
+    spec:
+      containers:
+      - name: {{ .Values.deployment.container.name }}
+        image: "{{ .Values.deployment.image.name }}"
+        imagePullPolicy: Always
+        ports:
+        - name: http
+          containerPort: 5001
+        - name: probe
+          containerPort: 8086
+      imagePullSecrets:
+        - name: aks-secret     
diff --git a/charts/abtest-model/templates/service.yaml b/charts/abtest-model/templates/service.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ .Values.svc.name }}-{{ .Values.deployment.bluegreen }}"
+  namespace: {{ .Values.namespace }}
+spec:
+  selector:
+    app: {{ .Values.appname }}   
+    model_version: {{ .Values.deployment.bluegreen }}
+  ports:
+    - port: {{ .Values.svc.port }}
+      targetPort: {{ .Values.deployment.container.port }}
+
diff --git a/charts/abtest-model/values.yaml b/charts/abtest-model/values.yaml
@@ -0,0 +1,13 @@
+namespace: abtesting
+appname: model
+
+deployment:
+  name: model-green  
+  bluegreen: green
+  container:
+    name: model  
+    port: 5001
+
+svc:
+  name: model-svc
+  port: 5001