diff --git a/.pipelines/azdo-abtest-pipeline.yml b/.pipelines/azdo-abtest-pipeline.yml new file mode 100644 index 00000000..a74cf330 --- /dev/null +++ b/.pipelines/azdo-abtest-pipeline.yml @@ -0,0 +1,162 @@ +pr: none +trigger: + branches: + include: + - master + paths: + exclude: + - docs/ + - environment_setup/ + - ml_service/util/create_scoring_image.* + - ml_service/util/smoke_test_scoring_service.py + +variables: +- group: 'devopsforai-aml-vg' +- name: 'helmVersion' + value: 'v3.0.1' +- name: 'helmDownloadURL' + value: 'https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz' +- name: 'blueReleaseName' + value: 'model-blue' +- name: 'greenReleaseName' + value: 'model-green' +- name: 'SCORE_SCRIPT' + value: 'scoreA.py' + + +stages: +- stage: 'Building' + jobs: + - job: "Build_Scoring_image" + timeoutInMinutes: 0 + pool: + vmImage: 'ubuntu-latest' + container: mcr.microsoft.com/mlops/python:latest + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt + displayName: 'Create Scoring Image' + name: 'buildscoringimage' + + - publish: image_location.txt + artifact: image_location + + - publish: $(System.DefaultWorkingDirectory)/charts + artifact: allcharts + + +- stage: 'Blue_Staging' + jobs: + - deployment: "Deploy_to_Staging" + timeoutInMinutes: 0 + environment: abtestenv + strategy: + runOnce: + deploy: + steps: + - script: | + IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)" + echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" + displayName: 'Get Image Location' + - template: azdo-helm-upgrade.yml + parameters: + chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model' + releaseName: $(blueReleaseName) + overrideValues: 'deployment.name=$(blueReleaseName),deployment.bluegreen=blue,deployment.image.name=$(IMAGE_LOCATION)' + +- stage: 'Blue_50' + jobs: + - job: 'Blue_Rollout_50' + displayName: 50 50 rollout to blue environment + timeoutInMinutes: 0 + steps: + - template: azdo-helm-upgrade.yml + parameters: + chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio' + releaseName: 'abtest-istio' + overrideValues: 'weight.blue=50,weight.green=50' + +- stage: 'Blue_100' + jobs: + - deployment: 'blue_Rollout_100' + timeoutInMinutes: 0 + environment: abtestenv + strategy: + runOnce: + deploy: + steps: + - template: azdo-helm-upgrade.yml + parameters: + chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio' + releaseName: 'abtest-istio' + overrideValues: 'weight.blue=100,weight.green=0' + +- stage: 'Rollback' + dependsOn: 'Blue_100' + condition: failed() + jobs: + - deployment: 'Roll_Back' + displayName: 'Roll Back after failure' + environment: abtestenv + strategy: + runOnce: + deploy: + steps: + - template: azdo-helm-upgrade.yml + parameters: + chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio' + releaseName: 'abtest-istio' + overrideValues: 'weight.blue=0,weight.green=100' + +- stage: 'Set_Production_Tag' + dependsOn: 'Blue_100' + condition: succeeded() + jobs: + - deployment: 'green_blue_tagging' + timeoutInMinutes: 0 + environment: abtestenv + strategy: + runOnce: + deploy: + steps: + - script: | + IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)" + echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" + displayName: 'Get Image Location' + - template: azdo-helm-upgrade.yml + parameters: + chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model' + releaseName: $(greenReleaseName) + overrideValues: 'deployment.name=$(greenReleaseName),deployment.bluegreen=green,deployment.image.name=$(IMAGE_LOCATION)' + +- stage: 'Green_100' + jobs: + - job: 'Prod_Rollout_100' + timeoutInMinutes: 0 + steps: + - template: azdo-helm-upgrade.yml + parameters: + chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio' + releaseName: 'abtest-istio' + overrideValues: 'weight.blue=0,weight.green=100' + +- stage: 'Disable_blue' + condition: always() + jobs: + - job: 'blue_disable' + timeoutInMinutes: 0 + steps: + - template: azdo-helm-install.yml + - task: HelmDeploy@0 + displayName: 'helm uninstall blue' + inputs: + connectionType: 'Kubernetes Service Connection' + kubernetesServiceConnection: $(K8S_AB_SERVICE_CONNECTION) + command: delete + arguments: $(blueReleaseName) --namespace $(K8S_AB_NAMESPACE) diff --git a/.pipelines/azdo-ci-image.yml b/.pipelines/azdo-ci-image.yml new file mode 100644 index 00000000..b5375ad5 --- /dev/null +++ b/.pipelines/azdo-ci-image.yml @@ -0,0 +1,36 @@ +pr: none +trigger: + branches: + include: + - master + paths: + include: + - ml_service/util/create_scoring_image.py + - ml_service/util/Dockerfile + - code/scoring/ + exclude: + - code/scoring/deployment_config_aci.yml + - code/scoring/deployment_config_aks.yml + +pool: + vmImage: 'ubuntu-latest' + +container: mcr.microsoft.com/mlops/python:latest + +variables: +- group: devopsforai-aml-vg +- name: 'SCORE_SCRIPT' + value: 'scoreB.py' + +steps: + +- task: AzureCLI@1 + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python3 $(Build.SourcesDirectory)/ml_service/util/create_scoring_image.py + displayName: 'Create Scoring Image' + diff --git a/.pipelines/azdo-helm-install.yml b/.pipelines/azdo-helm-install.yml new file mode 100644 index 00000000..fc5eac6f --- /dev/null +++ b/.pipelines/azdo-helm-install.yml @@ -0,0 +1,9 @@ +steps: +- task: Bash@3 + displayName: 'Install Helm $(helmVersion)' + inputs: + targetType: inline + script: wget -q $(helmDownloadURL) -O /tmp/$FILENAME && tar -zxvf /tmp/$FILENAME -C /tmp && sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm + env: + HELM_VERSION: $(helmVersion) + FILENAME: helm-$(helmVersion)-linux-amd64.tar.gz diff --git a/.pipelines/azdo-helm-upgrade.yml b/.pipelines/azdo-helm-upgrade.yml new file mode 100644 index 00000000..949ddde7 --- /dev/null +++ b/.pipelines/azdo-helm-upgrade.yml @@ -0,0 +1,19 @@ +parameters: + chartPath: '' + releaseName: '' + overrideValues: '' + +steps: +- template: azdo-helm-install.yml +- task: HelmDeploy@0 + displayName: 'helm upgrade' + inputs: + connectionType: 'Kubernetes Service Connection' + kubernetesServiceConnection: $(K8S_AB_SERVICE_CONNECTION) + command: upgrade + chartType: FilePath + chartPath: ${{ parameters.chartPath }} + releaseName: ${{ parameters.releaseName }} + overrideValues: ${{ parameters.overrideValues }} + install: true + arguments: --namespace $(K8S_AB_NAMESPACE) diff --git a/.pipelines/azdo-variables.yml b/.pipelines/azdo-variables.yml index 5d7da750..fcf67c2b 100644 --- a/.pipelines/azdo-variables.yml +++ b/.pipelines/azdo-variables.yml @@ -37,4 +37,6 @@ variables: value: 'mltrained' # Optional. Used by a training pipeline with R on Databricks - name: DB_CLUSTER_ID - value: '' \ No newline at end of file + value: '' +- name: SCORE_SCRIPT + value: score.py \ No newline at end of file diff --git a/charts/abtest-istio/Chart.yaml b/charts/abtest-istio/Chart.yaml new file mode 100644 index 00000000..bfcf8584 --- /dev/null +++ b/charts/abtest-istio/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: abtest-istio +version: 0.1.0 diff --git a/charts/abtest-istio/templates/istio-canary.yaml b/charts/abtest-istio/templates/istio-canary.yaml new file mode 100644 index 00000000..a030fd0d --- /dev/null +++ b/charts/abtest-istio/templates/istio-canary.yaml @@ -0,0 +1,60 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: mlmodel-gateway + namespace: abtesting +spec: + selector: + istio: ingressgateway + servers: + - port: + number: {{ .Values.ingress.port }} + name: http + protocol: HTTP + hosts: + - "*" +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: mlmodel-virtualservice + namespace: abtesting +spec: + gateways: + - mlmodel-gateway + hosts: + - '*' + http: + - match: + - uri: + prefix: /score + headers: + x-api-version: + exact: 'blue' + route: + - destination: + host: {{ .Values.svc.name }}-blue.abtesting.svc.cluster.local + port: + number: {{ .Values.svc.port }} + - match: + - uri: + prefix: /score + headers: + x-api-version: + exact: 'green' + route: + - destination: + host: {{ .Values.svc.name }}-green.abtesting.svc.cluster.local + port: + number: {{ .Values.svc.port }} + - route: + - destination: + host: {{ .Values.svc.name }}-green.abtesting.svc.cluster.local + port: + number: {{ .Values.svc.port }} + weight: {{ .Values.weight.green }} + - destination: + host: {{ .Values.svc.name }}-blue.abtesting.svc.cluster.local + port: + number: {{ .Values.svc.port }} + weight: {{ .Values.weight.blue }} \ No newline at end of file diff --git a/charts/abtest-istio/values.yaml b/charts/abtest-istio/values.yaml new file mode 100644 index 00000000..014845bc --- /dev/null +++ b/charts/abtest-istio/values.yaml @@ -0,0 +1,15 @@ +ingress: + port: 80 + +svc: + port: 5001 + name: model-svc + + +weight: + green: 50 + blue: 50 + +uri: + prefix: /score + diff --git a/charts/abtest-model/Chart.yaml b/charts/abtest-model/Chart.yaml new file mode 100644 index 00000000..eeaa24bf --- /dev/null +++ b/charts/abtest-model/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: abtest-model +version: 0.1.0 diff --git a/charts/abtest-model/templates/deployment.yaml b/charts/abtest-model/templates/deployment.yaml new file mode 100644 index 00000000..13572b36 --- /dev/null +++ b/charts/abtest-model/templates/deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.deployment.name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Values.appname }} + model_version: {{ .Values.deployment.bluegreen }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.appname }} + model_version: {{ .Values.deployment.bluegreen }} + template: + metadata: + labels: + app: {{ .Values.appname }} + model_version: {{ .Values.deployment.bluegreen }} + spec: + containers: + - name: {{ .Values.deployment.container.name }} + image: "{{ .Values.deployment.image.name }}" + imagePullPolicy: Always + ports: + - name: http + containerPort: 5001 + - name: probe + containerPort: 8086 + imagePullSecrets: + - name: aks-secret diff --git a/charts/abtest-model/templates/service.yaml b/charts/abtest-model/templates/service.yaml new file mode 100644 index 00000000..a4a6ed8b --- /dev/null +++ b/charts/abtest-model/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: "{{ .Values.svc.name }}-{{ .Values.deployment.bluegreen }}" + namespace: {{ .Values.namespace }} +spec: + selector: + app: {{ .Values.appname }} + model_version: {{ .Values.deployment.bluegreen }} + ports: + - port: {{ .Values.svc.port }} + targetPort: {{ .Values.deployment.container.port }} + \ No newline at end of file diff --git a/charts/abtest-model/values.yaml b/charts/abtest-model/values.yaml new file mode 100644 index 00000000..c3ab1b60 --- /dev/null +++ b/charts/abtest-model/values.yaml @@ -0,0 +1,13 @@ +namespace: abtesting +appname: model + +deployment: + name: model-green + bluegreen: green + container: + name: model + port: 5001 + +svc: + name: model-svc + port: 5001 \ No newline at end of file diff --git a/charts/load_test.sh b/charts/load_test.sh new file mode 100755 index 00000000..25a06452 --- /dev/null +++ b/charts/load_test.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +for ((i=1;i<=$1;i++)) +do + curl --header "x-api-version: $3" $2 + echo + sleep .2 +done \ No newline at end of file diff --git a/code/scoring/scoreA.py b/code/scoring/scoreA.py new file mode 100644 index 00000000..ac4a6100 --- /dev/null +++ b/code/scoring/scoreA.py @@ -0,0 +1,6 @@ +def init(): + global model + + +def run(raw_data): + return "New Model A" diff --git a/code/scoring/scoreB.py b/code/scoring/scoreB.py new file mode 100644 index 00000000..c0865269 --- /dev/null +++ b/code/scoring/scoreB.py @@ -0,0 +1,6 @@ +def init(): + global model + + +def run(raw_data): + return "New Model B" diff --git a/docs/canary_ab_deployment.md b/docs/canary_ab_deployment.md new file mode 100644 index 00000000..b114a308 --- /dev/null +++ b/docs/canary_ab_deployment.md @@ -0,0 +1,106 @@ +## Model deployment to AKS cluster with Canary deployemnt + +[![Build Status](https://aidemos.visualstudio.com/MLOps/_apis/build/status/microsoft.MLOpsPython-Canary?branchName=master)](https://aidemos.visualstudio.com/MLOps/_build/latest?definitionId=133&branchName=master) + +If your target deployment environment is a K8s cluster and you want to implement [Canary and/or A/B testing deployemnt strategies](http://adfpractice-fedor.blogspot.com/2019/04/deployment-strategies-with-kubernetes.html) you can follow this sample guidance. + +**Note:** It is assumed that you have an AKS instance and configured ***kubectl*** to communicate with the cluster. + +#### 1. Install Istio on a K8s cluster. + +This guidance uses [Istio](https://istio.io) service mesh implememtation to control traffic routing between model versions. The instruction on installing Istio is available [here](https://docs.microsoft.com/en-us/azure/aks/servicemesh-istio-install?pivots=client-operating-system-linux). + +Having the Istio installed, figure out the Istio gateway endpoint on your K8s cluster: + +```bash +GATEWAY_IP=$(kubectl get svc istio-ingressgateway -n istio-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +``` + +#### 2. Set up variables + +There are some extra variables that you need to setup in ***devopsforai-aml-vg*** variable group (see [getting started](./getting_started.md)): + +| Variable Name | Suggested Value | +| --------------------------- | -----------------------------------------------------| +| K8S_AB_SERVICE_CONNECTION | AzDo service connection to a K8s cluster | +| K8S_AB_NAMESPACE | Namespace in a K8s cluster to deploy the model | +| IMAGE_REPO_NAME | Image reposiory name (e.g. mlopspyciamlcr.azurecr.io)| + + +#### 3. Configure a pipeline to build and deploy a scoring Image + +Import and run the [azdo-abtest-pipeline.yml](./.pipelines/azdo-abtest-pipeline.yml) multistage deployment pipeline. + +The result of the pipeline will be a registered Docker image in the ACR repository attached to the AML Service: + +![scoring image](./images/scoring_image.png) + +The pipeline will also deploy the scoring image to the Kubernetes cluster. + +```bash +kubectl get deployments --namespace abtesting +NAME READY UP-TO-DATE AVAILABLE AGE +model-green 1/1 1 1 19h +``` + +#### 4. Build a new Scoring Image. + +Change value of the ***SCORE_SCRIPT*** variable in the [azdo-abtest-pipeline.yml](./.pipelines/azdo-abtest-pipeline.yml) to point to ***scoreA.py*** and merge it to the master branch. + +**Note:** ***scoreA.py*** and ***scoreB.py*** files used in this tutorial are just mockups returning either "New Model A" or "New Model B" respectively. They are used to demonstrate the concept of testing two scoring images with different models or scoring code. In real life you would implement a scoring file similar to [score.py](./../code/scoring/score.py) (see [getting started](./getting_started.md)). + +It will automatically trigger the pipeline and deploy a new scoring image with the following stages implementing ***Canary*** deployment strategy: + +| Stage | Green Weight| Blue Weight| Description | +| ------------------- |-------------|------------|-----------------------------------------------------------------| +| Blue_0 |100 |0 |New image (blue) is deployed.
But all traffic (100%) is still routed to the old (green) image.| +| Blue_50 |50 |50 |Traffic is split between old (green) and new (blue) images 50/50.| +| Blue_100 |0 |100 |All traffic (100%) is routed to the blue image.| +| Blue_Green |0 |100 |Old green image is removed. The new blue image is copied as green.
Blue and Green images are equal.
All traffic (100%) is routed to the blue image.| +| Green_100 |100 |0 |All traffic (100%) is routed to the green image.
The blue image is removed + + +**Note:** The pipeline performs the rollout without any pausing. You may want to configure [Approvals and Checks](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/approvals?view=azure-devops&tabs=check-pass) for the stages on your environment for better experience of the model testing. The environment ***abtestenv*** will be added automatically to your AzDo project after the first pipeline run. + +At each stage you can verify how the traffic is routed sending requests to $GATEWAY_IP/score with ***Postman*** or with ***curl***: + +```bash +curl $GATEWAY_IP/score +``` + +You can also emulate a simple load test on the gateway with the ***load_test.sh***: + +```bash +./charts/load_test.sh 10 $GATEWAY_IP/score +``` + +The command above sends 10 requests to the gateway. So if the pipeline has completted stage Blue_50, the result will look like this: + +```bash +"New Model A" +"New Model A" +"New Model A" +"New Model B" +"New Model A" +"New Model B" +"New Model B" +"New Model A" +"New Model A" +"New Model A" +``` + +Despite what blue/green weights are configured now on the cluster, you can perform ***A/B testing*** and send requests directly to either blue or green images: + +```bash +curl --header "x-api-version: blue" $GATEWAY_IP/score +curl --header "x-api-version: green" $GATEWAY_IP/score +``` + +or with the load_test.sh: + +```bash +./charts/load_test.sh 10 $GATEWAY_IP/score blue +./charts/load_test.sh 10 $GATEWAY_IP/score green +``` + +In this case the Istio Virtual Service analyzes the request header and routes the traffic directly to the specified model version. diff --git a/docs/getting_started.md b/docs/getting_started.md index b5e92902..b24545f4 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -199,6 +199,8 @@ The final stage is to deploy the model to the production environment running on tutorial, but you can find set up information in the docs [here](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal#create-an-aks-cluster). +**Note:** If your target deployment environment is a K8s cluster and you want to implement Canary and/or A/B testing deployemnt strategies check out this [tutorial](./canary_ab_deployment.md). + In the Variables tab, edit your variable group (`devopsforai-aml-vg`). In the variable group definition, add the following variables: | Variable Name | Suggested Value | diff --git a/docs/images/scoring_image.png b/docs/images/scoring_image.png new file mode 100644 index 00000000..ecb1c245 Binary files /dev/null and b/docs/images/scoring_image.png differ diff --git a/ml_service/util/create_scoring_image.py b/ml_service/util/create_scoring_image.py index 0968b6c4..edc57bf1 100644 --- a/ml_service/util/create_scoring_image.py +++ b/ml_service/util/create_scoring_image.py @@ -29,7 +29,7 @@ os.chdir("./code/scoring") image_config = ContainerImage.image_configuration( - execution_script="score.py", + execution_script=e.score_script, runtime="python", conda_file="conda_dependencies.yml", description="Image with ridge regression model", diff --git a/ml_service/util/env_variables.py b/ml_service/util/env_variables.py index 5ed59866..ed7f25f3 100644 --- a/ml_service/util/env_variables.py +++ b/ml_service/util/env_variables.py @@ -38,6 +38,7 @@ def __init__(self): self._image_name = os.environ.get('IMAGE_NAME') self._model_path = os.environ.get('MODEL_PATH') self._db_cluster_id = os.environ.get("DB_CLUSTER_ID") + self._score_script = os.environ.get("SCORE_SCRIPT") @property def workspace_name(self): @@ -130,3 +131,7 @@ def image_name(self): @property def model_path(self): return self._model_path + + @property + def score_script(self): + return self._score_script