- Azure subscription
- Azure CLI 2.0
- Docker
- Helm 2.12.3 or later
- JQ
Note: in linux systems, it is possible to run the docker command without prefacing with sudo. For more information, please refer to the Post-installation steps for linux
Clone or download this repo locally.
git clone https://github.com/mspnp/microservices-reference-implementation.git
The deployment steps shown here use Bash shell commands. On Windows, you can use the Windows Subsystem for Linux to run Bash.
the SSH rsa key pair can be generated using ssh-keygen, among other tools, on Linux, Mac, or Windows. If you already have an ~/.ssh/id_rsa.pub file, you could provide the same later on. If you need to create an SSH key pair, see How to create and use an SSH key pair.
Note: the SSH rsa public key will be requested when deploying your Kubernetes cluster in Azure.
Set environment variables.
export SSH_PUBLIC_KEY_FILE=[YOUR_RECENTLY_GENERATED_SSH_RSA_PUBLIC_KEY_FILE_HERE]
export LOCATION=[YOUR_LOCATION_HERE]
export RESOURCE_GROUP=[YOUR_RESOURCE_GROUP_HERE]
export SUBSCRIPTION_ID=$(az account show --query id --output tsv)
export SUBSCRIPTION_NAME=$(az account show --query name --output tsv)
export TENANT_ID=$(az account show --query tenantId --output tsv)
export PROJECT_ROOT=./microservices-reference-implementation
export K8S=$PROJECT_ROOT/k8s
export HELM_CHARTS=$PROJECT_ROOT/charts
Infrastructure Prerequisites
# Log in to Azure
az login
# Create service principal for AKS
export SP_DETAILS=$(az ad sp create-for-rbac --role="Contributor" -o json) && \
export SP_APP_ID=$(echo $SP_DETAILS | jq ".appId" -r) && \
export SP_CLIENT_SECRET=$(echo $SP_DETAILS | jq ".password" -r) && \
export SP_OBJECT_ID=$(az ad sp show --id $SP_APP_ID -o tsv --query objectId)
Add CI/CD to Drone Delivery using Azure Pipelines with YAML.
Important: If you don't want to set up the CI/CD pipelines, you can manually deploy the application for development as follows.
Note: this deployment might take up to 20 minutes
Infrastructure
# Deploy the resource groups and managed identities
# These are deployed first in a separate template to avoid propagation delays with AAD
az deployment create \
--name azuredeploy-prereqs-dev \
--location $LOCATION \
--template-file ${PROJECT_ROOT}/azuredeploy-prereqs.json \
--parameters resourceGroupName=$RESOURCE_GROUP \
resourceGroupLocation=$LOCATION
export IDENTITIES_DEPLOYMENT_NAME=$(az deployment show -n azuredeploy-prereqs-dev --query properties.outputs.identitiesDeploymentName.value -o tsv) && \
export DELIVERY_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.deliveryIdName.value -o tsv) && \
export DELIVERY_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DELIVERY_ID_NAME --query principalId -o tsv) && \
export DRONESCHEDULER_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerIdName.value -o tsv) && \
export DRONESCHEDULER_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query principalId -o tsv) && \
export WORKFLOW_ID_NAME=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowIdName.value -o tsv) && \
export WORKFLOW_ID_PRINCIPAL_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query principalId -o tsv) && \
export RESOURCE_GROUP_ACR=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.acrResourceGroupName.value -o tsv)
# Wait for AAD propagation
until az ad sp show --id ${DELIVERY_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done
until az ad sp show --id ${DRONESCHEDULER_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done
until az ad sp show --id ${WORKFLOW_ID_PRINCIPAL_ID} &> /dev/null ; do echo "Waiting for AAD propagation" && sleep 5; done
# Export the kubernetes cluster version
export KUBERNETES_VERSION=$(az aks get-versions -l $LOCATION --query "orchestrators[?default!=null].orchestratorVersion" -o tsv)
# Deploy all other resources
az group deployment create -g $RESOURCE_GROUP --name azuredeploy-dev --template-file ${PROJECT_ROOT}/azuredeploy.json \
--parameters servicePrincipalClientId=${SP_APP_ID} \
servicePrincipalClientSecret=${SP_CLIENT_SECRET} \
servicePrincipalId=${SP_OBJECT_ID} \
kubernetesVersion=${KUBERNETES_VERSION} \
sshRSAPublicKey="$(cat ${SSH_PUBLIC_KEY_FILE})" \
deliveryIdName=${DELIVERY_ID_NAME} \
deliveryPrincipalId=${DELIVERY_ID_PRINCIPAL_ID} \
droneSchedulerIdName=${DRONESCHEDULER_ID_NAME} \
droneSchedulerPrincipalId=${DRONESCHEDULER_ID_PRINCIPAL_ID} \
workflowIdName=${WORKFLOW_ID_NAME} \
workflowPrincipalId=${WORKFLOW_ID_PRINCIPAL_ID} \
acrResourceGroupName=${RESOURCE_GROUP_ACR} \
acrResourceGroupLocation=$LOCATION
Get outputs from Azure Deploy
# Shared
export ACR_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.acrName.value -o tsv) && \
export ACR_SERVER=$(az acr show -n $ACR_NAME --query loginServer -o tsv) && \
export CLUSTER_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.aksClusterName.value -o tsv)
Enable Azure Monitoring for the AKS cluster
az aks enable-addons -a monitoring --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME
Download kubectl and create a k8s namespace
# Install kubectl
sudo az aks install-cli
# Get the Kubernetes cluster credentials
az aks get-credentials --resource-group=$RESOURCE_GROUP --name=$CLUSTER_NAME
# Create namespaces
kubectl create namespace backend-dev
Setup Helm in the container
kubectl apply -f $K8S/tiller-rbac.yaml
helm init --service-account tiller
Integrate Application Insights instance
# Acquire Instrumentation Key
export AI_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.appInsightsName.value -o tsv)
export AI_IKEY=$(az resource show \
-g $RESOURCE_GROUP \
-n $AI_NAME \
--resource-type "Microsoft.Insights/components" \
--query properties.InstrumentationKey \
-o tsv)
# add RBAC for AppInsights
kubectl apply -f $K8S/k8s-rbac-ai.yaml
Complete instructions can be found at https://github.com/Azure/kubernetes-keyvault-flexvol
Note: the tested nmi version was 1.4. It enables namespaced pod identity.
# setup AAD pod identity
kubectl create -f https://raw.githubusercontent.com/Azure/aad-pod-identity/master/deploy/infra/deployment-rbac.yaml
kubectl create -f https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml
# Deploy the ngnix ingress controller
helm install stable/nginx-ingress --name nginx-ingress-dev --namespace ingress-controllers --set rbac.create=true --set controller.ingressClass=nginx-dev
# Obtain the load balancer ip address and assign a domain name
until export INGRESS_LOAD_BALANCER_IP=$(kubectl get services/nginx-ingress-dev-controller -n ingress-controllers -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2> /dev/null) && test -n "$INGRESS_LOAD_BALANCER_IP"; do echo "Waiting for load balancer deployment" && sleep 20; done
export INGRESS_LOAD_BALANCER_IP_ID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$INGRESS_LOAD_BALANCER_IP')].[id]" --output tsv)
export EXTERNAL_INGEST_DNS_NAME="${RESOURCE_GROUP}-ingest-dev"
export EXTERNAL_INGEST_FQDN=$(az network public-ip update --ids $INGRESS_LOAD_BALANCER_IP_ID --dns-name $EXTERNAL_INGEST_DNS_NAME --query "dnsSettings.fqdn" --output tsv)
# Create a self-signed certificate for TLS
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
-out ingestion-ingress-tls.crt \
-keyout ingestion-ingress-tls.key \
-subj "/CN=${EXTERNAL_INGEST_FQDN}/O=fabrikam"
kubectl apply -f $K8S/k8s-resource-quotas-dev.yaml
Extract resource details from deployment
export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.deliveryCosmosDbName.value -o tsv) && \
export DATABASE_NAME="${COSMOSDB_NAME}-db" && \
export COLLECTION_NAME="${DATABASE_NAME}-col" && \
export DELIVERY_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.deliveryKeyVaultUri.value -o tsv)
Build the Delivery service
export DELIVERY_PATH=$PROJECT_ROOT/src/shipping/delivery
Build and publish the container image
# Build the Docker image
docker build --pull --compress -t $ACR_SERVER/delivery:0.1.0 $DELIVERY_PATH/.
# Push the image to ACR
az acr login --name $ACR_NAME
docker push $ACR_SERVER/delivery:0.1.0
Deploy the Delivery service:
# Extract pod identity outputs from deployment
export DELIVERY_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.deliveryPrincipalResourceId.value -o tsv) && \
export DELIVERY_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $DELIVERY_ID_NAME --query clientId -o tsv)
export DELIVERY_INGRESS_TLS_SECRET_NAME=delivery-ingress-tls
# Deploy the service
helm install $HELM_CHARTS/delivery/ \
--set image.tag=0.1.0 \
--set image.repository=delivery \
--set dockerregistry=$ACR_SERVER \
--set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \
--set ingress.hosts[0].serviceName=delivery \
--set ingress.hosts[0].tls=true \
--set ingress.hosts[0].tlsSecretName=$DELIVERY_INGRESS_TLS_SECRET_NAME \
--set ingress.tls.secrets[0].name=$DELIVERY_INGRESS_TLS_SECRET_NAME \
--set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \
--set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \
--set identity.clientid=$DELIVERY_PRINCIPAL_CLIENT_ID \
--set identity.resourceid=$DELIVERY_PRINCIPAL_RESOURCE_ID \
--set cosmosdb.id=$DATABASE_NAME \
--set cosmosdb.collectionid=$COLLECTION_NAME \
--set keyvault.uri=$DELIVERY_KEYVAULT_URI \
--set reason="Initial deployment" \
--set tags.dev=true \
--namespace backend-dev \
--name delivery-v0.1.0-dev \
--dep-up
# Verify the pod is created
helm status delivery-v0.1.0-dev
Extract resource details from deployment
export COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.packageMongoDbName.value -o tsv)
Build the Package service
export PACKAGE_PATH=$PROJECT_ROOT/src/shipping/package
# Build the docker image
docker build -f $PACKAGE_PATH/Dockerfile -t $ACR_SERVER/package:0.1.0 $PACKAGE_PATH
# Push the docker image to ACR
az acr login --name $ACR_NAME
docker push $ACR_SERVER/package:0.1.0
Deploy the Package service
# Create secret
# Note: Connection strings cannot be exported as outputs in ARM deployments
export COSMOSDB_CONNECTION=$(az cosmosdb list-connection-strings --name $COSMOSDB_NAME --resource-group $RESOURCE_GROUP --query "connectionStrings[0].connectionString" -o tsv | sed 's/==/%3D%3D/g') && \
export COSMOSDB_COL_NAME=packages
# Deploy service
helm install $HELM_CHARTS/package/ \
--set image.tag=0.1.0 \
--set image.repository=package \
--set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \
--set ingress.hosts[0].serviceName=package \
--set ingress.hosts[0].tls=false \
--set secrets.appinsights.ikey=$AI_IKEY \
--set secrets.mongo.pwd=$COSMOSDB_CONNECTION \
--set cosmosDb.collectionName=$COSMOSDB_COL_NAME \
--set dockerregistry=$ACR_SERVER \
--set reason="Initial deployment" \
--set tags.dev=true \
--namespace backend-dev \
--name package-v0.1.0-dev \
--dep-up
# Verify the pod is created
helm status package-v0.1.0-dev
Extract resource details from deployment
export WORKFLOW_KEYVAULT_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.workflowKeyVaultName.value -o tsv)
Build the workflow service
export WORKFLOW_PATH=$PROJECT_ROOT/src/shipping/workflow
# Build the Docker image
docker build --pull --compress -t $ACR_SERVER/workflow:0.1.0 $WORKFLOW_PATH/.
# Push the image to ACR
az acr login --name $ACR_NAME
docker push $ACR_SERVER/workflow:0.1.0
Create and set up pod identity
# Extract outputs from deployment
export WORKFLOW_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.workflowPrincipalResourceId.value -o tsv) && \
export WORKFLOW_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $WORKFLOW_ID_NAME --query clientId -o tsv)
Deploy the Workflow service:
# Deploy the service
helm install $HELM_CHARTS/workflow/ \
--set image.tag=0.1.0 \
--set image.repository=workflow \
--set dockerregistry=$ACR_SERVER \
--set identity.clientid=$WORKFLOW_PRINCIPAL_CLIENT_ID \
--set identity.resourceid=$WORKFLOW_PRINCIPAL_RESOURCE_ID \
--set keyvault.name=$WORKFLOW_KEYVAULT_NAME \
--set keyvault.resourcegroup=$RESOURCE_GROUP \
--set keyvault.subscriptionid=$SUBSCRIPTION_ID \
--set keyvault.tenantid=$TENANT_ID \
--set reason="Initial deployment" \
--set tags.dev=true \
--namespace backend-dev \
--name workflow-v0.1.0-dev \
--dep-up
# Verify the pod is created
helm status workflow-v0.1.0-dev
Extract resource details from deployment
export INGESTION_QUEUE_NAMESPACE=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.ingestionQueueNamespace.value -o tsv) && \
export INGESTION_QUEUE_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.ingestionQueueName.value -o tsv)
export INGESTION_ACCESS_KEY_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.ingestionServiceAccessKeyName.value -o tsv)
export INGESTION_ACCESS_KEY_VALUE=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --namespace-name $INGESTION_QUEUE_NAMESPACE --name $INGESTION_ACCESS_KEY_NAME --query primaryKey -o tsv)
Build the Ingestion service
export INGESTION_PATH=$PROJECT_ROOT/src/shipping/ingestion
# Build the docker image
docker build -f $INGESTION_PATH/Dockerfile -t $ACR_SERVER/ingestion:0.1.0 $INGESTION_PATH
# Push the docker image to ACR
az acr login --name $ACR_NAME
docker push $ACR_SERVER/ingestion:0.1.0
Deploy the Ingestion service
# Set secreat name
export INGRESS_TLS_SECRET_NAME=ingestion-ingress-tls
# Deploy service
helm install $HELM_CHARTS/ingestion/ \
--set image.tag=0.1.0 \
--set image.repository=ingestion \
--set dockerregistry=$ACR_SERVER \
--set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \
--set ingress.hosts[0].serviceName=ingestion \
--set ingress.hosts[0].tls=true \
--set ingress.hosts[0].tlsSecretName=$INGRESS_TLS_SECRET_NAME \
--set ingress.tls.secrets[0].name=$INGRESS_TLS_SECRET_NAME \
--set ingress.tls.secrets[0].key="$(cat ingestion-ingress-tls.key)" \
--set ingress.tls.secrets[0].certificate="$(cat ingestion-ingress-tls.crt)" \
--set secrets.appinsights.ikey=${AI_IKEY} \
--set secrets.queue.keyname=IngestionServiceAccessKey \
--set secrets.queue.keyvalue=${INGESTION_ACCESS_KEY_VALUE} \
--set secrets.queue.name=${INGESTION_QUEUE_NAME} \
--set secrets.queue.namespace=${INGESTION_QUEUE_NAMESPACE} \
--set reason="Initial deployment" \
--set tags.dev=true \
--namespace backend-dev \
--name ingestion-v0.1.0-dev \
--dep-up
# Verify the pod is created
helm status ingestion-v0.1.0-dev
Extract resource details from deployment
export DRONESCHEDULER_KEYVAULT_URI=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.droneSchedulerKeyVaultUri.value -o tsv)
export DRONESCHEDULER_COSMOSDB_NAME=$(az group deployment show -g $RESOURCE_GROUP -n azuredeploy-dev --query properties.outputs.droneSchedulerCosmosDbName.value -o tsv) && \
export ENDPOINT_URL=$(az cosmosdb show -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query documentEndpoint -o tsv) && \
export AUTH_KEY=$(az cosmosdb list-keys -n $DRONESCHEDULER_COSMOSDB_NAME -g $RESOURCE_GROUP --query primaryMasterKey -o tsv) && \
export DATABASE_NAME="invoicing" && \
export COLLECTION_NAME="utilization"
Build the dronescheduler services
export DRONE_PATH=$PROJECT_ROOT/src/shipping/dronescheduler
Create and set up pod identity
# Extract outputs from deployment
export DRONESCHEDULER_PRINCIPAL_RESOURCE_ID=$(az group deployment show -g $RESOURCE_GROUP -n $IDENTITIES_DEPLOYMENT_NAME --query properties.outputs.droneSchedulerPrincipalResourceId.value -o tsv) && \
export DRONESCHEDULER_PRINCIPAL_CLIENT_ID=$(az identity show -g $RESOURCE_GROUP -n $DRONESCHEDULER_ID_NAME --query clientId -o tsv)
Build and publish the container image
# Build the Docker image
docker build -f $DRONE_PATH/Dockerfile -t $ACR_SERVER/dronescheduler:0.1.0 $DRONE_PATH/../
# Push the images to ACR
az acr login --name $ACR_NAME
docker push $ACR_SERVER/dronescheduler:0.1.0
Deploy the dronescheduler service:
# Deploy the service
helm install $HELM_CHARTS/dronescheduler/ \
--set image.tag=0.1.0 \
--set image.repository=dronescheduler \
--set dockerregistry=$ACR_SERVER \
--set ingress.hosts[0].name=$EXTERNAL_INGEST_FQDN \
--set ingress.hosts[0].serviceName=dronescheduler \
--set ingress.hosts[0].tls=false \
--set identity.clientid=$DRONESCHEDULER_PRINCIPAL_CLIENT_ID \
--set identity.resourceid=$DRONESCHEDULER_PRINCIPAL_RESOURCE_ID \
--set keyvault.uri=$DRONESCHEDULER_KEYVAULT_URI \
--set cosmosdb.id=$DATABASE_NAME \
--set cosmosdb.collectionid=$COLLECTION_NAME \
--set reason="Initial deployment" \
--set tags.dev=true \
--namespace backend-dev \
--name dronescheduler-v0.1.0-dev \
--dep-up
# Verify the pod is created
helm status dronescheduler-v0.1.0-dev
You can send delivery requests and check their statuses using curl.
Since the certificate used for TLS is self-signed, the request disables TLS validation using the '-k' option.
curl -X POST "https://$EXTERNAL_INGEST_FQDN/api/deliveryrequests" --header 'Content-Type: application/json' --header 'Accept: application/json' -k -i -d '{
"confirmationRequired": "None",
"deadline": "",
"deliveryId": "mydelivery",
"dropOffLocation": "drop off",
"expedited": true,
"ownerId": "myowner",
"packageInfo": {
"packageId": "mypackage",
"size": "Small",
"tag": "mytag",
"weight": 10
},
"pickupLocation": "my pickup",
"pickupTime": "2019-05-08T20:00:00.000Z"
}'
curl "https://$EXTERNAL_INGEST_FQDN/api/deliveries/mydelivery" --header 'Accept: application/json' -k -i
To run load testing against the solution, follow the steps listed here.
Follow these steps to add logging and monitoring capabilities to the solution.
Deploy Elasticsearch. For more information, see https://github.com/kubernetes/examples/tree/master/staging/elasticsearch
kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/service-account.yaml && \
kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-svc.yaml && \
kubectl --namespace kube-system apply -f https://raw.githubusercontent.com/kubernetes/examples/master/staging/elasticsearch/es-rc.yaml
Deploy Fluentd. For more information, see https://docs.fluentd.org/v0.12/articles/kubernetes-fluentd
# The example elasticsearch yaml files deploy a service named "elasticsearch"
wget https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/fluentd-daemonset-elasticsearch.yaml && \
sed -i "s/elasticsearch-logging/elasticsearch/" fluentd-daemonset-elasticsearch.yaml
# Commenting out X-Pack credentials for demo purposes.
# Make sure to configure X-Pack in elasticsearch and provide credentials here for production workloads
sed -i "s/- name: FLUENT_ELASTICSEARCH_USER/#- name: FLUENT_ELASTICSEARCH_USER/" fluentd-daemonset-elasticsearch.yaml && \
sed -i 's/ value: "elastic"/# value: "elastic"/' fluentd-daemonset-elasticsearch.yaml && \
sed -i "s/- name: FLUENT_ELASTICSEARCH_PASSWORD/#- name: FLUENT_ELASTICSEARCH_PASSWORD/" fluentd-daemonset-elasticsearch.yaml && \
sed -i 's/ value: "changeme"/# value: "changeme"/' fluentd-daemonset-elasticsearch.yaml && \
kubectl --namespace kube-system apply -f fluentd-daemonset-elasticsearch.yaml