From 9dadef4216ae7cf3c1417191394eeba36a046c6a Mon Sep 17 00:00:00 2001 From: "andrebriggs@users.noreply.github.com" Date: Wed, 9 Jan 2019 19:43:05 -0800 Subject: [PATCH 1/8] Fixed issue with retrieving jaeger pod name --- tools/jaeger | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/jaeger b/tools/jaeger index 3421d32..023870c 100755 --- a/tools/jaeger +++ b/tools/jaeger @@ -1,3 +1,3 @@ -export JAEGER_POD=$(kubectl get pods -l "jaeger-component=query" -o jsonpath="{.items[0].metadata.name}") +export JAEGER_POD=$(kubectl get pods -n jaeger -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}" | grep jaeger-query) `sleep 1 && open http://localhost:16686/` & -kubectl port-forward $JAEGER_POD 16686 +kubectl port-forward -n jaeger $JAEGER_POD 16686 From c1ec55317382357041b77cc1435e3e476a312d83 Mon Sep 17 00:00:00 2001 From: Bhargav Nookala Date: Thu, 10 Jan 2019 14:27:51 -0800 Subject: [PATCH 2/8] Better readme instructions (#20) Better readme instructions --- README.md | 8 +++++++- infra/environments/dev/init | 1 + infra/environments/prod/init | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cad7c65..718fbb8 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,15 @@ To deploy a cluster, 1. Ensure you have the [az cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) installed, in your path, and logged in to your subscription. 2. Edit cluster/environments/azure-aks/main.tf and adjust the name of the cluster and, if desired, any of the sizing or network parameters. -3. Deploy the cluster using: +3. Create a service principal (optional - feel free to use an existing service principal) for your cluster using the Azure CLI: `az ad sp create-for-rbac` - note down the `appId` and `password`. + +4. Deploy the cluster using: ``` $ cd cluster/environments/azure-aks +$ export TF_VAR_client_id="" # the `appId` from `az ad sp create-for-rbac`, or some other existing appId +$ export TF_VAR_client_secret="" # the `password` from `az ad sp create-for-rbac` or some other existing service principal's secret. +$ export TF_VAR_ssh_public_key="" # the contents of your ssh public key. $ ./init $ ./apply ``` @@ -78,6 +83,7 @@ $ export TF_VAR_grafana_admin_password="SECRETpass" 5. Deploy the dev configuration: ``` +bash-4.4# helm repo update bash-4.4# cd infra/environments/dev bash-4.4# ./init bash-4.4# ./apply diff --git a/infra/environments/dev/init b/infra/environments/dev/init index f74a9be..237b7d3 100755 --- a/infra/environments/dev/init +++ b/infra/environments/dev/init @@ -4,5 +4,6 @@ rm -rf .terraform rm terraform.tfstate rm terraform.tfstate.backup +helm repo update helm init terraform init diff --git a/infra/environments/prod/init b/infra/environments/prod/init index 0b1ae5c..295d647 100755 --- a/infra/environments/prod/init +++ b/infra/environments/prod/init @@ -4,5 +4,6 @@ rm -rf .terraform rm terraform.tfstate rm terraform.tfstate.backup +helm repo update helm init terraform init -var-file="../common/common.tfvars" \ No newline at end of file From 107d4ddcb1f189c6d505b5092ee883d02ed4687f Mon Sep 17 00:00:00 2001 From: Tim Park Date: Thu, 10 Jan 2019 14:32:22 -0800 Subject: [PATCH 3/8] Remove now obsolete Traefik tool --- tools/traefik | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 tools/traefik diff --git a/tools/traefik b/tools/traefik deleted file mode 100755 index 294029b..0000000 --- a/tools/traefik +++ /dev/null @@ -1,3 +0,0 @@ -export TRAEFIK_POD=$(kubectl get pods -n kube-system -l "app=traefik" -o jsonpath="{.items[0].metadata.name}") -`sleep 1 && open http://localhost:8080` & -kubectl --namespace kube-system port-forward $TRAEFIK_POD 8080 From 702808837b7d1668efe02d2b57e3515d6f2f568c Mon Sep 17 00:00:00 2001 From: nathanielrose Date: Tue, 15 Jan 2019 13:58:08 -0800 Subject: [PATCH 4/8] Random key identifier added to cluster resources for deployments --- cluster/environments/azure-aks/inputs.tf | 3 +++ cluster/providers/azure-acs-engine/acs-engine.tf | 12 ++++++++---- cluster/providers/azure-aks/aks.tf | 16 +++++++++++----- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cluster/environments/azure-aks/inputs.tf b/cluster/environments/azure-aks/inputs.tf index d598515..f43f8eb 100644 --- a/cluster/environments/azure-aks/inputs.tf +++ b/cluster/environments/azure-aks/inputs.tf @@ -17,12 +17,15 @@ variable "aad_tenant_id" { */ variable "client_id" { type = "string" + default = "{ENTER-CLIENT-ID}" } variable "client_secret" { type = "string" + default = "{ENTER-CLIENT-SECRET}" } variable "ssh_public_key" { type = "string" + default = "{ENTER-SSH-PUBLIC-KEY-HERE}" } diff --git a/cluster/providers/azure-acs-engine/acs-engine.tf b/cluster/providers/azure-acs-engine/acs-engine.tf index 29365d4..30c18d7 100644 --- a/cluster/providers/azure-acs-engine/acs-engine.tf +++ b/cluster/providers/azure-acs-engine/acs-engine.tf @@ -1,17 +1,21 @@ +resource "random_integer" "ri" { + min = 10000 + max = 99999 +} resource "azurerm_resource_group" "cluster" { - name = "${var.cluster_name}-rg" + name = "${var.cluster_name}-${random_integer.ri.result}-rg" location = "${var.location}" } resource "azurerm_virtual_network" "cluster" { - name = "${var.cluster_name}-vnet" + name = "${var.cluster_name}-${random_integer.ri.result}-vnet" address_space = ["${var.vnet_address_space}"] location = "${var.location}" resource_group_name = "${azurerm_resource_group.cluster.name}" } resource "azurerm_subnet" "cluster" { - name = "${var.cluster_name}-subnet" + name = "${var.cluster_name}-${random_integer.ri.result}-subnet" resource_group_name = "${azurerm_resource_group.cluster.name}" address_prefix = "${var.subnet_address_space}" virtual_network_name = "${azurerm_virtual_network.cluster.name}" @@ -61,7 +65,7 @@ resource "null_resource" "generate_acs_engine_deployment" { # Locally run the Azure 2.0 CLI to create the resource deployment resource "null_resource" "cluster" { provisioner "local-exec" { - command = "az group deployment create --name ${var.cluster_name} --resource-group ${var.cluster_name}-rg --template-file ./deployment/acs-engine/azuredeploy.json --parameters @./deployment/acs-engine/azuredeploy.parameters.json" + command = "az group deployment create --name ${var.cluster_name}-${random_integer.ri.result} --resource-group ${azurerm_resource_group.cluster.name} --template-file ./deployment/acs-engine/azuredeploy.json --parameters @./deployment/acs-engine/azuredeploy.parameters.json" } depends_on = ["null_resource.generate_acs_engine_deployment"] diff --git a/cluster/providers/azure-aks/aks.tf b/cluster/providers/azure-aks/aks.tf index 3b2381d..74dcff0 100644 --- a/cluster/providers/azure-aks/aks.tf +++ b/cluster/providers/azure-aks/aks.tf @@ -1,22 +1,28 @@ +resource "random_integer" "ri" { + min = 10000 + max = 99999 +} resource "azurerm_resource_group" "cluster" { - name = "${var.cluster_name}-rg" + name = "${var.cluster_name}-${random_integer.ri.result}-rg" location = "${var.location}" } resource "azurerm_virtual_network" "cluster" { - name = "${var.cluster_name}-vnet" + name = "${var.cluster_name}-${random_integer.ri.result}-vnet" address_space = ["${var.vnet_address_space}"] location = "${var.location}" resource_group_name = "${azurerm_resource_group.cluster.name}" } resource "azurerm_subnet" "cluster" { - name = "${var.cluster_name}-subnet" + name = "${var.cluster_name}-${random_integer.ri.result}-subnet" resource_group_name = "${azurerm_resource_group.cluster.name}" address_prefix = "${var.subnet_address_space}" virtual_network_name = "${azurerm_virtual_network.cluster.name}" } + + /* NOTE: Currently you can not enable RBAC without a backing AAD service principal. In the meantime, use the az command line. @@ -69,7 +75,7 @@ resource "azurerm_kubernetes_cluster" "cluster" { resource "null_resource" "create_cluster" { provisioner "local-exec" { - command = "az aks create -g ${azurerm_resource_group.cluster.name} -n ${var.cluster_name} -l ${azurerm_resource_group.cluster.location} --kubernetes-version ${var.kubernetes_version} --node-count ${var.agent_vm_count} --node-vm-size ${var.agent_vm_size} --network-plugin azure --vnet-subnet-id ${azurerm_subnet.cluster.id}" + command = "az aks create -g ${azurerm_resource_group.cluster.name} -n ${var.cluster_name}-${random_integer.ri.result} -l ${azurerm_resource_group.cluster.location} --kubernetes-version ${var.kubernetes_version} --node-count ${var.agent_vm_count} --node-vm-size ${var.agent_vm_size} --network-plugin azure --vnet-subnet-id ${azurerm_subnet.cluster.id}" } depends_on = ["azurerm_subnet.cluster"] @@ -77,7 +83,7 @@ resource "null_resource" "create_cluster" { resource "null_resource" "cluster_credentials" { provisioner "local-exec" { - command = "az aks get-credentials --resource-group ${azurerm_resource_group.cluster.name} --name ${var.cluster_name} --overwrite-existing" + command = "az aks get-credentials --resource-group ${azurerm_resource_group.cluster.name} --name ${var.cluster_name}-${random_integer.ri.result} --overwrite-existing" } //depends_on = ["azurerm_kubernetes_cluster.cluster"] From b16668cb95ced6503112a36fb09eb8da836a9dc4 Mon Sep 17 00:00:00 2001 From: nathanielrose Date: Tue, 15 Jan 2019 14:55:18 -0800 Subject: [PATCH 5/8] Small edits to README for user clarity --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cad7c65..e2de3c0 100644 --- a/README.md +++ b/README.md @@ -39,12 +39,13 @@ If you already have a Kubernetes cluster running and its context is the default, We've included Terraform scripts for building a Kubernetes cluster with Azure AKS or ACS Engine, but would welcome pull requests for other cloud providers. -To deploy a cluster, +To deploy a cluster: -1. Ensure you have the [az cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) installed, in your path, and logged in to your subscription. -2. Edit cluster/environments/azure-aks/main.tf and adjust the name of the cluster and, if desired, any of the sizing or network parameters. - -3. Deploy the cluster using: +1. Ensure you have the latest [az cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) installed, in your path, and logged in to your subscription. +2. Ensure you have [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) and [helm](https://github.com/helm/helm/blob/master/docs/install.md) installed. +3. Generate [ssh keys](https://confluence.atlassian.com/bitbucketserver054/creating-ssh-keys-939508421.html) to be deployed with your cluster. +4. Edit cluster/environments/azure-aks/main.tf and adjust the name of the cluster and, if desired, any of the sizing or network parameters. +5. Deploy the cluster using: ``` $ cd cluster/environments/azure-aks From 291491ca2b4a68fbfb6e020c379f22eb4bc1b4fe Mon Sep 17 00:00:00 2001 From: nathanielrose Date: Wed, 16 Jan 2019 14:22:53 -0800 Subject: [PATCH 6/8] :bug: Closes #30 - Removed workaround script with RBAC provider --- README.md | 5 +-- cluster/providers/azure-aks/aks.tf | 54 +++++++----------------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index e2de3c0..79aa40c 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,9 @@ To deploy a cluster: 1. Ensure you have the latest [az cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) installed, in your path, and logged in to your subscription. 2. Ensure you have [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) and [helm](https://github.com/helm/helm/blob/master/docs/install.md) installed. 3. Generate [ssh keys](https://confluence.atlassian.com/bitbucketserver054/creating-ssh-keys-939508421.html) to be deployed with your cluster. -4. Edit cluster/environments/azure-aks/main.tf and adjust the name of the cluster and, if desired, any of the sizing or network parameters. -5. Deploy the cluster using: +4. Generate a [service principal](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli?view=azure-cli-latest) and update cluster/environments/azure-aks/inputs.tf with the configurations +5. Edit cluster/environments/azure-aks/main.tf and adjust the name of the cluster and, if desired, any of the sizing or network parameters. +6. Deploy the cluster using: ``` $ cd cluster/environments/azure-aks diff --git a/cluster/providers/azure-aks/aks.tf b/cluster/providers/azure-aks/aks.tf index 74dcff0..01fd5de 100644 --- a/cluster/providers/azure-aks/aks.tf +++ b/cluster/providers/azure-aks/aks.tf @@ -21,18 +21,11 @@ resource "azurerm_subnet" "cluster" { virtual_network_name = "${azurerm_virtual_network.cluster.name}" } - - -/* - -NOTE: Currently you can not enable RBAC without a backing AAD service principal. In the meantime, use the az command line. -TODO: This support is expected within a couple of releases -- switch back when its available. - resource "azurerm_kubernetes_cluster" "cluster" { - name = "${var.cluster_name}" + name = "${var.cluster_name}-${random_integer.ri.result}" location = "${azurerm_resource_group.cluster.location}" resource_group_name = "${azurerm_resource_group.cluster.name}" - dns_prefix = "${var.cluster_name}" + dns_prefix = "${var.cluster_name}-${random_integer.ri.result}" kubernetes_version = "${var.kubernetes_version}" linux_profile { @@ -56,44 +49,21 @@ resource "azurerm_kubernetes_cluster" "cluster" { network_plugin = "azure" } + service_principal { + client_id = "${var.client_id}" + client_secret = "${var.client_secret}" + } + role_based_access_control { + enabled = true + /* # Use for AAD backed RBAC azure_active_directory { server_app_id = "${var.aad_server_app_id}" server_app_secret = "${var.aad_server_app_secret}" client_app_id = "${var.aad_client_app_id}" tenant_id = "${var.aad_tenant_id}" } - } - - service_principal { - client_id = "${var.client_id}" - client_secret = "${var.client_secret}" - } -} - -*/ + }*/ -resource "null_resource" "create_cluster" { - provisioner "local-exec" { - command = "az aks create -g ${azurerm_resource_group.cluster.name} -n ${var.cluster_name}-${random_integer.ri.result} -l ${azurerm_resource_group.cluster.location} --kubernetes-version ${var.kubernetes_version} --node-count ${var.agent_vm_count} --node-vm-size ${var.agent_vm_size} --network-plugin azure --vnet-subnet-id ${azurerm_subnet.cluster.id}" - } - - depends_on = ["azurerm_subnet.cluster"] -} - -resource "null_resource" "cluster_credentials" { - provisioner "local-exec" { - command = "az aks get-credentials --resource-group ${azurerm_resource_group.cluster.name} --name ${var.cluster_name}-${random_integer.ri.result} --overwrite-existing" - } - - //depends_on = ["azurerm_kubernetes_cluster.cluster"] - depends_on = ["null_resource.create_cluster"] -} - -resource "null_resource" "helm" { - provisioner "local-exec" { - command = "kubectl apply -f ${path.module}/tiller.yaml && helm init --service-account tiller --upgrade --wait" - } - - depends_on = ["null_resource.cluster_credentials"] -} + } +} \ No newline at end of file From 1c89313b57e16ff6cf782b87b94617df988dc350 Mon Sep 17 00:00:00 2001 From: nathanielrose Date: Thu, 17 Jan 2019 13:17:16 -0800 Subject: [PATCH 7/8] Working flux support --- cluster/providers/azure-aks/aks.tf | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cluster/providers/azure-aks/aks.tf b/cluster/providers/azure-aks/aks.tf index 89d4f97..bdd432e 100644 --- a/cluster/providers/azure-aks/aks.tf +++ b/cluster/providers/azure-aks/aks.tf @@ -64,6 +64,24 @@ resource "azurerm_kubernetes_cluster" "cluster" { tenant_id = "${var.aad_tenant_id}" } }*/ - } + } } + + resource "null_resource" "cluster_credentials" { + provisioner "local-exec" { + command = "az aks get-credentials --resource-group ${azurerm_resource_group.cluster.name} --name ${var.cluster_name}-${random_integer.ri.result} --overwrite-existing" + } + depends_on = ["azurerm_kubernetes_cluster.cluster"] + } + + resource "null_resource" "deploy_flux" { + provisioner "local-exec" { + command = "./deploy-flux.sh -f ${var.flux_repo_url} -g ${var.gitops_url} -k ${var.gitops_ssh_key}" + } + + depends_on = ["null_resource.cluster_credentials"] + } + + + From 6a08fb20a41d65088c971c3c1dd7b324471b8710 Mon Sep 17 00:00:00 2001 From: nathanielrose Date: Thu, 17 Jan 2019 18:57:24 -0800 Subject: [PATCH 8/8] Revised unique key and tfvars template for cluster configuration --- cluster/environments/azure-aks/inputs.tf | 17 ++++++++++++++--- cluster/environments/azure-aks/main.tf | 2 +- cluster/environments/common/common.tfvars | 7 +++++++ cluster/providers/azure-aks/aks.tf | 16 ++++++---------- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cluster/environments/azure-aks/inputs.tf b/cluster/environments/azure-aks/inputs.tf index 6ac76dd..80e0ce3 100644 --- a/cluster/environments/azure-aks/inputs.tf +++ b/cluster/environments/azure-aks/inputs.tf @@ -15,19 +15,20 @@ variable "aad_tenant_id" { type = "string" } */ +variable "cluster_id" { + type = "string" + default = "my-dev-cluster" +} variable "client_id" { type = "string" - default = "{ENTER-CLIENT-ID}" } variable "client_secret" { type = "string" - default = "{ENTER-CLIENT-SECRET}" } variable "ssh_public_key" { type = "string" - default = "{ENTER-SSH-PUBLIC-KEY-HERE}" } # URL to get flux which will be installed in the Kubernetes cluster @@ -51,3 +52,13 @@ variable "gitops_ssh_key" { type = "string" default = "./identity" } + +# generate a random unique key to be apended to cluster name +locals { + key_id = "${random_integer.ri.result}" +} + +resource "random_integer" "ri" { + min = 10000 + max = 99999 +} diff --git a/cluster/environments/azure-aks/main.tf b/cluster/environments/azure-aks/main.tf index a3910cb..2d1f349 100644 --- a/cluster/environments/azure-aks/main.tf +++ b/cluster/environments/azure-aks/main.tf @@ -1,7 +1,7 @@ module "azure_aks" { source = "../../providers/azure-aks" - cluster_name = "my-dev-cluster" + cluster_name = "${var.cluster_id}-${local.key_id}" agent_vm_count = "3" agent_vm_size = "Standard_DS3_v2" diff --git a/cluster/environments/common/common.tfvars b/cluster/environments/common/common.tfvars index e69de29..16d953d 100644 --- a/cluster/environments/common/common.tfvars +++ b/cluster/environments/common/common.tfvars @@ -0,0 +1,7 @@ +cluster_id = "cluster-name" +client_id = "client-id" +client_secret = "client-secret" +ssh_public_key = "ssh-key" +flux_repo_url = "https://github.com/weaveworks/flux.git" +gitops_url = "git@github.com:sarath-p/flux-get-started.git" +gitops_ssh_key = "./identity" \ No newline at end of file diff --git a/cluster/providers/azure-aks/aks.tf b/cluster/providers/azure-aks/aks.tf index bdd432e..f5ebbfd 100644 --- a/cluster/providers/azure-aks/aks.tf +++ b/cluster/providers/azure-aks/aks.tf @@ -1,31 +1,27 @@ -resource "random_integer" "ri" { - min = 10000 - max = 99999 -} resource "azurerm_resource_group" "cluster" { - name = "${var.cluster_name}-${random_integer.ri.result}-rg" + name = "${var.cluster_name}-rg" location = "${var.location}" } resource "azurerm_virtual_network" "cluster" { - name = "${var.cluster_name}-${random_integer.ri.result}-vnet" + name = "${var.cluster_name}-vnet" address_space = ["${var.vnet_address_space}"] location = "${var.location}" resource_group_name = "${azurerm_resource_group.cluster.name}" } resource "azurerm_subnet" "cluster" { - name = "${var.cluster_name}-${random_integer.ri.result}-subnet" + name = "${var.cluster_name}-subnet" resource_group_name = "${azurerm_resource_group.cluster.name}" address_prefix = "${var.subnet_address_space}" virtual_network_name = "${azurerm_virtual_network.cluster.name}" } resource "azurerm_kubernetes_cluster" "cluster" { - name = "${var.cluster_name}-${random_integer.ri.result}" + name = "${var.cluster_name}" location = "${azurerm_resource_group.cluster.location}" resource_group_name = "${azurerm_resource_group.cluster.name}" - dns_prefix = "${var.cluster_name}-${random_integer.ri.result}" + dns_prefix = "${var.cluster_name}" kubernetes_version = "${var.kubernetes_version}" linux_profile { @@ -69,7 +65,7 @@ resource "azurerm_kubernetes_cluster" "cluster" { resource "null_resource" "cluster_credentials" { provisioner "local-exec" { - command = "az aks get-credentials --resource-group ${azurerm_resource_group.cluster.name} --name ${var.cluster_name}-${random_integer.ri.result} --overwrite-existing" + command = "az aks get-credentials --resource-group ${azurerm_resource_group.cluster.name} --name ${var.cluster_name} --overwrite-existing" } depends_on = ["azurerm_kubernetes_cluster.cluster"] }