From 7ac8837cc0b1e6fe906c97d2a06b60954c7cb6b4 Mon Sep 17 00:00:00 2001 From: Annie Hedgpeth Date: Thu, 8 Jun 2017 11:38:34 -0500 Subject: [PATCH] provider/azurerm: Add example of Spark and Cassrandra on CentOS (#15123) * initial commit - 101-vm-from-user-image * changed branch name * not deploying - storage problems * provisions vm but image not properly prepared * storage not correct * provisions properly * changed main.tf to azuredeploy.tf * added tfvars and info for README * tfvars ignored and corrected file ext * added CI config; added sane defaults for variables; updated deployment script, added mac specific deployment for local testing * deploy.sh to be executable * executable deploy files * added CI files; changed vars * prep for PR * removal of old folder * prep for PR * wrong args for travis * more PR prep * updated README * commented out variables in terraform.tfvars * Topic 101 vm from user image (#2) * initial commit - 101-vm-from-user-image * added tfvars and info for README * added CI config; added sane defaults for variables; updated deployment script, added mac specific deployment for local testing * prep for PR * added new template * oops, left off master * prep for PR * correct repository for destination * renamed scripts to be more intuitive; added check for docker * merge vm simple; vm from image * initial commit * deploys locally * updated deploy * consolidated deploy and after_deploy into a single script; simplified ci process; added os_profile_linux_config * added terraform show * changed to allow http & https (like ARM tmplt) * changed host_name & host_name variable desc * added az cli check * on this branch, only build test_dir; master will aggregate all the examples * merge master * added new constructs/naming for deploy scripts, etc. * suppress az login output * suppress az login output * forgot about line breaks * breaking build as an example * fixing broken build example * merge of CI config * fixed grammar in readme * prep for PR * took out armviz button and minor README changes * changed host_name * fixed merge conflicts * changed host_name variable * updating Hashicorp's changes to merged simple linux branch * updating files to merge w/master and prep for Hashicorp pr * Revert "updating files to merge w/master and prep for Hashicorp pr" This reverts commit b850cd5d2a858eff073fc5a1097a6813d0f8b362. * Revert "updating Hashicorp's changes to merged simple linux branch" This reverts commit dbaf8d14a9cdfcef0281919671357f6171ebd4e6. * removing vm from user image example from this branch * removed old branch * azure-2-vms-loadbalancer-lbrules (#13) * initial commit * need to change lb_rule & nic * deploys locally * updated README * updated travis and deploy scripts for Hari's repo * renamed deploy script * clean up * prep for PR * updated readme * fixing conflict in .travis.yml * initial commit; in progress * in progress * in progress; encryption fails * in progress * deploys successfully locally * clean up; deploy typo fixed * merging hashi master into this branch * troubleshooting deploy * added missing vars to deploy script * updated README, outputs, and added graph * simplified outputs * provisions locally * cleaned up vars * fixed chart on README * prepping for pr * fixed merge conflict * switching to Hashicorp's .travis.yml * edited comments * removed graph * reverting travis.yml to original added return line at 45 --- .travis.yml | 1 + .../README.md | 67 +++ .../deploy.ci.sh | 43 ++ .../deploy.mac.sh | 15 + .../main.tf | 405 ++++++++++++++++++ .../outputs.tf | 15 + .../variables.tf | 234 ++++++++++ 7 files changed, 780 insertions(+) create mode 100644 examples/azure-spark-and-cassandra-on-centos/README.md create mode 100755 examples/azure-spark-and-cassandra-on-centos/deploy.ci.sh create mode 100755 examples/azure-spark-and-cassandra-on-centos/deploy.mac.sh create mode 100644 examples/azure-spark-and-cassandra-on-centos/main.tf create mode 100644 examples/azure-spark-and-cassandra-on-centos/outputs.tf create mode 100644 examples/azure-spark-and-cassandra-on-centos/variables.tf diff --git a/.travis.yml b/.travis.yml index 1f29c4cd142a..1e54217ba4df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,3 +42,4 @@ matrix: fast_finish: true allow_failures: - go: tip + diff --git a/examples/azure-spark-and-cassandra-on-centos/README.md b/examples/azure-spark-and-cassandra-on-centos/README.md new file mode 100644 index 000000000000..9d8492b08bfe --- /dev/null +++ b/examples/azure-spark-and-cassandra-on-centos/README.md @@ -0,0 +1,67 @@ +# Spark & Cassandra on CentOS 7.x + +This Terraform template was based on [this](https://github.com/Azure/azure-quickstart-templates/tree/master/spark-and-cassandra-on-centos) Azure Quickstart Template. Changes to the ARM template that may have occurred since the creation of this example may not be reflected here. + +This project configures a Spark cluster (1 master and n-slave nodes) and a single node Cassandra on Azure using CentOS 7.x. The base image starts with CentOS 7.3, and it is updated to the latest version as part of the provisioning steps. + +Please note that [Azure Resource Manager][3] is used to provision the environment. + +### Software ### + +| Category | Software | Version | Notes | +| --- | --- | --- | --- | +| Operating System | CentOS | 7.x | Based on CentOS 7.1 but it will be auto upgraded to the lastest point release | +| Java | OpenJDK | 1.8.0 | Installed on all servers | +| Spark | Spark | 1.6.0 with Hadoop 2.6 | The installation contains libraries needed for Hadoop 2.6 | +| Cassandra | Cassandra | 3.2 | Installed through DataStax's YUM repository | + + +### Defaults ### + +| Component | Setting | Default | Notes | +| --- | --- | --- | --- | +| Spark - Master | VM Size | Standard D1 V2 | | +| Spark - Master | Storage | Standard LRS | | +| Spark - Master | Internal IP | 10.0.0.5 | | +| Spark - Master | Service User Account | spark | Password-less access | +| | | | +| Spark - Slave | VM Size | Standard D3 V2 | | +| Spark - Slave | Storage | Standard LRS | | +| Spark - Slave | Internal IP Range | 10.0.1.5 - 10.0.1.255 | | +| Spark - Slave | # of Nodes | 2 | Maximum of 200 | +| Spark - Slave | Availability | 2 fault domains, 5 update domains | | +| Spark - Slave | Service User Account | spark | Password-less access | +| | | | +| Cassandra | VM Size | Standard D3 V2 | | +| Cassandra | Storage | Standard LRS | | +| Cassandra | Internal IP | 10.2.0.5 | | +| Cassandra | Service User Account | cassandra | Password-less access | + +## Prerequisites + +1. Ensure you have an Azure subscription. +2. Ensure you have enough available vCPU cores on your subscription. Otherwise, you will receive an error during the process. The number of cores can be increased through a support ticket in Azure Portal. + +## main.tf +The `main.tf` file contains the actual resources that will be deployed. It also contains the Azure Resource Group definition and any defined variables. + +## outputs.tf +This data is outputted when `terraform apply` is called, and can be queried using the `terraform output` command. + +## provider.tf +Azure requires that an application is added to Azure Active Directory to generate the `client_id`, `client_secret`, and `tenant_id` needed by Terraform (`subscription_id` can be recovered from your Azure account details). Please go [here](https://www.terraform.io/docs/providers/azurerm/) for full instructions on how to create this to populate your `provider.tf` file. + +## terraform.tfvars +If a `terraform.tfvars` file is present in the current directory, Terraform automatically loads it to populate variables. We don't recommend saving usernames and password to version control, but you can create a local secret variables file and use `-var-file` to load it. + +If you are committing this template to source control, please insure that you add this file to your `.gitignore` file. + +## variables.tf +The `variables.tf` file contains all of the input parameters that the user can specify when deploying this Terraform template. + +## Post-Deployment + +1. All servers will have a public IP and SSH port enabled by default. These can be disabled or modified in the template or by using Azure Portal. +2. All servers are configured with the same username and password. You may SSH into each server and ensure connectivity. +3. Spark WebUI is running on **port 8080**. Access it using MASTER_WEB_UI_PUBLIC_IP:8080 on your browser. Public IP is available in the outputs as well as through Azure Portal. +4. Delete the Resource Group that was created to stage the provisioning scripts. diff --git a/examples/azure-spark-and-cassandra-on-centos/deploy.ci.sh b/examples/azure-spark-and-cassandra-on-centos/deploy.ci.sh new file mode 100755 index 000000000000..ce4cd1fad323 --- /dev/null +++ b/examples/azure-spark-and-cassandra-on-centos/deploy.ci.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -o errexit -o nounset + +docker run --rm -it \ + -e ARM_CLIENT_ID \ + -e ARM_CLIENT_SECRET \ + -e ARM_SUBSCRIPTION_ID \ + -e ARM_TENANT_ID \ + -v $(pwd):/data \ + --workdir=/data \ + --entrypoint "/bin/sh" \ + hashicorp/terraform:light \ + -c "/bin/terraform get; \ + /bin/terraform validate; \ + /bin/terraform plan -out=out.tfplan \ + -var resource_group=$KEY \ + -var unique_prefix=$KEY \ + -var vm_admin_username=$KEY \ + -var vm_admin_password=$PASSWORD; \ + /bin/terraform apply out.tfplan" + +# cleanup deployed azure resources via azure-cli +docker run --rm -it \ + azuresdk/azure-cli-python \ + sh -c "az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID > /dev/null; \ + az vm list -g $KEY" + +# cleanup deployed azure resources via terraform +docker run --rm -it \ + -e ARM_CLIENT_ID \ + -e ARM_CLIENT_SECRET \ + -e ARM_SUBSCRIPTION_ID \ + -e ARM_TENANT_ID \ + -v $(pwd):/data \ + --workdir=/data \ + --entrypoint "/bin/sh" \ + hashicorp/terraform:light \ + -c "/bin/terraform destroy -force \ + -var resource_group=$KEY \ + -var unique_prefix=$KEY \ + -var vm_admin_username=$KEY \ + -var vm_admin_password=$PASSWORD;" \ No newline at end of file diff --git a/examples/azure-spark-and-cassandra-on-centos/deploy.mac.sh b/examples/azure-spark-and-cassandra-on-centos/deploy.mac.sh new file mode 100755 index 000000000000..cf5cdc32279f --- /dev/null +++ b/examples/azure-spark-and-cassandra-on-centos/deploy.mac.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -o errexit -o nounset + +if docker -v; then + + # generate a unique string for CI deployment + export KEY=$(cat /dev/urandom | env LC_CTYPE=C tr -cd 'a-z' | head -c 12) + export PASSWORD=$KEY$(cat /dev/urandom | env LC_CTYPE=C tr -cd 'A-Z' | head -c 2)$(cat /dev/urandom | env LC_CTYPE=C tr -cd '0-9' | head -c 2) + +/bin/sh ./deploy.ci.sh + +else + echo "Docker is used to run terraform commands, please install before run: https://docs.docker.com/docker-for-mac/install/" +fi \ No newline at end of file diff --git a/examples/azure-spark-and-cassandra-on-centos/main.tf b/examples/azure-spark-and-cassandra-on-centos/main.tf new file mode 100644 index 000000000000..18d0678b2820 --- /dev/null +++ b/examples/azure-spark-and-cassandra-on-centos/main.tf @@ -0,0 +1,405 @@ +# provider "azurerm" { +# subscription_id = "${var.subscription_id}" +# client_id = "${var.client_id}" +# client_secret = "${var.client_secret}" +# tenant_id = "${var.tenant_id}" +# } + +resource "azurerm_resource_group" "rg" { + name = "${var.resource_group}" + location = "${var.location}" +} + +# ********************** NETWORK SECURITY GROUPS ********************** # +resource "azurerm_network_security_group" "master" { + name = "${var.nsg_spark_master_name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + + security_rule { + name = "ssh" + description = "Allow SSH" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "Internet" + destination_address_prefix = "*" + } + + security_rule { + name = "http_webui_spark" + description = "Allow Web UI Access to Spark" + priority = 101 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "8080" + source_address_prefix = "Internet" + destination_address_prefix = "*" + } + + security_rule { + name = "http_rest_spark" + description = "Allow REST API Access to Spark" + priority = 102 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "6066" + source_address_prefix = "Internet" + destination_address_prefix = "*" + } +} + +resource "azurerm_network_security_group" "slave" { + name = "${var.nsg_spark_slave_name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + + security_rule { + name = "ssh" + description = "Allow SSH" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "Internet" + destination_address_prefix = "*" + } +} + +resource "azurerm_network_security_group" "cassandra" { + name = "${var.nsg_cassandra_name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + + security_rule { + name = "ssh" + description = "Allow SSH" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "Internet" + destination_address_prefix = "*" + } +} + +# ********************** VNET / SUBNETS ********************** # +resource "azurerm_virtual_network" "spark" { + name = "vnet-spark" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + address_space = ["${var.vnet_spark_prefix}"] +} + +resource "azurerm_subnet" "subnet1" { + name = "${var.vnet_spark_subnet1_name}" + virtual_network_name = "${azurerm_virtual_network.spark.name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + address_prefix = "${var.vnet_spark_subnet1_prefix}" + network_security_group_id = "${azurerm_network_security_group.master.id}" + depends_on = ["azurerm_virtual_network.spark"] +} + +resource "azurerm_subnet" "subnet2" { + name = "${var.vnet_spark_subnet2_name}" + virtual_network_name = "${azurerm_virtual_network.spark.name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + address_prefix = "${var.vnet_spark_subnet2_prefix}" +} + +resource "azurerm_subnet" "subnet3" { + name = "${var.vnet_spark_subnet3_name}" + virtual_network_name = "${azurerm_virtual_network.spark.name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + address_prefix = "${var.vnet_spark_subnet3_prefix}" +} + +# ********************** PUBLIC IP ADDRESSES ********************** # +resource "azurerm_public_ip" "master" { + name = "${var.public_ip_master_name}" + location = "${azurerm_resource_group.rg.location}" + resource_group_name = "${azurerm_resource_group.rg.name}" + public_ip_address_allocation = "Static" +} + +resource "azurerm_public_ip" "slave" { + name = "${var.public_ip_slave_name_prefix}${count.index}" + location = "${azurerm_resource_group.rg.location}" + resource_group_name = "${azurerm_resource_group.rg.name}" + public_ip_address_allocation = "Static" + count = "${var.vm_number_of_slaves}" +} + +resource "azurerm_public_ip" "cassandra" { + name = "${var.public_ip_cassandra_name}" + location = "${azurerm_resource_group.rg.location}" + resource_group_name = "${azurerm_resource_group.rg.name}" + public_ip_address_allocation = "Static" +} + +# ********************** NETWORK INTERFACE ********************** # +resource "azurerm_network_interface" "master" { + name = "${var.nic_master_name}" + location = "${azurerm_resource_group.rg.location}" + resource_group_name = "${azurerm_resource_group.rg.name}" + network_security_group_id = "${azurerm_network_security_group.master.id}" + depends_on = ["azurerm_virtual_network.spark", "azurerm_public_ip.master", "azurerm_network_security_group.master"] + + ip_configuration { + name = "ipconfig1" + subnet_id = "${azurerm_subnet.subnet1.id}" + private_ip_address_allocation = "Static" + private_ip_address = "${var.nic_master_node_ip}" + public_ip_address_id = "${azurerm_public_ip.master.id}" + } +} + +resource "azurerm_network_interface" "slave" { + name = "${var.nic_slave_name_prefix}${count.index}" + location = "${azurerm_resource_group.rg.location}" + resource_group_name = "${azurerm_resource_group.rg.name}" + network_security_group_id = "${azurerm_network_security_group.slave.id}" + count = "${var.vm_number_of_slaves}" + depends_on = ["azurerm_virtual_network.spark", "azurerm_public_ip.slave", "azurerm_network_security_group.slave"] + + ip_configuration { + name = "ipconfig1" + subnet_id = "${azurerm_subnet.subnet2.id}" + private_ip_address_allocation = "Static" + private_ip_address = "${var.nic_slave_node_ip_prefix}${5 + count.index}" + public_ip_address_id = "${element(azurerm_public_ip.slave.*.id, count.index)}" + } +} + +resource "azurerm_network_interface" "cassandra" { + name = "${var.nic_cassandra_name}" + location = "${azurerm_resource_group.rg.location}" + resource_group_name = "${azurerm_resource_group.rg.name}" + network_security_group_id = "${azurerm_network_security_group.cassandra.id}" + depends_on = ["azurerm_virtual_network.spark", "azurerm_public_ip.cassandra", "azurerm_network_security_group.cassandra"] + + ip_configuration { + name = "ipconfig1" + subnet_id = "${azurerm_subnet.subnet3.id}" + private_ip_address_allocation = "Static" + private_ip_address = "${var.nic_cassandra_node_ip}" + public_ip_address_id = "${azurerm_public_ip.cassandra.id}" + } +} + +# ********************** AVAILABILITY SET ********************** # +resource "azurerm_availability_set" "slave" { + name = "${var.availability_slave_name}" + location = "${azurerm_resource_group.rg.location}" + resource_group_name = "${azurerm_resource_group.rg.name}" + platform_update_domain_count = 5 + platform_fault_domain_count = 2 +} + +# ********************** STORAGE ACCOUNTS ********************** # +resource "azurerm_storage_account" "master" { + name = "master${var.unique_prefix}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + account_type = "${var.storage_master_type}" +} + +resource "azurerm_storage_container" "master" { + name = "${var.vm_master_storage_account_container_name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + storage_account_name = "${azurerm_storage_account.master.name}" + container_access_type = "private" + depends_on = ["azurerm_storage_account.master"] +} + +resource "azurerm_storage_account" "slave" { + name = "slave${var.unique_prefix}${count.index}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + count = "${var.vm_number_of_slaves}" + account_type = "${var.storage_slave_type}" +} + +resource "azurerm_storage_container" "slave" { + name = "${var.vm_slave_storage_account_container_name}${count.index}" + resource_group_name = "${azurerm_resource_group.rg.name}" + storage_account_name = "${element(azurerm_storage_account.slave.*.name, count.index)}" + container_access_type = "private" + depends_on = ["azurerm_storage_account.slave"] +} + +resource "azurerm_storage_account" "cassandra" { + name = "cassandra${var.unique_prefix}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + account_type = "${var.storage_cassandra_type}" +} + +resource "azurerm_storage_container" "cassandra" { + name = "${var.vm_cassandra_storage_account_container_name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + storage_account_name = "${azurerm_storage_account.cassandra.name}" + container_access_type = "private" + depends_on = ["azurerm_storage_account.cassandra"] +} + +# ********************** MASTER VIRTUAL MACHINE ********************** # +resource "azurerm_virtual_machine" "master" { + name = "${var.vm_master_name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + vm_size = "${var.vm_master_vm_size}" + network_interface_ids = ["${azurerm_network_interface.master.id}"] + depends_on = ["azurerm_storage_account.master", "azurerm_network_interface.master", "azurerm_storage_container.master"] + + storage_image_reference { + publisher = "${var.os_image_publisher}" + offer = "${var.os_image_offer}" + sku = "${var.os_version}" + version = "latest" + } + + storage_os_disk { + name = "${var.vm_master_os_disk_name}" + vhd_uri = "http://${azurerm_storage_account.master.name}.blob.core.windows.net/${azurerm_storage_container.master.name}/${var.vm_master_os_disk_name}.vhd" + create_option = "FromImage" + caching = "ReadWrite" + } + + os_profile { + computer_name = "${var.vm_master_name}" + admin_username = "${var.vm_admin_username}" + admin_password = "${var.vm_admin_password}" + } + + os_profile_linux_config { + disable_password_authentication = false + } + + connection { + type = "ssh" + host = "${azurerm_public_ip.master.ip_address}" + user = "${var.vm_admin_username}" + password = "${var.vm_admin_password}" + } + + provisioner "remote-exec" { + inline = [ + "wget ${var.artifacts_location}${var.script_spark_provisioner_script_file_name}", + "echo ${var.vm_admin_password} | sudo -S sh ./${var.script_spark_provisioner_script_file_name} -runas=master -master=${var.nic_master_node_ip}", + ] + } +} + +# ********************** SLAVE VIRTUAL MACHINES ********************** # +resource "azurerm_virtual_machine" "slave" { + name = "${var.vm_slave_name_prefix}${count.index}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + vm_size = "${var.vm_slave_vm_size}" + network_interface_ids = ["${element(azurerm_network_interface.slave.*.id, count.index)}"] + count = "${var.vm_number_of_slaves}" + availability_set_id = "${azurerm_availability_set.slave.id}" + depends_on = ["azurerm_storage_account.slave", "azurerm_network_interface.slave", "azurerm_storage_container.slave"] + + + storage_image_reference { + publisher = "${var.os_image_publisher}" + offer = "${var.os_image_offer}" + sku = "${var.os_version}" + version = "latest" + } + + + storage_os_disk { + name = "${var.vm_slave_os_disk_name_prefix}${count.index}" + vhd_uri = "http://${element(azurerm_storage_account.slave.*.name, count.index)}.blob.core.windows.net/${element(azurerm_storage_container.slave.*.name, count.index)}/${var.vm_slave_os_disk_name_prefix}.vhd" + create_option = "FromImage" + caching = "ReadWrite" + } + + + os_profile { + computer_name = "${var.vm_slave_name_prefix}${count.index}" + admin_username = "${var.vm_admin_username}" + admin_password = "${var.vm_admin_password}" + } + + + os_profile_linux_config { + disable_password_authentication = false + } + + connection { + type = "ssh" + host = "${element(azurerm_public_ip.slave.*.ip_address, count.index)}" + user = "${var.vm_admin_username}" + password = "${var.vm_admin_password}" + } + + provisioner "remote-exec" { + inline = [ + "wget ${var.artifacts_location}${var.script_spark_provisioner_script_file_name}", + "echo ${var.vm_admin_password} | sudo -S sh ./${var.script_spark_provisioner_script_file_name} -runas=slave -master=${var.nic_master_node_ip}", + ] + } +} + +# ********************** CASSANDRA VIRTUAL MACHINE ********************** # +resource "azurerm_virtual_machine" "cassandra" { + name = "${var.vm_cassandra_name}" + resource_group_name = "${azurerm_resource_group.rg.name}" + location = "${azurerm_resource_group.rg.location}" + vm_size = "${var.vm_cassandra_vm_size}" + network_interface_ids = ["${azurerm_network_interface.cassandra.id}"] + depends_on = ["azurerm_storage_account.cassandra", "azurerm_network_interface.cassandra", "azurerm_storage_container.cassandra"] + + storage_image_reference { + publisher = "${var.os_image_publisher}" + offer = "${var.os_image_offer}" + sku = "${var.os_version}" + version = "latest" + } + + storage_os_disk { + name = "${var.vm_cassandra_os_disk_name}" + vhd_uri = "http://${azurerm_storage_account.cassandra.name}.blob.core.windows.net/${azurerm_storage_container.cassandra.name}/${var.vm_cassandra_os_disk_name}.vhd" + create_option = "FromImage" + caching = "ReadWrite" + } + + os_profile { + computer_name = "${var.vm_cassandra_name}" + admin_username = "${var.vm_admin_username}" + admin_password = "${var.vm_admin_password}" + } + + os_profile_linux_config { + disable_password_authentication = false + } + + connection { + type = "ssh" + host = "${azurerm_public_ip.cassandra.ip_address}" + user = "${var.vm_admin_username}" + password = "${var.vm_admin_password}" + } + + provisioner "remote-exec" { + inline = [ + "wget ${var.artifacts_location}${var.script_cassandra_provisioner_script_file_name}", + "echo ${var.vm_admin_password} | sudo -S sh ./${var.script_cassandra_provisioner_script_file_name}", + ] + } +} diff --git a/examples/azure-spark-and-cassandra-on-centos/outputs.tf b/examples/azure-spark-and-cassandra-on-centos/outputs.tf new file mode 100644 index 000000000000..5ff3225e4d01 --- /dev/null +++ b/examples/azure-spark-and-cassandra-on-centos/outputs.tf @@ -0,0 +1,15 @@ +output "resource_group" { + value = "${var.resource_group}" +} + +output "master_ip_address" { + value = "${azurerm_public_ip.master.ip_address}" +} + +output "master_ssh_command" { + value = "ssh ${var.vm_admin_username}@${azurerm_public_ip.master.ip_address}" +} + +output "master_web_ui_public_ip" { + value = "${azurerm_public_ip.master.ip_address}:8080" +} diff --git a/examples/azure-spark-and-cassandra-on-centos/variables.tf b/examples/azure-spark-and-cassandra-on-centos/variables.tf new file mode 100644 index 000000000000..0988c70b963d --- /dev/null +++ b/examples/azure-spark-and-cassandra-on-centos/variables.tf @@ -0,0 +1,234 @@ +variable "resource_group" { + description = "Resource group name into which your Spark and Cassandra deployment will go." +} + +variable "location" { + description = "The location/region where the virtual network is created. Changing this forces a new resource to be created." + default = "southcentralus" +} + +variable "unique_prefix" { + description = "This prefix is used for names which need to be globally unique." +} + +variable "storage_master_type" { + description = "Storage type that is used for master Spark node. This storage account is used to store VM disks. Allowed values: Standard_LRS, Standard_ZRS, Standard_GRS, Standard_RAGRS, Premium_LRS" + default = "Standard_LRS" +} + +variable "storage_slave_type" { + description = "Storage type that is used for each of the slave Spark node. This storage account is used to store VM disks. Allowed values : Standard_LRS, Standard_ZRS, Standard_GRS, Standard_RAGRS, Premium_LRS" + default = "Standard_LRS" +} + +variable "storage_cassandra_type" { + description = "Storage type that is used for Cassandra. This storage account is used to store VM disks. Allowed values: Standard_LRS, Standard_ZRS, Standard_GRS, Standard_RAGRS, Premium_LRS" + default = "Standard_LRS" +} + +variable "vm_master_vm_size" { + description = "VM size for master Spark node. This VM can be sized smaller. Allowed values: Standard_D1_v2, Standard_D2_v2, Standard_D3_v2, Standard_D4_v2, Standard_D5_v2, Standard_D11_v2, Standard_D12_v2, Standard_D13_v2, Standard_D14_v2, Standard_A8, Standard_A9, Standard_A10, Standard_A11" + default = "Standard_D1_v2" +} + +variable "vm_number_of_slaves" { + description = "Number of VMs to create to support the slaves. Each slave is created on it's own VM. Minimum of 2 & Maximum of 200 VMs. min = 2, max = 200" + default = 2 +} + +variable "vm_slave_vm_size" { + description = "VM size for slave Spark nodes. This VM should be sized based on workloads. Allowed values: Standard_D1_v2, Standard_D2_v2, Standard_D3_v2, Standard_D4_v2, Standard_D5_v2, Standard_D11_v2, Standard_D12_v2, Standard_D13_v2, Standard_D14_v2, Standard_A8, Standard_A9, Standard_A10, Standard_A11" + default = "Standard_D3_v2" +} + +variable "vm_cassandra_vm_size" { + description = "VM size for Cassandra node. This VM should be sized based on workloads. Allowed values: Standard_D1_v2, Standard_D2_v2, Standard_D3_v2, Standard_D4_v2, Standard_D5_v2, Standard_D11_v2, Standard_D12_v2, Standard_D13_v2, Standard_D14_v2, Standard_A8, Standard_A9, Standard_A10, Standard_A11" + default = "Standard_D3_v2" +} + +variable "vm_admin_username" { + description = "Specify an admin username that should be used to login to the VM. Min length: 1" +} + +variable "vm_admin_password" { + description = "Specify an admin password that should be used to login to the VM. Must be between 6-72 characters long and must satisfy at least 3 of password complexity requirements from the following: 1) Contains an uppercase character 2) Contains a lowercase character 3) Contains a numeric digit 4) Contains a special character" +} + +variable "os_image_publisher" { + description = "name of the publisher of the image (az vm image list)" + default = "OpenLogic" +} + +variable "os_image_offer" { + description = "the name of the offer (az vm image list)" + default = "CentOS" +} + +variable "os_version" { + description = "version of the image to apply (az vm image list)" + default = "7.3" +} + +variable "api_version" { + default = "2015-06-15" +} + +variable "artifacts_location" { + description = "The base URI where artifacts required by this template are located." + default = "https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/spark-and-cassandra-on-centos/CustomScripts/" +} + +variable "vnet_spark_prefix" { + description = "The address space that is used by the virtual network. You can supply more than one address space. Changing this forces a new resource to be created." + default = "10.0.0.0/16" +} + +variable "vnet_spark_subnet1_name" { + description = "The name used for the Master subnet." + default = "Subnet-Master" +} + +variable "vnet_spark_subnet1_prefix" { + description = "The address prefix to use for the Master subnet." + default = "10.0.0.0/24" +} + +variable "vnet_spark_subnet2_name" { + description = "The name used for the slave/agent subnet." + default = "Subnet-Slave" +} + +variable "vnet_spark_subnet2_prefix" { + description = "The address prefix to use for the slave/agent subnet." + default = "10.0.1.0/24" +} + +variable "vnet_spark_subnet3_name" { + description = "The name used for the subnet used by Cassandra." + default = "Subnet-Cassandra" +} + +variable "vnet_spark_subnet3_prefix" { + description = "The address prefix to use for the subnet used by Cassandra." + default = "10.0.2.0/24" +} + +variable "nsg_spark_master_name" { + description = "The name of the network security group for Spark's Master" + default = "nsg-spark-master" +} + +variable "nsg_spark_slave_name" { + description = "The name of the network security group for Spark's slave/agent nodes" + default = "nsg-spark-slave" +} + +variable "nsg_cassandra_name" { + description = "The name of the network security group for Cassandra" + default = "nsg-cassandra" +} + +variable "nic_master_name" { + description = "The name of the network interface card for Master" + default = "nic-master" +} + +variable "nic_master_node_ip" { + description = "The private IP address used by the Master's network interface card" + default = "10.0.0.5" +} + +variable "nic_cassandra_name" { + description = "The name of the network interface card used by Cassandra" + default = "nic-cassandra" +} + +variable "nic_cassandra_node_ip" { + description = "The private IP address of Cassandra's network interface card" + default = "10.0.2.5" +} + +variable "nic_slave_name_prefix" { + description = "The prefix used to constitute the slave/agents' names" + default = "nic-slave-" +} + +variable "nic_slave_node_ip_prefix" { + description = "The prefix of the private IP address used by the network interface card of the slave/agent nodes" + default = "10.0.1." +} + +variable "public_ip_master_name" { + description = "The name of the master node's public IP address" + default = "public-ip-master" +} + +variable "public_ip_slave_name_prefix" { + description = "The prefix to the slave/agent nodes' IP address names" + default = "public-ip-slave-" +} + +variable "public_ip_cassandra_name" { + description = "The name of Cassandra's node's public IP address" + default = "public-ip-cassandra" +} + +variable "vm_master_name" { + description = "The name of Spark's Master virtual machine" + default = "spark-master" +} + +variable "vm_master_os_disk_name" { + description = "The name of the os disk used by Spark's Master virtual machine" + default = "vmMasterOSDisk" +} + +variable "vm_master_storage_account_container_name" { + description = "The name of the storage account container used by Spark's master" + default = "vhds" +} + +variable "vm_slave_name_prefix" { + description = "The name prefix used by Spark's slave/agent nodes" + default = "spark-slave-" +} + +variable "vm_slave_os_disk_name_prefix" { + description = "The prefix used to constitute the names of the os disks used by the slave/agent nodes" + default = "vmSlaveOSDisk-" +} + +variable "vm_slave_storage_account_container_name" { + description = "The name of the storage account container used by the slave/agent nodes" + default = "vhds" +} + +variable "vm_cassandra_name" { + description = "The name of the virtual machine used by Cassandra" + default = "cassandra" +} + +variable "vm_cassandra_os_disk_name" { + description = "The name of the os disk used by the Cassandra virtual machine" + default = "vmCassandraOSDisk" +} + +variable "vm_cassandra_storage_account_container_name" { + description = "The name of the storage account container used by the Cassandra node" + default = "vhds" +} + +variable "availability_slave_name" { + description = "The name of the availability set for the slave/agent machines" + default = "availability-slave" +} + +variable "script_spark_provisioner_script_file_name" { + description = "The name of the script kept in version control which will provision Spark" + default = "scriptSparkProvisioner.sh" +} + +variable "script_cassandra_provisioner_script_file_name" { + description = "The name of the script kept in version control which will provision Cassandra" + default = "scriptCassandraProvisioner.sh" +}