OpenStack Cross Cloud update

README.md

@@ -103,6 +103,28 @@ docker run \
   -ti registry.cncf.ci/cncf/cross-cloud/provisioning:production
 ```
 
+##### Quick start for OpenStack
+
+You will need a full set of credentials for an OpenStack cloud, including
+authentication endpoint.
+
+**Run the following to provision an OpenStack cluster:**
+``` bash
+docker run \
+  -v $(pwd)/data:/cncf/data \
+  -e NAME=cross-cloud \
+  -e CLOUD=openstack \
+  -e COMMAND=deploy \
+  -e BACKEND=file \
+  -e TF_VAR_os_auth_url=$OS_AUTH_URL \
+  -e TF_VAR_os_region_name=$OS_REGION_NAME \
+  -e TF_VAR_os_user_domain_name=$OS_USER_DOMAIN_NAME \
+  -e TF_VAR_os_username=$OS_USERNAME \
+  -e TF_VAR_os_project_name=$OS_PROJECT_NAME \
+  -e TF_VAR_os_password=$OS_PASSWORD \
+  -ti registry.cncf.ci/cncf/cross-cloud/provisioning:ci-stable-v0-2-0
+```
+
 #### General usage and configuration
 
 Minimum required configuration to use Cross-cloud to deploy a Kubernetes cluster on Cloud X.

build.sh

@@ -0,0 +1 @@
+docker build . --tag provisioning

openstack/README.md

@@ -5,9 +5,8 @@
 ### Prerequisites
 
 Building the CNCF/CICD cross-cloud container requires a recent version
-of Docker. You will also need credentials for an OpenStack
-cloud with Keystone, Nova, Neutron, and Octavia (LBaaSv2 is possibile,
-but strongly discouraged).
+of Docker. You will also need credentials for an OpenStack cloud with
+Keystone, Nova, Neutron. LBaaSv2 is not yet supported in this module.
 
 ### Building the container
 
@@ -18,9 +17,8 @@ In the top-level directory,
 ### Configuring your cloud
 
 You will need the following credentials for your OpenStack cloud.
-Since the credentials will be used in the Kubernetes deployment, and
-you will need to set the Terraform variables directly when calling
-the deploy command in the container.
+These credentials will used to provision the Kubernetes cluster,
+and will also be used for the OpenStack provider to Kubernetes.
 In your environment, set the following OpenStack environment variables:
 
 * `OS_AUTH_URL`
@@ -38,28 +36,21 @@ Then assign them to Terraform variables like this:
 * `TF_VAR_os_project_name=$OS_PROJECT_NAME`
 * `TF_VAR_os_password=$OS_PASSWORD`
 
-Optionally, you can tell Terraform to not use Octavia in favor of
-the Neutron LbaaSv2 service by setting the environment variable
-`TF_VAR_use_octavia=false`. It's strongly recommended to use Octavia,
-as using the default Neutron proxy interface can introduce some race
-conditions that will make it difficult to destroy a number of network
-resources created by Terraform.
-
-To log in remotely, your will need to have a keypair fixture in your
-cloud. The default name is `K8s`, but you may set it with the
+This deployment assumes you have a keypair available to inject into all
+of the instances. The default name is `K8s`, but you may set it with the
 variable `TF_VAR_keypair_name`.
 
 This project assumes the existence of recent CoreOS release.
 It defaults to the image named `CorsOS 1520.8.0`, but can be configured
 with the variables:
 
-* `TF_VAR_bastion_name_name`
+* `TF_VAR_lb_image_name`
 * `TF_VAR_master_image_name`
 * `TF_VAR_worker_image_name`.
 
-Similarly flavors can be set with:
+Similarly flavors (defaulting to `v1-standard-1`) can be set with:
 
-* `TF_VAR_bastion_flavor_name`
+* `TF_VAR_lb_flavor_name`
 * `TF_VAR_master_flavor_name`
 * `TF_VAR_worker_flavor_name`
 
@@ -74,6 +65,12 @@ Network parameters include:
 * `TF_VAR_bastion_floating_ip_pool`
 * `TF_VAR_external_network_id`
 
+As this is a multi-master deployment, a single load balancer is created
+to proxy non-terminated TLS requests to the K8s master nodes. Given the
+sporadic availability of LBaaSv2/Octavia across OpenStack clouds, the
+load balancer is created manually rather than relying on the LBaaSv2
+API.
+
 ### Deploying
 
 The following script will deploy Kubernetes into your cloud:

openstack/input.tf

@@ -16,13 +16,9 @@ variable keypair_name { default = "K8s" }
 # TLS settings
 variable "cloud_location" { default = "vexxhost.com" }
 
-variable "coreos_image_name" { default = "CoreOS 1520.8.0" }
-#variable "coreos_image_name" { default = "CoreOS 1298.6.0 (MoreOS) [2017-03-15]" }
-
-# Bastion Configuration
-variable "bastion_flavor_name" { default = "v1-standard-1" }
-variable "bastion_image_name" { default = "CoreOS 1520.8.0" }
-variable "bastion_floating_ip_pool" { default = "public" }
+# Load Balancer Configuration
+variable "lb_flavor_name" { default = "v1-standard-1" }
+variable "lb_image_name" { default = "CoreOS 1520.8.0" }
 
 # Master Configuration
 variable "master_flavor_name" { default = "v1-standard-1" }
@@ -35,11 +31,10 @@ variable "worker_image_name"  { default = "CoreOS 1520.8.0" }
 variable "worker_node_count" { default = "3" }
 
 # Network resources
+variable "public_floating_ip_pool" { default = "public" }
 variable "external_network_id" { default = "6d6357ac-0f70-4afa-8bd7-c274cc4ea235" }
 variable "external_lb_subnet_id" { default = "4083e5c2-41ef-4838-8844-d2d300d2fb06" }
 variable "internal_network_cidr" { default = "10.240.0.0/16" }
-variable "internal_lb_ip" { default = "10.240.0.103" }
-variable "use_octavia" { default = "true" }
 
 # Kubernetes configuration
 variable "etcd_endpoint" {default = "127.0.0.1"}

openstack/modules.tf

@@ -1,71 +1,59 @@
 module "master" {
   source = "./modules/master"
 
-  name = "${ var.name }"
-  master_flavor_name = "${ var.master_flavor_name }"
-  master_image_name = "${ var.master_image_name }"
-  master_node_count = "${ var.master_node_count }"
-  internal_network_id = "${ module.network.internal_network_id }"
-  internal_network_subnet_id = "${ module.network.internal_network_subnet_id }"
-  internal_lb_https_pool_id = "${ module.network.internal_lb_https_pool_id }"
-  master_cloud_init = "${ module.master_templates.master_cloud_init }"
-  security_group_name = "${ module.network.cross_cloud_security_group_name }"
-  keypair_name = "${ var.keypair_name }"
+  name           = "${ var.name }"
+  flavor         = "${ var.master_flavor_name }"
+  image          = "${ var.master_image_name }"
+  count          = "${ var.master_node_count }"
+  network_id     = "${ module.network.network_id }"
+  cloud_init     = "${ module.master_templates.master_cloud_init }"
+  security_group = "${ module.network.security_group_name }"
+  keypair        = "${ var.keypair_name }"
 }
 
 module "network" {
   source = "./modules/network"
 
   external_network_id = "${ var.external_network_id }"
   internal_network_cidr = "${ var.internal_network_cidr }"
-  internal_lb_ip = "${ var.internal_lb_ip }"
-  floating_ip_pool = "${ var.bastion_floating_ip_pool }"
-}
-
-module "bastion" {
-  source = "./modules/bastion"
-
-  name = "bastion"
-  bastion_image_name = "${ var.bastion_image_name }"
-  bastion_flavor_name = "${ var.bastion_flavor_name }"
-  internal_network_id = "${ module.network.internal_network_id }"
-  floating_ip_pool = "${ var.bastion_floating_ip_pool }"
+  floating_ip_pool = "${ var.public_floating_ip_pool }"
 }
 
 module "master_templates" {
   source = "/cncf/master_templates-v1.9.0"
 
   master_node_count = "${ var.master_node_count }"
-  name = "${ var.name }"
-  etcd_endpoint = "${ var.etcd_endpoint }"
-  etcd_bootstrap = ""
-
-  kubelet_artifact = "${ var.kubelet_artifact }"
-  cni_artifact = "${ var.cni_artifact }"
-  etcd_image = "${ var.etcd_image }"
-  etcd_tag = "${ var.etcd_tag }"
-  kube_apiserver_image = "${ var.kube_apiserver_image }"
-  kube_apiserver_tag = "${ var.kube_apiserver_tag }"
+  name              = "${ var.name }"
+  etcd_endpoint     = "${ var.etcd_endpoint }"
+  etcd_bootstrap    = ""
+
+  kubelet_artifact              = "${ var.kubelet_artifact }"
+  cni_artifact                  = "${ var.cni_artifact }"
+  etcd_image                    = "${ var.etcd_image }"
+  etcd_tag                      = "${ var.etcd_tag }"
+  kube_apiserver_image          = "${ var.kube_apiserver_image }"
+  kube_apiserver_tag            = "${ var.kube_apiserver_tag }"
   kube_controller_manager_image = "${ var.kube_controller_manager_image }"
-  kube_controller_manager_tag = "${ var.kube_controller_manager_tag }"
-  kube_scheduler_image = "${ var.kube_scheduler_image }"
-  kube_scheduler_tag = "${ var.kube_scheduler_tag }"
-  kube_proxy_image = "${ var.kube_proxy_image }"
-  kube_proxy_tag = "${ var.kube_proxy_tag }"
-
-  cloud_provider = "${ var.cloud_provider }"
-  cloud_config = "${ var.cloud_config }"
-  cluster_domain = "${ var.cluster_domain }"
-  cluster_name = "${ var.cluster_name }"
-  pod_cidr = "${ var.pod_cidr }"
-  service_cidr = "${ var.service_cidr }"
+  kube_controller_manager_tag   = "${ var.kube_controller_manager_tag }"
+  kube_scheduler_image          = "${ var.kube_scheduler_image }"
+  kube_scheduler_tag            = "${ var.kube_scheduler_tag }"
+  kube_proxy_image              = "${ var.kube_proxy_image }"
+  kube_proxy_tag                = "${ var.kube_proxy_tag }"
+
+  cloud_provider      = "${ var.cloud_provider }"
+  cloud_config        = "${ var.cloud_config }"
+  cluster_domain      = "${ var.cluster_domain }"
+  cluster_name        = "${ var.cluster_name }"
+  pod_cidr            = "${ var.pod_cidr }"
+  service_cidr        = "${ var.service_cidr }"
   non_masquerade_cidr = "${ var.non_masquerade_cidr }"
-  dns_service_ip = "${ var.dns_service_ip }"
+  dns_service_ip      = "${ var.dns_service_ip }"
 
-  ca = "${ module.tls.ca }"
-  ca_key = "${ module.tls.ca_key }"
-  apiserver = "${ module.tls.apiserver }"
+  ca            = "${ module.tls.ca }"
+  ca_key        = "${ module.tls.ca_key }"
+  apiserver     = "${ module.tls.apiserver }"
   apiserver_key = "${ module.tls.apiserver_key }"
+
   cloud_config_file = "${ data.template_file.cloud_conf.rendered }"
 
   dns_master = ""
@@ -75,41 +63,42 @@ module "master_templates" {
 module "worker" {
   source = "./modules/worker"
 
-  name = "${ var.name }"
-  worker_flavor_name = "${ var.worker_flavor_name }"
-  worker_image_name = "${ var.worker_image_name }"
-  worker_node_count = "${ var.worker_node_count }"
-  internal_network_id = "${ module.network.internal_network_id }"
-  security_group_name = "${ module.network.cross_cloud_security_group_name }"
-  worker_cloud_init = "${ module.worker_templates.worker_cloud_init }"
-  keypair_name = "${ var.keypair_name }"
+  name           = "${ var.name }"
+  flavor         = "${ var.worker_flavor_name }"
+  image          = "${ var.worker_image_name }"
+  count          = "${ var.worker_node_count }"
+  network_id     = "${ module.network.network_id }"
+  security_group = "${ module.network.security_group_name }"
+  cloud_init     = "${ module.worker_templates.worker_cloud_init }"
+  keypair        = "${ var.keypair_name }"
 }
 
 module "worker_templates" {
   source = "/cncf/worker_templates-v1.9.0"
 
   worker_node_count = "${ var.worker_node_count }"
-  name = "${ var.name }"
+  name              = "${ var.name }"
 
   kubelet_artifact = "${ var.kubelet_artifact }"
-  cni_artifact = "${ var.cni_artifact }"
+  cni_artifact     = "${ var.cni_artifact }"
   kube_proxy_image = "${ var.kube_proxy_image }"
-  kube_proxy_tag = "${ var.kube_proxy_tag }"
+  kube_proxy_tag   = "${ var.kube_proxy_tag }"
 
-  cloud_provider = "${ var.cloud_provider }"
-  cloud_config = "${ var.cloud_config }"
-  cluster_domain = "${ var.cluster_domain }"
-  pod_cidr = "${ var.pod_cidr }"
+  cloud_provider      = "${ var.cloud_provider }"
+  cloud_config        = "${ var.cloud_config }"
+  cluster_domain      = "${ var.cluster_domain }"
+  pod_cidr            = "${ var.pod_cidr }"
   non_masquerade_cidr = "${ var.non_masquerade_cidr }"
-  dns_service_ip = "${ var.dns_service_ip }"
-  internal_lb_ip = "${ var.internal_lb_ip }"
+  dns_service_ip      = "${ var.dns_service_ip }"
+  internal_lb_ip      = "${ module.network.lb_ip }"
 
-  ca = "${ module.tls.ca }"
-  worker = "${ module.tls.worker }"
+  ca         = "${ module.tls.ca }"
+  worker     = "${ module.tls.worker }"
   worker_key = "${ module.tls.worker_key }"
+
   cloud_config_file = "${ data.template_file.cloud_conf.rendered }"
 
-  dns_conf = ""
+  dns_conf   = ""
 }
 
 
@@ -135,9 +124,8 @@ module "tls" {
   tls_apiserver_cert_subject_common_name = "kubernetes-master"
   tls_apiserver_cert_validity_period_hours = 1000
   tls_apiserver_cert_early_renewal_hours = 100
-# TODO determine proper cert settings here
   tls_apiserver_cert_dns_names = "kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local,*.${ var.cloud_location }"
-  tls_apiserver_cert_ip_addresses = "127.0.0.1,10.0.0.1,100.64.0.1,${ var.internal_lb_ip },${ module.network.external_lb_fip }"
+  tls_apiserver_cert_ip_addresses = "127.0.0.1,10.0.0.1,100.64.0.1,${ var.dns_service_ip },${ module.network.fip },${ module.network.lb_ip }"
 
   tls_worker_cert_subject_common_name = "kubernetes-worker"
   tls_worker_cert_validity_period_hours = 1000
@@ -150,9 +138,24 @@ module "kubeconfig" {
   source = "../kubeconfig"
 
   data_dir = "${ var.data_dir }"
-  endpoint = "${ module.network.external_lb_fip }"
+  endpoint = "${ module.network.fip }"
   name = "${ var.name }"
   ca = "${ module.tls.ca }"
   client = "${ module.tls.client }"
   client_key = "${ module.tls.client_key }"
 }
+
+module "loadbalancer" {
+  source = "./modules/loadbalancer"
+
+  name              = "${ var.name }"
+  flavor            = "${ var.lb_flavor_name }"
+  image             = "${ var.lb_image_name }"
+  network_id        = "${ module.network.network_id }"
+  fip               = "${ module.network.fip }"
+  lb_port           = "${ module.network.lb_port }"
+  master_count      = "${ var.master_node_count }"
+  master_ips        = "${ module.master.ips }"
+  security_group    = "${ module.network.security_group_name }"
+  keypair           = "${ var.keypair_name }"
+}