ocp4 create and destroy on aws

README.md

@@ -10,6 +10,7 @@ This will deploy one or more clusters in the cloud, with optional post-install t
  * Docker EE
  * Openshift 3.11
  * Openshift 3.11 with CRI-O
+ * Openshift 4 (only on AWS at this time)
 
 ## Cloud
  * AWS
@@ -121,7 +122,7 @@ The `defaults.yml` file sets a number of deployment variables:
  * `quiet` - if "true", hide provisioning output
  * `auto_destroy` - if set to `true`, destroy deployment immediately after deploying (usually used with a `post_script` to output the results of a test or benchmark)
  * `nodes` - the number of worker nodes on each cluster
- * `platform` - can be set to either k8s, k3s, none, dockeree, ocp3 or ocp3c (OCPv3 with CRI-O)
+ * `platform` - can be set to either k8s, k3s, none, dockeree, ocp3, ocp3c (OCPv3 with CRI-O) or ocp4
  * `px_version` - the version of Portworx to install
  * `gcp_disks` - similar to aws_ebs, for example: `"pd-standard:20 pd-ssd:30"`
  * `gcp_region` - GCP region
@@ -140,6 +141,9 @@ The `defaults.yml` file sets a number of deployment variables:
  * `vsphere_network` - vSwitch or dvPortGroup for cluster ex: Team-SE-120
  * `vsphere_memory` - RAM in GB
  * `vsphere_cpu` - number of vCPUs
+ * `ocp4_domain` - domain that has been delegated to route53
+ * `ocp4_version` - eg `4.3.0`
+ * `ocp4_pull_secret` - the pull secret `'{"auths" ... }'`
 
 There are two ways to override these variables. The first is to specify a template with the `--template=...` parameter. For example:
 ```
@@ -211,6 +215,19 @@ $ px-deploy vsphere-init
 ```
 will read the vsphere variables from `defaults.yml` and provision a template at the path defined in `vsphere_template`.
 
+# Notes for OCP4 + AWS
+
+A "master" node will be provisioned for each cluster. This is not really a master node - it is just where `openshift-install` is run. The root user will have a kubeconfig, so it can be treated as a master node for the purposes of the scripts used in the templates.
+
+A subdomain must be delegated to Route53 on the same AWS account. There should be 4 NS records for the domain, for example:
+```
+$ host -t ns openshift.example.com
+openshift.example.com name server ns-1386.awsdns-45.org.
+openshift.example.com name server ns-1845.awsdns-38.co.uk.
+openshift.example.com name server ns-282.awsdns-35.com.
+openshift.example.com name server ns-730.awsdns-27.net.
+```
+
 # Bugs
 
  * The Azure Vagrant plugin will [fail when provisioning VMs in parallel](https://github.com/Azure/vagrant-azure/issues/229), so px-deploy disables parallel provisioning. This is really slow, and if a template uses a script that will not terminate until another VM is up, then it will never finish provisioning.

defaults.yml

@@ -37,5 +37,9 @@ vsphere_network: XXX
 vsphere_memory: 8
 vsphere_cpu: 2
 
+ocp4_domain: openshift.portworx.com
+ocp4_version: 4.3.0
+ocp4_pull_secret: XXX
+
 #env:
 #    licenses: "XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX ..."

px-deploy.go

@@ -60,6 +60,9 @@ type Config struct {
 	Description              string
 	Env                      map[string]string
 	Cluster                  []Config_Cluster
+	Ocp4_Version             string
+	Ocp4_Pull_Secret         string
+	Ocp4_Domain              string
 	Aws__Vpc                 string `yaml:"aws__vpc,omitempty"`
 	Aws__Sg                  string `yaml:"aws__sg,omitempty"`
 	Aws__Subnet              string `yaml:"aws__subnet,omitempty"`
@@ -135,7 +138,7 @@ func main() {
 				}
 			}
 			if createPlatform != "" {
-				if createPlatform != "k8s" && createPlatform != "k3s" && createPlatform != "none" && createPlatform != "dockeree" && createPlatform != "ocp3" && createPlatform != "ocp3c" {
+				if createPlatform != "k8s" && createPlatform != "k3s" && createPlatform != "none" && createPlatform != "dockeree" && createPlatform != "ocp3" && createPlatform != "ocp3c" && createPlatform != "ocp4" {
 					die("Invalid platform '" + createPlatform + "'")
 				}
 				config.Platform = createPlatform
@@ -265,6 +268,7 @@ func main() {
 				destroy_deployment(config.Name)
 				die("Aborted")
 			}
+			if config.Platform == "ocp4" && config.Cloud != "aws" { die("Openshift 4 only supported on AWS (not " + createCloud + ")") }
 			os.Chdir("/px-deploy/vagrant")
 			os.Setenv("deployment", config.Name)
 			var provider string
@@ -319,7 +323,7 @@ func main() {
 	}
 
 	cmdConnect := &cobra.Command{
-		Use:   "connect name [ command ]",
+		Use:   "connect -n name [ command ]",
 		Short: "Connects to a deployment",
 		Long:  "Connects to the first master node as root, and executes optional command",
 		Run: func(cmd *cobra.Command, args []string) {
@@ -365,7 +369,7 @@ func main() {
 
 				return nil
 			})
-			print_table([]string{"Deployment", "Cloud", "Region", "Platform", "Template", "Clusters", "Nodes", "Created"}, data)
+			print_table([]string{"Deployment", "Cloud", "Region", "Platform", "Template", "Clusters", "Nodes/Cl", "Created"}, data)
 		},
 	}
 
@@ -418,7 +422,7 @@ func main() {
 
 	defaults := parse_yaml("defaults.yml")
 	cmdCreate.Flags().StringVarP(&createName, "name", "n", "", "name of deployment to be created (if blank, generate UUID)")
-	cmdCreate.Flags().StringVarP(&createPlatform, "platform", "p", "", "k8s | dockeree | none | k3s | ocp3 | ocp3c (default "+defaults.Platform+")")
+	cmdCreate.Flags().StringVarP(&createPlatform, "platform", "p", "", "k8s | dockeree | none | k3s | ocp3 | ocp3c | ocp4 (default "+defaults.Platform+")")
 	cmdCreate.Flags().StringVarP(&createClusters, "clusters", "c", "", "number of clusters to be deployed (default "+defaults.Clusters+")")
 	cmdCreate.Flags().StringVarP(&createNodes, "nodes", "N", "", "number of nodes to be deployed in each cluster (default "+defaults.Nodes+")")
 	cmdCreate.Flags().StringVarP(&createK8sVer, "k8s_version", "k", "", "Kubernetes version to be deployed (default "+defaults.K8s_Version+")")
@@ -558,6 +562,15 @@ func destroy_deployment(name string) {
 	ip := get_ip(config.Name)
 	fmt.Println("Destroying deployment '" + config.Name + "'...")
 	if config.Cloud == "aws" {
+		if config.Platform == "ocp4" {
+			fmt.Println("Destroying OCP4, wait about 5 minutes (per cluster)...")
+			err := exec.Command("/usr/bin/ssh", "-oStrictHostKeyChecking=no", "-i", "keys/id_rsa."+config.Cloud+"."+config.Name, "root@"+ip, `
+				for i in $(seq 1 ` + config.Clusters + `); do
+				  ssh master-$i "cd /root/ocp4 ; openshift-install destroy cluster --log-level=debug"
+				done
+			`).Run()
+			if (err != nil) { die("Failed to destroy OCP4: " + err.Error()) }
+		}
 		c, _ := strconv.Atoi(config.Clusters)
 		n, _ := strconv.Atoi(config.Nodes)
 		if c < 3 && n < 5 {
@@ -568,7 +581,7 @@ func destroy_deployment(name string) {
         wait
         poweroff --force --force
         done
-      `).Start()
+			`).Start()
 			time.Sleep(5 * time.Second)
 		}
 		output, _ = exec.Command("bash", "-c", `

scripts/show-ip

@@ -1,3 +1,3 @@
 # post_script
 # Output external IP on master-1
-[ $cluster -eq 1 ] && echo IP is $(curl -s https://ipinfo.io/ip) || true
+[ $cluster -eq 1 ] && echo SUCCESS - IP is $(curl -s https://ipinfo.io/ip) || true

vagrant/Vagrantfile

@@ -19,6 +19,7 @@ node_script = "mkdir /var/log/px-deploy\n"
 node_scripts.each do |i|
   if File.exist?("/px-deploy/vagrant/#{i}") then node_script << "(\n" + File.read("/px-deploy/vagrant/#{i}") + ") >&/var/log/px-deploy/#{i}\n" end
 end
+if yaml['platform'] == "ocp4" then yaml['nodes'] = 0 end
 
 cluster = Array.new
 if !yaml['cluster'].empty?

vagrant/ocp4-master

@@ -1,66 +1,48 @@
-#https://www.openshift.com/blog/openshift-4-bare-metal-install-quickstart
-
-curl -s https://mirror.go-repo.io/centos/go-repo.repo >/etc/yum.repos.d/go-repo.repo
-dnf install -y bind git golang
-
-cat <<EOF >/var/named/db.ocp4
-\$TTL 3D
-@       IN      SOA     ocp4. root.ocp4. (
-                        202009290       ; serial, todays date + todays serial #
-                        8H              ; refresh, seconds
-                        2H              ; retry, seconds
-                        4W              ; expire, seconds
-                        1D )            ; minimum, seconds
-                NS      master-$c.ocp4.
-
-localhost       A       127.0.0.1
-api		A	192.168.10$c.90
-api-int		A	192.168.10$c.90
-*.apps		A	192.168.10$c.90
-bootstrap	A	192.168.10$c.90
-master-$c	A	192.168.10$c.90
-EOF
-
-cat <<EOF >/var/named/db.192.168.10$c
-\$TTL 3D
-10$c.168.192.in-addr.arpa.       IN      SOA     ocp4. root.ocp4. (
-                        202009290       ; serial, todays date + todays serial #
-                        8H              ; refresh, seconds
-                        2H              ; retry, seconds
-                        4W              ; expire, seconds
-                        1D )            ; minimum, seconds
-                NS      master-$c.ocp4.
-
-90	IN	PTR	master-$c.ocp4.
+eval $(ssh-agent)
+cd /tmp
+curl -sO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/$ocp4_version/openshift-install-linux-$ocp4_version.tar.gz
+curl -sO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/$ocp4_version/openshift-client-linux-$ocp4_version.tar.gz
+tar xzf openshift-install-linux-$ocp4_version.tar.gz
+tar xzf openshift-client-linux-$ocp4_version.tar.gz
+mv openshift-install oc kubectl /usr/bin
+rm -f openshift-install-linux-$ocp4_version.tar.gz openshift-client-linux-$ocp4_version.tar.gz
+
+mkdir /root/ocp4
+cd /root/ocp4
+cat <<EOF >install-config.yaml
+apiVersion: v1
+baseDomain: $ocp4_domain
+compute:
+- hyperthreading: Enabled
+  name: worker
+  platform:
+    aws:
+      type: t2.xlarge
+  replicas: $nodes
+controlPlane:
+  hyperthreading: Enabled
+  name: master
+  platform: {}
+  replicas: 3
+metadata:
+  creationTimestamp: null
+  name: $name
+networking:
+  clusterNetwork:
+  - cidr: 10.128.0.0/14
+    hostPrefix: 23
+  machineCIDR: 10.0.0.0/16
+  networkType: OpenShiftSDN
+  serviceNetwork:
+  - 172.30.0.0/16
+platform:
+  aws:
+    region: $aws_region
+pullSecret: '$ocp4_pull_secret'
+sshKey: |
 EOF
-
-for i in $(seq 1 $nodes); do
-  echo "node-$c-$i		A	192.168.1$(printf %.2d $c).1$(printf %.2d $i)" >>/var/named/db.ocp4
-  echo "10$i		PTR	node-$c-$i.ocp4." >>/var/named/db.192.168.10$c
-done
-
-cat <<EOF >>/etc/named.conf
-zone "ocp4" {
-type master;
-   file "/var/named/db.ocp4";
-};
-
-zone "1$(printf %.2d $c).168.192.in-addr.arpa" {
-type master;
-   file "/var/named/db.192.168.1$(printf %.2d $c)";
-};
-EOF
-
-systemctl enable bind
-systemctl start bind
-
-cd /root
-git clone https://github.com/openshift/installer
-cd installer
-./hack/build.sh
-#./bin/openshift-install create install-config
-#curl -sLO https://github.com/openshift/okd/releases/download/4.5.0-0.okd-2020-09-04-180756/openshift-install-linux-4.5.0-0.okd-2020-09-04-180756.tar.gz
-#tar -C /usr/bin -xzf openshift-install-linux-4.5.0-0.okd-2020-09-04-180756.tar.gz openshift-install
-#curl -sLO https://github.com/openshift/okd/releases/download/4.5.0-0.okd-2020-09-04-180756/openshift-client-linux-4.5.0-0.okd-2020-09-04-180756.tar.gz
-#tar -C /usr/bin -xzf openshift-client-linux-4.5.0-0.okd-2020-09-04-180756.tar.gz oc
-#ln -s /usr/bin/oc /usr/bin/kubectl
+echo -n "  " >>install-config.yaml
+ssh-keygen -y -f /root/.ssh/id_rsa >>install-config.yaml
+openshift-install create cluster --log-level=debug
+mkdir /root/.kube
+cp /root/ocp4/auth/kubeconfig /root/.kube/config