From 7afe5d7d2540402c00e3699bd2be83dec47d3043 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 14:45:36 -0700 Subject: [PATCH 01/13] asset/ignition: fix master and worker configs This fixes a few issues with the configs: - the Ignition spec version needed to be set - the SSH key needed to be added to the masters and workers - the CA was set to the root key instead of the root cert --- pkg/asset/ignition/bootstrap.go | 6 +++++- pkg/asset/ignition/master.go | 2 +- pkg/asset/ignition/node.go | 8 ++++++++ pkg/asset/ignition/worker.go | 2 +- pkg/asset/tls/certkey.go | 10 ++++++++++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/asset/ignition/bootstrap.go b/pkg/asset/ignition/bootstrap.go index 013c0d834f7..e34477570d8 100644 --- a/pkg/asset/ignition/bootstrap.go +++ b/pkg/asset/ignition/bootstrap.go @@ -137,7 +137,11 @@ func (a *bootstrap) Generate(dependencies map[asset.Asset]*asset.State) (*asset. return nil, err } - config := ignition.Config{} + config := ignition.Config{ + Ignition: ignition.Ignition{ + Version: ignition.MaxVersion.String(), + }, + } a.addBootstrapFiles(&config, dependencies) a.addBootkubeFiles(&config, dependencies, templateData) diff --git a/pkg/asset/ignition/master.go b/pkg/asset/ignition/master.go index 35262a44023..09388f37513 100644 --- a/pkg/asset/ignition/master.go +++ b/pkg/asset/ignition/master.go @@ -47,7 +47,7 @@ func (a *master) Generate(dependencies map[asset.Asset]*asset.State) (*asset.Sta } for i := range state.Contents { state.Contents[i].Name = fmt.Sprintf("master-%d.ign", i) - state.Contents[i].Data = pointerIgnitionConfig(installConfig, dependencies[a.rootCA].Contents[0].Data, "master", fmt.Sprintf("etcd_index=%d", i)) + state.Contents[i].Data = pointerIgnitionConfig(installConfig, dependencies[a.rootCA].Contents[tls.CertIndex].Data, "master", fmt.Sprintf("etcd_index=%d", i)) } return state, nil diff --git a/pkg/asset/ignition/node.go b/pkg/asset/ignition/node.go index 1bbc481394c..960ea5c56b4 100644 --- a/pkg/asset/ignition/node.go +++ b/pkg/asset/ignition/node.go @@ -67,6 +67,7 @@ func masterCount(installConfig *types.InstallConfig) int { func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string, query string) []byte { data, err := json.Marshal(ignition.Config{ Ignition: ignition.Ignition{ + Version: ignition.MaxVersion.String(), Config: ignition.IgnitionConfig{ Append: []ignition.ConfigReference{{ Source: func() *url.URL { @@ -87,6 +88,13 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro }, }, }, + // XXX: Remove this once MCO supports injecting SSH keys. + Passwd: ignition.Passwd{ + Users: []ignition.PasswdUser{{ + Name: "core", + SSHAuthorizedKeys: []ignition.SSHAuthorizedKey{ignition.SSHAuthorizedKey(installConfig.Admin.SSHKey)}, + }}, + }, }) if err != nil { panic(fmt.Sprintf("Failed to marshal pointer Ignition config: %v", err)) diff --git a/pkg/asset/ignition/worker.go b/pkg/asset/ignition/worker.go index 51351ce9183..8988be184cc 100644 --- a/pkg/asset/ignition/worker.go +++ b/pkg/asset/ignition/worker.go @@ -43,7 +43,7 @@ func (a *worker) Generate(dependencies map[asset.Asset]*asset.State) (*asset.Sta return &asset.State{ Contents: []asset.Content{{ Name: "worker.ign", - Data: pointerIgnitionConfig(installConfig, dependencies[a.rootCA].Contents[0].Data, "worker", ""), + Data: pointerIgnitionConfig(installConfig, dependencies[a.rootCA].Contents[tls.CertIndex].Data, "worker", ""), }}, }, nil } diff --git a/pkg/asset/tls/certkey.go b/pkg/asset/tls/certkey.go index 7ef90db73b8..bd83a77acf3 100644 --- a/pkg/asset/tls/certkey.go +++ b/pkg/asset/tls/certkey.go @@ -14,6 +14,16 @@ import ( "github.com/openshift/installer/pkg/types" ) +const ( + // KeyIndex is the index into a CertKey asset's contents at which the key + // can be found. + KeyIndex = 0 + + // CertIndex is the index into a CertKey asset's contents at which the + // certificate can be found. + CertIndex = 1 +) + // CertKey contains the private key and the cert that's // signed by the parent CA. type CertKey struct { From 979ec87a87808d2b7fe11fbed46765406713c15c Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 14:50:11 -0700 Subject: [PATCH 02/13] asset/ignition: update images and manifests The KubeCoreRenderer needed to be bumped, kubelet.service needed an update, and tectonic.sh needed a few assets removed. --- pkg/asset/ignition/bootstrap.go | 4 ++-- pkg/asset/ignition/content/kubelet.go | 9 ++++++--- pkg/asset/ignition/content/tectonic.go | 11 ----------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/pkg/asset/ignition/bootstrap.go b/pkg/asset/ignition/bootstrap.go index e34477570d8..cd21e4b14cf 100644 --- a/pkg/asset/ignition/bootstrap.go +++ b/pkg/asset/ignition/bootstrap.go @@ -193,7 +193,7 @@ func (a *bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootst CloudProvider: getCloudProvider(installConfig), CloudProviderConfig: getCloudProviderConfig(installConfig), DebugConfig: "", - KubeCoreRenderImage: "quay.io/coreos/kube-core-renderer-dev:436b1b4395ae54d866edc88864c9b01797cebac1", + KubeCoreRenderImage: "quay.io/coreos/kube-core-renderer-dev:3b6952f5a1ba89bb32dd0630faddeaf2779c9a85", MachineConfigOperatorImage: "docker.io/openshift/origin-machine-config-operator:v4.0.0", EtcdCertSignerImage: "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6", EtcdctlImage: "quay.io/coreos/etcd:v3.2.14", @@ -207,7 +207,7 @@ func (a *bootstrap) addBootstrapFiles(config *ignition.Config, dependencies map[ config.Storage.Files = append( config.Storage.Files, fileFromBytes("/etc/kubernetes/kubeconfig", 0600, dependencies[a.kubeconfigKubelet].Contents[0].Data), - fileFromBytes("/var/lib/kubeconfig", 0600, dependencies[a.kubeconfigKubelet].Contents[0].Data), + fileFromBytes("/var/lib/kubelet/kubeconfig", 0600, dependencies[a.kubeconfigKubelet].Contents[0].Data), ) config.Storage.Files = append( config.Storage.Files, diff --git a/pkg/asset/ignition/content/kubelet.go b/pkg/asset/ignition/content/kubelet.go index d665bbaada8..52e881f2dff 100644 --- a/pkg/asset/ignition/content/kubelet.go +++ b/pkg/asset/ignition/content/kubelet.go @@ -14,15 +14,17 @@ Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests ExecStartPre=/usr/bin/bash -c "gawk '/certificate-authority-data/ {print $2}' /etc/kubernetes/kubeconfig | base64 --decode > /etc/kubernetes/ca.crt" +Environment=KUBELET_RUNTIME_REQUEST_TIMEOUT=10m +EnvironmentFile=-/etc/kubernetes/kubelet-env ExecStart=/usr/bin/hyperkube \ kubelet \ --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --rotate-certificates \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --cni-bin-dir=/var/lib/cni/bin \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=/var/run/crio/crio.sock \ + --runtime-request-timeout=${KUBELET_RUNTIME_REQUEST_TIMEOUT} \ --lock-file=/var/run/lock/kubelet.lock \ --exit-on-lock-contention \ --pod-manifest-path=/etc/kubernetes/manifests \ @@ -36,6 +38,7 @@ ExecStart=/usr/bin/hyperkube \ --cloud-provider={{.CloudProvider}} \ --anonymous-auth=false \ --cgroup-driver=systemd \ + --serialize-image-pulls=false \ {{.CloudProviderConfig}} \ {{.DebugConfig}} \ diff --git a/pkg/asset/ignition/content/tectonic.go b/pkg/asset/ignition/content/tectonic.go index a56ff21a2a5..2cfb1e5979e 100644 --- a/pkg/asset/ignition/content/tectonic.go +++ b/pkg/asset/ignition/content/tectonic.go @@ -101,27 +101,16 @@ kubectl create --filename ingress/pull.json echo "Creating operators..." kubectl create --filename security/priviledged-scc-tectonic.yaml -kubectl create --filename updater/tectonic-channel-operator-kind.yaml kubectl create --filename updater/app-version-kind.yaml kubectl create --filename updater/migration-status-kind.yaml -kubectl --namespace=tectonic-system get customresourcedefinition channeloperatorconfigs.tco.coreos.com -kubectl create --filename updater/tectonic-channel-operator-config.yaml - kubectl create --filename updater/operators/kube-core-operator.yaml -kubectl create --filename updater/operators/tectonic-channel-operator.yaml kubectl create --filename updater/operators/kube-addon-operator.yaml -kubectl create --filename updater/operators/tectonic-alm-operator.yaml -kubectl create --filename updater/operators/tectonic-utility-operator.yaml -kubectl create --filename updater/operators/tectonic-ingress-controller-operator.yaml kubectl --namespace=tectonic-system get customresourcedefinition appversions.tco.coreos.com kubectl create --filename updater/app_versions/app-version-tectonic-cluster.yaml kubectl create --filename updater/app_versions/app-version-kube-core.yaml kubectl create --filename updater/app_versions/app-version-kube-addon.yaml -kubectl create --filename updater/app_versions/app-version-tectonic-alm.yaml -kubectl create --filename updater/app_versions/app-version-tectonic-utility.yaml -kubectl create --filename updater/app_versions/app-version-tectonic-ingress.yaml # Wait for Tectonic pods wait_for_pods tectonic-system From a79473f6024d44a478689e48bb53344005967c75 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:15:24 -0700 Subject: [PATCH 03/13] types: move node count methods to install config Many other assets will need to use master and worker counts. --- pkg/asset/ignition/bootstrap.go | 2 +- pkg/asset/ignition/master.go | 2 +- pkg/asset/ignition/node.go | 11 ----------- pkg/types/installconfig.go | 11 +++++++++++ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/asset/ignition/bootstrap.go b/pkg/asset/ignition/bootstrap.go index cd21e4b14cf..7d8468bac1e 100644 --- a/pkg/asset/ignition/bootstrap.go +++ b/pkg/asset/ignition/bootstrap.go @@ -184,7 +184,7 @@ func (a *bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootst if err != nil { return nil, err } - etcdEndpoints := make([]string, masterCount(installConfig)) + etcdEndpoints := make([]string, installConfig.MasterCount()) for i := range etcdEndpoints { etcdEndpoints[i] = fmt.Sprintf("https://%s-etcd-%d.%s:2379", installConfig.Name, i, installConfig.BaseDomain) } diff --git a/pkg/asset/ignition/master.go b/pkg/asset/ignition/master.go index 09388f37513..e12444dd267 100644 --- a/pkg/asset/ignition/master.go +++ b/pkg/asset/ignition/master.go @@ -43,7 +43,7 @@ func (a *master) Generate(dependencies map[asset.Asset]*asset.State) (*asset.Sta } state := &asset.State{ - Contents: make([]asset.Content, masterCount(installConfig)), + Contents: make([]asset.Content, installConfig.MasterCount()), } for i := range state.Contents { state.Contents[i].Name = fmt.Sprintf("master-%d.ign", i) diff --git a/pkg/asset/ignition/node.go b/pkg/asset/ignition/node.go index 960ea5c56b4..2ddf578d58f 100644 --- a/pkg/asset/ignition/node.go +++ b/pkg/asset/ignition/node.go @@ -51,17 +51,6 @@ func fileFromBytes(path string, mode int, contents []byte) ignition.File { } } -// masterCount determines the number of master nodes from the install config, -// defaulting to one if it is unspecified. -func masterCount(installConfig *types.InstallConfig) int { - for _, m := range installConfig.Machines { - if m.Name == "master" && m.Replicas != nil { - return int(*m.Replicas) - } - } - return 1 -} - // pointerIgnitionConfig generates a config which references the remote config // served by the machine config server. func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string, query string) []byte { diff --git a/pkg/types/installconfig.go b/pkg/types/installconfig.go index cfb538d8bab..faba2d4ccbd 100644 --- a/pkg/types/installconfig.go +++ b/pkg/types/installconfig.go @@ -37,6 +37,17 @@ type InstallConfig struct { PullSecret string `json:"pullSecret"` } +// MasterCount returns the number of replicas in the master machine pool, +// defaulting to one if no machine pool was found. +func (c *InstallConfig) MasterCount() int { + for _, m := range c.Machines { + if m.Name == "master" && m.Replicas != nil { + return int(*m.Replicas) + } + } + return 1 +} + // Admin is the configuration for the admin user. type Admin struct { // Email is the email address of the admin user. From aa9cc20ccda567548739c3c1f8e9a530de69e596 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:23:50 -0700 Subject: [PATCH 04/13] asset/manifests: clean up content formatting Some of the manifests had a trailing tab character, which isn't valid YAML. --- pkg/asset/manifests/content/bootkube/01-tectonic-namespace.go | 2 +- pkg/asset/manifests/content/bootkube/02-ingress-namespace.go | 2 +- .../content/bootkube/03-openshift-web-console-namespace.go | 2 +- .../content/bootkube/04-openshift-machine-config-operator.go | 2 +- .../content/bootkube/05-openshift-cluster-api-namespace.go | 2 +- pkg/asset/manifests/content/bootkube/app-version-kind.go | 2 +- .../manifests/content/bootkube/cluster-apiserver-certs.go | 2 +- pkg/asset/manifests/content/bootkube/ign-config.go | 2 +- pkg/asset/manifests/content/bootkube/kube-apiserver-secret.go | 2 +- pkg/asset/manifests/content/bootkube/kube-cloud-config.go | 2 +- .../content/bootkube/kube-controller-manager-secret.go | 2 +- pkg/asset/manifests/content/bootkube/machine-api-operator.go | 3 +-- .../content/bootkube/machine-config-operator-00-config-crd.go | 2 +- .../bootkube/machine-config-operator-01-images-configmap.go | 2 +- .../content/bootkube/machine-config-operator-02-rbac.go | 2 +- .../content/bootkube/machine-config-operator-03-deployment.go | 2 +- .../content/bootkube/machine-config-server-tls-secret.go | 2 +- .../manifests/content/bootkube/openshift-apiserver-secret.go | 2 +- pkg/asset/manifests/content/bootkube/operatorstatus-crd.go | 2 +- .../manifests/content/bootkube/tectonic-network-operator.go | 2 +- .../manifests/content/tectonic/updater/app-version-kind.go | 2 +- .../updater/appversions/app-version-tectonic-cluster.go | 2 +- .../content/tectonic/updater/migration-status-kind.go | 2 +- 23 files changed, 23 insertions(+), 24 deletions(-) diff --git a/pkg/asset/manifests/content/bootkube/01-tectonic-namespace.go b/pkg/asset/manifests/content/bootkube/01-tectonic-namespace.go index 67a93added7..106679ff6a8 100644 --- a/pkg/asset/manifests/content/bootkube/01-tectonic-namespace.go +++ b/pkg/asset/manifests/content/bootkube/01-tectonic-namespace.go @@ -10,5 +10,5 @@ metadata: labels: # network policy can only select by labels name: tectonic-system openshift.io/run-level: "1" - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/02-ingress-namespace.go b/pkg/asset/manifests/content/bootkube/02-ingress-namespace.go index 4e5a9380900..852cee1684c 100644 --- a/pkg/asset/manifests/content/bootkube/02-ingress-namespace.go +++ b/pkg/asset/manifests/content/bootkube/02-ingress-namespace.go @@ -13,5 +13,5 @@ metadata: kubernetes.io/ingress.class: tectonic name: openshift-ingress openshift.io/run-level: "1" - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/03-openshift-web-console-namespace.go b/pkg/asset/manifests/content/bootkube/03-openshift-web-console-namespace.go index a0b818e1f66..e7c8f643fde 100644 --- a/pkg/asset/manifests/content/bootkube/03-openshift-web-console-namespace.go +++ b/pkg/asset/manifests/content/bootkube/03-openshift-web-console-namespace.go @@ -11,5 +11,5 @@ metadata: name: openshift-web-console labels: name: openshift-web-console - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/04-openshift-machine-config-operator.go b/pkg/asset/manifests/content/bootkube/04-openshift-machine-config-operator.go index dd7dcfd50f0..91811be1dd3 100644 --- a/pkg/asset/manifests/content/bootkube/04-openshift-machine-config-operator.go +++ b/pkg/asset/manifests/content/bootkube/04-openshift-machine-config-operator.go @@ -10,5 +10,5 @@ metadata: labels: name: openshift-machine-config-operator openshift.io/run-level: "1" - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/05-openshift-cluster-api-namespace.go b/pkg/asset/manifests/content/bootkube/05-openshift-cluster-api-namespace.go index 52fbb034591..9fee418676d 100644 --- a/pkg/asset/manifests/content/bootkube/05-openshift-cluster-api-namespace.go +++ b/pkg/asset/manifests/content/bootkube/05-openshift-cluster-api-namespace.go @@ -11,5 +11,5 @@ metadata: labels: name: openshift-cluster-api openshift.io/run-level: "1" - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/app-version-kind.go b/pkg/asset/manifests/content/bootkube/app-version-kind.go index c8a37301b41..dcec739cb89 100644 --- a/pkg/asset/manifests/content/bootkube/app-version-kind.go +++ b/pkg/asset/manifests/content/bootkube/app-version-kind.go @@ -13,5 +13,5 @@ spec: names: plural: "appversions" kind: "AppVersion" - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/cluster-apiserver-certs.go b/pkg/asset/manifests/content/bootkube/cluster-apiserver-certs.go index 0dfb7a6e0c9..a47430d876b 100644 --- a/pkg/asset/manifests/content/bootkube/cluster-apiserver-certs.go +++ b/pkg/asset/manifests/content/bootkube/cluster-apiserver-certs.go @@ -19,5 +19,5 @@ metadata: data: tls.crt: {{.ClusterapiCaCert}} tls.key: {{.ClusterapiCaKey}} - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/ign-config.go b/pkg/asset/manifests/content/bootkube/ign-config.go index 0b9fe358223..1222a9d40d0 100644 --- a/pkg/asset/manifests/content/bootkube/ign-config.go +++ b/pkg/asset/manifests/content/bootkube/ign-config.go @@ -15,5 +15,5 @@ metadata: type: Opaque data: userData: {{.WorkerIgnConfig}} - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/kube-apiserver-secret.go b/pkg/asset/manifests/content/bootkube/kube-apiserver-secret.go index b52c90c0820..5f54761659f 100644 --- a/pkg/asset/manifests/content/bootkube/kube-apiserver-secret.go +++ b/pkg/asset/manifests/content/bootkube/kube-apiserver-secret.go @@ -31,5 +31,5 @@ data: service-serving-ca.crt: {{.ServiceServingCaCert}} service-serving-ca.key: {{.ServiceServingCaKey}} kubeconfig: {{.OpenshiftLoopbackKubeconfig}} - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/kube-cloud-config.go b/pkg/asset/manifests/content/bootkube/kube-cloud-config.go index 85ce2d7b98f..5463525d053 100644 --- a/pkg/asset/manifests/content/bootkube/kube-cloud-config.go +++ b/pkg/asset/manifests/content/bootkube/kube-cloud-config.go @@ -15,5 +15,5 @@ metadata: type: Opaque data: config: "" - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/kube-controller-manager-secret.go b/pkg/asset/manifests/content/bootkube/kube-controller-manager-secret.go index 8d9fb805622..cd88bd018eb 100644 --- a/pkg/asset/manifests/content/bootkube/kube-controller-manager-secret.go +++ b/pkg/asset/manifests/content/bootkube/kube-controller-manager-secret.go @@ -18,5 +18,5 @@ data: root-ca.crt: {{.RootCaCert}} kube-ca.crt: {{.KubeCaCert}} kube-ca.key: {{.KubeCaKey}} - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/machine-api-operator.go b/pkg/asset/manifests/content/bootkube/machine-api-operator.go index 48e4765eb88..2deea3e8247 100644 --- a/pkg/asset/manifests/content/bootkube/machine-api-operator.go +++ b/pkg/asset/manifests/content/bootkube/machine-api-operator.go @@ -56,6 +56,5 @@ spec: items: - key: mao-config path: config - - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/machine-config-operator-00-config-crd.go b/pkg/asset/manifests/content/bootkube/machine-config-operator-00-config-crd.go index 44254564406..69805aefa72 100644 --- a/pkg/asset/manifests/content/bootkube/machine-config-operator-00-config-crd.go +++ b/pkg/asset/manifests/content/bootkube/machine-config-operator-00-config-crd.go @@ -27,5 +27,5 @@ spec: singular: mcoconfig # kind is normally the CamelCased singular type. Your resource manifests use this. kind: MCOConfig - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/machine-config-operator-01-images-configmap.go b/pkg/asset/manifests/content/bootkube/machine-config-operator-01-images-configmap.go index fbea072d20d..fa27836473c 100644 --- a/pkg/asset/manifests/content/bootkube/machine-config-operator-01-images-configmap.go +++ b/pkg/asset/manifests/content/bootkube/machine-config-operator-01-images-configmap.go @@ -10,5 +10,5 @@ metadata: namespace: openshift-machine-config-operator data: images.json: '{"machineConfigController": "docker.io/openshift/origin-machine-config-controller:v4.0.0", "machineConfigDaemon": "docker.io/openshift/origin-machine-config-daemon:v4.0.0", "machineConfigServer": "docker.io/openshift/origin-machine-config-server:v4.0.0"}' - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/machine-config-operator-02-rbac.go b/pkg/asset/manifests/content/bootkube/machine-config-operator-02-rbac.go index 78c9f5691cc..3cb2b0ac1c1 100644 --- a/pkg/asset/manifests/content/bootkube/machine-config-operator-02-rbac.go +++ b/pkg/asset/manifests/content/bootkube/machine-config-operator-02-rbac.go @@ -16,5 +16,5 @@ roleRef: kind: ClusterRole name: cluster-admin apiGroup: rbac.authorization.k8s.io - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/machine-config-operator-03-deployment.go b/pkg/asset/manifests/content/bootkube/machine-config-operator-03-deployment.go index 425e651503c..d3b51f24e59 100644 --- a/pkg/asset/manifests/content/bootkube/machine-config-operator-03-deployment.go +++ b/pkg/asset/manifests/content/bootkube/machine-config-operator-03-deployment.go @@ -64,5 +64,5 @@ spec: - name: root-ca hostPath: path: /etc/kubernetes/ca.crt - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/machine-config-server-tls-secret.go b/pkg/asset/manifests/content/bootkube/machine-config-server-tls-secret.go index 2b4d5bdfcdc..5fe0c6c9e89 100644 --- a/pkg/asset/manifests/content/bootkube/machine-config-server-tls-secret.go +++ b/pkg/asset/manifests/content/bootkube/machine-config-server-tls-secret.go @@ -16,5 +16,5 @@ type: Opaque data: tls.crt: {{.McsTLSCert}} tls.key: {{.McsTLSKey}} - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/openshift-apiserver-secret.go b/pkg/asset/manifests/content/bootkube/openshift-apiserver-secret.go index f272d117cd2..59eb37cbdfe 100644 --- a/pkg/asset/manifests/content/bootkube/openshift-apiserver-secret.go +++ b/pkg/asset/manifests/content/bootkube/openshift-apiserver-secret.go @@ -33,5 +33,5 @@ data: service-serving-ca.crt: {{.ServiceServingCaCert}} service-serving-ca.key: {{.ServiceServingCaKey}} kubeconfig: {{.OpenshiftLoopbackKubeconfig}} - `)) +`)) ) diff --git a/pkg/asset/manifests/content/bootkube/operatorstatus-crd.go b/pkg/asset/manifests/content/bootkube/operatorstatus-crd.go index 78894ca6f90..610ec898434 100644 --- a/pkg/asset/manifests/content/bootkube/operatorstatus-crd.go +++ b/pkg/asset/manifests/content/bootkube/operatorstatus-crd.go @@ -27,5 +27,5 @@ spec: singular: operatorstatus # kind is normally the CamelCased singular type. Your resource manifests use this. kind: OperatorStatus - ` +` ) diff --git a/pkg/asset/manifests/content/bootkube/tectonic-network-operator.go b/pkg/asset/manifests/content/bootkube/tectonic-network-operator.go index e3ff2bb2ee4..91d1ce9b5cd 100644 --- a/pkg/asset/manifests/content/bootkube/tectonic-network-operator.go +++ b/pkg/asset/manifests/content/bootkube/tectonic-network-operator.go @@ -66,5 +66,5 @@ spec: rollingUpdate: maxUnavailable: 1 type: RollingUpdate - `)) +`)) ) diff --git a/pkg/asset/manifests/content/tectonic/updater/app-version-kind.go b/pkg/asset/manifests/content/tectonic/updater/app-version-kind.go index 6b391526a10..d59de2549ab 100644 --- a/pkg/asset/manifests/content/tectonic/updater/app-version-kind.go +++ b/pkg/asset/manifests/content/tectonic/updater/app-version-kind.go @@ -13,5 +13,5 @@ spec: names: plural: "appversions" kind: "AppVersion" - ` +` ) diff --git a/pkg/asset/manifests/content/tectonic/updater/appversions/app-version-tectonic-cluster.go b/pkg/asset/manifests/content/tectonic/updater/appversions/app-version-tectonic-cluster.go index 05de50302ac..69f887b7a5c 100644 --- a/pkg/asset/manifests/content/tectonic/updater/appversions/app-version-tectonic-cluster.go +++ b/pkg/asset/manifests/content/tectonic/updater/appversions/app-version-tectonic-cluster.go @@ -20,5 +20,5 @@ spec: status: currentVersion: {{.TectonicVersion}} paused: false - `)) +`)) ) diff --git a/pkg/asset/manifests/content/tectonic/updater/migration-status-kind.go b/pkg/asset/manifests/content/tectonic/updater/migration-status-kind.go index 87344ac1f53..06b771bcc55 100644 --- a/pkg/asset/manifests/content/tectonic/updater/migration-status-kind.go +++ b/pkg/asset/manifests/content/tectonic/updater/migration-status-kind.go @@ -13,5 +13,5 @@ spec: names: plural: "migrationstatuses" kind: "MigrationStatus" - ` +` ) From 0dff9d98c366d76e0b14d607833e04f7bdeceb9a Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:25:38 -0700 Subject: [PATCH 05/13] asset/manifests: fix etcd url generation There are multiple etcd servers in the cluster. --- pkg/asset/manifests/kube-core-operator.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/asset/manifests/kube-core-operator.go b/pkg/asset/manifests/kube-core-operator.go index 26b4cd2cf26..8e5c7c0f210 100644 --- a/pkg/asset/manifests/kube-core-operator.go +++ b/pkg/asset/manifests/kube-core-operator.go @@ -3,6 +3,7 @@ package manifests import ( "fmt" "net" + "strings" "github.com/ghodss/yaml" @@ -99,7 +100,7 @@ func (kco *kubeCoreOperator) coreConfig() (*kubecore.OperatorConfig, error) { coreConfig.NetworkConfig.ClusterCIDR = kco.installConfig.Networking.PodCIDR.String() coreConfig.NetworkConfig.ServiceCIDR = kco.installConfig.Networking.ServiceCIDR.String() coreConfig.NetworkConfig.AdvertiseAddress = networkConfigAdvertiseAddress - coreConfig.NetworkConfig.EtcdServers = kco.getEtcdServersURLs() + coreConfig.NetworkConfig.EtcdServers = strings.Join(kco.getEtcdServersURLs(), ",") return &coreConfig, nil } @@ -108,8 +109,12 @@ func (kco *kubeCoreOperator) getAPIServerURL() string { return fmt.Sprintf("https://%s-api.%s:6443", kco.installConfig.Name, kco.installConfig.BaseDomain) } -func (kco *kubeCoreOperator) getEtcdServersURLs() string { - return fmt.Sprintf("https://%s-etcd.%s:2379", kco.installConfig.Name, kco.installConfig.BaseDomain) +func (kco *kubeCoreOperator) getEtcdServersURLs() []string { + var urls []string + for i := 0; i < kco.installConfig.MasterCount(); i++ { + urls = append(urls, fmt.Sprintf("https://%s-etcd-%d.%s:2379", kco.installConfig.Name, i, kco.installConfig.BaseDomain)) + } + return urls } func (kco *kubeCoreOperator) getOicdIssuerURL() string { From ee20594d4aa9b21ca289352a116c8bb8f2487c80 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:31:47 -0700 Subject: [PATCH 06/13] asset/kubeconfig: fix kubeconfig name Bootkube expects the name of the kubeconfig to be "kubeconfig", not "kubeconfig-admin". --- pkg/asset/kubeconfig/kubeconfig.go | 5 +++-- pkg/asset/kubeconfig/kubeconfig_test.go | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/asset/kubeconfig/kubeconfig.go b/pkg/asset/kubeconfig/kubeconfig.go index fd02b1e3bd0..4e44a25ee26 100644 --- a/pkg/asset/kubeconfig/kubeconfig.go +++ b/pkg/asset/kubeconfig/kubeconfig.go @@ -47,12 +47,13 @@ func (k *Kubeconfig) Generate(parents map[asset.Asset]*asset.State) (*asset.Stat return nil, err } - var keyFilename, certFilename string + var keyFilename, certFilename, kubeconfigSuffix string switch k.userName { case KubeconfigUserNameAdmin: keyFilename, certFilename = tls.AdminKeyName, tls.AdminCertName case KubeconfigUserNameKubelet: keyFilename, certFilename = tls.KubeletKeyName, tls.KubeletCertName + kubeconfigSuffix = fmt.Sprintf("-%s", KubeconfigUserNameKubelet) } clientKeyData, err := asset.GetDataByFilename(k.certKey, parents, keyFilename) if err != nil { @@ -107,7 +108,7 @@ func (k *Kubeconfig) Generate(parents map[asset.Asset]*asset.State) (*asset.Stat Contents: []asset.Content{ { // E.g. generated/auth/kubeconfig-admin. - Name: filepath.Join("auth", fmt.Sprintf("kubeconfig-%s", k.userName)), + Name: filepath.Join("auth", "kubeconfig"+kubeconfigSuffix), Data: data, }, }, diff --git a/pkg/asset/kubeconfig/kubeconfig_test.go b/pkg/asset/kubeconfig/kubeconfig_test.go index 631bb30bcd2..e2b3afe9063 100644 --- a/pkg/asset/kubeconfig/kubeconfig_test.go +++ b/pkg/asset/kubeconfig/kubeconfig_test.go @@ -2,7 +2,6 @@ package kubeconfig import ( "fmt" - "path/filepath" "testing" "github.com/openshift/installer/pkg/asset" @@ -210,8 +209,6 @@ users: t.Errorf("expect error %v, saw nil", tt.errString) } - filename := filepath.Join("auth", fmt.Sprintf("kubeconfig-%s", tt.userName)) - assert.Equal(t, filename, st.Contents[0].Name, "unexpected filename") assert.Equal(t, tt.expectedData, st.Contents[0].Data, "unexpected data in kubeconfig") }) } From edc121c73333568ffb679983549589990a7b8ee0 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:33:51 -0700 Subject: [PATCH 07/13] asset/manifests: fix up secret generation All secrets need to be base64 encoded. This also fixes the key/cert pair used by MCS. --- pkg/asset/manifests/operators.go | 165 +++++++++++++------------------ pkg/asset/manifests/stock.go | 1 - pkg/asset/manifests/tectonic.go | 56 +++++------ 3 files changed, 93 insertions(+), 129 deletions(-) diff --git a/pkg/asset/manifests/operators.go b/pkg/asset/manifests/operators.go index 795ca0247b7..461398fdc38 100644 --- a/pkg/asset/manifests/operators.go +++ b/pkg/asset/manifests/operators.go @@ -3,6 +3,7 @@ package manifests import ( "bytes" + "encoding/base64" "path/filepath" "text/template" @@ -32,7 +33,6 @@ type manifests struct { apiServerCertKey asset.Asset openshiftAPIServerCertKey asset.Asset apiServerProxyCertKey asset.Asset - adminCertKey asset.Asset kubeletCertKey asset.Asset mcsCertKey asset.Asset serviceAccountKeyPair asset.Asset @@ -69,7 +69,7 @@ func (m *manifests) Dependencies() []asset.Asset { m.apiServerCertKey, m.openshiftAPIServerCertKey, m.apiServerProxyCertKey, - m.adminCertKey, + m.mcsCertKey, m.kubeletCertKey, m.serviceAccountKeyPair, m.kubeconfig, @@ -104,8 +104,6 @@ func (m *manifests) Generate(dependencies map[asset.Asset]*asset.State) (*asset. return nil, err } - bootkubeContents := m.generateBootKubeManifests(dependencies) - state := &asset.State{ Contents: []asset.Content{ { @@ -118,7 +116,7 @@ func (m *manifests) Generate(dependencies map[asset.Asset]*asset.State) (*asset. }, }, } - state.Contents = append(state.Contents, bootkubeContents...) + state.Contents = append(state.Contents, m.generateBootKubeManifests(dependencies)...) return state, nil } @@ -127,111 +125,82 @@ func (m *manifests) generateBootKubeManifests(dependencies map[asset.Asset]*asse if err != nil { return nil } - assetContents := make([]asset.Content, 0) templateData := &bootkubeTemplateData{ - AggregatorCaCert: string(dependencies[m.aggregatorCA].Contents[certIndex].Data), - AggregatorCaKey: string(dependencies[m.aggregatorCA].Contents[keyIndex].Data), - ApiserverCert: string(dependencies[m.apiServerCertKey].Contents[certIndex].Data), - ApiserverKey: string(dependencies[m.apiServerCertKey].Contents[keyIndex].Data), - ApiserverProxyCert: string(dependencies[m.apiServerProxyCertKey].Contents[certIndex].Data), - ApiserverProxyKey: string(dependencies[m.apiServerProxyCertKey].Contents[keyIndex].Data), + AggregatorCaCert: base64.StdEncoding.EncodeToString(dependencies[m.aggregatorCA].Contents[certIndex].Data), + AggregatorCaKey: base64.StdEncoding.EncodeToString(dependencies[m.aggregatorCA].Contents[keyIndex].Data), + ApiserverCert: base64.StdEncoding.EncodeToString(dependencies[m.apiServerCertKey].Contents[certIndex].Data), + ApiserverKey: base64.StdEncoding.EncodeToString(dependencies[m.apiServerCertKey].Contents[keyIndex].Data), + ApiserverProxyCert: base64.StdEncoding.EncodeToString(dependencies[m.apiServerProxyCertKey].Contents[certIndex].Data), + ApiserverProxyKey: base64.StdEncoding.EncodeToString(dependencies[m.apiServerProxyCertKey].Contents[keyIndex].Data), Base64encodeCloudProviderConfig: "", // FIXME - ClusterapiCaCert: string(dependencies[m.clusterAPIServerCertKey].Contents[certIndex].Data), - ClusterapiCaKey: string(dependencies[m.clusterAPIServerCertKey].Contents[keyIndex].Data), - EtcdCaCert: string(dependencies[m.etcdCA].Contents[certIndex].Data), - EtcdClientCert: string(dependencies[m.etcdClientCertKey].Contents[certIndex].Data), - EtcdClientKey: string(dependencies[m.etcdClientCertKey].Contents[keyIndex].Data), - KubeCaCert: string(dependencies[m.kubeCA].Contents[certIndex].Data), - KubeCaKey: string(dependencies[m.kubeCA].Contents[keyIndex].Data), + ClusterapiCaCert: base64.StdEncoding.EncodeToString(dependencies[m.clusterAPIServerCertKey].Contents[certIndex].Data), + ClusterapiCaKey: base64.StdEncoding.EncodeToString(dependencies[m.clusterAPIServerCertKey].Contents[keyIndex].Data), + EtcdCaCert: base64.StdEncoding.EncodeToString(dependencies[m.etcdCA].Contents[certIndex].Data), + EtcdClientCert: base64.StdEncoding.EncodeToString(dependencies[m.etcdClientCertKey].Contents[certIndex].Data), + EtcdClientKey: base64.StdEncoding.EncodeToString(dependencies[m.etcdClientCertKey].Contents[keyIndex].Data), + KubeCaCert: base64.StdEncoding.EncodeToString(dependencies[m.kubeCA].Contents[certIndex].Data), + KubeCaKey: base64.StdEncoding.EncodeToString(dependencies[m.kubeCA].Contents[keyIndex].Data), MachineConfigOperatorImage: "docker.io/openshift/origin-machine-config-operator:v4.0.0", - McsTLSCert: string(dependencies[m.adminCertKey].Contents[certIndex].Data), - McsTLSKey: string(dependencies[m.adminCertKey].Contents[keyIndex].Data), - OidcCaCert: string(dependencies[m.kubeCA].Contents[certIndex].Data), - OpenshiftApiserverCert: string(dependencies[m.openshiftAPIServerCertKey].Contents[certIndex].Data), - OpenshiftApiserverKey: string(dependencies[m.openshiftAPIServerCertKey].Contents[keyIndex].Data), - OpenshiftLoopbackKubeconfig: string(dependencies[m.kubeconfig].Contents[0].Data), - PullSecret: string(ic.PullSecret), - RootCaCert: string(dependencies[m.rootCA].Contents[certIndex].Data), - ServiceaccountKey: string(dependencies[m.serviceAccountKeyPair].Contents[keyIndex].Data), - ServiceaccountPub: string(dependencies[m.serviceAccountKeyPair].Contents[certIndex].Data), - ServiceServingCaCert: string(dependencies[m.serviceServingCA].Contents[certIndex].Data), - ServiceServingCaKey: string(dependencies[m.serviceServingCA].Contents[keyIndex].Data), + McsTLSCert: base64.StdEncoding.EncodeToString(dependencies[m.mcsCertKey].Contents[certIndex].Data), + McsTLSKey: base64.StdEncoding.EncodeToString(dependencies[m.mcsCertKey].Contents[keyIndex].Data), + OidcCaCert: base64.StdEncoding.EncodeToString(dependencies[m.kubeCA].Contents[certIndex].Data), + OpenshiftApiserverCert: base64.StdEncoding.EncodeToString(dependencies[m.openshiftAPIServerCertKey].Contents[certIndex].Data), + OpenshiftApiserverKey: base64.StdEncoding.EncodeToString(dependencies[m.openshiftAPIServerCertKey].Contents[keyIndex].Data), + OpenshiftLoopbackKubeconfig: base64.StdEncoding.EncodeToString(dependencies[m.kubeconfig].Contents[0].Data), + PullSecret: base64.StdEncoding.EncodeToString([]byte(ic.PullSecret)), + RootCaCert: base64.StdEncoding.EncodeToString(dependencies[m.rootCA].Contents[certIndex].Data), + ServiceaccountKey: base64.StdEncoding.EncodeToString(dependencies[m.serviceAccountKeyPair].Contents[keyIndex].Data), + ServiceaccountPub: base64.StdEncoding.EncodeToString(dependencies[m.serviceAccountKeyPair].Contents[certIndex].Data), + ServiceServingCaCert: base64.StdEncoding.EncodeToString(dependencies[m.serviceServingCA].Contents[certIndex].Data), + ServiceServingCaKey: base64.StdEncoding.EncodeToString(dependencies[m.serviceServingCA].Contents[keyIndex].Data), TectonicNetworkOperatorImage: "quay.io/coreos/tectonic-network-operator-dev:3b6952f5a1ba89bb32dd0630faddeaf2779c9a85", WorkerIgnConfig: "", // FIXME: this means depending on ignition assets (risk of cyclical dependencies) } - // belongs to machine api operator - data := applyTemplateData(bootkube.ClusterApiserverCerts, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "cluster-apiserver-certs.yaml"), Data: []byte(data)}) - - // machine api operator - data = applyTemplateData(bootkube.IgnConfig, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "ign-config.yaml"), Data: []byte(data)}) - - // kco - data = applyTemplateData(bootkube.KubeApiserverSecret, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "kube-apiserver-secret.yaml"), Data: []byte(data)}) - - // kco - data = applyTemplateData(bootkube.KubeCloudConfig, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "kube-cloud-config.yaml"), Data: []byte(data)}) - - // kco - data = applyTemplateData(bootkube.KubeControllerManagerSecret, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "kube-controller-manager-secret.yaml"), Data: []byte(data)}) - - // mco - data = applyTemplateData(bootkube.MachineConfigOperator03Deployment, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "machine-config-operator-03-deployment.yaml"), Data: []byte(data)}) - - // mco - data = applyTemplateData(bootkube.MachineConfigServerTLSSecret, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "machine-config-server-tls-secret.yaml"), Data: []byte(data)}) - - // kube core - data = applyTemplateData(bootkube.OpenshiftApiserverSecret, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "openshift-apiserver-secret.yaml"), Data: []byte(data)}) - - // common - data = applyTemplateData(bootkube.Pull, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "pull.json"), Data: []byte(data)}) - - // network operator - data = applyTemplateData(bootkube.TectonicNetworkOperator, templateData) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "tectonic-network-operator.yaml"), Data: []byte(data)}) - - // common - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "01-tectonic-namespace.yaml"), Data: []byte(bootkube.TectonicNamespace)}) - // ingress - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "02-ingress-namespace.yaml"), Data: []byte(bootkube.IngressNamespace)}) - // kao - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "03-openshift-web-console-namespace.yaml"), Data: []byte(bootkube.OpenshiftWebConsoleNamespace)}) - // mco - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "openshift-machine-config-operator.yaml"), Data: []byte(bootkube.OpenshiftMachineConfigOperator)}) - // machine api operator - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "openshift-cluster-api-namespace.yaml"), Data: []byte(bootkube.OpenshiftClusterAPINamespace)}) - // common - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "app-version-kind.yaml"), Data: []byte(bootkube.AppVersionKind)}) - // cmacine api operator - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "app-version-mao.yaml"), Data: []byte(bootkube.AppVersionMao)}) - // network - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "app-version-tectonic-network.yaml"), Data: []byte(bootkube.AppVersionTectonicNetwork)}) - // machine api operator - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "machine-api-operator.yaml"), Data: []byte(bootkube.MachineAPIOperator)}) - - // mco - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "machine-config-operator-00-config-crd.yaml"), Data: []byte(bootkube.MachineConfigOperator00ConfigCrd)}) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "machine-config-operator-01-images-configmap.yaml"), Data: []byte(bootkube.MachineConfigOperator01ImagesConfigmap)}) - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "machine-config-operator-02-rbac.yaml"), Data: []byte(bootkube.MachineConfigOperator02Rbac)}) - // common/cvo - assetContents = append(assetContents, asset.Content{Name: filepath.Join(manifestDir, "operatorstatus-crd.yaml"), Data: []byte(bootkube.OperatorstatusCrd)}) + assetData := map[string][]byte{ + "cluster-apiserver-certs.yaml": applyTemplateData(bootkube.ClusterApiserverCerts, templateData), + "ign-config.yaml": applyTemplateData(bootkube.IgnConfig, templateData), + "kube-apiserver-secret.yaml": applyTemplateData(bootkube.KubeApiserverSecret, templateData), + "kube-cloud-config.yaml": applyTemplateData(bootkube.KubeCloudConfig, templateData), + "kube-controller-manager-secret.yaml": applyTemplateData(bootkube.KubeControllerManagerSecret, templateData), + "machine-config-operator-03-deployment.yaml": applyTemplateData(bootkube.MachineConfigOperator03Deployment, templateData), + "machine-config-server-tls-secret.yaml": applyTemplateData(bootkube.MachineConfigServerTLSSecret, templateData), + "openshift-apiserver-secret.yaml": applyTemplateData(bootkube.OpenshiftApiserverSecret, templateData), + "pull.json": applyTemplateData(bootkube.Pull, templateData), + "tectonic-network-operator.yaml": applyTemplateData(bootkube.TectonicNetworkOperator, templateData), + + "01-tectonic-namespace.yaml": []byte(bootkube.TectonicNamespace), + "02-ingress-namespace.yaml": []byte(bootkube.IngressNamespace), + "03-openshift-web-console-namespace.yaml": []byte(bootkube.OpenshiftWebConsoleNamespace), + "04-openshift-machine-config-operator.yaml": []byte(bootkube.OpenshiftMachineConfigOperator), + "05-openshift-cluster-api-namespace.yaml": []byte(bootkube.OpenshiftClusterAPINamespace), + "openshift-machine-config-operator.yaml": []byte(bootkube.OpenshiftMachineConfigOperator), + "openshift-cluster-api-namespace.yaml": []byte(bootkube.OpenshiftClusterAPINamespace), + "app-version-kind.yaml": []byte(bootkube.AppVersionKind), + "app-version-mao.yaml": []byte(bootkube.AppVersionMao), + "app-version-tectonic-network.yaml": []byte(bootkube.AppVersionTectonicNetwork), + "machine-api-operator.yaml": []byte(bootkube.MachineAPIOperator), + "machine-config-operator-00-config-crd.yaml": []byte(bootkube.MachineConfigOperator00ConfigCrd), + "machine-config-operator-01-images-configmap.yaml": []byte(bootkube.MachineConfigOperator01ImagesConfigmap), + "machine-config-operator-02-rbac.yaml": []byte(bootkube.MachineConfigOperator02Rbac), + "operatorstatus-crd.yaml": []byte(bootkube.OperatorstatusCrd), + } + + var assetContents []asset.Content + for name, data := range assetData { + assetContents = append(assetContents, asset.Content{ + Name: filepath.Join(manifestDir, name), + Data: data, + }) + } + return assetContents } -func applyTemplateData(template *template.Template, templateData interface{}) string { +func applyTemplateData(template *template.Template, templateData interface{}) []byte { buf := &bytes.Buffer{} if err := template.Execute(buf, templateData); err != nil { panic(err) } - return buf.String() + return buf.Bytes() } diff --git a/pkg/asset/manifests/stock.go b/pkg/asset/manifests/stock.go index 96aee2fbe84..3ceee350a9b 100644 --- a/pkg/asset/manifests/stock.go +++ b/pkg/asset/manifests/stock.go @@ -60,7 +60,6 @@ func (s *StockImpl) EstablishStock(stock installconfig.Stock, tlsStock tls.Stock apiServerCertKey: tlsStock.APIServerCertKey(), openshiftAPIServerCertKey: tlsStock.OpenshiftAPIServerCertKey(), apiServerProxyCertKey: tlsStock.APIServerProxyCertKey(), - adminCertKey: tlsStock.AdminCertKey(), kubeletCertKey: tlsStock.KubeletCertKey(), mcsCertKey: tlsStock.MCSCertKey(), serviceAccountKeyPair: tlsStock.ServiceAccountKeyPair(), diff --git a/pkg/asset/manifests/tectonic.go b/pkg/asset/manifests/tectonic.go index 8073bec66f2..df62152f7ca 100644 --- a/pkg/asset/manifests/tectonic.go +++ b/pkg/asset/manifests/tectonic.go @@ -1,6 +1,7 @@ package manifests import ( + "encoding/base64" "path/filepath" "github.com/openshift/installer/pkg/asset" @@ -44,25 +45,22 @@ func (t *tectonic) Generate(dependencies map[asset.Asset]*asset.State) (*asset.S if err != nil { return nil, err } - manifestDir := "tectonic" - assetContents := make([]asset.Content, 0) - ingressContents := dependencies[t.ingressCertKey].Contents templateData := &tectonicTemplateData{ - IngressCaCert: string(dependencies[t.kubeCA].Contents[certIndex].Data), + IngressCaCert: base64.StdEncoding.EncodeToString(dependencies[t.kubeCA].Contents[certIndex].Data), IngressKind: "haproxy-router", IngressStatusPassword: ic.Admin.Password, // FIXME: generate a new random one instead? - IngressTLSBundle: string(ingressContents[certIndex].Data), - IngressTLSCert: string(ingressContents[certIndex].Data), - IngressTLSKey: string(ingressContents[keyIndex].Data), + IngressTLSBundle: base64.StdEncoding.EncodeToString(ingressContents[certIndex].Data), + IngressTLSCert: base64.StdEncoding.EncodeToString(ingressContents[certIndex].Data), + IngressTLSKey: base64.StdEncoding.EncodeToString(ingressContents[keyIndex].Data), KubeAddonOperatorImage: "quay.io/coreos/kube-addon-operator-dev:3b6952f5a1ba89bb32dd0630faddeaf2779c9a85", KubeCoreOperatorImage: "quay.io/coreos/kube-core-operator-dev:3b6952f5a1ba89bb32dd0630faddeaf2779c9a85", - PullSecret: ic.PullSecret, + PullSecret: base64.StdEncoding.EncodeToString([]byte(ic.PullSecret)), TectonicIngressControllerOperatorImage: "quay.io/coreos/tectonic-ingress-controller-operator-dev:3b6952f5a1ba89bb32dd0630faddeaf2779c9a85", TectonicVersion: "1.8.4-tectonic.2", } - assetContentMap := map[string]string{ + assetData := map[string][]byte{ // template files "secrets/ingress-tls.yaml": applyTemplateData(secrets.IngressTLS, templateData), "secrets/ca-cert.yaml": applyTemplateData(secrets.CaCert, templateData), @@ -75,28 +73,26 @@ func (t *tectonic) Generate(dependencies map[asset.Asset]*asset.State) (*asset.S "ingress/cluster-config.yaml": applyTemplateData(ingress.ClusterConfig, templateData), // constant files - "security/priviledged-scc-tectonic.yaml": security.PriviledgedSccTectonic, - "rbac/role-admin.yaml": rbac.RoleAdmin, - "rbac/binding-admin.yaml": rbac.BindingAdmin, - "rbac/binding-discovery.yaml": rbac.BindingDiscovery, - "rbac/role-user.yaml": rbac.RoleUser, - "updater/migration-status-kind.yaml": updater.MigrationStatusKind, - "updater/app_versions/app-version-kube-addon.yaml": appversions.AppVersionKubeAddon, - "updater/app_versions/app-version-tectonic-ingress.yaml": appversions.AppVersionTectonicIngress, - "updater/app_versions/app-version-kube-core.yaml": appversions.AppVersionKubeCore, - "updater/app-version-kind.yaml": updater.AppVersionKind, - "ingress/svc-account.yaml": ingress.SvcAccount, + "security/priviledged-scc-tectonic.yaml": []byte(security.PriviledgedSccTectonic), + "rbac/role-admin.yaml": []byte(rbac.RoleAdmin), + "rbac/binding-admin.yaml": []byte(rbac.BindingAdmin), + "rbac/binding-discovery.yaml": []byte(rbac.BindingDiscovery), + "rbac/role-user.yaml": []byte(rbac.RoleUser), + "updater/migration-status-kind.yaml": []byte(updater.MigrationStatusKind), + "updater/app_versions/app-version-kube-addon.yaml": []byte(appversions.AppVersionKubeAddon), + "updater/app_versions/app-version-tectonic-ingress.yaml": []byte(appversions.AppVersionTectonicIngress), + "updater/app_versions/app-version-kube-core.yaml": []byte(appversions.AppVersionKubeCore), + "updater/app-version-kind.yaml": []byte(updater.AppVersionKind), + "ingress/svc-account.yaml": []byte(ingress.SvcAccount), } - for k, v := range assetContentMap { - assetContent := asset.Content{ - Name: filepath.Join(manifestDir, k), - Data: []byte(v), - } - assetContents = append(assetContents, assetContent) - } - state := &asset.State{ - Contents: assetContents, + var assetContents []asset.Content + for name, data := range assetData { + assetContents = append(assetContents, asset.Content{ + Name: filepath.Join("tectonic", name), + Data: data, + }) } - return state, nil + + return &asset.State{Contents: assetContents}, nil } From 45ba4569594e3c818c3641e0884e2c833c1667b2 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:39:47 -0700 Subject: [PATCH 08/13] asset/tls: fix key and certificate names --- pkg/asset/tls/stock.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/asset/tls/stock.go b/pkg/asset/tls/stock.go index 3731c38b453..d02300547ca 100644 --- a/pkg/asset/tls/stock.go +++ b/pkg/asset/tls/stock.go @@ -18,9 +18,9 @@ const ( // KubeCACertName is the filename of the KubeCACert. KubeCACertName = "kube-ca.crt" // EtcdCAKeyName is the filename of the EtcdCAKey. - EtcdCAKeyName = "etcd-ca.key" + EtcdCAKeyName = "etcd-client-ca.key" // EtcdCACertName is the filename of the EtcdCACert. - EtcdCACertName = "etcd-ca.crt" + EtcdCACertName = "etcd-client-ca.crt" // AggregatorCAKeyName is the filename of the AggregatorCAKey. AggregatorCAKeyName = "aggregator-ca.key" // AggregatorCACertName is the filename of the AggregatorCACert. @@ -58,9 +58,9 @@ const ( // KubeletCertName is the filename of the KubeletCert. KubeletCertName = "kubelet.crt" // MCSKeyName is the filename of the MCSKey. - MCSKeyName = "mcs.key" + MCSKeyName = "machine-config-server.key" // MCSCertName is the filename of the MCSCert. - MCSCertName = "mcs.crt" + MCSCertName = "machine-config-server.crt" // ClusterAPIServerCAKeyName is the filename of the ClusterAPIServerCAKey. ClusterAPIServerCAKeyName = "cluster-apiserver-ca.key" // ClusterAPIServerCACertName is the filename of the ClusterAPIServerCACert. @@ -283,7 +283,7 @@ func (s *StockImpl) EstablishStock(stock installconfig.Stock) { s.clusterAPIServerCertKey = &CertKey{ installConfig: stock.InstallConfig(), - Subject: pkix.Name{CommonName: "cluster-apiserver", OrganizationalUnit: []string{"bootkube"}}, + Subject: pkix.Name{CommonName: "clusterapi.openshift-cluster-api.svc", OrganizationalUnit: []string{"bootkube"}}, KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, Validity: ValidityTenYears, KeyFileName: ClusterAPIServerCAKeyName, From 5465b272a5824f04fd3f7e913e3fe741350a12e4 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:49:58 -0700 Subject: [PATCH 09/13] asset/ignition: write etcd ca to correct location The certificate also belongs in /etc/ssl/etcd. --- pkg/asset/ignition/bootstrap.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/asset/ignition/bootstrap.go b/pkg/asset/ignition/bootstrap.go index 7d8468bac1e..5911facf0bf 100644 --- a/pkg/asset/ignition/bootstrap.go +++ b/pkg/asset/ignition/bootstrap.go @@ -260,6 +260,11 @@ func (a *bootstrap) addTLSCertFiles(config *ignition.Config, dependencies map[as } { config.Storage.Files = append(config.Storage.Files, filesFromContents(rootDir, 0600, dependencies[asset].Contents)...) } + + config.Storage.Files = append( + config.Storage.Files, + fileFromBytes("/etc/ssl/etcd/ca.crt", 0600, dependencies[a.etcdClientCertKey].Contents[tls.CertIndex].Data), + ) } func getCloudProvider(installConfig *types.InstallConfig) string { From 94531423e68c9b057b804b060e8852ebd43b06ca Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sun, 23 Sep 2018 09:29:36 -0700 Subject: [PATCH 10/13] asset/cluster: use cache for libvirt images This swaps out the URL of the OS image right before the cluster is created so that the cached image is used instead of fetching it from the network. --- pkg/asset/cluster/tfvars.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go index 3ce0c964ae2..a0709896ae1 100644 --- a/pkg/asset/cluster/tfvars.go +++ b/pkg/asset/cluster/tfvars.go @@ -64,6 +64,12 @@ func (t *TerraformVariables) Generate(parents map[asset.Asset]*asset.State) (*as return nil, err } + if cluster.Platform == config.PlatformLibvirt { + if err := cluster.Libvirt.UseCachedImage(); err != nil { + return nil, err + } + } + data, err := cluster.TFVars() if err != nil { return nil, err From 0104b055f3def2d437ec31daedf104cbf0d1387c Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sun, 23 Sep 2018 15:04:58 -0700 Subject: [PATCH 11/13] asset/installconfig: use a default image url Instead of asking the user for an image URL, this just defaults to the internal Red Hat CoreOS build server. --- pkg/asset/installconfig/installconfig.go | 4 ++-- pkg/asset/installconfig/platform.go | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/asset/installconfig/installconfig.go b/pkg/asset/installconfig/installconfig.go index 9a9d80bcfb6..a5686160e41 100644 --- a/pkg/asset/installconfig/installconfig.go +++ b/pkg/asset/installconfig/installconfig.go @@ -21,6 +21,7 @@ var ( defaultLibvirtNetworkIfName = "tt0" defaultLibvirtNetworkIPRange = "192.168.124.0/24" + defaultLibvirtImageURL = "http://aos-ostree.rhev-ci-vms.eng.rdu2.redhat.com/rhcos/images/cloud/latest/rhcos-qemu.qcow2.gz" ) // installConfig generates the install-config.yml file. @@ -113,7 +114,6 @@ func (a *installConfig) Generate(dependencies map[asset.Asset]*asset.State) (*as } case LibvirtPlatformType: uri := string(platformState.Contents[1].Data) - image := string(platformState.Contents[2].Data) installConfig.Libvirt = &types.LibvirtPlatform{ URI: uri, @@ -123,7 +123,7 @@ func (a *installConfig) Generate(dependencies map[asset.Asset]*asset.State) (*as IPRange: defaultLibvirtNetworkIPRange, }, DefaultMachinePlatform: &types.LibvirtMachinePoolPlatform{ - Image: image, + Image: defaultLibvirtImageURL, }, } // Set the default master and worker nodes to 1 for AWS. diff --git a/pkg/asset/installconfig/platform.go b/pkg/asset/installconfig/platform.go index 0caee73253a..7abeca3cdd4 100644 --- a/pkg/asset/installconfig/platform.go +++ b/pkg/asset/installconfig/platform.go @@ -102,23 +102,16 @@ func (a *Platform) awsPlatform() (*asset.State, error) { } func (a *Platform) libvirtPlatform() (*asset.State, error) { - var uri, image string + var uri string survey.AskOne(&survey.Input{ Message: "URI", Help: "The libvirt connection URI to be used. This must be accessible from the running cluster.", Default: "qemu+tcp://192.168.122.1/system", }, &uri, nil) - survey.AskOne(&survey.Input{ - Message: "Image", - Help: "The URL to the OS image.", - Default: "", - }, &image, nil) - return assetStateForStringContents( LibvirtPlatformType, uri, - image, ), nil } From 762d577740f3947a573bdfa436820116be717175 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sun, 23 Sep 2018 15:08:13 -0700 Subject: [PATCH 12/13] asset/installconfig: remove unneeded code --- pkg/asset/installconfig/installconfig.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pkg/asset/installconfig/installconfig.go b/pkg/asset/installconfig/installconfig.go index a5686160e41..256991ae50a 100644 --- a/pkg/asset/installconfig/installconfig.go +++ b/pkg/asset/installconfig/installconfig.go @@ -80,16 +80,6 @@ func (a *installConfig) Generate(dependencies map[asset.Asset]*asset.State) (*as }, }, PullSecret: pullSecret, - Machines: []types.MachinePool{ - { - Name: "master", - Replicas: func(x int64) *int64 { return &x }(3), - }, - { - Name: "worker", - Replicas: func(x int64) *int64 { return &x }(3), - }, - }, } platformState := dependencies[a.assetStock.Platform()] @@ -101,7 +91,6 @@ func (a *installConfig) Generate(dependencies map[asset.Asset]*asset.State) (*as Region: region, VPCCIDRBlock: defaultVPCCIDR, } - // Set the default master and worker nodes to 3 for AWS. installConfig.Machines = []types.MachinePool{ { Name: "master", @@ -126,7 +115,6 @@ func (a *installConfig) Generate(dependencies map[asset.Asset]*asset.State) (*as Image: defaultLibvirtImageURL, }, } - // Set the default master and worker nodes to 1 for AWS. installConfig.Machines = []types.MachinePool{ { Name: "master", From ba9d7e44dd417c6de4ff8c8e8a21ae27b6be9608 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Sat, 22 Sep 2018 15:09:08 -0700 Subject: [PATCH 13/13] asset/*: fix ip address calculations The issue was a result of IPNet's ParseCIDR() always returning a 16-byte address, while some third-party libraries assume that the address length corresponds to whether or not IPv6 is in use. --- pkg/asset/manifests/kube-core-operator.go | 4 +--- pkg/ipnet/ipnet.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/asset/manifests/kube-core-operator.go b/pkg/asset/manifests/kube-core-operator.go index 8e5c7c0f210..ccaff3983b0 100644 --- a/pkg/asset/manifests/kube-core-operator.go +++ b/pkg/asset/manifests/kube-core-operator.go @@ -2,7 +2,6 @@ package manifests import ( "fmt" - "net" "strings" "github.com/ghodss/yaml" @@ -85,8 +84,7 @@ func (kco *kubeCoreOperator) coreConfig() (*kubecore.OperatorConfig, error) { coreConfig.AuthConfig.OIDCGroupsClaim = authConfigOIDCGroupsClaim coreConfig.AuthConfig.OIDCUsernameClaim = authConfigOIDCUsernameClaim - svcCidr := kco.installConfig.Networking.ServiceCIDR - ip, err := cidr.Host(&net.IPNet{IP: svcCidr.IP, Mask: svcCidr.Mask}, 10) + ip, err := cidr.Host(&kco.installConfig.Networking.ServiceCIDR.IPNet, 10) if err != nil { return nil, err } diff --git a/pkg/ipnet/ipnet.go b/pkg/ipnet/ipnet.go index e6a4c3e4f5e..59c20d976a6 100644 --- a/pkg/ipnet/ipnet.go +++ b/pkg/ipnet/ipnet.go @@ -52,7 +52,19 @@ func (ipnet *IPNet) UnmarshalJSON(b []byte) (err error) { if err != nil { return err } - ipnet.IP = ip + + // This check is needed in order to work around a strange quirk in the Go + // standard library. All of the addresses returned by net.ParseCIDR() are + // 16-byte addresses. This does _not_ imply that they are IPv6 addresses, + // which is what some libraries (e.g. github.com/apparentlymart/go-cidr) + // assume. By forcing the address to be the expected length, we can work + // around these bugs. + if ip.To4() != nil { + ipnet.IP = ip.To4() + } else { + ipnet.IP = ip + } ipnet.Mask = net.Mask + return nil }