Fix volume attachments while upgrade (v12 v13) (#999)

* Fix etcd volume attachment - Set proper dependencies (vm depends on volume and not opposite) - Set docker disk size to 50GB for master (prevent unnecessary rezizing) * Add nvme udev hack * Update vendor
giantswarm · Jun 11, 2018 · f103a27 · f103a27
1 parent 0fb3b83
commit f103a27
Show file tree

Hide file tree

Showing 23 changed files with 151 additions and 50 deletions.
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/service/controller/v12/cloudconfig/master_template.go b/service/controller/v12/cloudconfig/master_template.go
@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/randomkeys"
 
@@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 			Owner:        FileOwner,
 			Permissions:  0644,
 		},
+		// NVME disks udev rules and script.
+		// Workaround for https://github.com/coreos/bugs/issues/2399
+		{
+			AssetContent: cloudconfig.NVMEUdevRule,
+			Path:         "/etc/udev/rules.d/10-ebs-nvme-mapping.rules",
+			Owner:        FileOwner,
+			Permissions:  0644,
+		},
+		{
+			AssetContent: cloudconfig.NVMEUdevScript,
+			Path:         "/opt/ebs-nvme-mapping",
+			Owner:        FileOwner,
+			Permissions:  0766,
+		},
 	}
 
 	var newFiles []k8scloudconfig.FileAsset
@@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 
 func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) {
 	unitsMeta := []k8scloudconfig.UnitMetadata{
+		// Create symlinks for nvme disks.
+		// This service should be started only on first boot.
+		{
+			AssetContent: cloudconfig.NVMEUdevTriggerUnit,
+			Name:         "ebs-nvme-udev-trigger.service",
+			Enable:       false,
+			Command:      "start",
+		},
 		{
 			AssetContent: cloudconfig.DecryptTLSAssetsService,
 			Name:         "decrypt-tls-assets.service",

diff --git a/service/controller/v12/cloudconfig/spec.go b/service/controller/v12/cloudconfig/spec.go
@@ -7,7 +7,7 @@ import (
 const (
 	// CloudConfigVersion defines the version of k8scloudconfig in use.
 	// It is used in the main stack output and S3 object paths.
-	CloudConfigVersion = "v_3_3_2"
+	CloudConfigVersion = "v_3_3_3"
 )
 
 type KMSClient interface {

diff --git a/service/controller/v12/cloudconfig/worker_template.go b/service/controller/v12/cloudconfig/worker_template.go
@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 
 	"github.com/giantswarm/aws-operator/service/controller/v12/templates/cloudconfig"

diff --git a/service/controller/v12/templates/cloudconfig/format_etcd_volume.go b/service/controller/v12/templates/cloudconfig/format_etcd_volume.go
@@ -11,13 +11,11 @@ RemainAfterExit=yes
 
 # Do not wipe the disk if it's already being used, so the etcd data is
 # persistent across reboots and updates.
-Environment=DEV=/dev/nvme2n1
+Environment=DEV=/dev/xvdh
 
-# line 1: For compatibility with m3.large that has xvdX disks.
-# line 2: Create filesystem if does not exist.
-# line 3: For compatibility with older clusters. Label existing filesystem with etcd label.
+# line 1: Create filesystem if does not exist.
+# line 2: For compatibility with older clusters. Label existing filesystem with etcd label.
 ExecStart=/bin/bash -c "\
-[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\
 if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\
 [ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd"
 

diff --git a/service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go b/service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go
@@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker
 
 [Service]
 Type=oneshot
-ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)"
+ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker"
 
 [Install]
 WantedBy=multi-user.target

diff --git a/service/controller/v12/templates/cloudconfig/nvme_udev_hack.go b/service/controller/v12/templates/cloudconfig/nvme_udev_hack.go
@@ -0,0 +1,26 @@
+package cloudconfig
+
+const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c"
+`
+
+const NVMEUdevScript = `#!/bin/bash
+vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
+vol=${vol#/dev/}
+if [[ -n "$vol" ]]; then
+    echo ${vol/xvd/sd} ${vol/sd/xvd}
+fi
+`
+
+const NVMEUdevTriggerUnit = `[Unit]
+Description=Reload AWS EBS NVMe rules
+Requires=coreos-setup-environment.service
+After=coreos-setup-environment.service
+Before=user-config.target
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=-/etc/environment
+ExecStart=/usr/bin/udevadm control --reload-rules
+ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*"
+ExecStart=/usr/bin/udevadm settle
+`
diff --git a/service/controller/v12/templates/cloudformation/guest/instance.go b/service/controller/v12/templates/cloudformation/guest/instance.go
@@ -4,6 +4,9 @@ const Instance = `{{define "instance"}}
   {{ .Instance.Master.Instance.ResourceName }}:
     Type: "AWS::EC2::Instance"
     Description: Master instance
+    DependsOn:
+    - DockerVolume
+    - EtcdVolume
     Properties:
       AvailabilityZone: {{ .Instance.Master.AZ }}
       IamInstanceProfile: !Ref MasterInstanceProfile
@@ -19,25 +22,21 @@ const Instance = `{{define "instance"}}
         Value: {{ .Instance.Cluster.ID }}-master
   DockerVolume:
     Type: AWS::EC2::Volume
-    DependsOn:
-    - {{ .Instance.Master.Instance.ResourceName }}
     Properties:
       Encrypted: true
-      Size: 100
+      Size: 50
       VolumeType: gp2
-      AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
+      AvailabilityZone: {{ .Instance.Master.AZ }}
       Tags:
       - Key: Name
         Value: {{ .Instance.Master.DockerVolume.Name }}
   EtcdVolume:
     Type: AWS::EC2::Volume
-    DependsOn:
-    - {{ .Instance.Master.Instance.ResourceName }}
     Properties:
       Encrypted: true
       Size: 100
       VolumeType: gp2
-      AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
+      AvailabilityZone: {{ .Instance.Master.AZ }}
       Tags:
       - Key: Name
         Value: {{ .Instance.Master.EtcdVolume.Name }}

diff --git a/service/controller/v12/version_bundle.go b/service/controller/v12/version_bundle.go
@@ -29,7 +29,17 @@ func VersionBundle() versionbundle.Bundle {
 			},
 			{
 				Component:   "kubernetes",
-				Description: "Updated to 1.10.2 due to regression in 1.10.3 with configmaps.",
+				Description: "Updated to 1.10.4 due to regression in 1.10.3 with configmaps.",
+				Kind:        versionbundle.KindChanged,
+			},
+			{
+				Component:   "cloudconfig",
+				Description: "Added udev rule for NVMe disks.",
+				Kind:        versionbundle.KindAdded,
+			},
+			{
+				Component:   "cloudconfig",
+				Description: "Remove Nginx version from Server header in Ingress Controller",
 				Kind:        versionbundle.KindChanged,
 			},
 		},
@@ -56,7 +66,7 @@ func VersionBundle() versionbundle.Bundle {
 			},
 			{
 				Name:    "kubernetes",
-				Version: "1.10.2",
+				Version: "1.10.4",
 			},
 			{
 				Name:    "nginx-ingress-controller",

diff --git a/service/controller/v13/cloudconfig/master_template.go b/service/controller/v13/cloudconfig/master_template.go
@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/randomkeys"
 
@@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 			Owner:        FileOwner,
 			Permissions:  0644,
 		},
+		// NVME disks udev rules and script.
+		// Workaround for https://github.com/coreos/bugs/issues/2399
+		{
+			AssetContent: cloudconfig.NVMEUdevRule,
+			Path:         "/etc/udev/rules.d/10-ebs-nvme-mapping.rules",
+			Owner:        FileOwner,
+			Permissions:  0644,
+		},
+		{
+			AssetContent: cloudconfig.NVMEUdevScript,
+			Path:         "/opt/ebs-nvme-mapping",
+			Owner:        FileOwner,
+			Permissions:  0766,
+		},
 	}
 
 	var newFiles []k8scloudconfig.FileAsset
@@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 
 func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) {
 	unitsMeta := []k8scloudconfig.UnitMetadata{
+		// Create symlinks for nvme disks.
+		// This service should be started only on first boot.
+		{
+			AssetContent: cloudconfig.NVMEUdevTriggerUnit,
+			Name:         "ebs-nvme-udev-trigger.service",
+			Enable:       false,
+			Command:      "start",
+		},
 		{
 			AssetContent: cloudconfig.DecryptTLSAssetsService,
 			Name:         "decrypt-tls-assets.service",

diff --git a/service/controller/v13/cloudconfig/spec.go b/service/controller/v13/cloudconfig/spec.go
@@ -7,7 +7,7 @@ import (
 const (
 	// CloudConfigVersion defines the version of k8scloudconfig in use.
 	// It is used in the main stack output and S3 object paths.
-	CloudConfigVersion = "v_3_3_2"
+	CloudConfigVersion = "v_3_3_3"
 )
 
 type KMSClient interface {

diff --git a/service/controller/v13/cloudconfig/worker_template.go b/service/controller/v13/cloudconfig/worker_template.go
@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 
 	"github.com/giantswarm/aws-operator/service/controller/v13/templates/cloudconfig"

diff --git a/service/controller/v13/templates/cloudconfig/format_etcd_volume.go b/service/controller/v13/templates/cloudconfig/format_etcd_volume.go
@@ -11,13 +11,11 @@ RemainAfterExit=yes
 
 # Do not wipe the disk if it's already being used, so the etcd data is
 # persistent across reboots and updates.
-Environment=DEV=/dev/nvme2n1
+Environment=DEV=/dev/xvdh
 
-# line 1: For compatibility with m3.large that has xvdX disks.
-# line 2: Create filesystem if does not exist.
-# line 3: For compatibility with older clusters. Label existing filesystem with etcd label.
+# line 1: Create filesystem if does not exist.
+# line 2: For compatibility with older clusters. Label existing filesystem with etcd label.
 ExecStart=/bin/bash -c "\
-[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\
 if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\
 [ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd"
 

diff --git a/service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go b/service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go
@@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker
 
 [Service]
 Type=oneshot
-ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)"
+ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker"
 
 [Install]
 WantedBy=multi-user.target

diff --git a/service/controller/v13/templates/cloudconfig/nvme_udev_hack.go b/service/controller/v13/templates/cloudconfig/nvme_udev_hack.go
@@ -0,0 +1,26 @@
+package cloudconfig
+
+const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c"
+`
+
+const NVMEUdevScript = `#!/bin/bash
+vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
+vol=${vol#/dev/}
+if [[ -n "$vol" ]]; then
+    echo ${vol/xvd/sd} ${vol/sd/xvd}
+fi
+`
+
+const NVMEUdevTriggerUnit = `[Unit]
+Description=Reload AWS EBS NVMe rules
+Requires=coreos-setup-environment.service
+After=coreos-setup-environment.service
+Before=user-config.target
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=-/etc/environment
+ExecStart=/usr/bin/udevadm control --reload-rules
+ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*"
+ExecStart=/usr/bin/udevadm settle
+`
diff --git a/service/controller/v13/templates/cloudformation/guest/instance.go b/service/controller/v13/templates/cloudformation/guest/instance.go
@@ -4,6 +4,9 @@ const Instance = `{{define "instance"}}
   {{ .Instance.Master.Instance.ResourceName }}:
     Type: "AWS::EC2::Instance"
     Description: Master instance
+    DependsOn:
+    - DockerVolume
+    - EtcdVolume
     Properties:
       AvailabilityZone: {{ .Instance.Master.AZ }}
       IamInstanceProfile: !Ref MasterInstanceProfile
@@ -19,25 +22,21 @@ const Instance = `{{define "instance"}}
         Value: {{ .Instance.Cluster.ID }}-master
   DockerVolume:
     Type: AWS::EC2::Volume
-    DependsOn:
-    - {{ .Instance.Master.Instance.ResourceName }}
     Properties:
       Encrypted: true
-      Size: 100
+      Size: 50
       VolumeType: gp2
-      AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
+      AvailabilityZone: {{ .Instance.Master.AZ }}
       Tags:
       - Key: Name
         Value: {{ .Instance.Master.DockerVolume.Name }}
   EtcdVolume:
     Type: AWS::EC2::Volume
-    DependsOn:
-    - {{ .Instance.Master.Instance.ResourceName }}
     Properties:
       Encrypted: true
       Size: 100
       VolumeType: gp2
-      AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
+      AvailabilityZone: {{ .Instance.Master.AZ }}
       Tags:
       - Key: Name
         Value: {{ .Instance.Master.EtcdVolume.Name }}

diff --git a/service/controller/v13/version_bundle.go b/service/controller/v13/version_bundle.go
@@ -36,7 +36,7 @@ func VersionBundle() versionbundle.Bundle {
 			},
 			{
 				Name:    "kubernetes",
-				Version: "1.10.2",
+				Version: "1.10.4",
 			},
 			{
 				Name:    "nginx-ingress-controller",

diff --git a/...arm/k8scloudconfig/v_3_3_2/cloudconfig.go → ...arm/k8scloudconfig/v_3_3_3/cloudconfig.go b/...arm/k8scloudconfig/v_3_3_2/cloudconfig.go → ...arm/k8scloudconfig/v_3_3_3/cloudconfig.go
diff --git a/...iantswarm/k8scloudconfig/v_3_3_2/error.go → ...iantswarm/k8scloudconfig/v_3_3_3/error.go b/...iantswarm/k8scloudconfig/v_3_3_2/error.go → ...iantswarm/k8scloudconfig/v_3_3_3/error.go