Add nvme udev hack

service/controller/v12/cloudconfig/master_template.go

@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/randomkeys"
 
@@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 			Owner:        FileOwner,
 			Permissions:  0644,
 		},
+		// NVME disks udev rules and script.
+		// Workaround for https://github.com/coreos/bugs/issues/2399
+		{
+			AssetContent: cloudconfig.NVMEUdevRule,
+			Path:         "/etc/udev/rules.d/10-ebs-nvme-mapping.rules",
+			Owner:        FileOwner,
+			Permissions:  0644,
+		},
+		{
+			AssetContent: cloudconfig.NVMEUdevScript,
+			Path:         "/opt/ebs-nvme-mapping",
+			Owner:        FileOwner,
+			Permissions:  0766,
+		},
 	}
 
 	var newFiles []k8scloudconfig.FileAsset
@@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 
 func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) {
 	unitsMeta := []k8scloudconfig.UnitMetadata{
+		// Create symlinks for nvme disks.
+		// This service should be started only on first boot.
+		{
+			AssetContent: cloudconfig.NVMEUdevTriggerUnit,
+			Name:         "ebs-nvme-udev-trigger.service",
+			Enable:       false,
+			Command:      "start",
+		},
 		{
 			AssetContent: cloudconfig.DecryptTLSAssetsService,
 			Name:         "decrypt-tls-assets.service",

service/controller/v12/cloudconfig/spec.go

@@ -7,7 +7,7 @@ import (
 const (
 	// CloudConfigVersion defines the version of k8scloudconfig in use.
 	// It is used in the main stack output and S3 object paths.
-	CloudConfigVersion = "v_3_3_2"
+	CloudConfigVersion = "v_3_3_3"
 )
 
 type KMSClient interface {

service/controller/v12/cloudconfig/worker_template.go

@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 
 	"github.com/giantswarm/aws-operator/service/controller/v12/templates/cloudconfig"

service/controller/v12/templates/cloudconfig/format_etcd_volume.go

@@ -11,13 +11,11 @@ RemainAfterExit=yes
 
 # Do not wipe the disk if it's already being used, so the etcd data is
 # persistent across reboots and updates.
-Environment=DEV=/dev/nvme2n1
+Environment=DEV=/dev/xvdh
 
-# line 1: For compatibility with m3.large that has xvdX disks.
-# line 2: Create filesystem if does not exist.
-# line 3: For compatibility with older clusters. Label existing filesystem with etcd label.
+# line 1: Create filesystem if does not exist.
+# line 2: For compatibility with older clusters. Label existing filesystem with etcd label.
 ExecStart=/bin/bash -c "\
-[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\
 if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\
 [ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd"
 

service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go

@@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker
 
 [Service]
 Type=oneshot
-ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)"
+ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker"
 
 [Install]
 WantedBy=multi-user.target

service/controller/v12/templates/cloudconfig/nvme_udev_hack.go

@@ -0,0 +1,26 @@
+package cloudconfig
+
+const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c"
+`
+
+const NVMEUdevScript = `#!/bin/bash
+vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
+vol=${vol#/dev/}
+if [[ -n "$vol" ]]; then
+    echo ${vol/xvd/sd} ${vol/sd/xvd}
+fi
+`
+
+const NVMEUdevTriggerUnit = `[Unit]
+Description=Reload AWS EBS NVMe rules
+Requires=coreos-setup-environment.service
+After=coreos-setup-environment.service
+Before=user-config.target
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=-/etc/environment
+ExecStart=/usr/bin/udevadm control --reload-rules
+ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*"
+ExecStart=/usr/bin/udevadm settle
+`

service/controller/v12/version_bundle.go

@@ -29,7 +29,17 @@ func VersionBundle() versionbundle.Bundle {
 			},
 			{
 				Component:   "kubernetes",
-				Description: "Updated to 1.10.2 due to regression in 1.10.3 with configmaps.",
+				Description: "Updated to 1.10.4 due to regression in 1.10.3 with configmaps.",
+				Kind:        versionbundle.KindChanged,
+			},
+			{
+				Component:   "cloudconfig",
+				Description: "Added udev rule for NVMe disks.",
+				Kind:        versionbundle.KindAdded,
+			},
+			{
+				Component:   "cloudconfig",
+				Description: "Remove Nginx version from Server header in Ingress Controller",
 				Kind:        versionbundle.KindChanged,
 			},
 		},
@@ -56,7 +66,7 @@ func VersionBundle() versionbundle.Bundle {
 			},
 			{
 				Name:    "kubernetes",
-				Version: "1.10.2",
+				Version: "1.10.4",
 			},
 			{
 				Name:    "nginx-ingress-controller",

service/controller/v13/cloudconfig/master_template.go

@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/randomkeys"
 
@@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 			Owner:        FileOwner,
 			Permissions:  0644,
 		},
+		// NVME disks udev rules and script.
+		// Workaround for https://github.com/coreos/bugs/issues/2399
+		{
+			AssetContent: cloudconfig.NVMEUdevRule,
+			Path:         "/etc/udev/rules.d/10-ebs-nvme-mapping.rules",
+			Owner:        FileOwner,
+			Permissions:  0644,
+		},
+		{
+			AssetContent: cloudconfig.NVMEUdevScript,
+			Path:         "/opt/ebs-nvme-mapping",
+			Owner:        FileOwner,
+			Permissions:  0766,
+		},
 	}
 
 	var newFiles []k8scloudconfig.FileAsset
@@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
 
 func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) {
 	unitsMeta := []k8scloudconfig.UnitMetadata{
+		// Create symlinks for nvme disks.
+		// This service should be started only on first boot.
+		{
+			AssetContent: cloudconfig.NVMEUdevTriggerUnit,
+			Name:         "ebs-nvme-udev-trigger.service",
+			Enable:       false,
+			Command:      "start",
+		},
 		{
 			AssetContent: cloudconfig.DecryptTLSAssetsService,
 			Name:         "decrypt-tls-assets.service",

service/controller/v13/cloudconfig/spec.go

@@ -7,7 +7,7 @@ import (
 const (
 	// CloudConfigVersion defines the version of k8scloudconfig in use.
 	// It is used in the main stack output and S3 object paths.
-	CloudConfigVersion = "v_3_3_2"
+	CloudConfigVersion = "v_3_3_3"
 )
 
 type KMSClient interface {

service/controller/v13/cloudconfig/worker_template.go

@@ -3,7 +3,7 @@ package cloudconfig
 import (
 	"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
 	"github.com/giantswarm/certs/legacy"
-	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
+	k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
 	"github.com/giantswarm/microerror"
 
 	"github.com/giantswarm/aws-operator/service/controller/v13/templates/cloudconfig"

service/controller/v13/templates/cloudconfig/format_etcd_volume.go

@@ -11,13 +11,11 @@ RemainAfterExit=yes
 
 # Do not wipe the disk if it's already being used, so the etcd data is
 # persistent across reboots and updates.
-Environment=DEV=/dev/nvme2n1
+Environment=DEV=/dev/xvdh
 
-# line 1: For compatibility with m3.large that has xvdX disks.
-# line 2: Create filesystem if does not exist.
-# line 3: For compatibility with older clusters. Label existing filesystem with etcd label.
+# line 1: Create filesystem if does not exist.
+# line 2: For compatibility with older clusters. Label existing filesystem with etcd label.
 ExecStart=/bin/bash -c "\
-[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\
 if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\
 [ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd"
 

service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go

@@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker
 
 [Service]
 Type=oneshot
-ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)"
+ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker"
 
 [Install]
 WantedBy=multi-user.target

service/controller/v13/templates/cloudconfig/nvme_udev_hack.go

@@ -0,0 +1,26 @@
+package cloudconfig
+
+const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c"
+`
+
+const NVMEUdevScript = `#!/bin/bash
+vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
+vol=${vol#/dev/}
+if [[ -n "$vol" ]]; then
+    echo ${vol/xvd/sd} ${vol/sd/xvd}
+fi
+`
+
+const NVMEUdevTriggerUnit = `[Unit]
+Description=Reload AWS EBS NVMe rules
+Requires=coreos-setup-environment.service
+After=coreos-setup-environment.service
+Before=user-config.target
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=-/etc/environment
+ExecStart=/usr/bin/udevadm control --reload-rules
+ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*"
+ExecStart=/usr/bin/udevadm settle
+`

service/controller/v13/version_bundle.go

@@ -36,7 +36,7 @@ func VersionBundle() versionbundle.Bundle {
 			},
 			{
 				Name:    "kubernetes",
-				Version: "1.10.2",
+				Version: "1.10.4",
 			},
 			{
 				Name:    "nginx-ingress-controller",