diff --git a/service/controller/v12/cloudconfig/master_template.go b/service/controller/v12/cloudconfig/master_template.go index 237e279d0f..0bb89dc956 100644 --- a/service/controller/v12/cloudconfig/master_template.go +++ b/service/controller/v12/cloudconfig/master_template.go @@ -3,7 +3,7 @@ package cloudconfig import ( "github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1" "github.com/giantswarm/certs/legacy" - k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2" + k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3" "github.com/giantswarm/microerror" "github.com/giantswarm/randomkeys" @@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) { Owner: FileOwner, Permissions: 0644, }, + // NVME disks udev rules and script. + // Workaround for https://github.com/coreos/bugs/issues/2399 + { + AssetContent: cloudconfig.NVMEUdevRule, + Path: "/etc/udev/rules.d/10-ebs-nvme-mapping.rules", + Owner: FileOwner, + Permissions: 0644, + }, + { + AssetContent: cloudconfig.NVMEUdevScript, + Path: "/opt/ebs-nvme-mapping", + Owner: FileOwner, + Permissions: 0766, + }, } var newFiles []k8scloudconfig.FileAsset @@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) { func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) { unitsMeta := []k8scloudconfig.UnitMetadata{ + // Create symlinks for nvme disks. + // This service should be started only on first boot. + { + AssetContent: cloudconfig.NVMEUdevTriggerUnit, + Name: "ebs-nvme-udev-trigger.service", + Enable: false, + Command: "start", + }, { AssetContent: cloudconfig.DecryptTLSAssetsService, Name: "decrypt-tls-assets.service", diff --git a/service/controller/v12/cloudconfig/spec.go b/service/controller/v12/cloudconfig/spec.go index e3f83b0d94..df52973ce9 100644 --- a/service/controller/v12/cloudconfig/spec.go +++ b/service/controller/v12/cloudconfig/spec.go @@ -7,7 +7,7 @@ import ( const ( // CloudConfigVersion defines the version of k8scloudconfig in use. // It is used in the main stack output and S3 object paths. - CloudConfigVersion = "v_3_3_2" + CloudConfigVersion = "v_3_3_3" ) type KMSClient interface { diff --git a/service/controller/v12/cloudconfig/worker_template.go b/service/controller/v12/cloudconfig/worker_template.go index e059946741..074c9ed310 100644 --- a/service/controller/v12/cloudconfig/worker_template.go +++ b/service/controller/v12/cloudconfig/worker_template.go @@ -3,7 +3,7 @@ package cloudconfig import ( "github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1" "github.com/giantswarm/certs/legacy" - k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2" + k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3" "github.com/giantswarm/microerror" "github.com/giantswarm/aws-operator/service/controller/v12/templates/cloudconfig" diff --git a/service/controller/v12/templates/cloudconfig/format_etcd_volume.go b/service/controller/v12/templates/cloudconfig/format_etcd_volume.go index f5db2528fd..ee225efe7c 100644 --- a/service/controller/v12/templates/cloudconfig/format_etcd_volume.go +++ b/service/controller/v12/templates/cloudconfig/format_etcd_volume.go @@ -11,13 +11,11 @@ RemainAfterExit=yes # Do not wipe the disk if it's already being used, so the etcd data is # persistent across reboots and updates. -Environment=DEV=/dev/nvme2n1 +Environment=DEV=/dev/xvdh -# line 1: For compatibility with m3.large that has xvdX disks. -# line 2: Create filesystem if does not exist. -# line 3: For compatibility with older clusters. Label existing filesystem with etcd label. +# line 1: Create filesystem if does not exist. +# line 2: For compatibility with older clusters. Label existing filesystem with etcd label. ExecStart=/bin/bash -c "\ -[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\ if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\ [ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd" diff --git a/service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go b/service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go index 13e1b1eb1d..da7e11a37a 100644 --- a/service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go +++ b/service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go @@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker [Service] Type=oneshot -ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)" +ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker" [Install] WantedBy=multi-user.target diff --git a/service/controller/v12/templates/cloudconfig/nvme_udev_hack.go b/service/controller/v12/templates/cloudconfig/nvme_udev_hack.go new file mode 100644 index 0000000000..027db74fd6 --- /dev/null +++ b/service/controller/v12/templates/cloudconfig/nvme_udev_hack.go @@ -0,0 +1,26 @@ +package cloudconfig + +const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c" +` + +const NVMEUdevScript = `#!/bin/bash +vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g') +vol=${vol#/dev/} +if [[ -n "$vol" ]]; then + echo ${vol/xvd/sd} ${vol/sd/xvd} +fi +` + +const NVMEUdevTriggerUnit = `[Unit] +Description=Reload AWS EBS NVMe rules +Requires=coreos-setup-environment.service +After=coreos-setup-environment.service +Before=user-config.target +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=-/etc/environment +ExecStart=/usr/bin/udevadm control --reload-rules +ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*" +ExecStart=/usr/bin/udevadm settle +` diff --git a/service/controller/v12/version_bundle.go b/service/controller/v12/version_bundle.go index 1f1683cb27..22a1f060aa 100644 --- a/service/controller/v12/version_bundle.go +++ b/service/controller/v12/version_bundle.go @@ -29,7 +29,17 @@ func VersionBundle() versionbundle.Bundle { }, { Component: "kubernetes", - Description: "Updated to 1.10.2 due to regression in 1.10.3 with configmaps.", + Description: "Updated to 1.10.4 due to regression in 1.10.3 with configmaps.", + Kind: versionbundle.KindChanged, + }, + { + Component: "cloudconfig", + Description: "Added udev rule for NVMe disks.", + Kind: versionbundle.KindAdded, + }, + { + Component: "cloudconfig", + Description: "Remove Nginx version from Server header in Ingress Controller", Kind: versionbundle.KindChanged, }, }, @@ -56,7 +66,7 @@ func VersionBundle() versionbundle.Bundle { }, { Name: "kubernetes", - Version: "1.10.2", + Version: "1.10.4", }, { Name: "nginx-ingress-controller", diff --git a/service/controller/v13/cloudconfig/master_template.go b/service/controller/v13/cloudconfig/master_template.go index a1b1afef96..94f041b8a6 100644 --- a/service/controller/v13/cloudconfig/master_template.go +++ b/service/controller/v13/cloudconfig/master_template.go @@ -3,7 +3,7 @@ package cloudconfig import ( "github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1" "github.com/giantswarm/certs/legacy" - k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2" + k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3" "github.com/giantswarm/microerror" "github.com/giantswarm/randomkeys" @@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) { Owner: FileOwner, Permissions: 0644, }, + // NVME disks udev rules and script. + // Workaround for https://github.com/coreos/bugs/issues/2399 + { + AssetContent: cloudconfig.NVMEUdevRule, + Path: "/etc/udev/rules.d/10-ebs-nvme-mapping.rules", + Owner: FileOwner, + Permissions: 0644, + }, + { + AssetContent: cloudconfig.NVMEUdevScript, + Path: "/opt/ebs-nvme-mapping", + Owner: FileOwner, + Permissions: 0766, + }, } var newFiles []k8scloudconfig.FileAsset @@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) { func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) { unitsMeta := []k8scloudconfig.UnitMetadata{ + // Create symlinks for nvme disks. + // This service should be started only on first boot. + { + AssetContent: cloudconfig.NVMEUdevTriggerUnit, + Name: "ebs-nvme-udev-trigger.service", + Enable: false, + Command: "start", + }, { AssetContent: cloudconfig.DecryptTLSAssetsService, Name: "decrypt-tls-assets.service", diff --git a/service/controller/v13/cloudconfig/spec.go b/service/controller/v13/cloudconfig/spec.go index e3f83b0d94..df52973ce9 100644 --- a/service/controller/v13/cloudconfig/spec.go +++ b/service/controller/v13/cloudconfig/spec.go @@ -7,7 +7,7 @@ import ( const ( // CloudConfigVersion defines the version of k8scloudconfig in use. // It is used in the main stack output and S3 object paths. - CloudConfigVersion = "v_3_3_2" + CloudConfigVersion = "v_3_3_3" ) type KMSClient interface { diff --git a/service/controller/v13/cloudconfig/worker_template.go b/service/controller/v13/cloudconfig/worker_template.go index ddd20b398b..1ca2dc4a6e 100644 --- a/service/controller/v13/cloudconfig/worker_template.go +++ b/service/controller/v13/cloudconfig/worker_template.go @@ -3,7 +3,7 @@ package cloudconfig import ( "github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1" "github.com/giantswarm/certs/legacy" - k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2" + k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3" "github.com/giantswarm/microerror" "github.com/giantswarm/aws-operator/service/controller/v13/templates/cloudconfig" diff --git a/service/controller/v13/templates/cloudconfig/format_etcd_volume.go b/service/controller/v13/templates/cloudconfig/format_etcd_volume.go index f5db2528fd..ee225efe7c 100644 --- a/service/controller/v13/templates/cloudconfig/format_etcd_volume.go +++ b/service/controller/v13/templates/cloudconfig/format_etcd_volume.go @@ -11,13 +11,11 @@ RemainAfterExit=yes # Do not wipe the disk if it's already being used, so the etcd data is # persistent across reboots and updates. -Environment=DEV=/dev/nvme2n1 +Environment=DEV=/dev/xvdh -# line 1: For compatibility with m3.large that has xvdX disks. -# line 2: Create filesystem if does not exist. -# line 3: For compatibility with older clusters. Label existing filesystem with etcd label. +# line 1: Create filesystem if does not exist. +# line 2: For compatibility with older clusters. Label existing filesystem with etcd label. ExecStart=/bin/bash -c "\ -[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\ if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\ [ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd" diff --git a/service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go b/service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go index 13e1b1eb1d..da7e11a37a 100644 --- a/service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go +++ b/service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go @@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker [Service] Type=oneshot -ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)" +ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker" [Install] WantedBy=multi-user.target diff --git a/service/controller/v13/templates/cloudconfig/nvme_udev_hack.go b/service/controller/v13/templates/cloudconfig/nvme_udev_hack.go new file mode 100644 index 0000000000..027db74fd6 --- /dev/null +++ b/service/controller/v13/templates/cloudconfig/nvme_udev_hack.go @@ -0,0 +1,26 @@ +package cloudconfig + +const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c" +` + +const NVMEUdevScript = `#!/bin/bash +vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g') +vol=${vol#/dev/} +if [[ -n "$vol" ]]; then + echo ${vol/xvd/sd} ${vol/sd/xvd} +fi +` + +const NVMEUdevTriggerUnit = `[Unit] +Description=Reload AWS EBS NVMe rules +Requires=coreos-setup-environment.service +After=coreos-setup-environment.service +Before=user-config.target +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=-/etc/environment +ExecStart=/usr/bin/udevadm control --reload-rules +ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*" +ExecStart=/usr/bin/udevadm settle +` diff --git a/service/controller/v13/version_bundle.go b/service/controller/v13/version_bundle.go index a2e8119ea2..d53c9df752 100644 --- a/service/controller/v13/version_bundle.go +++ b/service/controller/v13/version_bundle.go @@ -36,7 +36,7 @@ func VersionBundle() versionbundle.Bundle { }, { Name: "kubernetes", - Version: "1.10.2", + Version: "1.10.4", }, { Name: "nginx-ingress-controller",