Skip to content
This repository has been archived by the owner on Oct 30, 2024. It is now read-only.

Fix volume attachments while upgrade (v12 v13) #999

Merged
merged 3 commits into from
Jun 11, 2018
Merged

Fix volume attachments while upgrade (v12 v13) #999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c639d10
Fix etcd volume attachment
Jun 7, 2018
81d4b75
Add nvme udev hack
Jun 8, 2018
2752d0e
Update vendor
Jun 8, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Gopkg.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 23 additions & 1 deletion service/controller/v12/cloudconfig/master_template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package cloudconfig
import (
"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
"github.com/giantswarm/certs/legacy"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
"github.com/giantswarm/microerror"
"github.com/giantswarm/randomkeys"

Expand Down Expand Up @@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
Owner: FileOwner,
Permissions: 0644,
},
// NVME disks udev rules and script.
// Workaround for https://github.com/coreos/bugs/issues/2399
{
AssetContent: cloudconfig.NVMEUdevRule,
Path: "/etc/udev/rules.d/10-ebs-nvme-mapping.rules",
Owner: FileOwner,
Permissions: 0644,
},
{
AssetContent: cloudconfig.NVMEUdevScript,
Path: "/opt/ebs-nvme-mapping",
Owner: FileOwner,
Permissions: 0766,
},
}

var newFiles []k8scloudconfig.FileAsset
Expand All @@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {

func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) {
unitsMeta := []k8scloudconfig.UnitMetadata{
// Create symlinks for nvme disks.
// This service should be started only on first boot.
{
AssetContent: cloudconfig.NVMEUdevTriggerUnit,
Name: "ebs-nvme-udev-trigger.service",
Enable: false,
Command: "start",
},
{
AssetContent: cloudconfig.DecryptTLSAssetsService,
Name: "decrypt-tls-assets.service",
Expand Down
2 changes: 1 addition & 1 deletion service/controller/v12/cloudconfig/spec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
const (
// CloudConfigVersion defines the version of k8scloudconfig in use.
// It is used in the main stack output and S3 object paths.
CloudConfigVersion = "v_3_3_2"
CloudConfigVersion = "v_3_3_3"
)

type KMSClient interface {
Expand Down
2 changes: 1 addition & 1 deletion service/controller/v12/cloudconfig/worker_template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package cloudconfig
import (
"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
"github.com/giantswarm/certs/legacy"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
"github.com/giantswarm/microerror"

"github.com/giantswarm/aws-operator/service/controller/v12/templates/cloudconfig"
Expand Down
8 changes: 3 additions & 5 deletions service/controller/v12/templates/cloudconfig/format_etcd_volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,11 @@ RemainAfterExit=yes

# Do not wipe the disk if it's already being used, so the etcd data is
# persistent across reboots and updates.
Environment=DEV=/dev/nvme2n1
Environment=DEV=/dev/xvdh

# line 1: For compatibility with m3.large that has xvdX disks.
# line 2: Create filesystem if does not exist.
# line 3: For compatibility with older clusters. Label existing filesystem with etcd label.
# line 1: Create filesystem if does not exist.
# line 2: For compatibility with older clusters. Label existing filesystem with etcd label.
ExecStart=/bin/bash -c "\
[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\
if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\
[ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd"

Expand Down
2 changes: 1 addition & 1 deletion service/controller/v12/templates/cloudconfig/master_format_var_lib_docker_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker

[Service]
Type=oneshot
ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)"
ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

-e file exists (any type), because /dev/xvdc is symlink in case NVMe and block device in case m3


[Install]
WantedBy=multi-user.target
Expand Down
26 changes: 26 additions & 0 deletions service/controller/v12/templates/cloudconfig/nvme_udev_hack.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package cloudconfig

const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

udev rule that calls script on any events with nvme that will create/delete sylinks with name that was specified in EBS e.g. /dev/xvdh

`

const NVMEUdevScript = `#!/bin/bash
vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
vol=${vol#/dev/}
if [[ -n "$vol" ]]; then
echo ${vol/xvd/sd} ${vol/sd/xvd}
fi
`

const NVMEUdevTriggerUnit = `[Unit]
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This unit only necessary on the first boot, because udev rule was just added and we need to retrigger udev.

Description=Reload AWS EBS NVMe rules
Requires=coreos-setup-environment.service
After=coreos-setup-environment.service
Before=user-config.target
[Service]
Type=oneshot
RemainAfterExit=yes
EnvironmentFile=-/etc/environment
ExecStart=/usr/bin/udevadm control --reload-rules
ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*"
ExecStart=/usr/bin/udevadm settle
`
13 changes: 6 additions & 7 deletions service/controller/v12/templates/cloudformation/guest/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ const Instance = `{{define "instance"}}
{{ .Instance.Master.Instance.ResourceName }}:
Type: "AWS::EC2::Instance"
Description: Master instance
DependsOn:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Problem was that VM did not depend on volumes to it was immediately created with etcd as first disk and in that time docker disk was resized (takes 3-5 minutes) and then docker disk attached as second disk and mounted to /var/lib/etcd.

This dependency makes sure that VM will wait to both volumes to be ready before get started.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Very good catch, thanks!

- DockerVolume
- EtcdVolume
Properties:
AvailabilityZone: {{ .Instance.Master.AZ }}
IamInstanceProfile: !Ref MasterInstanceProfile
Expand All @@ -19,25 +22,21 @@ const Instance = `{{define "instance"}}
Value: {{ .Instance.Cluster.ID }}-master
DockerVolume:
Type: AWS::EC2::Volume
DependsOn:
- {{ .Instance.Master.Instance.ResourceName }}
Properties:
Encrypted: true
Size: 100
Size: 50
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm rolling master docker disk size to 50GB, because we don't really need 100GB on master node. 100GB only needed for workers.

This prevents unnecessary delay (3-5 minutes caused by resizing) while upgrade.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes makes sense to leave the volume at 50GB for masters. Great that it avoids the delay.

VolumeType: gp2
AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
AvailabilityZone: {{ .Instance.Master.AZ }}
Tags:
- Key: Name
Value: {{ .Instance.Master.DockerVolume.Name }}
EtcdVolume:
Type: AWS::EC2::Volume
DependsOn:
- {{ .Instance.Master.Instance.ResourceName }}
Properties:
Encrypted: true
Size: 100
VolumeType: gp2
AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
AvailabilityZone: {{ .Instance.Master.AZ }}
Tags:
- Key: Name
Value: {{ .Instance.Master.EtcdVolume.Name }}
Expand Down
14 changes: 12 additions & 2 deletions service/controller/v12/version_bundle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,17 @@ func VersionBundle() versionbundle.Bundle {
},
{
Component: "kubernetes",
Description: "Updated to 1.10.2 due to regression in 1.10.3 with configmaps.",
Description: "Updated to 1.10.4 due to regression in 1.10.3 with configmaps.",
Kind: versionbundle.KindChanged,
},
{
Component: "cloudconfig",
Description: "Added udev rule for NVMe disks.",
Kind: versionbundle.KindAdded,
},
{
Component: "cloudconfig",
Description: "Remove Nginx version from Server header in Ingress Controller",
Kind: versionbundle.KindChanged,
},
},
Expand All @@ -56,7 +66,7 @@ func VersionBundle() versionbundle.Bundle {
},
{
Name: "kubernetes",
Version: "1.10.2",
Version: "1.10.4",
},
{
Name: "nginx-ingress-controller",
Expand Down
24 changes: 23 additions & 1 deletion service/controller/v13/cloudconfig/master_template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package cloudconfig
import (
"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
"github.com/giantswarm/certs/legacy"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
"github.com/giantswarm/microerror"
"github.com/giantswarm/randomkeys"

Expand Down Expand Up @@ -208,6 +208,20 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {
Owner: FileOwner,
Permissions: 0644,
},
// NVME disks udev rules and script.
// Workaround for https://github.com/coreos/bugs/issues/2399
{
AssetContent: cloudconfig.NVMEUdevRule,
Path: "/etc/udev/rules.d/10-ebs-nvme-mapping.rules",
Owner: FileOwner,
Permissions: 0644,
},
{
AssetContent: cloudconfig.NVMEUdevScript,
Path: "/opt/ebs-nvme-mapping",
Owner: FileOwner,
Permissions: 0766,
},
}

var newFiles []k8scloudconfig.FileAsset
Expand All @@ -231,6 +245,14 @@ func (e *MasterExtension) Files() ([]k8scloudconfig.FileAsset, error) {

func (e *MasterExtension) Units() ([]k8scloudconfig.UnitAsset, error) {
unitsMeta := []k8scloudconfig.UnitMetadata{
// Create symlinks for nvme disks.
// This service should be started only on first boot.
{
AssetContent: cloudconfig.NVMEUdevTriggerUnit,
Name: "ebs-nvme-udev-trigger.service",
Enable: false,
Command: "start",
},
{
AssetContent: cloudconfig.DecryptTLSAssetsService,
Name: "decrypt-tls-assets.service",
Expand Down
2 changes: 1 addition & 1 deletion service/controller/v13/cloudconfig/spec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
const (
// CloudConfigVersion defines the version of k8scloudconfig in use.
// It is used in the main stack output and S3 object paths.
CloudConfigVersion = "v_3_3_2"
CloudConfigVersion = "v_3_3_3"
)

type KMSClient interface {
Expand Down
2 changes: 1 addition & 1 deletion service/controller/v13/cloudconfig/worker_template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package cloudconfig
import (
"github.com/giantswarm/apiextensions/pkg/apis/provider/v1alpha1"
"github.com/giantswarm/certs/legacy"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_2"
k8scloudconfig "github.com/giantswarm/k8scloudconfig/v_3_3_3"
"github.com/giantswarm/microerror"

"github.com/giantswarm/aws-operator/service/controller/v13/templates/cloudconfig"
Expand Down
8 changes: 3 additions & 5 deletions service/controller/v13/templates/cloudconfig/format_etcd_volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,11 @@ RemainAfterExit=yes

# Do not wipe the disk if it's already being used, so the etcd data is
# persistent across reboots and updates.
Environment=DEV=/dev/nvme2n1
Environment=DEV=/dev/xvdh

# line 1: For compatibility with m3.large that has xvdX disks.
# line 2: Create filesystem if does not exist.
# line 3: For compatibility with older clusters. Label existing filesystem with etcd label.
# line 1: Create filesystem if does not exist.
# line 2: For compatibility with older clusters. Label existing filesystem with etcd label.
ExecStart=/bin/bash -c "\
[ -b /dev/xvdh ] && export DEV=/dev/xvdh ;\
if ! blkid $DEV; then mkfs.ext4 -L etcd $DEV; fi ;\
[ -L /dev/disk/by-label/etcd ] || e2label $DEV etcd"

Expand Down
2 changes: 1 addition & 1 deletion service/controller/v13/templates/cloudconfig/master_format_var_lib_docker_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ ConditionPathExists=!/var/lib/docker

[Service]
Type=oneshot
ExecStart=/bin/bash -c "([ -b "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker) || ([ -b "/dev/nvme1n1" ] && /usr/sbin/mkfs.xfs -f /dev/nvme1n1 -L docker)"
ExecStart=/bin/bash -c "[ -e "/dev/xvdc" ] && /usr/sbin/mkfs.xfs -f /dev/xvdc -L docker"

[Install]
WantedBy=multi-user.target
Expand Down
26 changes: 26 additions & 0 deletions service/controller/v13/templates/cloudconfig/nvme_udev_hack.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package cloudconfig

const NVMEUdevRule = `KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/opt/ebs-nvme-mapping /dev/%k", SYMLINK+="%c"
`

const NVMEUdevScript = `#!/bin/bash
vol=$(nvme id-ctrl --raw-binary "$1" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
vol=${vol#/dev/}
if [[ -n "$vol" ]]; then
echo ${vol/xvd/sd} ${vol/sd/xvd}
fi
`

const NVMEUdevTriggerUnit = `[Unit]
Description=Reload AWS EBS NVMe rules
Requires=coreos-setup-environment.service
After=coreos-setup-environment.service
Before=user-config.target
[Service]
Type=oneshot
RemainAfterExit=yes
EnvironmentFile=-/etc/environment
ExecStart=/usr/bin/udevadm control --reload-rules
ExecStart=/usr/bin/udevadm trigger -y "nvme[0-9]*n[0-9]*"
ExecStart=/usr/bin/udevadm settle
`
13 changes: 6 additions & 7 deletions service/controller/v13/templates/cloudformation/guest/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ const Instance = `{{define "instance"}}
{{ .Instance.Master.Instance.ResourceName }}:
Type: "AWS::EC2::Instance"
Description: Master instance
DependsOn:
- DockerVolume
- EtcdVolume
Properties:
AvailabilityZone: {{ .Instance.Master.AZ }}
IamInstanceProfile: !Ref MasterInstanceProfile
Expand All @@ -19,25 +22,21 @@ const Instance = `{{define "instance"}}
Value: {{ .Instance.Cluster.ID }}-master
DockerVolume:
Type: AWS::EC2::Volume
DependsOn:
- {{ .Instance.Master.Instance.ResourceName }}
Properties:
Encrypted: true
Size: 100
Size: 50
VolumeType: gp2
AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
AvailabilityZone: {{ .Instance.Master.AZ }}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is because the master resource now depends on the volumes?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, otherwise it has curcular error :(

Tags:
- Key: Name
Value: {{ .Instance.Master.DockerVolume.Name }}
EtcdVolume:
Type: AWS::EC2::Volume
DependsOn:
- {{ .Instance.Master.Instance.ResourceName }}
Properties:
Encrypted: true
Size: 100
VolumeType: gp2
AvailabilityZone: !GetAtt {{ .Instance.Master.Instance.ResourceName }}.AvailabilityZone
AvailabilityZone: {{ .Instance.Master.AZ }}
Tags:
- Key: Name
Value: {{ .Instance.Master.EtcdVolume.Name }}
Expand Down
2 changes: 1 addition & 1 deletion service/controller/v13/version_bundle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func VersionBundle() versionbundle.Bundle {
},
{
Name: "kubernetes",
Version: "1.10.2",
Version: "1.10.4",
},
{
Name: "nginx-ingress-controller",
Expand Down
2 changes: 1 addition & 1 deletion ...arm/k8scloudconfig/v_3_3_2/cloudconfig.go → ...arm/k8scloudconfig/v_3_3_3/cloudconfig.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion ...iantswarm/k8scloudconfig/v_3_3_2/error.go → ...iantswarm/k8scloudconfig/v_3_3_3/error.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading