Add Support for Pre bootstrap commands in Windows Node Groups (#2004)

Allows the specification of preBootstrapCommands in the node group spec for windows. Users can run custom Powershell commands on instance start before the instance is bootstrapped for EKS.

Issue #2003

pkg/apis/eksctl.io/v1alpha5/validation.go

@@ -211,8 +211,10 @@ func ValidateNodeGroup(i int, ng *NodeGroup) error {
 		if ng.KubeletExtraConfig != nil {
 			return fieldNotSupported("kubeletExtraConfig")
 		}
-		if ng.PreBootstrapCommands != nil {
-			return fieldNotSupported("preBootstrapCommands")
+		if ng.AMIFamily == NodeImageFamilyBottlerocket {
+			if ng.PreBootstrapCommands != nil {
+				return fieldNotSupported("preBootstrapCommands")
+			}
 		}
 		if ng.OverrideBootstrapCommand != nil {
 			return fieldNotSupported("overrideBootstrapCommand")

pkg/apis/eksctl.io/v1alpha5/validation_test.go

@@ -744,6 +744,41 @@ var _ = Describe("ClusterConfig validation", func() {
 			errSubstr: "secretsEncryption.keyARN is required",
 		}),
 	)
+
+	Describe("Windows node groups", func() {
+		It("returns an error with unsupported fields", func() {
+			cmd := "start /wait msiexec.exe"
+			doc := InlineDocument{
+				"cgroupDriver": "systemd",
+			}
+
+			ngs := map[string]*NodeGroup{
+				"OverrideBootstrapCommand": {OverrideBootstrapCommand: &cmd},
+				"KubeletExtraConfig":       {KubeletExtraConfig: &doc},
+			}
+
+			for name, ng := range ngs {
+				ng.AMIFamily = NodeImageFamilyWindowsServer2019CoreContainer
+				err := ValidateNodeGroup(0, ng)
+				Expect(err).To(HaveOccurred(), "Expected an error when provided %s", name)
+			}
+		})
+
+		It("has no error with supported fields", func() {
+			x := 32
+			ngs := []*NodeGroup{
+				{Labels: map[string]string{"label": "label-value"}},
+				{MaxPodsPerNode: x},
+				{MinSize: &x},
+				{PreBootstrapCommands: []string{"start /wait msiexec.exe"}},
+			}
+
+			for i, ng := range ngs {
+				ng.AMIFamily = NodeImageFamilyWindowsServer2019CoreContainer
+				Expect(ValidateNodeGroup(i, ng)).To(Succeed())
+			}
+		})
+	})
 })
 
 func checkItDetectsError(SSHConfig *NodeGroupSSH) {

pkg/nodebootstrap/userdata.go

@@ -2,6 +2,7 @@ package nodebootstrap
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -147,6 +148,7 @@ func toCLIArgs(values map[string]string) string {
 	for k, v := range values {
 		args = append(args, fmt.Sprintf("--%s=%s", k, v))
 	}
+	sort.Strings(args)
 	return strings.Join(args, " ")
 }
 
@@ -189,7 +191,7 @@ func NewUserData(spec *api.ClusterConfig, ng *api.NodeGroup) (string, error) {
 	case api.NodeImageFamilyBottlerocket:
 		return NewUserDataForBottlerocket(spec, ng)
 	case api.NodeImageFamilyWindowsServer2019FullContainer, api.NodeImageFamilyWindowsServer2019CoreContainer:
-		return newUserDataForWindows(spec, ng)
+		return NewUserDataForWindows(spec, ng)
 	default:
 		return "", nil
 	}

pkg/nodebootstrap/userdata_win.go

@@ -9,10 +9,13 @@ import (
 	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
 )
 
-func newUserDataForWindows(spec *api.ClusterConfig, ng *api.NodeGroup) (string, error) {
+func NewUserDataForWindows(spec *api.ClusterConfig, ng *api.NodeGroup) (string, error) {
 	bootstrapScript := `<powershell>
 [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
 `
+	for _, command := range ng.PreBootstrapCommands {
+		bootstrapScript += fmt.Sprintf("%s\n", command)
+	}
 
 	kubeletOptions := map[string]string{
 		"node-labels":          kvs(ng.Labels),

pkg/nodebootstrap/userdata_win_test.go

@@ -0,0 +1,103 @@
+package nodebootstrap
+
+import (
+	"encoding/base64"
+	"strings"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
+)
+
+var _ = Describe("Windows", func() {
+	var (
+		clusterConfig *api.ClusterConfig
+		ng            *api.NodeGroup
+	)
+
+	BeforeEach(func() {
+		clusterConfig = api.NewClusterConfig()
+		clusterConfig.Status = &api.ClusterStatus{
+			Endpoint:                 "unit-test.example.com",
+			CertificateAuthorityData: []byte(`CertificateAuthorityData`),
+		}
+		clusterConfig.Metadata = &api.ClusterMeta{
+			Name: "unit-test",
+		}
+		ng = &api.NodeGroup{
+			AMIFamily: api.NodeImageFamilyWindowsServer2019CoreContainer,
+		}
+	})
+
+	Describe("with single pre bootstrap script", func() {
+		It("produces correct userdata", func() {
+			ng.PreBootstrapCommands = []string{
+				"wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi",
+			}
+			userdata, err := NewUserDataForWindows(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			decodedBytes, err := base64.StdEncoding.DecodeString(userdata)
+			Expect(err).ToNot(HaveOccurred())
+
+			decodedString := string(decodedBytes)
+
+			Expect(decodedString).ToNot(Equal(""))
+			Expect(decodedString).To(Equal(strings.TrimSpace(`
+<powershell>
+[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
+wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi
+& $EKSBootstrapScriptFile -EKSClusterName "unit-test" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
+</powershell>
+`)))
+		})
+	})
+
+	Describe("with multiple pre bootstrap script", func() {
+		It("produces correct userdata", func() {
+			ng.PreBootstrapCommands = []string{
+				"wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi",
+				"start /wait msiexec.exe /qb /i \"amazon-cloudwatch-agent.msi\"",
+			}
+			userdata, err := NewUserDataForWindows(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			decodedBytes, err := base64.StdEncoding.DecodeString(userdata)
+			Expect(err).ToNot(HaveOccurred())
+
+			decodedString := string(decodedBytes)
+
+			Expect(decodedString).ToNot(Equal(""))
+			Expect(decodedString).To(Equal(strings.TrimSpace(`
+<powershell>
+[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
+wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi
+start /wait msiexec.exe /qb /i "amazon-cloudwatch-agent.msi"
+& $EKSBootstrapScriptFile -EKSClusterName "unit-test" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
+</powershell>
+`)))
+		})
+	})
+
+	Describe("without pre bootstrap scripts", func() {
+		It("produces correct userdata", func() {
+			ng.PreBootstrapCommands = nil
+			userdata, err := NewUserDataForWindows(clusterConfig, ng)
+			Expect(err).ToNot(HaveOccurred())
+
+			decodedBytes, err := base64.StdEncoding.DecodeString(userdata)
+			Expect(err).ToNot(HaveOccurred())
+
+			decodedString := string(decodedBytes)
+
+			Expect(decodedString).ToNot(Equal(""))
+			Expect(decodedString).To(Equal(strings.TrimSpace(`
+<powershell>
+[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
+& $EKSBootstrapScriptFile -EKSClusterName "unit-test" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
+</powershell>
+`)))
+		})
+	})
+})