Skip to content

Commit

Permalink
feat: Stop passing in ClusterCA and Endpoint information for Windows …
Browse files Browse the repository at this point in the history 
…nodes (#337)

* fix: lookup EKS cluster on Windows nodes to workaround default logic for ServiceCidr calculation

Signed-off-by: Jonah Back <jonah@jonahback.com>

* fix: add flag to disable windows cluster info injection

Signed-off-by: Jonah Back <jonah@jonahback.com>
  • Loading branch information
@backjo
backjo authored Nov 22, 2021
1 parent 5215605 commit 306c823
Show file tree
Hide file tree
Showing 6 changed files with 148 additions and 62 deletions.
38 changes: 20 additions & 18 deletions controllers/instancegroup_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,18 @@ import (
// InstanceGroupReconciler reconciles an InstanceGroup object
type InstanceGroupReconciler struct {
client.Client
SpotRecommendationTime float64
ConfigNamespace string
NodeRelabel bool
Log logr.Logger
MaxParallel int
Auth *InstanceGroupAuthenticator
ConfigMap *corev1.ConfigMap
Namespaces map[string]corev1.Namespace
NamespacesLock *sync.RWMutex
ConfigRetention int
Metrics *common.MetricsCollector
SpotRecommendationTime float64
ConfigNamespace string
NodeRelabel bool
Log logr.Logger
MaxParallel int
Auth *InstanceGroupAuthenticator
ConfigMap *corev1.ConfigMap
Namespaces map[string]corev1.Namespace
NamespacesLock *sync.RWMutex
ConfigRetention int
Metrics *common.MetricsCollector
DisableWinClusterInjection bool
}

type InstanceGroupAuthenticator struct {
Expand Down Expand Up @@ -134,13 +135,14 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque
r.SetFinalizer(instanceGroup)

input := provisioners.ProvisionerInput{
AwsWorker: r.Auth.Aws,
Kubernetes: r.Auth.Kubernetes,
Configuration: r.ConfigMap,
InstanceGroup: instanceGroup,
Log: r.Log,
ConfigRetention: r.ConfigRetention,
Metrics: r.Metrics,
AwsWorker: r.Auth.Aws,
Kubernetes: r.Auth.Kubernetes,
Configuration: r.ConfigMap,
InstanceGroup: instanceGroup,
Log: r.Log,
ConfigRetention: r.ConfigRetention,
Metrics: r.Metrics,
DisableWinClusterInjection: r.DisableWinClusterInjection,
}

var (
Expand Down
34 changes: 18 additions & 16 deletions controllers/provisioners/eks/eks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,14 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext {
)

ctx := &EksInstanceGroupContext{
InstanceGroup: instanceGroup,
KubernetesClient: p.Kubernetes,
AwsWorker: p.AwsWorker,
Log: p.Log.WithName("eks"),
ResourcePrefix: fmt.Sprintf("%v-%v-%v", configuration.GetClusterName(), instanceGroup.GetNamespace(), instanceGroup.GetName()),
ConfigRetention: p.ConfigRetention,
Metrics: p.Metrics,
InstanceGroup: instanceGroup,
KubernetesClient: p.Kubernetes,
AwsWorker: p.AwsWorker,
Log: p.Log.WithName("eks"),
ResourcePrefix: fmt.Sprintf("%v-%v-%v", configuration.GetClusterName(), instanceGroup.GetNamespace(), instanceGroup.GetName()),
ConfigRetention: p.ConfigRetention,
Metrics: p.Metrics,
DisableWinClusterInjection: p.DisableWinClusterInjection,
}

ctx.SetState(v1alpha1.ReconcileInit)
Expand All @@ -89,15 +90,16 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext {

type EksInstanceGroupContext struct {
sync.Mutex
InstanceGroup *v1alpha1.InstanceGroup
KubernetesClient kubeprovider.KubernetesClientSet
AwsWorker awsprovider.AwsWorker
DiscoveredState *DiscoveredState
Log logr.Logger
Configuration *provisioners.ProvisionerConfiguration
ConfigRetention int
ResourcePrefix string
Metrics *common.MetricsCollector
InstanceGroup *v1alpha1.InstanceGroup
KubernetesClient kubeprovider.KubernetesClientSet
AwsWorker awsprovider.AwsWorker
DiscoveredState *DiscoveredState
Log logr.Logger
Configuration *provisioners.ProvisionerConfiguration
ConfigRetention int
ResourcePrefix string
Metrics *common.MetricsCollector
DisableWinClusterInjection bool
}

type UserDataPayload struct {
Expand Down
2 changes: 1 addition & 1 deletion controllers/provisioners/eks/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ func (ctx *EksInstanceGroupContext) GetBootstrapArgs() string {
var sb strings.Builder
switch strings.ToLower(osFamily) {
case OsFamilyWindows:
if state.Cluster != nil {
if state.Cluster != nil && !ctx.DisableWinClusterInjection {
sb.WriteString(fmt.Sprintf("-Base64ClusterCA %v ", aws.StringValue(state.Cluster.CertificateAuthority.Data)))
sb.WriteString(fmt.Sprintf("-APIServerEndpoint %v ", aws.StringValue(state.Cluster.Endpoint)))
}
Expand Down
77 changes: 77 additions & 0 deletions controllers/provisioners/eks/helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,83 @@ func TestGetBasicUserDataWindows(t *testing.T) {
}
}

func TestGetBasicUserDataWindowsWithInjectionDisabled(t *testing.T) {
var (
k = MockKubernetesClientSet()
ig = MockInstanceGroup()
asgMock = NewAutoScalingMocker()
iamMock = NewIamMocker()
eksMock = NewEksMocker()
ec2Mock = NewEc2Mocker()
ssmMock = NewSsmMocker()
configuration = ig.GetEKSConfiguration()
)

w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock)
ctx := MockContext(ig, k, w)

configuration.BootstrapOptions = &v1alpha1.BootstrapOptions{
MaxPods: 4,
}
configuration.Labels = map[string]string{
"foo": "bar",
}
configuration.Taints = []corev1.Taint{
{
Key: "foo",
Value: "bar",
Effect: "NoSchedule",
},
}

configuration.BootstrapArguments = "--eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2"
configuration.UserData = []v1alpha1.UserDataStage{
{
Stage: "PreBootstrap",
Data: "foo",
},
{
Stage: "PostBootstrap",
Data: "bar",
},
}

ig.Annotations[OsFamilyAnnotation] = OsFamilyWindows

expectedDataWindows := `
<powershell>
foo
[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS"
[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1'
[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
[string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
[string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
[string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
if ($Lifecycle -like "*Warmed*") {
Echo "Not starting Kubelet due to warmed state."
& C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule
} else {
& $EKSBootstrapScriptFile -EKSClusterName foo -KubeletExtraArgs '--node-labels=foo=bar,instancemgr.keikoproj.io/image=ami-123456789012,node.kubernetes.io/role=instance-group-1 --register-with-taints=foo=bar:NoSchedule --eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2 --max-pods=4' 3>&1 4>&1 5>&1 6>&1
bar
}
</powershell>`

ctx.DisableWinClusterInjection = true
var (
args = ctx.GetBootstrapArgs()
kubeletArgs = ctx.GetKubeletExtraArgs()
userDataPayload = ctx.GetUserDataStages()
mounts = ctx.GetMountOpts()
)

userData := ctx.GetBasicUserData("foo", args, kubeletArgs, userDataPayload, mounts)
basicUserDataDecoded, _ := base64.StdEncoding.DecodeString(userData)
basicUserDataString := string(basicUserDataDecoded)
if basicUserDataString != expectedDataWindows {
t.Fatalf("\nExpected: START>%v<END\n Got: START>%v<END", expectedDataWindows, basicUserDataString)
}
}

func TestCustomNetworkingMaxPods(t *testing.T) {
var (
k = MockKubernetesClientSet()
Expand Down
15 changes: 8 additions & 7 deletions controllers/provisioners/provisioners.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,14 @@ const (
)

type ProvisionerInput struct {
AwsWorker awsprovider.AwsWorker
Kubernetes kubeprovider.KubernetesClientSet
InstanceGroup *v1alpha1.InstanceGroup
Configuration *corev1.ConfigMap
Log logr.Logger
ConfigRetention int
Metrics *common.MetricsCollector
AwsWorker awsprovider.AwsWorker
Kubernetes kubeprovider.KubernetesClientSet
InstanceGroup *v1alpha1.InstanceGroup
Configuration *corev1.ConfigMap
Log logr.Logger
ConfigRetention int
Metrics *common.MetricsCollector
DisableWinClusterInjection bool
}

var (
Expand Down
44 changes: 24 additions & 20 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,16 @@ func main() {
printVersion()

var (
metricsAddr string
configNamespace string
spotRecommendationTime float64
enableLeaderElection bool
nodeRelabel bool
maxParallel int
maxAPIRetries int
configRetention int
err error
metricsAddr string
configNamespace string
spotRecommendationTime float64
enableLeaderElection bool
nodeRelabel bool
disableWinClusterInjection bool
maxParallel int
maxAPIRetries int
configRetention int
err error
)

flag.IntVar(&maxParallel, "max-workers", 5, "The number of maximum parallel reconciles")
Expand All @@ -85,6 +86,8 @@ func main() {
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&nodeRelabel, "node-relabel", true, "relabel nodes as they join with kubernetes.io/role label via controller")
flag.BoolVar(&disableWinClusterInjection, "disable-windows-cluster-ca-injection", false, "Setting this to true will cause the ClusterCA and Endpoint to not be injected for Windows nodes")

flag.Parse()
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))

Expand Down Expand Up @@ -147,17 +150,18 @@ func main() {
}

err = (&controllers.InstanceGroupReconciler{
Metrics: controllerCollector,
ConfigMap: cm,
ConfigRetention: configRetention,
SpotRecommendationTime: spotRecommendationTime,
ConfigNamespace: configNamespace,
Namespaces: make(map[string]corev1.Namespace),
NamespacesLock: &sync.RWMutex{},
NodeRelabel: nodeRelabel,
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("instancegroup"),
MaxParallel: maxParallel,
Metrics: controllerCollector,
ConfigMap: cm,
ConfigRetention: configRetention,
SpotRecommendationTime: spotRecommendationTime,
ConfigNamespace: configNamespace,
Namespaces: make(map[string]corev1.Namespace),
NamespacesLock: &sync.RWMutex{},
NodeRelabel: nodeRelabel,
DisableWinClusterInjection: disableWinClusterInjection,
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("instancegroup"),
MaxParallel: maxParallel,
Auth: &controllers.InstanceGroupAuthenticator{
Aws: awsWorker,
Kubernetes: kube,
Expand Down

0 comments on commit 306c823

Please sign in to comment.