From 724623c26164245b1be399ba8527f8486e20b811 Mon Sep 17 00:00:00 2001 From: Michal Jura Date: Sun, 9 May 2021 19:13:59 +0200 Subject: [PATCH 1/2] Add HTTP Application Routing Add new option to addon profile HTTPApplicationRouting which configures an Ingress controller in AKS cluster. --- charts/aks-operator-crd/templates/crds.yaml | 3 +++ controller/aks-cluster-config-handler.go | 19 +++++++++++++++-- internal/aks/client.go | 13 ++++++++++++ internal/aks/create.go | 21 +++++++++++++++++-- pkg/apis/aks.cattle.io/v1/types.go | 1 + .../aks.cattle.io/v1/zz_generated_deepcopy.go | 5 +++++ 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/charts/aks-operator-crd/templates/crds.yaml b/charts/aks-operator-crd/templates/crds.yaml index da7d7e1f..c26e196e 100755 --- a/charts/aks-operator-crd/templates/crds.yaml +++ b/charts/aks-operator-crd/templates/crds.yaml @@ -46,6 +46,9 @@ spec: dockerBridgeCidr: nullable: true type: string + httpApplicationRouting: + nullable: true + type: boolean imported: type: boolean kubernetesVersion: diff --git a/controller/aks-cluster-config-handler.go b/controller/aks-cluster-config-handler.go index b10ef120..85a03592 100644 --- a/controller/aks-cluster-config-handler.go +++ b/controller/aks-cluster-config-handler.go @@ -581,6 +581,12 @@ func BuildUpstreamClusterState(ctx context.Context, secretsCache wranglerv1.Secr upstreamSpec.LinuxSSHPublicKey = sshKeys[0].KeyData } + // set addons profile + addonProfile := clusterState.AddonProfiles + if addonProfile != nil && addonProfile["httpApplicationRouting"] != nil { + upstreamSpec.HTTPApplicationRouting = addonProfile["httpApplicationRouting"].Enabled + } + // set API server access profile upstreamSpec.PrivateCluster = to.BoolPtr(false) if clusterState.APIServerAccessProfile != nil { @@ -597,7 +603,8 @@ func BuildUpstreamClusterState(ctx context.Context, secretsCache wranglerv1.Secr // updateUpstreamClusterState compares the upstream spec with the config spec, then updates the upstream AKS cluster to // match the config spec. Function returns after a update is finished. -func (h *Handler) updateUpstreamClusterState(ctx context.Context, secretsCache wranglerv1.SecretCache, config *aksv1.AKSClusterConfig, upstreamSpec *aksv1.AKSClusterConfigSpec) (*aksv1.AKSClusterConfig, error) { +func (h *Handler) updateUpstreamClusterState(ctx context.Context, secretsCache wranglerv1.SecretCache, + config *aksv1.AKSClusterConfig, upstreamSpec *aksv1.AKSClusterConfigSpec) (*aksv1.AKSClusterConfig, error) { credentials, err := aks.GetSecrets(secretsCache, &config.Spec) if err != nil { return config, err @@ -697,6 +704,14 @@ func (h *Handler) updateUpstreamClusterState(ctx context.Context, secretsCache w } } + // check addon HTTP Application Routing + if config.Spec.HTTPApplicationRouting != nil { + if to.Bool(config.Spec.HTTPApplicationRouting) != to.Bool(upstreamSpec.HTTPApplicationRouting) { + logrus.Infof("Updating HTTP application routing for cluster [%s]", config.Spec.ClusterName) + updateAksCluster = true + } + } + if updateAksCluster { resourceGroupsClient, err := aks.NewResourceGroupClient(credentials) if err != nil { @@ -705,7 +720,7 @@ func (h *Handler) updateUpstreamClusterState(ctx context.Context, secretsCache w if !aks.ExistsResourceGroup(ctx, resourceGroupsClient, config.Spec.ResourceGroup) { logrus.Infof("Resource group [%s] does not exist, creating", config.Spec.ResourceGroup) - if err := aks.CreateResourceGroup(ctx, resourceGroupsClient, &config.Spec); err != nil { + if err = aks.CreateResourceGroup(ctx, resourceGroupsClient, &config.Spec); err != nil { return config, fmt.Errorf("error during updating resource group %v", err) } logrus.Infof("Resource group [%s] updated successfully", config.Spec.ResourceGroup) diff --git a/internal/aks/client.go b/internal/aks/client.go index 72453d56..7a42e2a9 100644 --- a/internal/aks/client.go +++ b/internal/aks/client.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-11-01/containerservice" + "github.com/Azure/azure-sdk-for-go/services/operationalinsights/mgmt/2020-08-01/operationalinsights" "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-10-01/resources" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" @@ -59,6 +60,18 @@ func NewAgentPoolClient(cred *Credentials) (*containerservice.AgentPoolsClient, return &agentProfile, nil } +func NewOperationInsightsWorkspaceClient(cred *Credentials) (*operationalinsights.WorkspacesClient, error) { + authorizer, err := NewClientAuthorizer(cred) + if err != nil { + return nil, err + } + + client := operationalinsights.NewWorkspacesClientWithBaseURI(to.String(cred.BaseURL), cred.SubscriptionID) + client.Authorizer = authorizer + + return &client, nil +} + func NewClientAuthorizer(cred *Credentials) (autorest.Authorizer, error) { if cred.AuthBaseURL == nil { cred.AuthBaseURL = to.StringPtr(azure.PublicCloud.ActiveDirectoryEndpoint) diff --git a/internal/aks/create.go b/internal/aks/create.go index dde47356..dd543faa 100644 --- a/internal/aks/create.go +++ b/internal/aks/create.go @@ -3,6 +3,7 @@ package aks import ( "context" "fmt" + "strings" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-11-01/containerservice" "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-10-01/resources" @@ -23,7 +24,8 @@ func CreateResourceGroup(ctx context.Context, groupsClient *resources.GroupsClie } // CreateOrUpdateCluster creates a new managed Kubernetes cluster -func CreateOrUpdateCluster(ctx context.Context, cred *Credentials, clusterClient *containerservice.ManagedClustersClient, spec *aksv1.AKSClusterConfigSpec) error { +func CreateOrUpdateCluster(ctx context.Context, cred *Credentials, clusterClient *containerservice.ManagedClustersClient, + spec *aksv1.AKSClusterConfigSpec) error { dnsPrefix := spec.DNSPrefix if dnsPrefix == nil { dnsPrefix = to.StringPtr(spec.ClusterName) @@ -120,6 +122,15 @@ func CreateOrUpdateCluster(ctx context.Context, cred *Credentials, clusterClient } } + var addonProfiles map[string]*containerservice.ManagedClusterAddonProfile + if hasHTTPApplicationRoutingSupport(spec) { + addonProfiles = map[string]*containerservice.ManagedClusterAddonProfile{ + "httpApplicationRouting": { + Enabled: spec.HTTPApplicationRouting, + }, + } + } + managedCluster := containerservice.ManagedCluster{ Name: to.StringPtr(spec.ClusterName), Location: to.StringPtr(spec.ResourceLocation), @@ -130,6 +141,7 @@ func CreateOrUpdateCluster(ctx context.Context, cred *Credentials, clusterClient AgentPoolProfiles: &agentPoolProfiles, LinuxProfile: linuxProfile, NetworkProfile: networkProfile, + AddonProfiles: addonProfiles, ServicePrincipalProfile: &containerservice.ManagedClusterServicePrincipalProfile{ ClientID: to.StringPtr(cred.ClientID), Secret: to.StringPtr(cred.ClientSecret), @@ -142,7 +154,7 @@ func CreateOrUpdateCluster(ctx context.Context, cred *Credentials, clusterClient AuthorizedIPRanges: spec.AuthorizedIPRanges, } } - if spec.PrivateCluster != nil && *spec.PrivateCluster { + if to.Bool(spec.PrivateCluster) { managedCluster.APIServerAccessProfile = &containerservice.ManagedClusterAPIServerAccessProfile{ EnablePrivateCluster: spec.PrivateCluster, } @@ -183,3 +195,8 @@ func hasCustomVirtualNetwork(spec *aksv1.AKSClusterConfigSpec) bool { func hasLinuxProfile(spec *aksv1.AKSClusterConfigSpec) bool { return spec.LinuxAdminUsername != nil && spec.LinuxSSHPublicKey != nil } + +func hasHTTPApplicationRoutingSupport(spec *aksv1.AKSClusterConfigSpec) bool { + // HttpApplicationRouting is not supported in azure china cloud + return !strings.HasPrefix(spec.ResourceLocation, "china") +} diff --git a/pkg/apis/aks.cattle.io/v1/types.go b/pkg/apis/aks.cattle.io/v1/types.go index b4a3f774..3805c792 100644 --- a/pkg/apis/aks.cattle.io/v1/types.go +++ b/pkg/apis/aks.cattle.io/v1/types.go @@ -58,6 +58,7 @@ type AKSClusterConfigSpec struct { NodePools []AKSNodePool `json:"nodePools"` PrivateCluster *bool `json:"privateCluster"` AuthorizedIPRanges *[]string `json:"authorizedIpRanges"` + HTTPApplicationRouting *bool `json:"httpApplicationRouting"` } type AKSClusterConfigStatus struct { diff --git a/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go b/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go index 4b31b9a0..8aa5b470 100644 --- a/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go +++ b/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go @@ -196,6 +196,11 @@ func (in *AKSClusterConfigSpec) DeepCopyInto(out *AKSClusterConfigSpec) { copy(*out, *in) } } + if in.HTTPApplicationRouting != nil { + in, out := &in.HTTPApplicationRouting, &out.HTTPApplicationRouting + *out = new(bool) + **out = **in + } return } From 426a4a5e67a0cba8b5b6f074d9aab4d10e967afc Mon Sep 17 00:00:00 2001 From: Michal Jura Date: Mon, 31 May 2021 11:51:08 +0200 Subject: [PATCH 2/2] Fix autoscaling for nodedools Fix enabling autoscaling for nodepools after AKS cluster was provisioned --- internal/aks/create.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/aks/create.go b/internal/aks/create.go index dd543faa..abafb830 100644 --- a/internal/aks/create.go +++ b/internal/aks/create.go @@ -178,7 +178,11 @@ func CreateOrUpdateAgentPool(ctx context.Context, agentPoolClient *containerserv OsType: containerservice.OSType(np.OsType), VMSize: containerservice.VMSizeTypes(np.VMSize), Mode: containerservice.AgentPoolMode(np.Mode), + Type: containerservice.VirtualMachineScaleSets, OrchestratorVersion: np.OrchestratorVersion, + EnableAutoScaling: np.EnableAutoScaling, + MinCount: np.MinCount, + MaxCount: np.MaxCount, } _, err := agentPoolClient.CreateOrUpdate(ctx, spec.ResourceGroup, spec.ClusterName, to.String(np.Name), containerservice.AgentPool{