diff --git a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go index a3e714fc07bc..01b82b3182d4 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go @@ -163,7 +163,7 @@ func (as *AgentPool) GetVMIndexes() ([]int, map[int]string, error) { } indexes = append(indexes, index) - resourceID, err := convertResourceGroupNameToLower("azure://" + *instance.ID) + resourceID, err := convertResourceGroupNameToLower(azurePrefix + *instance.ID) if err != nil { return nil, nil, err } @@ -489,7 +489,7 @@ func (as *AgentPool) Nodes() ([]cloudprovider.Instance, error) { // To keep consistent with providerID from kubernetes cloud provider, convert // resourceGroupName in the ID to lower case. - resourceID, err := convertResourceGroupNameToLower("azure://" + *instance.ID) + resourceID, err := convertResourceGroupNameToLower(azurePrefix + *instance.ID) if err != nil { return nil, err } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go index 7890cdde72ef..444f2f9f235c 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_agent_pool_test.go @@ -35,8 +35,8 @@ import ( "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-09-01/storage" "github.com/Azure/go-autorest/autorest/date" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" ) var ( @@ -185,7 +185,8 @@ func TestGetVMsFromCache(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) testAS.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), testAS.manager.config.ResourceGroup).Return(expectedVMs, nil) - ac, err := newAzureCache(testAS.manager.azClient, refreshInterval, testAS.manager.config.ResourceGroup, vmTypeStandard, false, "") + testAS.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(testAS.manager.azClient, refreshInterval, *testAS.manager.config) assert.NoError(t, err) testAS.manager.azureCache = ac @@ -203,7 +204,8 @@ func TestGetVMIndexes(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) as.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - ac, err := newAzureCache(as.manager.azClient, refreshInterval, as.manager.config.ResourceGroup, vmTypeStandard, false, "") + as.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -242,7 +244,8 @@ func TestGetCurSize(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) as.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - ac, err := newAzureCache(as.manager.azClient, refreshInterval, as.manager.config.ResourceGroup, vmTypeStandard, false, "") + as.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -266,7 +269,8 @@ func TestAgentPoolTargetSize(t *testing.T) { as.manager.azClient.virtualMachinesClient = mockVMClient expectedVMs := getExpectedVMs() mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - ac, err := newAzureCache(as.manager.azClient, refreshInterval, as.manager.config.ResourceGroup, vmTypeStandard, false, "") + as.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -285,7 +289,8 @@ func TestAgentPoolIncreaseSize(t *testing.T) { as.manager.azClient.virtualMachinesClient = mockVMClient expectedVMs := getExpectedVMs() mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil).MaxTimes(2) - ac, err := newAzureCache(as.manager.azClient, refreshInterval, as.manager.config.ResourceGroup, vmTypeStandard, false, "") + as.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -313,7 +318,8 @@ func TestDecreaseTargetSize(t *testing.T) { as.manager.azClient.virtualMachinesClient = mockVMClient expectedVMs := getExpectedVMs() mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil).MaxTimes(3) - ac, err := newAzureCache(as.manager.azClient, refreshInterval, as.manager.config.ResourceGroup, vmTypeStandard, false, "") + as.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac @@ -431,7 +437,9 @@ func TestAgentPoolDeleteNodes(t *testing.T) { mockSAClient := mockstorageaccountclient.NewMockInterface(ctrl) as.manager.azClient.storageAccountsClient = mockSAClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - ac, err := newAzureCache(as.manager.azClient, refreshInterval, as.manager.config.ResourceGroup, vmTypeStandard, false, "") + as.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) + as.manager.config.VMType = vmTypeVMSS assert.NoError(t, err) as.manager.azureCache = ac @@ -497,7 +505,8 @@ func TestAgentPoolNodes(t *testing.T) { mockVMClient := mockvmclient.NewMockInterface(ctrl) as.manager.azClient.virtualMachinesClient = mockVMClient mockVMClient.EXPECT().List(gomock.Any(), as.manager.config.ResourceGroup).Return(expectedVMs, nil) - ac, err := newAzureCache(as.manager.azClient, refreshInterval, as.manager.config.ResourceGroup, vmTypeStandard, false, "") + as.manager.config.VMType = vmTypeStandard + ac, err := newAzureCache(as.manager.azClient, refreshInterval, *as.manager.config) assert.NoError(t, err) as.manager.azureCache = ac diff --git a/cluster-autoscaler/cloudprovider/azure/azure_autodiscovery.go b/cluster-autoscaler/cloudprovider/azure/azure_autodiscovery.go index a218dfe637e9..b505149dea07 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_autodiscovery.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_autodiscovery.go @@ -128,7 +128,7 @@ func matchDiscoveryConfig(labels map[string]*string, configs []labelAutoDiscover return nil } - if len(v) > 0 { + if v != "" { if value == nil || *value != v { return nil } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cache.go b/cluster-autoscaler/cloudprovider/azure/azure_cache.go index 982f3d1eb99b..7f1b487046f1 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_cache.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_cache.go @@ -39,35 +39,72 @@ var ( // azureCache is used for caching cluster resources state. // // It is needed to: -// - keep track of node groups (VM and VMSS types) in the cluster, -// - keep track of instances and which node group they belong to, -// - limit repetitive Azure API calls. +// - keep track of node groups (VM and VMSS types) in the cluster, +// - keep track of instances and which node group they belong to, +// (for VMSS it only keeps track of instanceid-to-nodegroup mapping) +// - limit repetitive Azure API calls. +// +// It backs efficient responds to +// - cloudprovider.NodeGroups() (= registeredNodeGroups) +// - cloudprovider.NodeGroupForNode (via azureManager.GetNodeGroupForInstance => FindForInstance, +// using instanceToNodeGroup and unownedInstances) +// +// CloudProvider.Refresh, called before every autoscaler loop (every 10s by defaul), +// is implemented by AzureManager.Refresh which makes the cache refresh decision, +// based on AzureManager.lastRefresh and azureCache.refreshInterval. type azureCache struct { - mutex sync.Mutex - interrupt chan struct{} - azClient *azClient + mutex sync.Mutex + interrupt chan struct{} + azClient *azClient + + // refreshInterval specifies how often azureCache needs to be refreshed. + // The value comes from AZURE_VMSS_CACHE_TTL env var (or 1min if not specified), + // and is also used by some other caches. Together with AzureManager.lastRefresh, + // it is uses to decide whether a refresh is needed. refreshInterval time.Duration // Cache content. - resourceGroup string - vmType string - vmsPoolSet map[string]struct{} // track the nodepools that're vms pool - scaleSets map[string]compute.VirtualMachineScaleSet - virtualMachines map[string][]compute.VirtualMachine + + // resourceGroup specifies the name of the resource group that this cache tracks + resourceGroup string + + // vmType can be one of vmTypeVMSS (default), vmTypeStandard + vmType string + + vmsPoolSet map[string]struct{} // track the nodepools that're vms pool + + // scaleSets keeps the set of all known scalesets in the resource group, populated/refreshed via VMSS.List() call. + // It is only used/populated if vmType is vmTypeVMSS (default). + scaleSets map[string]compute.VirtualMachineScaleSet + // virtualMachines keeps the set of all VMs in the resource group. + // It is only used/populated if vmType is vmTypeStandard. + virtualMachines map[string][]compute.VirtualMachine + + // registeredNodeGroups represents all known NodeGroups. registeredNodeGroups []cloudprovider.NodeGroup - instanceToNodeGroup map[azureRef]cloudprovider.NodeGroup - unownedInstances map[azureRef]bool - autoscalingOptions map[azureRef]map[string]string - skus map[string]*skewer.Cache + + // instanceToNodeGroup maintains a mapping from instance Ids to nodegroups. + // It is populated from the results of calling Nodes() on each nodegroup. + // It is used (together with unownedInstances) when looking up the nodegroup + // for a given instance id (see FindForInstance). + instanceToNodeGroup map[azureRef]cloudprovider.NodeGroup + + // unownedInstance maintains a set of instance ids not belonging to any nodegroup. + // It is used (together with instanceToNodeGroup) when looking up the nodegroup for a given instance id. + // It is reset by invalidateUnownedInstanceCache(). + unownedInstances map[azureRef]bool + + autoscalingOptions map[azureRef]map[string]string + skus map[string]*skewer.Cache } -func newAzureCache(client *azClient, cacheTTL time.Duration, resourceGroup, vmType string, enableDynamicInstanceList bool, defaultLocation string) (*azureCache, error) { +func newAzureCache(client *azClient, cacheTTL time.Duration, config Config) (*azureCache, error) { cache := &azureCache{ interrupt: make(chan struct{}), azClient: client, refreshInterval: cacheTTL, - resourceGroup: resourceGroup, - vmType: vmType, + resourceGroup: config.ResourceGroup, + vmType: config.VMType, vmsPoolSet: make(map[string]struct{}), scaleSets: make(map[string]compute.VirtualMachineScaleSet), virtualMachines: make(map[string][]compute.VirtualMachine), @@ -78,8 +115,8 @@ func newAzureCache(client *azClient, cacheTTL time.Duration, resourceGroup, vmTy skus: make(map[string]*skewer.Cache), } - if enableDynamicInstanceList { - cache.skus[defaultLocation] = &skewer.Cache{} + if config.EnableDynamicInstanceList { + cache.skus[config.Location] = &skewer.Cache{} } if err := cache.regenerate(); err != nil { diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client.go b/cluster-autoscaler/cloudprovider/azure/azure_client.go index cdbb23ad14a1..2bf337a4e8d4 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client.go @@ -302,7 +302,7 @@ func newServicePrincipalTokenFromCredentials(config *Config, env *azure.Environm if err != nil { return nil, fmt.Errorf("getting the managed service identity endpoint: %v", err) } - if len(config.UserAssignedIdentityID) > 0 { + if config.UserAssignedIdentityID != "" { klog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token") return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, env.ServiceManagementEndpoint, @@ -314,7 +314,7 @@ func newServicePrincipalTokenFromCredentials(config *Config, env *azure.Environm env.ServiceManagementEndpoint) } - if len(config.AADClientSecret) > 0 { + if config.AADClientSecret != "" { klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") return adal.NewServicePrincipalToken( *oauthConfig, @@ -323,13 +323,13 @@ func newServicePrincipalTokenFromCredentials(config *Config, env *azure.Environm env.ServiceManagementEndpoint) } - if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 { + if config.AADClientCertPath != "" { klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") certData, err := ioutil.ReadFile(config.AADClientCertPath) if err != nil { return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err) } - certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword) + certificate, privateKey, err := adal.DecodePfxCertificateData(certData, config.AADClientCertPassword) if err != nil { return nil, fmt.Errorf("decoding the client certificate: %v", err) } @@ -399,6 +399,7 @@ func newAzClient(cfg *Config, env *azure.Environment) (*azClient, error) { // https://github.com/Azure/go-autorest/blob/main/autorest/azure/environments.go skuClient := compute.NewResourceSkusClientWithBaseURI(azClientConfig.ResourceManagerEndpoint, cfg.SubscriptionID) skuClient.Authorizer = azClientConfig.Authorizer + skuClient.UserAgent = azClientConfig.UserAgent klog.V(5).Infof("Created sku client with authorizer: %v", skuClient) agentPoolClient, err := newAgentpoolClient(cfg) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client_test.go b/cluster-autoscaler/cloudprovider/azure/azure_client_test.go new file mode 100644 index 000000000000..7ed0dd4c01f7 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/azure/azure_client_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "os" + "testing" + + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/stretchr/testify/assert" +) + +func TestGetServicePrincipalTokenFromCertificate(t *testing.T) { + config := &Config{ + TenantID: "TenantID", + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/test.pfx", + AADClientCertPassword: "id", + } + env := &azure.PublicCloud + token, err := newServicePrincipalTokenFromCredentials(config, env) + assert.NoError(t, err) + + oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) + assert.NoError(t, err) + pfxContent, err := os.ReadFile("./testdata/test.pfx") + assert.NoError(t, err) + certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "id") + assert.NoError(t, err) + spt, err := adal.NewServicePrincipalTokenFromCertificate( + *oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) + assert.NoError(t, err) + assert.Equal(t, token, spt) +} + +func TestGetServicePrincipalTokenFromCertificateWithoutPassword(t *testing.T) { + config := &Config{ + TenantID: "TenantID", + AADClientID: "AADClientID", + AADClientCertPath: "./testdata/testnopassword.pfx", + } + env := &azure.PublicCloud + token, err := newServicePrincipalTokenFromCredentials(config, env) + assert.NoError(t, err) + + oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) + assert.NoError(t, err) + pfxContent, err := os.ReadFile("./testdata/testnopassword.pfx") + assert.NoError(t, err) + certificate, privateKey, err := adal.DecodePfxCertificateData(pfxContent, "") + assert.NoError(t, err) + spt, err := adal.NewServicePrincipalTokenFromCertificate( + *oauthConfig, config.AADClientID, certificate, privateKey, env.ServiceManagementEndpoint) + assert.NoError(t, err) + assert.Equal(t, token, spt) +} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go index 744873284e6d..b4c57f9e3981 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider.go @@ -32,7 +32,8 @@ import ( const ( // GPULabel is the label added to nodes with GPU resource. - GPULabel = "accelerator" + GPULabel = AKSLabelKeyPrefixValue + "accelerator" + legacyGPULabel = "accelerator" ) var ( @@ -73,7 +74,7 @@ func (azure *AzureCloudProvider) Name() string { // GPULabel returns the label added to nodes with GPU resource. func (azure *AzureCloudProvider) GPULabel() string { - return GPULabel + return legacyGPULabel // Use legacy to avoid breaking, for now } // GetAvailableGPUTypes return all available GPU types cloud provider supports diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go index 29bdde9fb352..24cb6a4e36da 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go @@ -30,20 +30,20 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssvmclient/mockvmssvmclient" "github.com/Azure/go-autorest/autorest/azure" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" ) func newTestAzureManager(t *testing.T) *AzureManager { ctrl := gomock.NewController(t) defer ctrl.Finish() - expectedScaleSets := newTestVMSSList(3, "test-vmss", "eastus", compute.Uniform) + expectedScaleSets := newTestVMSSList(3, "test-asg", "eastus", compute.Uniform) expectedVMSSVMs := newTestVMSSVMList(3) mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) mockVMSSClient.EXPECT().List(gomock.Any(), "rg").Return(expectedScaleSets, nil).AnyTimes() mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl) - mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg", "test-vmss", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg", "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() mockVMClient := mockvmclient.NewMockInterface(ctrl) expectedVMs := newTestVMList(3) mockVMClient.EXPECT().List(gomock.Any(), "rg").Return(expectedVMs, nil).AnyTimes() @@ -57,6 +57,7 @@ func newTestAzureManager(t *testing.T) *AzureManager { MaxDeploymentsCount: 2, Deployment: "deployment", EnableForceDelete: true, + Location: "eastus", }, azClient: &azClient{ virtualMachineScaleSetsClient: mockVMSSClient, @@ -82,7 +83,7 @@ func newTestAzureManager(t *testing.T) *AzureManager { }, } - cache, error := newAzureCache(manager.azClient, refreshInterval, manager.config.ResourceGroup, vmTypeVMSS, false, "") + cache, error := newAzureCache(manager.azClient, refreshInterval, *manager.config) assert.NoError(t, error) manager.azureCache = cache @@ -193,7 +194,6 @@ func TestNodeGroupForNode(t *testing.T) { expectedVMs := newTestVMList(3) for _, orchMode := range orchestrationModes { - expectedScaleSets := newTestVMSSList(3, "test-asg", "eastus", orchMode) provider := newTestProvider(t) mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) @@ -204,7 +204,6 @@ func TestNodeGroupForNode(t *testing.T) { mockVMClient.EXPECT().List(gomock.Any(), provider.azureManager.config.ResourceGroup).Return(expectedVMs, nil).AnyTimes() if orchMode == compute.Uniform { - mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl) mockVMSSVMClient.EXPECT().List(gomock.Any(), provider.azureManager.config.ResourceGroup, "test-asg", gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() provider.azureManager.azClient.virtualMachineScaleSetVMsClient = mockVMSSVMClient @@ -216,8 +215,8 @@ func TestNodeGroupForNode(t *testing.T) { } registered := provider.azureManager.RegisterNodeGroup( - newTestScaleSet(provider.azureManager, "test-asg")) - provider.azureManager.explicitlyConfigured["test-asg"] = true + newTestScaleSet(provider.azureManager, testASG)) + provider.azureManager.explicitlyConfigured[testASG] = true assert.True(t, registered) assert.Equal(t, len(provider.NodeGroups()), 1) @@ -227,21 +226,20 @@ func TestNodeGroupForNode(t *testing.T) { group, err := provider.NodeGroupForNode(node) assert.NoError(t, err) assert.NotNil(t, group, "Group should not be nil") - assert.Equal(t, group.Id(), "test-asg") + assert.Equal(t, group.Id(), testASG) assert.Equal(t, group.MinSize(), 1) assert.Equal(t, group.MaxSize(), 5) // test node in cluster that is not in a group managed by cluster autoscaler nodeNotInGroup := &apiv1.Node{ Spec: apiv1.NodeSpec{ - ProviderID: "azure:///subscriptions/subscripion/resourceGroups/test-resource-group/providers/Microsoft.Compute/virtualMachines/test-instance-id-not-in-group", + ProviderID: azurePrefix + "/subscriptions/subscripion/resourceGroups/test-resource-group/providers/Microsoft.Compute/virtualMachines/test-instance-id-not-in-group", }, } group, err = provider.NodeGroupForNode(nodeNotInGroup) assert.NoError(t, err) assert.Nil(t, group) } - } func TestNodeGroupForNodeWithNoProviderId(t *testing.T) { diff --git a/cluster-autoscaler/cloudprovider/azure/azure_config.go b/cluster-autoscaler/cloudprovider/azure/azure_config.go index 096d95e2f4e0..6c354c2a23e4 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_config.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_config.go @@ -131,9 +131,6 @@ type Config struct { // Jitter in seconds subtracted from the VMSS cache TTL before the first refresh VmssVmsCacheJitter int `json:"vmssVmsCacheJitter" yaml:"vmssVmsCacheJitter"` - // GetVmssSizeRefreshPeriod (seconds) defines how frequently to call GET VMSS API to fetch VMSS info per nodegroup instance - GetVmssSizeRefreshPeriod int `json:"getVmssSizeRefreshPeriod" yaml:"getVmssSizeRefreshPeriod"` - // number of latest deployments that will not be deleted MaxDeploymentsCount int64 `json:"maxDeploymentsCount" yaml:"maxDeploymentsCount"` @@ -152,6 +149,12 @@ type Config struct { // EnableVmssFlex defines whether to enable Vmss Flex support or not EnableVmssFlex bool `json:"enableVmssFlex,omitempty" yaml:"enableVmssFlex,omitempty"` + + // (DEPRECATED, DO NOT USE) EnableDetailedCSEMessage defines whether to emit error messages in the CSE error body info + EnableDetailedCSEMessage bool `json:"enableDetailedCSEMessage,omitempty" yaml:"enableDetailedCSEMessage,omitempty"` + + // (DEPRECATED, DO NOT USE) GetVmssSizeRefreshPeriod (seconds) defines how frequently to call GET VMSS API to fetch VMSS info per nodegroup instance + GetVmssSizeRefreshPeriod int `json:"getVmssSizeRefreshPeriod,omitempty" yaml:"getVmssSizeRefreshPeriod,omitempty"` } // BuildAzureConfig returns a Config object for the Azure clients diff --git a/cluster-autoscaler/cloudprovider/azure/azure_config_test.go b/cluster-autoscaler/cloudprovider/azure/azure_config_test.go index 5adeba8c0687..65c74a5c90a1 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_config_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_config_test.go @@ -105,6 +105,7 @@ func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitAlreadyS assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets) } +// nolint: goconst func TestInitializeCloudProviderRateLimitConfigWithInvalidReadAndWriteRateLimitSettingsFromEnv(t *testing.T) { emptyConfig := &CloudProviderRateLimitConfig{} var rateLimitReadQPS float32 = 3.0 diff --git a/cluster-autoscaler/cloudprovider/azure/azure_instance_gpu_sku.go b/cluster-autoscaler/cloudprovider/azure/azure_instance_gpu_sku.go index aea520027248..577bf47845e0 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_instance_gpu_sku.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_instance_gpu_sku.go @@ -17,9 +17,10 @@ limitations under the License. package azure import ( + "strings" + "github.com/Azure/skewer" "github.com/pkg/errors" - "strings" ) var ( diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager.go b/cluster-autoscaler/cloudprovider/azure/azure_manager.go index 73c4d0999d55..2787c703e5b6 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_manager.go @@ -36,6 +36,8 @@ import ( ) const ( + azurePrefix = "azure://" + vmTypeVMSS = "vmss" vmTypeStandard = "standard" @@ -50,8 +52,19 @@ type AzureManager struct { azClient *azClient env azure.Environment - azureCache *azureCache - lastRefresh time.Time + // azureCache is used for caching Azure resources. + // It keeps track of nodegroups and instances + // (and of which nodegroup instances belong to) + azureCache *azureCache + // lastRefresh is the time azureCache was last refreshed. + // Together with azureCache.refreshInterval is it used to decide whether + // it is time to refresh the cache from Azure resources. + // + // Cache invalidation can also be requested via invalidateCache() + // (used by both AzureManager and ScaleSet), which manipulates + // lastRefresh to force refresh on the next check. + lastRefresh time.Time + autoDiscoverySpecs []labelAutoDiscoveryConfig explicitlyConfigured map[string]bool } @@ -93,7 +106,7 @@ func createAzureManagerInternal(configReader io.Reader, discoveryOpts cloudprovi if cfg.VmssCacheTTL != 0 { cacheTTL = time.Duration(cfg.VmssCacheTTL) * time.Second } - cache, err := newAzureCache(azClient, cacheTTL, cfg.ResourceGroup, cfg.VMType, cfg.EnableDynamicInstanceList, cfg.Location) + cache, err := newAzureCache(azClient, cacheTTL, *cfg) if err != nil { return nil, err } @@ -170,7 +183,7 @@ func (m *AzureManager) buildNodeGroupFromSpec(spec string) (cloudprovider.NodeGr case vmTypeStandard: return NewAgentPool(s, m) case vmTypeVMSS: - return NewScaleSet(s, m, -1) + return NewScaleSet(s, m, -1, false) default: return nil, fmt.Errorf("vmtype %s not supported", m.config.VMType) } @@ -198,6 +211,8 @@ func (m *AzureManager) forceRefresh() error { return nil } +// invalidateCache forces cache reload on the next check +// by manipulating lastRefresh timestamp func (m *AzureManager) invalidateCache() { m.lastRefresh = time.Now().Add(-1 * m.azureCache.refreshInterval) klog.V(2).Infof("Invalidated Azure cache") @@ -366,7 +381,9 @@ func (m *AzureManager) getFilteredScaleSets(filter []labelAutoDiscoveryConfig) ( curSize = *scaleSet.Sku.Capacity } - vmss, err := NewScaleSet(spec, m, curSize) + dedicatedHost := scaleSet.VirtualMachineScaleSetProperties != nil && scaleSet.VirtualMachineScaleSetProperties.HostGroup != nil + + vmss, err := NewScaleSet(spec, m, curSize, dedicatedHost) if err != nil { klog.Warningf("ignoring vmss %q %s", *scaleSet.Name, err) continue diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go b/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go index eb5eda27a8ac..942af43a65c4 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go @@ -29,8 +29,8 @@ import ( "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources" "github.com/Azure/go-autorest/autorest/date" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/config" azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients" @@ -136,7 +136,10 @@ const validAzureCfgForStandardVMTypeWithoutDeploymentParameters = `{ "deployment":"cluster-autoscaler-0001" }` -const invalidAzureCfg = `{{}"cloud": "AzurePublicCloud",}` +const ( + invalidAzureCfg = `{{}"cloud": "AzurePublicCloud",}` + testASG = "test-asg" +) func TestCreateAzureManagerValidConfig(t *testing.T) { ctrl := gomock.NewController(t) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go index a506d54df869..9d2c9fcdcac3 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go @@ -84,10 +84,13 @@ type ScaleSet struct { instanceMutex sync.Mutex instanceCache []cloudprovider.Instance lastInstanceRefresh time.Time + + // uses Azure Dedicated Host + dedicatedHost bool } // NewScaleSet creates a new NewScaleSet. -func NewScaleSet(spec *dynamic.NodeGroupSpec, az *AzureManager, curSize int64) (*ScaleSet, error) { +func NewScaleSet(spec *dynamic.NodeGroupSpec, az *AzureManager, curSize int64, dedicatedHost bool) (*ScaleSet, error) { scaleSet := &ScaleSet{ azureRef: azureRef{ Name: spec.Name, @@ -100,6 +103,7 @@ func NewScaleSet(spec *dynamic.NodeGroupSpec, az *AzureManager, curSize int64) ( enableDynamicInstanceList: az.config.EnableDynamicInstanceList, instancesRefreshJitter: az.config.VmssVmsCacheJitter, enableForceDelete: az.config.EnableForceDelete, + dedicatedHost: dedicatedHost, } if az.config.VmssVmsCacheTTL != 0 { @@ -586,7 +590,7 @@ func (scaleSet *ScaleSet) TemplateNodeInfo() (*schedulerframework.NodeInfo, erro return nil, err } - node, err := buildNodeFromTemplate(scaleSet.Name, template, scaleSet.manager) + node, err := buildNodeFromTemplate(scaleSet.Name, template, scaleSet.manager, scaleSet.enableDynamicInstanceList) if err != nil { return nil, err } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go index c9ba730852f9..cda0c77bb230 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go @@ -23,8 +23,8 @@ import ( "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmclient/mockvmclient" @@ -33,7 +33,6 @@ import ( ) const ( - testASG = "test-asg" testLocation = "eastus" ) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_template.go b/cluster-autoscaler/cloudprovider/azure/azure_template.go index 719762078fed..fdee130eabac 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_template.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_template.go @@ -36,46 +36,15 @@ import ( const ( azureDiskTopologyKey string = "topology.disk.csi.azure.com/zone" + // AKSLabelPrefixValue represents the constant prefix for AKSLabelKeyPrefixValue + AKSLabelPrefixValue = "kubernetes.azure.com" + // AKSLabelKeyPrefixValue represents prefix for AKS Labels + AKSLabelKeyPrefixValue = AKSLabelPrefixValue + "/" ) -func buildInstanceOS(template compute.VirtualMachineScaleSet) string { - instanceOS := cloudprovider.DefaultOS - if template.VirtualMachineProfile != nil && template.VirtualMachineProfile.OsProfile != nil && template.VirtualMachineProfile.OsProfile.WindowsConfiguration != nil { - instanceOS = "windows" - } - - return instanceOS -} - -func buildGenericLabels(template compute.VirtualMachineScaleSet, nodeName string) map[string]string { - result := make(map[string]string) - - result[apiv1.LabelArchStable] = cloudprovider.DefaultArch - result[apiv1.LabelOSStable] = buildInstanceOS(template) - - result[apiv1.LabelInstanceTypeStable] = *template.Sku.Name - result[apiv1.LabelTopologyRegion] = strings.ToLower(*template.Location) - - if template.Zones != nil && len(*template.Zones) > 0 { - failureDomains := make([]string, len(*template.Zones)) - for k, v := range *template.Zones { - failureDomains[k] = strings.ToLower(*template.Location) + "-" + v - } - - result[apiv1.LabelTopologyZone] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) - result[azureDiskTopologyKey] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) - } else { - result[apiv1.LabelTopologyZone] = "0" - result[azureDiskTopologyKey] = "" - } - - result[apiv1.LabelHostname] = nodeName - return result -} - -func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineScaleSet, manager *AzureManager) (*apiv1.Node, error) { +func buildNodeFromTemplate(nodeGroupName string, template compute.VirtualMachineScaleSet, manager *AzureManager, enableDynamicInstanceList bool) (*apiv1.Node, error) { node := apiv1.Node{} - nodeName := fmt.Sprintf("%s-asg-%d", scaleSetName, rand.Int63()) + nodeName := fmt.Sprintf("%s-asg-%d", nodeGroupName, rand.Int63()) node.ObjectMeta = metav1.ObjectMeta{ Name: nodeName, @@ -91,7 +60,7 @@ func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineS // Fetching SKU information from SKU API if enableDynamicInstanceList is true. var dynamicErr error - if manager.config.EnableDynamicInstanceList { + if enableDynamicInstanceList { var vmssTypeDynamic InstanceType klog.V(1).Infof("Fetching instance information for SKU: %s from SKU API", *template.Sku.Name) vmssTypeDynamic, dynamicErr = GetVMSSTypeDynamically(template, manager.azureCache) @@ -103,7 +72,7 @@ func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineS klog.Errorf("Dynamically fetching of instance information from SKU api failed with error: %v", dynamicErr) } } - if !manager.config.EnableDynamicInstanceList || dynamicErr != nil { + if !enableDynamicInstanceList || dynamicErr != nil { klog.V(1).Infof("Falling back to static SKU list for SKU: %s", *template.Sku.Name) // fall-back on static list of vmss if dynamic workflow fails. vmssTypeStatic, staticErr := GetVMSSTypeStatically(template) @@ -128,11 +97,6 @@ func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineS node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(memoryMb*1024*1024, resource.DecimalSI) - resourcesFromTags := extractAllocatableResourcesFromScaleSet(template.Tags) - for resourceName, val := range resourcesFromTags { - node.Status.Capacity[apiv1.ResourceName(resourceName)] = *val - } - // TODO: set real allocatable. node.Status.Allocatable = node.Status.Capacity @@ -153,6 +117,11 @@ func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineS // Labels from the Scale Set's Tags node.Labels = cloudprovider.JoinStringMaps(node.Labels, extractLabelsFromScaleSet(template.Tags)) + resourcesFromTags := extractAllocatableResourcesFromScaleSet(template.Tags) + for resourceName, val := range resourcesFromTags { + node.Status.Capacity[apiv1.ResourceName(resourceName)] = *val + } + // Taints from the Scale Set's Tags node.Spec.Taints = extractTaintsFromScaleSet(template.Tags) @@ -160,6 +129,41 @@ func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineS return &node, nil } +func buildInstanceOS(template compute.VirtualMachineScaleSet) string { + instanceOS := cloudprovider.DefaultOS + if template.VirtualMachineProfile != nil && template.VirtualMachineProfile.OsProfile != nil && template.VirtualMachineProfile.OsProfile.WindowsConfiguration != nil { + instanceOS = "windows" + } + + return instanceOS +} + +func buildGenericLabels(template compute.VirtualMachineScaleSet, nodeName string) map[string]string { + result := make(map[string]string) + + result[apiv1.LabelArchStable] = cloudprovider.DefaultArch + result[apiv1.LabelOSStable] = buildInstanceOS(template) + + result[apiv1.LabelInstanceTypeStable] = *template.Sku.Name + result[apiv1.LabelTopologyRegion] = strings.ToLower(*template.Location) + + if template.Zones != nil && len(*template.Zones) > 0 { + failureDomains := make([]string, len(*template.Zones)) + for k, v := range *template.Zones { + failureDomains[k] = strings.ToLower(*template.Location) + "-" + v + } + + result[apiv1.LabelTopologyZone] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) + result[azureDiskTopologyKey] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) + } else { + result[apiv1.LabelTopologyZone] = "0" + result[azureDiskTopologyKey] = "" + } + + result[apiv1.LabelHostname] = nodeName + return result +} + func extractLabelsFromScaleSet(tags map[string]*string) map[string]string { result := make(map[string]string) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_template_test.go b/cluster-autoscaler/cloudprovider/azure/azure_template_test.go index 3eb8295f662b..9ca887fc6cc7 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_template_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_template_test.go @@ -18,11 +18,12 @@ package azure import ( "fmt" + "testing" + "github.com/Azure/go-autorest/autorest/to" "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "testing" ) func TestExtractLabelsFromScaleSet(t *testing.T) { diff --git a/cluster-autoscaler/cloudprovider/azure/azure_util.go b/cluster-autoscaler/cloudprovider/azure/azure_util.go index f42271692623..9f4a44284ed2 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_util.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_util.go @@ -18,8 +18,6 @@ package azure import ( "context" - "crypto/rsa" - "crypto/x509" "encoding/json" "fmt" "io/ioutil" @@ -37,8 +35,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" - "golang.org/x/crypto/pkcs12" - + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/version" klog "k8s.io/klog/v2" "sigs.k8s.io/cloud-provider-azure/pkg/retry" @@ -64,6 +61,12 @@ const ( vmResourceType = "Microsoft.Compute/virtualMachines" vmExtensionType = "Microsoft.Compute/virtualMachines/extensions" + // CSE Extension checks + vmssCSEExtensionName = "vmssCSE" + vmssExtensionProvisioningFailed = "VMExtensionProvisioningFailed" + // vmExtensionProvisioningErrorClass represents a Vm extension provisioning error + vmExtensionProvisioningErrorClass cloudprovider.InstanceErrorClass = 103 + // resource ids nsgID = "nsgID" rtID = "routeTableID" @@ -181,7 +184,7 @@ func (util *AzUtil) DeleteVirtualMachine(rg string, name string) error { } klog.V(2).Infof("VirtualMachine %s/%s removed", rg, name) - if len(nicName) > 0 { + if nicName != "" { klog.Infof("deleting nic: %s/%s", rg, nicName) interfaceCtx, interfaceCancel := getContextWithCancel() defer interfaceCancel() @@ -225,25 +228,9 @@ func (util *AzUtil) DeleteVirtualMachine(rg string, name string) error { klog.V(2).Infof("disk %s/%s removed", rg, *osDiskName) } } - return nil } -// decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and -// the private RSA key -func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) { - privateKey, certificate, err := pkcs12.Decode(pkcs, password) - if err != nil { - return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %v", err) - } - rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey) - if !isRsaKey { - return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key") - } - - return certificate, rsaPrivateKey, nil -} - func getUserAgentExtension() string { suffix := os.Getenv("AZURE_CLUSTER_AUTOSCALER_USER_AGENT_SUFFIX") return fmt.Sprintf("cluster-autoscaler%s/v%s", suffix, version.ClusterAutoscalerVersion) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_util_test.go b/cluster-autoscaler/cloudprovider/azure/azure_util_test.go index 983693d49b9f..a1d5313c4797 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_util_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_util_test.go @@ -215,13 +215,15 @@ func TestConvertResourceGroupNameToLower(t *testing.T) { }, { desc: "providerID not in Azure format should report error", - resourceID: "azure://invalid-id", + resourceID: azurePrefix + "invalid-id", expectError: true, }, { - desc: "resource group name in VM providerID should be converted", - resourceID: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", - expected: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + desc: "resource group name in VM providerID should be converted", + resourceID: azurePrefix + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName" + + "/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + expected: azurePrefix + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname" + + "/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", }, { desc: "resource group name in VM resourceID should be converted", @@ -229,9 +231,11 @@ func TestConvertResourceGroupNameToLower(t *testing.T) { expected: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", }, { - desc: "resource group name in VMSS providerID should be converted", - resourceID: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", - expected: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + desc: "resource group name in VMSS providerID should be converted", + resourceID: azurePrefix + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName" + + "/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + expected: azurePrefix + "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname" + + "/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", }, { desc: "resource group name in VMSS resourceID should be converted", diff --git a/cluster-autoscaler/cloudprovider/azure/testdata/test.pfx b/cluster-autoscaler/cloudprovider/azure/testdata/test.pfx new file mode 100644 index 000000000000..693133363fbd Binary files /dev/null and b/cluster-autoscaler/cloudprovider/azure/testdata/test.pfx differ diff --git a/cluster-autoscaler/cloudprovider/azure/testdata/testnopassword.pfx b/cluster-autoscaler/cloudprovider/azure/testdata/testnopassword.pfx new file mode 100644 index 000000000000..0b32730ed8c7 Binary files /dev/null and b/cluster-autoscaler/cloudprovider/azure/testdata/testnopassword.pfx differ diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index ee9983c1cf74..7341cbd08217 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -8,9 +8,9 @@ require ( cloud.google.com/go/compute/metadata v0.3.0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/azure-sdk-for-go-extensions v0.1.6 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0-beta.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.9.0-beta.1 github.com/Azure/go-autorest/autorest v0.11.29 github.com/Azure/go-autorest/autorest/adal v0.9.23 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 @@ -34,7 +34,6 @@ require ( github.com/stretchr/testify v1.9.0 github.com/vburenin/ifacemaker v1.2.1 go.uber.org/mock v0.4.0 - golang.org/x/crypto v0.24.0 golang.org/x/net v0.26.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sys v0.21.0 @@ -57,12 +56,21 @@ require ( k8s.io/kubelet v0.31.0-beta.0 k8s.io/kubernetes v1.31.0-beta.0 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 - sigs.k8s.io/cloud-provider-azure v1.28.0 + sigs.k8s.io/cloud-provider-azure v1.29.4 sigs.k8s.io/yaml v1.4.0 ) require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 // indirect github.com/Azure/go-armbalancer v0.0.2 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect @@ -70,7 +78,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 // indirect github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect github.com/Microsoft/go-winio v0.6.0 // indirect @@ -111,7 +119,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/cadvisor v0.49.0 // indirect @@ -149,7 +157,7 @@ require ( github.com/opencontainers/runc v1.1.13 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 // indirect github.com/opencontainers/selinux v1.11.0 // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect @@ -179,12 +187,13 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect @@ -207,6 +216,8 @@ require ( k8s.io/kubectl v0.28.0 // indirect k8s.io/mount-utils v0.26.0-alpha.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect + sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13 // indirect + sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index b4083ce7dca1..d00b52672e1d 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -8,20 +8,38 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go-extensions v0.1.6 h1:EXGvDcj54u98XfaI/Cy65Ds6vNsIJeGKYf0eNLB1y4Q= github.com/Azure/azure-sdk-for-go-extensions v0.1.6/go.mod h1:27StPiXJp6Xzkq2AQL7gPK7VC0hgmCnUKlco1dO1jaM= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 h1:ui3YNbxfW7J3tTFIZMH6LIGRjCngp+J+nIFlnizfNTE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0/go.mod h1:gZmgV+qBqygoznvqo2J9oKZAFziqhLZ2xE/WVUmzkHA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v2 v2.4.0 h1:1u/K2BFv0MwkG6he8RYuUcbbeK22rkoZbg4lKa/msZU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v2 v2.4.0/go.mod h1:U5gpsREQZE6SLk1t/cFfc1eMhYAlYpEzvaYXuDfefy8= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0-beta.1 h1:6RFNcR7iE8Ka8j76gE0a/b28eAX6AZF4zqSw0XnFWbg= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0-beta.1/go.mod h1:gYq8wyDgv6JLhGbAU6gg8amCPgQWRE+aCvrV2gyzdfs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.9.0-beta.1 h1:iqhrjj9w9/AQZsHjaOVyloamkeAFRbWI0iHNy6INMYk= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.9.0-beta.1/go.mod h1:gYq8wyDgv6JLhGbAU6gg8amCPgQWRE+aCvrV2gyzdfs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 h1:HlZMUZW8S4P9oob1nCHxCCKrytxyLc+24nUJGssoEto= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 h1:9Eih8XcEeQnFD0ntMlUDleKMzfeCeUfa+VbnDCI4AZs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0/go.mod h1:wGPyTi+aURdqPAGMZDQqnNs9IrShADF8w2WZb6bKeq0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= github.com/Azure/go-armbalancer v0.0.2 h1:NVnxsTWHI5/fEzL6k6TjxPUfcB/3Si3+HFOZXOu0QtA= github.com/Azure/go-armbalancer v0.0.2/go.mod h1:yTg7MA/8YnfKQc9o97tzAJ7fbdVkod1xGsIvKmhYPRE= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -59,8 +77,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/skewer v0.0.14 h1:0mzUJhspECkajYyynYsOCp//E2PSnYXrgP45bcskqfQ= github.com/Azure/skewer v0.0.14/go.mod h1:6WTecuPyfGtuvS8Mh4JYWuHhO4kcWycGfsUBB+XTFG4= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 h1:lwL1vLWmdBJ5h+StMEN6+GMz1J/Y0yUU3RDv+QBy+Q4= github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0/go.mod h1:UTfhBnADaj2rybPT049NScSh7Eall3u2ib43wmz3deg= @@ -223,8 +241,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -387,8 +405,8 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78/go.m github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -550,8 +568,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -616,7 +634,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -640,8 +657,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -703,6 +720,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= +gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= @@ -780,8 +799,12 @@ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/cloud-provider-azure v1.28.0 h1:LkvvDQ2u0rCr1lhFBoyjvKhYazhpYnAohOqQKN060H8= -sigs.k8s.io/cloud-provider-azure v1.28.0/go.mod h1:ubvg4F58jePO4Z7C4XfgJkFFGpqhVeogpzOdc1X4dyk= +sigs.k8s.io/cloud-provider-azure v1.29.4 h1:lW/mqq9fofs52/T+Crs6JNzzEhz0NjzQUtSXMseh67M= +sigs.k8s.io/cloud-provider-azure v1.29.4/go.mod h1:73KgMVXVtMqG/JlhRRezonfirvoO2ldsxC6H+1FKEPg= +sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13 h1:dxpo41/N6m2R//9fmqKgqYZL2k0rQSE1NvQteIQ9pGA= +sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.13/go.mod h1:tN2BDTM6RDyQsae6JRvaaA14LVxDsRaLU3Ea2MRUBjg= +sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.4 h1:CFMHYo6/OQpLTycJGQIze2pchNeJQ7L2TQC6fDo4JGY= +sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.4/go.mod h1:PvXgFxPcfve6yBiWNIO/fqAMvGVC9W7qN6M2vIj4zmY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=