diff --git a/cloud/scope/managed_control_plane.go b/cloud/scope/managed_control_plane.go index 5bf63f1c..5e15e3c3 100644 --- a/cloud/scope/managed_control_plane.go +++ b/cloud/scope/managed_control_plane.go @@ -591,6 +591,7 @@ func setControlPlaneSpecDefaults(spec *infrav2exp.OCIManagedControlPlaneSpec) { if spec.ClusterType == "" { spec.ClusterType = infrav2exp.BasicClusterType } + spec.Addons = nil if spec.ImagePolicyConfig == nil { spec.ImagePolicyConfig = &infrav2exp.ImagePolicyConfig{ IsPolicyEnabled: common.Bool(false), @@ -615,6 +616,7 @@ func (s *ManagedControlPlaneScope) getSpecFromActual(cluster *oke.Cluster) *infr Version: cluster.KubernetesVersion, KmsKeyId: cluster.KmsKeyId, ID: cluster.Id, + Addons: nil, } if cluster.ImagePolicyConfig != nil { keys := make([]infrav2exp.KeyDetails, 0) @@ -673,6 +675,183 @@ func (s *ManagedControlPlaneScope) getSpecFromActual(cluster *oke.Cluster) *infr return &spec } +// ReconcileAddons reconciles addons which have been specified in the spec on the OKE cluster +func (s *ManagedControlPlaneScope) ReconcileAddons(ctx context.Context, okeCluster *oke.Cluster) error { + addonSpec := s.OCIManagedControlPlane.Spec.Addons + // go through the list of addons present in the spec and reconcile them, reconcile can be 2 ways, either + // install the addon if it has not been installed till now, or update the addon + for _, addon := range addonSpec { + resp, err := s.ContainerEngineClient.GetAddon(ctx, oke.GetAddonRequest{ + ClusterId: okeCluster.Id, + AddonName: addon.Name, + }) + if err != nil { + // addon is not present, hence install it + if ociutil.IsNotFound(err) { + s.Info(fmt.Sprintf("Install addon %s", *addon.Name)) + _, err = s.ContainerEngineClient.InstallAddon(ctx, oke.InstallAddonRequest{ + ClusterId: okeCluster.Id, + InstallAddonDetails: oke.InstallAddonDetails{ + Version: addon.Version, + Configurations: getAddonConfigurations(addon.Configurations), + AddonName: addon.Name, + }, + }) + if err != nil { + return err + } + // add it to status, details will be reconciled in next loop + status := infrav2exp.AddonStatus{ + LifecycleState: common.String(string(oke.AddonLifecycleStateCreating)), + } + s.OCIManagedControlPlane.SetAddonStatus(*addon.Name, status) + } else { + return err + } + } else { + s.OCIManagedControlPlane.SetAddonStatus(*addon.Name, s.getStatus(resp.Addon)) + // addon present, update it + err = s.handleExistingAddon(ctx, okeCluster, resp.Addon, addon) + if err != nil { + return err + } + } + } + // for addons which are present in the status object but not in the spec, the possibility + // is that user deleted it from the spec. Hence disable the addon + for k, _ := range s.OCIManagedControlPlane.Status.AddonStatus { + // present in status but not in spec + if getAddon(addonSpec, k) == nil { + err := s.handleDeletedAddon(ctx, okeCluster, k) + if err != nil { + return err + } + } + } + return nil +} + +func (s *ManagedControlPlaneScope) getStatus(addon oke.Addon) infrav2exp.AddonStatus { + // update status of the addon + status := infrav2exp.AddonStatus{ + LifecycleState: common.String(string(addon.LifecycleState)), + CurrentlyInstalledVersion: addon.CurrentInstalledVersion, + } + if addon.AddonError != nil { + status.AddonError = &infrav2exp.AddonError{ + Status: addon.AddonError.Status, + Code: addon.AddonError.Code, + Message: addon.AddonError.Message, + } + } + return status +} + +func (s *ManagedControlPlaneScope) handleExistingAddon(ctx context.Context, okeCluster *oke.Cluster, addon oke.Addon, addonInSpec infrav2exp.Addon) error { + // if the addon can be updated do so + // if the addon is already in updating state, or in failed state, do not update + s.Info(fmt.Sprintf("Reconciling addon %s with lifecycle state %s", *addon.Name, string(addon.LifecycleState))) + if !(addon.LifecycleState == oke.AddonLifecycleStateUpdating || + addon.LifecycleState == oke.AddonLifecycleStateFailed) { + addonConfigurationsActual := getActualAddonConfigurations(addon.Configurations) + // if the version changed or the configuration changed, update the addon + // if the lifecycle state is needs attention, try to update + if addon.LifecycleState == oke.AddonLifecycleStateNeedsAttention || + !reflect.DeepEqual(addonInSpec.Version, addon.Version) || + !reflect.DeepEqual(addonConfigurationsActual, addonInSpec.Configurations) { + s.Info(fmt.Sprintf("Updating addon %s", *addon.Name)) + _, err := s.ContainerEngineClient.UpdateAddon(ctx, oke.UpdateAddonRequest{ + ClusterId: okeCluster.Id, + AddonName: addon.Name, + UpdateAddonDetails: oke.UpdateAddonDetails{ + Version: addonInSpec.Version, + Configurations: getAddonConfigurations(addonInSpec.Configurations), + }, + }) + if err != nil { + return err + } + } + } + return nil +} + +func (s *ManagedControlPlaneScope) handleDeletedAddon(ctx context.Context, okeCluster *oke.Cluster, addonName string) error { + resp, err := s.ContainerEngineClient.GetAddon(ctx, oke.GetAddonRequest{ + ClusterId: okeCluster.Id, + AddonName: common.String(addonName), + }) + if err != nil { + if ociutil.IsNotFound(err) { + s.OCIManagedControlPlane.RemoveAddonStatus(addonName) + return nil + } else { + return err + } + } + addonState := resp.LifecycleState + switch addonState { + // nothing to do if addon is in deleting state + case oke.AddonLifecycleStateDeleting: + s.Info(fmt.Sprintf("Addon %s is in deleting state", addonName)) + break + case oke.AddonLifecycleStateDeleted: + // delete addon from status if addon has been deleted + s.Info(fmt.Sprintf("Addon %s is in deleted state", addonName)) + s.OCIManagedControlPlane.RemoveAddonStatus(addonName) + break + default: + // else delete the addon + // delete addon is called disable addon with remove flag turned on + _, err := s.ContainerEngineClient.DisableAddon(ctx, oke.DisableAddonRequest{ + ClusterId: okeCluster.Id, + AddonName: common.String(addonName), + IsRemoveExistingAddOn: common.Bool(true), + }) + if err != nil { + return err + } + } + return nil +} + +func getAddonConfigurations(configurations []infrav2exp.AddonConfiguration) []oke.AddonConfiguration { + if len(configurations) == 0 { + return nil + } + config := make([]oke.AddonConfiguration, len(configurations)) + for i, c := range configurations { + config[i] = oke.AddonConfiguration{ + Key: c.Key, + Value: c.Value, + } + } + return config +} + +func getActualAddonConfigurations(addonConfigurations []oke.AddonConfiguration) []infrav2exp.AddonConfiguration { + if len(addonConfigurations) == 0 { + return nil + } + config := make([]infrav2exp.AddonConfiguration, len(addonConfigurations)) + for i, c := range addonConfigurations { + config[i] = infrav2exp.AddonConfiguration{ + Key: c.Key, + Value: c.Value, + } + } + return config +} + +func getAddon(addons []infrav2exp.Addon, name string) *infrav2exp.Addon { + for i, addon := range addons { + if *addon.Name == name { + return &addons[i] + } + } + return nil +} + func getKubeConfigUserName(clusterName string, isUser bool) string { if isUser { return fmt.Sprintf("%s-user", clusterName) diff --git a/cloud/scope/managed_control_plane_test.go b/cloud/scope/managed_control_plane_test.go index 4f9d985a..17d9be02 100644 --- a/cloud/scope/managed_control_plane_test.go +++ b/cloud/scope/managed_control_plane_test.go @@ -18,6 +18,7 @@ package scope import ( "context" + "errors" "io" "strings" "testing" @@ -794,3 +795,358 @@ func TestControlPlaneKubeconfigReconcile(t *testing.T) { }) } } + +func TestAddonReconcile(t *testing.T) { + var ( + cs *ManagedControlPlaneScope + mockCtrl *gomock.Controller + okeClient *mock_containerengine.MockClient + baseClient *mock_base.MockBaseClient + ) + + setup := func(t *testing.T, g *WithT) { + var err error + mockCtrl = gomock.NewController(t) + okeClient = mock_containerengine.NewMockClient(mockCtrl) + baseClient = mock_base.NewMockBaseClient(mockCtrl) + ociClusterAccessor := OCIManagedCluster{ + &infrav2exp.OCIManagedCluster{}, + } + ociClusterAccessor.OCIManagedCluster.Spec.OCIResourceIdentifier = "resource_uid" + cs, err = NewManagedControlPlaneScope(ManagedControlPlaneScopeParams{ + ContainerEngineClient: okeClient, + BaseClient: baseClient, + OCIManagedControlPlane: &infrav2exp.OCIManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + OCIClusterAccessor: ociClusterAccessor, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + }, + }) + g.Expect(err).To(BeNil()) + } + teardown := func(t *testing.T, g *WithT) { + mockCtrl.Finish() + } + + tests := []struct { + name string + errorExpected bool + objects []client.Object + expectedEvent string + eventNotExpected string + matchError error + errorSubStringMatch bool + okeCluster oke.Cluster + matchStatus map[string]infrav2exp.AddonStatus + testSpecificSetup func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) + }{ + { + name: "install addon", + errorExpected: false, + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Spec.Addons = []infrav2exp.Addon{ + { + Name: common.String("dashboard"), + }, + } + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{}, ociutil.ErrNotFound) + okeClient.EXPECT().InstallAddon(gomock.Any(), gomock.Eq(oke.InstallAddonRequest{ + ClusterId: common.String("id"), + InstallAddonDetails: oke.InstallAddonDetails{ + AddonName: common.String("dashboard"), + }, + })). + Return(oke.InstallAddonResponse{}, nil) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + }, + { + name: "install addon with config and version", + errorExpected: false, + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Spec.Addons = []infrav2exp.Addon{ + { + Name: common.String("dashboard"), + Version: common.String("v0.1.0"), + Configurations: []infrav2exp.AddonConfiguration{ + { + Key: common.String("k1"), + Value: common.String("v1"), + }, + { + Key: common.String("k2"), + Value: common.String("v2"), + }, + }, + }, + } + cs.OCIManagedControlPlane.Status.AddonStatus = nil + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{}, ociutil.ErrNotFound) + okeClient.EXPECT().InstallAddon(gomock.Any(), gomock.Eq(oke.InstallAddonRequest{ + ClusterId: common.String("id"), + InstallAddonDetails: oke.InstallAddonDetails{ + AddonName: common.String("dashboard"), + Version: common.String("v0.1.0"), + Configurations: []oke.AddonConfiguration{ + { + Key: common.String("k1"), + Value: common.String("v1"), + }, + { + Key: common.String("k2"), + Value: common.String("v2"), + }, + }, + }, + })). + Return(oke.InstallAddonResponse{}, nil) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + }, + { + name: "update addon", + errorExpected: false, + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Spec.Addons = []infrav2exp.Addon{ + { + Name: common.String("dashboard"), + Configurations: []infrav2exp.AddonConfiguration{ + { + Key: common.String("k1"), + Value: common.String("v1"), + }, + { + Key: common.String("k2"), + Value: common.String("v2"), + }, + }, + }, + } + cs.OCIManagedControlPlane.Status.AddonStatus = nil + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{ + Addon: oke.Addon{ + Name: common.String("dashboard"), + }, + }, nil) + okeClient.EXPECT().UpdateAddon(gomock.Any(), gomock.Eq(oke.UpdateAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + UpdateAddonDetails: oke.UpdateAddonDetails{ + Configurations: []oke.AddonConfiguration{ + { + Key: common.String("k1"), + Value: common.String("v1"), + }, + { + Key: common.String("k2"), + Value: common.String("v2"), + }, + }, + }, + })). + Return(oke.UpdateAddonResponse{}, nil) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + }, + { + name: "delete addon", + errorExpected: false, + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Status.AddonStatus = map[string]infrav2exp.AddonStatus{ + "dashboard": { + LifecycleState: common.String("ACTIVE"), + }, + } + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{ + Addon: oke.Addon{ + Name: common.String("dashboard"), + LifecycleState: oke.AddonLifecycleStateActive, + }, + }, nil) + okeClient.EXPECT().DisableAddon(gomock.Any(), gomock.Eq(oke.DisableAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + IsRemoveExistingAddOn: common.Bool(true), + })). + Return(oke.DisableAddonResponse{}, nil) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + }, + { + name: "delete addon, already deleted", + errorExpected: false, + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Status.AddonStatus = map[string]infrav2exp.AddonStatus{ + "dashboard": { + LifecycleState: common.String("ACTIVE"), + }, + } + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{}, ociutil.ErrNotFound) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + }, + { + name: "addon in deleting state", + errorExpected: false, + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Status.AddonStatus = map[string]infrav2exp.AddonStatus{ + "dashboard": { + LifecycleState: common.String("ACTIVE"), + }, + } + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{ + Addon: oke.Addon{ + LifecycleState: oke.AddonLifecycleStateDeleting, + }, + }, nil) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + }, + { + name: "install addon error", + errorExpected: true, + matchError: errors.New("install error"), + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Spec.Addons = []infrav2exp.Addon{ + { + Name: common.String("dashboard"), + }, + } + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{}, ociutil.ErrNotFound) + okeClient.EXPECT().InstallAddon(gomock.Any(), gomock.Eq(oke.InstallAddonRequest{ + ClusterId: common.String("id"), + InstallAddonDetails: oke.InstallAddonDetails{ + AddonName: common.String("dashboard"), + }, + })). + Return(oke.InstallAddonResponse{}, errors.New("install error")) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + }, + { + name: "addon status error", + errorExpected: false, + testSpecificSetup: func(cs *ManagedControlPlaneScope, okeClient *mock_containerengine.MockClient) { + cs.OCIManagedControlPlane.Spec.Addons = []infrav2exp.Addon{ + { + Name: common.String("dashboard"), + }, + } + okeClient.EXPECT().GetAddon(gomock.Any(), gomock.Eq(oke.GetAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + })). + Return(oke.GetAddonResponse{ + Addon: oke.Addon{ + Name: common.String("dashboard"), + LifecycleState: oke.AddonLifecycleStateNeedsAttention, + AddonError: &oke.AddonError{ + Code: common.String("32"), + Message: common.String("error"), + Status: common.String("status"), + }, + }, + }, nil) + okeClient.EXPECT().UpdateAddon(gomock.Any(), gomock.Eq(oke.UpdateAddonRequest{ + ClusterId: common.String("id"), + AddonName: common.String("dashboard"), + UpdateAddonDetails: oke.UpdateAddonDetails{}, + })). + Return(oke.UpdateAddonResponse{}, nil) + }, + okeCluster: oke.Cluster{ + Id: common.String("id"), + Name: common.String("test"), + }, + matchStatus: map[string]infrav2exp.AddonStatus{ + "dashboard": { + LifecycleState: common.String("NEEDS_ATTENTION"), + AddonError: &infrav2exp.AddonError{ + Code: common.String("32"), + Message: common.String("error"), + Status: common.String("status"), + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + defer teardown(t, g) + setup(t, g) + tc.testSpecificSetup(cs, okeClient) + err := cs.ReconcileAddons(context.Background(), &tc.okeCluster) + if tc.errorExpected { + g.Expect(err).To(Not(BeNil())) + if tc.errorSubStringMatch { + g.Expect(err.Error()).To(ContainSubstring(tc.matchError.Error())) + } else { + g.Expect(err.Error()).To(Equal(tc.matchError.Error())) + } + } else { + g.Expect(err).To(BeNil()) + } + if tc.matchStatus != nil { + g.Expect(cs.OCIManagedControlPlane.Status.AddonStatus).To(Equal(tc.matchStatus)) + } + }) + } +} diff --git a/cloud/services/containerengine/client.go b/cloud/services/containerengine/client.go index fdadb4ac..a92aa371 100644 --- a/cloud/services/containerengine/client.go +++ b/cloud/services/containerengine/client.go @@ -48,4 +48,11 @@ type Client interface { //Work Request GetWorkRequest(ctx context.Context, request containerengine.GetWorkRequestRequest) (response containerengine.GetWorkRequestResponse, err error) + + // Addons + ListAddons(ctx context.Context, request containerengine.ListAddonsRequest) (response containerengine.ListAddonsResponse, err error) + InstallAddon(ctx context.Context, request containerengine.InstallAddonRequest) (response containerengine.InstallAddonResponse, err error) + UpdateAddon(ctx context.Context, request containerengine.UpdateAddonRequest) (response containerengine.UpdateAddonResponse, err error) + DisableAddon(ctx context.Context, request containerengine.DisableAddonRequest) (response containerengine.DisableAddonResponse, err error) + GetAddon(ctx context.Context, request containerengine.GetAddonRequest) (response containerengine.GetAddonResponse, err error) } diff --git a/cloud/services/containerengine/mock_containerengine/client_mock.go b/cloud/services/containerengine/mock_containerengine/client_mock.go index 45f29b7b..cbdce76a 100644 --- a/cloud/services/containerengine/mock_containerengine/client_mock.go +++ b/cloud/services/containerengine/mock_containerengine/client_mock.go @@ -140,6 +140,36 @@ func (mr *MockClientMockRecorder) DeleteVirtualNodePool(ctx, request interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVirtualNodePool", reflect.TypeOf((*MockClient)(nil).DeleteVirtualNodePool), ctx, request) } +// DisableAddon mocks base method. +func (m *MockClient) DisableAddon(ctx context.Context, request containerengine.DisableAddonRequest) (containerengine.DisableAddonResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DisableAddon", ctx, request) + ret0, _ := ret[0].(containerengine.DisableAddonResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DisableAddon indicates an expected call of DisableAddon. +func (mr *MockClientMockRecorder) DisableAddon(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DisableAddon", reflect.TypeOf((*MockClient)(nil).DisableAddon), ctx, request) +} + +// GetAddon mocks base method. +func (m *MockClient) GetAddon(ctx context.Context, request containerengine.GetAddonRequest) (containerengine.GetAddonResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAddon", ctx, request) + ret0, _ := ret[0].(containerengine.GetAddonResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAddon indicates an expected call of GetAddon. +func (mr *MockClientMockRecorder) GetAddon(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAddon", reflect.TypeOf((*MockClient)(nil).GetAddon), ctx, request) +} + // GetCluster mocks base method. func (m *MockClient) GetCluster(ctx context.Context, request containerengine.GetClusterRequest) (containerengine.GetClusterResponse, error) { m.ctrl.T.Helper() @@ -215,6 +245,36 @@ func (mr *MockClientMockRecorder) GetWorkRequest(ctx, request interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkRequest", reflect.TypeOf((*MockClient)(nil).GetWorkRequest), ctx, request) } +// InstallAddon mocks base method. +func (m *MockClient) InstallAddon(ctx context.Context, request containerengine.InstallAddonRequest) (containerengine.InstallAddonResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallAddon", ctx, request) + ret0, _ := ret[0].(containerengine.InstallAddonResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InstallAddon indicates an expected call of InstallAddon. +func (mr *MockClientMockRecorder) InstallAddon(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallAddon", reflect.TypeOf((*MockClient)(nil).InstallAddon), ctx, request) +} + +// ListAddons mocks base method. +func (m *MockClient) ListAddons(ctx context.Context, request containerengine.ListAddonsRequest) (containerengine.ListAddonsResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListAddons", ctx, request) + ret0, _ := ret[0].(containerengine.ListAddonsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListAddons indicates an expected call of ListAddons. +func (mr *MockClientMockRecorder) ListAddons(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAddons", reflect.TypeOf((*MockClient)(nil).ListAddons), ctx, request) +} + // ListClusters mocks base method. func (m *MockClient) ListClusters(ctx context.Context, request containerengine.ListClustersRequest) (containerengine.ListClustersResponse, error) { m.ctrl.T.Helper() @@ -275,6 +335,21 @@ func (mr *MockClientMockRecorder) ListVirtualNodes(ctx, request interface{}) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVirtualNodes", reflect.TypeOf((*MockClient)(nil).ListVirtualNodes), ctx, request) } +// UpdateAddon mocks base method. +func (m *MockClient) UpdateAddon(ctx context.Context, request containerengine.UpdateAddonRequest) (containerengine.UpdateAddonResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateAddon", ctx, request) + ret0, _ := ret[0].(containerengine.UpdateAddonResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateAddon indicates an expected call of UpdateAddon. +func (mr *MockClientMockRecorder) UpdateAddon(ctx, request interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateAddon", reflect.TypeOf((*MockClient)(nil).UpdateAddon), ctx, request) +} + // UpdateCluster mocks base method. func (m *MockClient) UpdateCluster(ctx context.Context, request containerengine.UpdateClusterRequest) (containerengine.UpdateClusterResponse, error) { m.ctrl.T.Helper() diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanes.yaml index 33e7e2eb..ce2b8109 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanes.yaml @@ -212,6 +212,39 @@ spec: description: OCIManagedControlPlaneSpec defines the desired state of OCIManagedControlPlane. The properties are generated from https://docs.oracle.com/en-us/iaas/api/#/en/containerengine/20180222/datatypes/CreateClusterDetails properties: + addons: + description: The list of addons to be applied to the OKE cluster. + items: + description: Addon defines the properties of an addon. + properties: + configurations: + description: Configurations defines a list of configurations + of the addon. + items: + description: AddonConfiguration defines a configuration of + an addon. + properties: + key: + description: The key of the configuration. + type: string + value: + description: The value of the configuration. + type: string + type: object + type: array + name: + description: Name represents the name of the addon. + type: string + version: + description: Version represents the version of the addon. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map clusterOptions: description: ClusterOptions defines Optional attributes for the cluster. properties: @@ -308,6 +341,37 @@ spec: description: OCIManagedControlPlaneStatus defines the observed state of OCIManagedControlPlane properties: + addonStatus: + additionalProperties: + description: AddonStatus defines the status of an Addon. + properties: + addonError: + description: AddonError defines the error encountered by the + Addon. + properties: + code: + description: Code defines a short error code that defines + the upstream error, meant for programmatic parsing. + type: string + message: + description: Message defines a human-readable error string + of the upstream error. + type: string + status: + description: Status defines the status of the HTTP response + encountered in the upstream error. + type: string + type: object + currentlyInstalledVersion: + description: Version represents the version of the addon. + type: string + lifecycleState: + description: LifecycleState defines the lifecycle state of the + addon. + type: string + type: object + description: AddonStatus represents the status of the addon. + type: object conditions: description: NetworkSpec encapsulates all things related to OCI network. items: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanetemplates.yaml index eaf9532a..13c5cb29 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ocimanagedcontrolplanetemplates.yaml @@ -183,6 +183,39 @@ spec: of OCIManagedControlPlane. The properties are generated from https://docs.oracle.com/en-us/iaas/api/#/en/containerengine/20180222/datatypes/CreateClusterDetails properties: + addons: + description: The list of addons to be applied to the OKE cluster. + items: + description: Addon defines the properties of an addon. + properties: + configurations: + description: Configurations defines a list of configurations + of the addon. + items: + description: AddonConfiguration defines a configuration + of an addon. + properties: + key: + description: The key of the configuration. + type: string + value: + description: The value of the configuration. + type: string + type: object + type: array + name: + description: Name represents the name of the addon. + type: string + version: + description: Version represents the version of the addon. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map clusterOptions: description: ClusterOptions defines Optional attributes for the cluster. diff --git a/exp/api/v1beta1/conversion.go b/exp/api/v1beta1/conversion.go index 47d9b0ad..ed1892f7 100644 --- a/exp/api/v1beta1/conversion.go +++ b/exp/api/v1beta1/conversion.go @@ -55,3 +55,7 @@ func Convert_v1beta2_OCIManagedControlPlaneSpec_To_v1beta1_OCIManagedControlPlan func Convert_v1beta2_OCIManagedMachinePoolSpec_To_v1beta1_OCIManagedMachinePoolSpec(in *v1beta2.OCIManagedMachinePoolSpec, out *OCIManagedMachinePoolSpec, s conversion.Scope) error { return autoConvert_v1beta2_OCIManagedMachinePoolSpec_To_v1beta1_OCIManagedMachinePoolSpec(in, out, s) } + +func Convert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedControlPlaneStatus(in *v1beta2.OCIManagedControlPlaneStatus, out *OCIManagedControlPlaneStatus, s conversion.Scope) error { + return autoConvert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedControlPlaneStatus(in, out, s) +} diff --git a/exp/api/v1beta1/ocimanagedcontrolplane_conversion.go b/exp/api/v1beta1/ocimanagedcontrolplane_conversion.go index 9246eaa8..c99359d6 100644 --- a/exp/api/v1beta1/ocimanagedcontrolplane_conversion.go +++ b/exp/api/v1beta1/ocimanagedcontrolplane_conversion.go @@ -34,6 +34,8 @@ func (src *OCIManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error { return err } dst.Spec.ClusterType = restored.Spec.ClusterType + dst.Spec.Addons = restored.Spec.Addons + dst.Status.AddonStatus = restored.Status.AddonStatus return nil } diff --git a/exp/api/v1beta1/zz_generated.conversion.go b/exp/api/v1beta1/zz_generated.conversion.go index 7ad59bc1..314cfa85 100644 --- a/exp/api/v1beta1/zz_generated.conversion.go +++ b/exp/api/v1beta1/zz_generated.conversion.go @@ -361,11 +361,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta2.OCIManagedControlPlaneStatus)(nil), (*OCIManagedControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedControlPlaneStatus(a.(*v1beta2.OCIManagedControlPlaneStatus), b.(*OCIManagedControlPlaneStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*OCIManagedControlPlaneTemplate)(nil), (*v1beta2.OCIManagedControlPlaneTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_OCIManagedControlPlaneTemplate_To_v1beta2_OCIManagedControlPlaneTemplate(a.(*OCIManagedControlPlaneTemplate), b.(*v1beta2.OCIManagedControlPlaneTemplate), scope) }); err != nil { @@ -626,6 +621,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta2.OCIManagedControlPlaneStatus)(nil), (*OCIManagedControlPlaneStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedControlPlaneStatus(a.(*v1beta2.OCIManagedControlPlaneStatus), b.(*OCIManagedControlPlaneStatus), scope) + }); err != nil { + return err + } return nil } @@ -1566,6 +1566,7 @@ func autoConvert_v1beta2_OCIManagedControlPlaneSpec_To_v1beta1_OCIManagedControl // WARNING: in.ClusterType requires manual conversion: does not exist in peer-type out.KmsKeyId = (*string)(unsafe.Pointer(in.KmsKeyId)) out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + // WARNING: in.Addons requires manual conversion: does not exist in peer-type out.Version = (*string)(unsafe.Pointer(in.Version)) return nil } @@ -1587,15 +1588,11 @@ func autoConvert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedContr out.Ready = in.Ready out.Conditions = *(*clusterapiapiv1beta1.Conditions)(unsafe.Pointer(&in.Conditions)) out.Version = (*string)(unsafe.Pointer(in.Version)) + // WARNING: in.AddonStatus requires manual conversion: does not exist in peer-type out.Initialized = in.Initialized return nil } -// Convert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedControlPlaneStatus is an autogenerated conversion function. -func Convert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedControlPlaneStatus(in *v1beta2.OCIManagedControlPlaneStatus, out *OCIManagedControlPlaneStatus, s conversion.Scope) error { - return autoConvert_v1beta2_OCIManagedControlPlaneStatus_To_v1beta1_OCIManagedControlPlaneStatus(in, out, s) -} - func autoConvert_v1beta1_OCIManagedControlPlaneTemplate_To_v1beta2_OCIManagedControlPlaneTemplate(in *OCIManagedControlPlaneTemplate, out *v1beta2.OCIManagedControlPlaneTemplate, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1beta1_OCIManagedControlPlaneTemplateSpec_To_v1beta2_OCIManagedControlPlaneTemplateSpec(&in.Spec, &out.Spec, s); err != nil { diff --git a/exp/api/v1beta2/ocimanagedcontrolplane_types.go b/exp/api/v1beta2/ocimanagedcontrolplane_types.go index 80a39a83..c7714eff 100644 --- a/exp/api/v1beta2/ocimanagedcontrolplane_types.go +++ b/exp/api/v1beta2/ocimanagedcontrolplane_types.go @@ -67,6 +67,12 @@ type OCIManagedControlPlaneSpec struct { // +optional ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` + // The list of addons to be applied to the OKE cluster. + // +optional + // +listType=map + // +listMapKey=name + Addons []Addon `json:"addons,omitempty"` + // Version represents the version of the Kubernetes Cluster Control Plane. Version *string `json:"version,omitempty"` } @@ -161,12 +167,68 @@ type OCIManagedControlPlaneStatus struct { // +optional Version *string `json:"version,omitempty"` + // AddonStatus represents the status of the addon. + // +optional + AddonStatus map[string]AddonStatus `json:"addonStatus,omitempty"` + // Initialized denotes whether or not the control plane has the // uploaded kubernetes config-map. // +optional Initialized bool `json:"initialized"` } +// Addon defines the properties of an addon. +type Addon struct { + // Name represents the name of the addon. + Name *string `json:"name"` + + // Version represents the version of the addon. + // +optional + Version *string `json:"version,omitempty"` + + // Configurations defines a list of configurations of the addon. + // +optional + Configurations []AddonConfiguration `json:"configurations,omitempty"` +} + +// AddonConfiguration defines a configuration of an addon. +type AddonConfiguration struct { + // The key of the configuration. + Key *string `json:"key,omitempty"` + + // The value of the configuration. + Value *string `json:"value,omitempty"` +} + +// AddonStatus defines the status of an Addon. +type AddonStatus struct { + // Version represents the version of the addon. + // +optional + CurrentlyInstalledVersion *string `json:"currentlyInstalledVersion,omitempty"` + + // AddonError defines the error encountered by the Addon. + // +optional + AddonError *AddonError `json:"addonError,omitempty"` + + // LifecycleState defines the lifecycle state of the addon. + // +optional + LifecycleState *string `json:"lifecycleState,omitempty"` +} + +type AddonError struct { + // Code defines a short error code that defines the upstream error, meant for programmatic parsing. + // +optional + Code *string `json:"code,omitempty"` + + // Message defines a human-readable error string of the upstream error. + // +optional + Message *string `json:"message,omitempty"` + + // Status defines the status of the HTTP response encountered in the upstream error. + // +optional + Status *string `json:"status,omitempty"` +} + //+kubebuilder:object:root=true //+kubebuilder:subresource:status // +kubebuilder:storageversion @@ -200,6 +262,22 @@ func (c *OCIManagedControlPlane) SetConditions(conditions clusterv1.Conditions) c.Status.Conditions = conditions } +// SetAddonStatus sets the addon status in the OCIManagedControlPlane +func (c *OCIManagedControlPlane) SetAddonStatus(name string, status AddonStatus) { + if c.Status.AddonStatus == nil { + c.Status.AddonStatus = make(map[string]AddonStatus) + } + c.Status.AddonStatus[name] = status +} + +// RemoveAddonStatus removes the addon status from OCIManagedControlPlane +func (c *OCIManagedControlPlane) RemoveAddonStatus(name string) { + if c.Status.AddonStatus == nil { + c.Status.AddonStatus = make(map[string]AddonStatus) + } + delete(c.Status.AddonStatus, name) +} + func init() { SchemeBuilder.Register(&OCIManagedControlPlane{}, &OCIManagedControlPlaneList{}) } diff --git a/exp/api/v1beta2/zz_generated.deepcopy.go b/exp/api/v1beta2/zz_generated.deepcopy.go index 1cf9577c..60851135 100644 --- a/exp/api/v1beta2/zz_generated.deepcopy.go +++ b/exp/api/v1beta2/zz_generated.deepcopy.go @@ -54,6 +54,123 @@ func (in *AddOnOptions) DeepCopy() *AddOnOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Addon) DeepCopyInto(out *Addon) { + *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } + if in.Version != nil { + in, out := &in.Version, &out.Version + *out = new(string) + **out = **in + } + if in.Configurations != nil { + in, out := &in.Configurations, &out.Configurations + *out = make([]AddonConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Addon. +func (in *Addon) DeepCopy() *Addon { + if in == nil { + return nil + } + out := new(Addon) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddonConfiguration) DeepCopyInto(out *AddonConfiguration) { + *out = *in + if in.Key != nil { + in, out := &in.Key, &out.Key + *out = new(string) + **out = **in + } + if in.Value != nil { + in, out := &in.Value, &out.Value + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonConfiguration. +func (in *AddonConfiguration) DeepCopy() *AddonConfiguration { + if in == nil { + return nil + } + out := new(AddonConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddonError) DeepCopyInto(out *AddonError) { + *out = *in + if in.Code != nil { + in, out := &in.Code, &out.Code + *out = new(string) + **out = **in + } + if in.Message != nil { + in, out := &in.Message, &out.Message + *out = new(string) + **out = **in + } + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonError. +func (in *AddonError) DeepCopy() *AddonError { + if in == nil { + return nil + } + out := new(AddonError) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddonStatus) DeepCopyInto(out *AddonStatus) { + *out = *in + if in.CurrentlyInstalledVersion != nil { + in, out := &in.CurrentlyInstalledVersion, &out.CurrentlyInstalledVersion + *out = new(string) + **out = **in + } + if in.AddonError != nil { + in, out := &in.AddonError, &out.AddonError + *out = new(AddonError) + (*in).DeepCopyInto(*out) + } + if in.LifecycleState != nil { + in, out := &in.LifecycleState, &out.LifecycleState + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonStatus. +func (in *AddonStatus) DeepCopy() *AddonStatus { + if in == nil { + return nil + } + out := new(AddonStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AdmissionControllerOptions) DeepCopyInto(out *AdmissionControllerOptions) { *out = *in @@ -1020,6 +1137,13 @@ func (in *OCIManagedControlPlaneSpec) DeepCopyInto(out *OCIManagedControlPlaneSp **out = **in } out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + if in.Addons != nil { + in, out := &in.Addons, &out.Addons + *out = make([]Addon, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Version != nil { in, out := &in.Version, &out.Version *out = new(string) @@ -1052,6 +1176,13 @@ func (in *OCIManagedControlPlaneStatus) DeepCopyInto(out *OCIManagedControlPlane *out = new(string) **out = **in } + if in.AddonStatus != nil { + in, out := &in.AddonStatus, &out.AddonStatus + *out = make(map[string]AddonStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCIManagedControlPlaneStatus. diff --git a/exp/controllers/ocimanagedcluster_controlplane_controller.go b/exp/controllers/ocimanagedcluster_controlplane_controller.go index 838c334b..921b5f13 100644 --- a/exp/controllers/ocimanagedcluster_controlplane_controller.go +++ b/exp/controllers/ocimanagedcluster_controlplane_controller.go @@ -232,6 +232,10 @@ func (r *OCIManagedClusterControlPlaneReconciler) reconcile(ctx context.Context, if isUpdated { return reconcile.Result{RequeueAfter: 30 * time.Second}, nil } + err = controlPlaneScope.ReconcileAddons(ctx, okeControlPlane) + if err != nil { + return ctrl.Result{}, err + } return reconcile.Result{RequeueAfter: 180 * time.Second}, nil default: conditions.MarkFalse(controlPlane, infrav2exp.ControlPlaneReadyCondition, infrav2exp.ControlPlaneProvisionFailedReason, clusterv1.ConditionSeverityError, "") diff --git a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-managed-virtual/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-managed-virtual/cluster.yaml index a7a64366..503dc7a8 100644 --- a/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-managed-virtual/cluster.yaml +++ b/test/e2e/data/infrastructure-oci/v1beta2/cluster-template-managed-virtual/cluster.yaml @@ -34,4 +34,6 @@ metadata: spec: version: "${OCI_MANAGED_KUBERNETES_VERSION}" clusterType: "ENHANCED_CLUSTER" + addons: + - name: KubernetesDashboard --- \ No newline at end of file diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 7cd40670..2e4e73c6 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -37,6 +37,7 @@ import ( oci_config "github.com/oracle/cluster-api-provider-oci/cloud/config" "github.com/oracle/cluster-api-provider-oci/cloud/scope" "github.com/oracle/cluster-api-provider-oci/cloud/services/compute" + "github.com/oracle/cluster-api-provider-oci/cloud/services/containerengine" nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer" "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" infrav1exp "github.com/oracle/cluster-api-provider-oci/exp/api/v1beta1" @@ -106,6 +107,8 @@ var ( lbClient nlb.NetworkLoadBalancerClient + okeClient containerengine.Client + adCount int ) @@ -240,6 +243,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { identityClient := ociClients.IdentityClient vcnClient = ociClients.VCNClient lbClient = ociClients.NetworkLoadBalancerClient + okeClient = ociClients.ContainerEngineClient Expect(identityClient).NotTo(BeNil()) Expect(lbClient).NotTo(BeNil()) diff --git a/test/e2e/managed_cluster_test.go b/test/e2e/managed_cluster_test.go index 56494532..206a2518 100644 --- a/test/e2e/managed_cluster_test.go +++ b/test/e2e/managed_cluster_test.go @@ -32,6 +32,8 @@ import ( . "github.com/onsi/gomega" infrav1exp "github.com/oracle/cluster-api-provider-oci/exp/api/v1beta1" infrav2exp "github.com/oracle/cluster-api-provider-oci/exp/api/v1beta2" + "github.com/oracle/oci-go-sdk/v65/common" + oke "github.com/oracle/oci-go-sdk/v65/containerengine" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -307,6 +309,17 @@ var _ = Describe("Managed Workload cluster creation", func() { } clusterctl.ApplyClusterTemplateAndWait(ctx, input, result) + + controlPlane := GetOCIManagedControlPlaneByCluster(ctx, bootstrapClusterProxy.GetClient(), clusterName, namespace.Name) + Expect(controlPlane).To(Not(BeNil())) + clusterOcid := controlPlane.Spec.ID + Eventually(func() error { + _, err := okeClient.GetAddon(ctx, oke.GetAddonRequest{ + ClusterId: clusterOcid, + AddonName: common.String("KubernetesDashboard"), + }) + return err + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to install Addon") }) })