From eb8bb48e43f323b791113839471adaa5e0f1c177 Mon Sep 17 00:00:00 2001 From: gaohj Date: Fri, 15 Feb 2019 12:01:40 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=9B=86=E7=BE=A4?= =?UTF-8?q?=E5=92=8C=E5=91=BD=E5=90=8D=E7=A9=BA=E9=97=B4=E7=BA=A7=E5=88=AB?= =?UTF-8?q?=E8=87=AA=E5=AE=9A=E4=B9=89=E6=B3=A8=E8=A7=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/controllers/common/deployment.go | 26 ++++++++++++++++ src/backend/controllers/common/ingress.go | 29 ++++++++++++++++++ src/backend/controllers/common/service.go | 30 +++++++++++++++++++ .../kubernetes/deployment/deployment.go | 25 +--------------- .../controllers/kubernetes/ingress/ingress.go | 19 ++++++++++++ .../controllers/kubernetes/service/service.go | 19 ++++++++++++ src/backend/models/cluster.go | 6 +++- src/backend/models/namespace.go | 4 +++ 8 files changed, 133 insertions(+), 25 deletions(-) create mode 100644 src/backend/controllers/common/ingress.go create mode 100644 src/backend/controllers/common/service.go diff --git a/src/backend/controllers/common/deployment.go b/src/backend/controllers/common/deployment.go index bf6b7b0de..0d9b8c0d8 100644 --- a/src/backend/controllers/common/deployment.go +++ b/src/backend/controllers/common/deployment.go @@ -1,6 +1,7 @@ package common import ( + "encoding/json" "strings" "k8s.io/api/apps/v1beta1" @@ -8,6 +9,8 @@ import ( "github.com/Qihoo360/wayne/src/backend/models" "github.com/Qihoo360/wayne/src/backend/util" + "github.com/Qihoo360/wayne/src/backend/util/logs" + "github.com/Qihoo360/wayne/src/backend/util/hack" ) func DeploymentPreDeploy(kubeDeployment *v1beta1.Deployment, deploy *models.Deployment, @@ -83,3 +86,26 @@ func DeploymentPreDeploy(kubeDeployment *v1beta1.Deployment, deploy *models.Depl kubeDeployment.Spec.Template.Annotations[util.PodAnnotationControllerKindLabelKey] = strings.ToLower(string(models.KubeApiTypeDeployment)) } + + +func GetNamespace(appId int64) (*models.Namespace, error) { + app, err := models.AppModel.GetById(appId) + if err != nil { + logs.Warning("get app by id (%d) error. %v", appId, err) + return nil, err + } + + ns, err := models.NamespaceModel.GetById(app.Namespace.Id) + if err != nil { + logs.Warning("get namespace by id (%d) error. %v", app.Namespace.Id, err) + return nil, err + } + var namespaceMetaData models.NamespaceMetaData + err = json.Unmarshal(hack.Slice(ns.MetaData), &namespaceMetaData) + if err != nil { + logs.Error("Unmarshal namespace metadata (%s) error. %v", ns.MetaData, err) + return nil, err + } + ns.MetaDataObj = namespaceMetaData + return ns, nil +} \ No newline at end of file diff --git a/src/backend/controllers/common/ingress.go b/src/backend/controllers/common/ingress.go new file mode 100644 index 000000000..4b28d321d --- /dev/null +++ b/src/backend/controllers/common/ingress.go @@ -0,0 +1,29 @@ +package common + +import ( + "github.com/Qihoo360/wayne/src/backend/models" + kapiv1beta1 "k8s.io/api/extensions/v1beta1" +) + +func IngressPreDeploy(kubeIngress *kapiv1beta1.Ingress, cluster *models.Cluster, namespace *models.Namespace) { + preDefinedAnnotationMap := make(map[string]string) + + annotationResult := make(map[string]string, 0) + // user defined + for k, v := range kubeIngress.Annotations { + preDefinedAnnotationMap[k] = v + } + // cluster defined, overwrite user defined + for k, v := range cluster.MetaDataObj.SvcAnnotations { + preDefinedAnnotationMap[k] = v + } + // namespace defined, overwrite cluster and user defined + for k, v := range namespace.MetaDataObj.SvcAnnotations { + preDefinedAnnotationMap[k] = v + } + for k, v := range preDefinedAnnotationMap { + annotationResult[k] = v + } + + kubeIngress.Annotations = annotationResult +} diff --git a/src/backend/controllers/common/service.go b/src/backend/controllers/common/service.go new file mode 100644 index 000000000..8559198b5 --- /dev/null +++ b/src/backend/controllers/common/service.go @@ -0,0 +1,30 @@ +package common + +import ( + "k8s.io/api/core/v1" + + "github.com/Qihoo360/wayne/src/backend/models" +) + +func ServicePreDeploy(kubeService *v1.Service, cluster *models.Cluster, namespace *models.Namespace) { + preDefinedAnnotationMap := make(map[string]string) + + annotationResult := make(map[string]string, 0) + // user defined + for k, v := range kubeService.Annotations { + preDefinedAnnotationMap[k] = v + } + // cluster defined, overwrite user defined + for k, v := range cluster.MetaDataObj.SvcAnnotations { + preDefinedAnnotationMap[k] = v + } + // namespace defined, overwrite cluster and user defined + for k, v := range namespace.MetaDataObj.SvcAnnotations { + preDefinedAnnotationMap[k] = v + } + for k, v := range preDefinedAnnotationMap { + annotationResult[k] = v + } + + kubeService.Annotations = annotationResult +} diff --git a/src/backend/controllers/kubernetes/deployment/deployment.go b/src/backend/controllers/kubernetes/deployment/deployment.go index 2a7d036b0..a6baca120 100644 --- a/src/backend/controllers/kubernetes/deployment/deployment.go +++ b/src/backend/controllers/kubernetes/deployment/deployment.go @@ -18,7 +18,6 @@ import ( "github.com/Qihoo360/wayne/src/backend/resources/deployment" "github.com/Qihoo360/wayne/src/backend/resources/namespace" "github.com/Qihoo360/wayne/src/backend/util" - "github.com/Qihoo360/wayne/src/backend/util/hack" "github.com/Qihoo360/wayne/src/backend/util/logs" "github.com/Qihoo360/wayne/src/backend/workers/webhook" ) @@ -133,7 +132,7 @@ func (c *KubeDeploymentController) Deploy() { cluster := c.Ctx.Input.Param(":cluster") cli, err := client.Client(cluster) if err == nil { - namespaceModel, err := getNamespace(c.AppId) + namespaceModel, err := common.GetNamespace(c.AppId) if err != nil { logs.Error("get getNamespaceMetaData error.%v", err) c.HandleError(err) @@ -261,28 +260,6 @@ func checkResourceAvailable(ns *models.Namespace, cli *kubernetes.Clientset, kub return nil } -func getNamespace(appId int64) (*models.Namespace, error) { - app, err := models.AppModel.GetById(appId) - if err != nil { - logs.Warning("get app by id (%d) error. %v", appId, err) - return nil, err - } - - ns, err := models.NamespaceModel.GetById(app.Namespace.Id) - if err != nil { - logs.Warning("get namespace by id (%d) error. %v", app.Namespace.Id, err) - return nil, err - } - var namespaceMetaData models.NamespaceMetaData - err = json.Unmarshal(hack.Slice(ns.MetaData), &namespaceMetaData) - if err != nil { - logs.Error("Unmarshal namespace metadata (%s) error. %v", ns.MetaData, err) - return nil, err - } - ns.MetaDataObj = namespaceMetaData - return ns, nil -} - // @Title Get // @Description find Deployment by cluster // @Param cluster path string true "the cluster name" diff --git a/src/backend/controllers/kubernetes/ingress/ingress.go b/src/backend/controllers/kubernetes/ingress/ingress.go index e83bc3949..eea271311 100644 --- a/src/backend/controllers/kubernetes/ingress/ingress.go +++ b/src/backend/controllers/kubernetes/ingress/ingress.go @@ -7,6 +7,7 @@ import ( "github.com/Qihoo360/wayne/src/backend/client" "github.com/Qihoo360/wayne/src/backend/controllers/base" + "github.com/Qihoo360/wayne/src/backend/controllers/common" "github.com/Qihoo360/wayne/src/backend/models" "github.com/Qihoo360/wayne/src/backend/models/response" "github.com/Qihoo360/wayne/src/backend/resources/ingress" @@ -92,6 +93,24 @@ func (c *KubeIngressController) Deploy() { c.AbortBadRequestFormat("Cluster") return } + + namespaceModel, err := common.GetNamespace(c.AppId) + if err != nil { + logs.Error("get getNamespaceMetaData error.%v", err) + c.HandleError(err) + return + } + + clusterModel, err := models.ClusterModel.GetParsedMetaDataByName(clusterName) + if err != nil { + logs.Error("get cluster error.%v", err) + c.HandleError(err) + return + } + + // add ingress predeploy + common.IngressPreDeploy(&kubeIngress, clusterModel, namespaceModel) + publishHistory := &models.PublishHistory{ Type: models.PublishTypeIngress, ResourceId: int64(ingressId), diff --git a/src/backend/controllers/kubernetes/service/service.go b/src/backend/controllers/kubernetes/service/service.go index e24634707..4458e1426 100644 --- a/src/backend/controllers/kubernetes/service/service.go +++ b/src/backend/controllers/kubernetes/service/service.go @@ -12,6 +12,7 @@ import ( "github.com/Qihoo360/wayne/src/backend/resources/service" "github.com/Qihoo360/wayne/src/backend/util/logs" "github.com/Qihoo360/wayne/src/backend/workers/webhook" + "github.com/Qihoo360/wayne/src/backend/controllers/common" ) type KubeServiceController struct { @@ -113,6 +114,24 @@ func (c *KubeServiceController) Deploy() { cluster := c.Ctx.Input.Param(":cluster") cli, err := client.Client(cluster) if err == nil { + namespaceModel, err := common.GetNamespace(c.AppId) + if err != nil { + logs.Error("get getNamespaceMetaData error.%v", err) + c.HandleError(err) + return + } + + clusterModel, err := models.ClusterModel.GetParsedMetaDataByName(cluster) + if err != nil { + logs.Error("get cluster error.%v", err) + c.HandleError(err) + return + } + + // add service predeploy + common.ServicePreDeploy(&kubeService, clusterModel, namespaceModel) + + publishHistory := &models.PublishHistory{ Type: models.PublishTypeService, ResourceId: int64(serviceId), diff --git a/src/backend/models/cluster.go b/src/backend/models/cluster.go index 02411db7e..4d40c99b2 100644 --- a/src/backend/models/cluster.go +++ b/src/backend/models/cluster.go @@ -79,10 +79,14 @@ type ClusterMetaData struct { RBD *v1.RBDVolumeSource `json:"rbd"` // cephfs默认配置，创建或修改cephfs类型的PV时会使用此配置填充 CephFS *v1.CephFSVolumeSource `json:"cephfs"` - // 默认添加环境变量，会在发布资源时在每个Container添加此环境变量, will be overwrite by namespace's ImagePullSecrets + // 默认添加环境变量，会在发布资源时在每个Container添加此环境变量, will be overwrite by namespace's Env Env []v1.EnvVar // current cluster image pull secrets, will be overwrite by namespace's ImagePullSecrets ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets"` + // 默认添加service注解，会在发布资源时在每个service添加此Annotations, will be overwrite by namespace's Annotations + SvcAnnotations map[string]string `json:"svcAnnotations,omitempty"` + // 默认添加ingress注解，会在发布资源时在每个ingress添加此Annotations, will be overwrite by namespace's Annotations + IngressAnnotations map[string]string `json:"ingressAnnotations,omitempty"` } type ClusterRobinMetaData struct { diff --git a/src/backend/models/namespace.go b/src/backend/models/namespace.go index 3cb437101..39d27fc42 100644 --- a/src/backend/models/namespace.go +++ b/src/backend/models/namespace.go @@ -49,6 +49,10 @@ type NamespaceMetaData struct { Env []v1.EnvVar `json:"env,omitempty"` // current namespace image pull secrets, will overwrite cluster's ImagePullSecrets ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets"` + // current namespace service annotation, will overwrite cluster service's Annotation + SvcAnnotations map[string]string `json:"svcAnnotations,omitempty"` + // current namespace ingress annotation, will overwrite cluster ingress's Annotation + IngressAnnotations map[string]string `json:"ingressAnnotations,omitempty"` } type ClusterMeta struct { From 213dee3821a0db2c215dc597f0b42b5895b62c3b Mon Sep 17 00:00:00 2001 From: gaohj Date: Fri, 15 Feb 2019 17:47:28 +0800 Subject: [PATCH 2/5] update SvcAnnotations to ServiceAnnotations --- src/backend/controllers/common/ingress.go | 5 +++-- src/backend/models/cluster.go | 2 +- src/backend/models/namespace.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backend/controllers/common/ingress.go b/src/backend/controllers/common/ingress.go index 4b28d321d..9bf16eb1d 100644 --- a/src/backend/controllers/common/ingress.go +++ b/src/backend/controllers/common/ingress.go @@ -2,6 +2,7 @@ package common import ( "github.com/Qihoo360/wayne/src/backend/models" + kapiv1beta1 "k8s.io/api/extensions/v1beta1" ) @@ -14,11 +15,11 @@ func IngressPreDeploy(kubeIngress *kapiv1beta1.Ingress, cluster *models.Cluster, preDefinedAnnotationMap[k] = v } // cluster defined, overwrite user defined - for k, v := range cluster.MetaDataObj.SvcAnnotations { + for k, v := range cluster.MetaDataObj.ServiceAnnotations { preDefinedAnnotationMap[k] = v } // namespace defined, overwrite cluster and user defined - for k, v := range namespace.MetaDataObj.SvcAnnotations { + for k, v := range namespace.MetaDataObj.ServiceAnnotations { preDefinedAnnotationMap[k] = v } for k, v := range preDefinedAnnotationMap { diff --git a/src/backend/models/cluster.go b/src/backend/models/cluster.go index 4d40c99b2..b51d9e929 100644 --- a/src/backend/models/cluster.go +++ b/src/backend/models/cluster.go @@ -84,7 +84,7 @@ type ClusterMetaData struct { // current cluster image pull secrets, will be overwrite by namespace's ImagePullSecrets ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets"` // 默认添加service注解，会在发布资源时在每个service添加此Annotations, will be overwrite by namespace's Annotations - SvcAnnotations map[string]string `json:"svcAnnotations,omitempty"` + ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` // 默认添加ingress注解，会在发布资源时在每个ingress添加此Annotations, will be overwrite by namespace's Annotations IngressAnnotations map[string]string `json:"ingressAnnotations,omitempty"` } diff --git a/src/backend/models/namespace.go b/src/backend/models/namespace.go index 39d27fc42..f51d219b0 100644 --- a/src/backend/models/namespace.go +++ b/src/backend/models/namespace.go @@ -50,7 +50,7 @@ type NamespaceMetaData struct { // current namespace image pull secrets, will overwrite cluster's ImagePullSecrets ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets"` // current namespace service annotation, will overwrite cluster service's Annotation - SvcAnnotations map[string]string `json:"svcAnnotations,omitempty"` + ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` // current namespace ingress annotation, will overwrite cluster ingress's Annotation IngressAnnotations map[string]string `json:"ingressAnnotations,omitempty"` } From 100e1f62d86fecc45880062d2917bb535e9fd952 Mon Sep 17 00:00:00 2001 From: gaohj Date: Fri, 15 Feb 2019 18:07:20 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=B0=86GetNamespace=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E5=88=B0model=E4=B8=AD=20GetNamespaceByAppId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/controllers/common/deployment.go | 26 ------------------ src/backend/controllers/common/service.go | 4 +-- .../kubernetes/deployment/deployment.go | 2 +- .../controllers/kubernetes/ingress/ingress.go | 2 +- .../controllers/kubernetes/service/service.go | 2 +- src/backend/models/namespace.go | 27 +++++++++++++++++++ 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/backend/controllers/common/deployment.go b/src/backend/controllers/common/deployment.go index 0d9b8c0d8..bf6b7b0de 100644 --- a/src/backend/controllers/common/deployment.go +++ b/src/backend/controllers/common/deployment.go @@ -1,7 +1,6 @@ package common import ( - "encoding/json" "strings" "k8s.io/api/apps/v1beta1" @@ -9,8 +8,6 @@ import ( "github.com/Qihoo360/wayne/src/backend/models" "github.com/Qihoo360/wayne/src/backend/util" - "github.com/Qihoo360/wayne/src/backend/util/logs" - "github.com/Qihoo360/wayne/src/backend/util/hack" ) func DeploymentPreDeploy(kubeDeployment *v1beta1.Deployment, deploy *models.Deployment, @@ -86,26 +83,3 @@ func DeploymentPreDeploy(kubeDeployment *v1beta1.Deployment, deploy *models.Depl kubeDeployment.Spec.Template.Annotations[util.PodAnnotationControllerKindLabelKey] = strings.ToLower(string(models.KubeApiTypeDeployment)) } - - -func GetNamespace(appId int64) (*models.Namespace, error) { - app, err := models.AppModel.GetById(appId) - if err != nil { - logs.Warning("get app by id (%d) error. %v", appId, err) - return nil, err - } - - ns, err := models.NamespaceModel.GetById(app.Namespace.Id) - if err != nil { - logs.Warning("get namespace by id (%d) error. %v", app.Namespace.Id, err) - return nil, err - } - var namespaceMetaData models.NamespaceMetaData - err = json.Unmarshal(hack.Slice(ns.MetaData), &namespaceMetaData) - if err != nil { - logs.Error("Unmarshal namespace metadata (%s) error. %v", ns.MetaData, err) - return nil, err - } - ns.MetaDataObj = namespaceMetaData - return ns, nil -} \ No newline at end of file diff --git a/src/backend/controllers/common/service.go b/src/backend/controllers/common/service.go index 8559198b5..d678e8e64 100644 --- a/src/backend/controllers/common/service.go +++ b/src/backend/controllers/common/service.go @@ -15,11 +15,11 @@ func ServicePreDeploy(kubeService *v1.Service, cluster *models.Cluster, namespac preDefinedAnnotationMap[k] = v } // cluster defined, overwrite user defined - for k, v := range cluster.MetaDataObj.SvcAnnotations { + for k, v := range cluster.MetaDataObj.ServiceAnnotations { preDefinedAnnotationMap[k] = v } // namespace defined, overwrite cluster and user defined - for k, v := range namespace.MetaDataObj.SvcAnnotations { + for k, v := range namespace.MetaDataObj.ServiceAnnotations { preDefinedAnnotationMap[k] = v } for k, v := range preDefinedAnnotationMap { diff --git a/src/backend/controllers/kubernetes/deployment/deployment.go b/src/backend/controllers/kubernetes/deployment/deployment.go index a6baca120..7c41ccc57 100644 --- a/src/backend/controllers/kubernetes/deployment/deployment.go +++ b/src/backend/controllers/kubernetes/deployment/deployment.go @@ -132,7 +132,7 @@ func (c *KubeDeploymentController) Deploy() { cluster := c.Ctx.Input.Param(":cluster") cli, err := client.Client(cluster) if err == nil { - namespaceModel, err := common.GetNamespace(c.AppId) + namespaceModel, err := models.NamespaceModel.GetNamespaceByAppId(c.AppId) if err != nil { logs.Error("get getNamespaceMetaData error.%v", err) c.HandleError(err) diff --git a/src/backend/controllers/kubernetes/ingress/ingress.go b/src/backend/controllers/kubernetes/ingress/ingress.go index eea271311..d53218810 100644 --- a/src/backend/controllers/kubernetes/ingress/ingress.go +++ b/src/backend/controllers/kubernetes/ingress/ingress.go @@ -94,7 +94,7 @@ func (c *KubeIngressController) Deploy() { return } - namespaceModel, err := common.GetNamespace(c.AppId) + namespaceModel, err := models.NamespaceModel.GetNamespaceByAppId(c.AppId) if err != nil { logs.Error("get getNamespaceMetaData error.%v", err) c.HandleError(err) diff --git a/src/backend/controllers/kubernetes/service/service.go b/src/backend/controllers/kubernetes/service/service.go index 4458e1426..cb0b88902 100644 --- a/src/backend/controllers/kubernetes/service/service.go +++ b/src/backend/controllers/kubernetes/service/service.go @@ -114,7 +114,7 @@ func (c *KubeServiceController) Deploy() { cluster := c.Ctx.Input.Param(":cluster") cli, err := client.Client(cluster) if err == nil { - namespaceModel, err := common.GetNamespace(c.AppId) + namespaceModel, err := models.NamespaceModel.GetNamespaceByAppId(c.AppId) if err != nil { logs.Error("get getNamespaceMetaData error.%v", err) c.HandleError(err) diff --git a/src/backend/models/namespace.go b/src/backend/models/namespace.go index f51d219b0..180252142 100644 --- a/src/backend/models/namespace.go +++ b/src/backend/models/namespace.go @@ -1,10 +1,15 @@ package models import ( + "encoding/json" "fmt" "time" "k8s.io/api/core/v1" + + "github.com/Qihoo360/wayne/src/backend/util/hack" + "github.com/Qihoo360/wayne/src/backend/util/logs" + ) const ( @@ -164,3 +169,25 @@ func (*namespaceModel) InitNamespace() (err error) { _, _, err = Ormer().ReadOrCreate(&defaultNS, "Name") return } + +func (*namespaceModel) GetNamespaceByAppId(appId int64) (*Namespace, error) { + app, err := AppModel.GetById(appId) + if err != nil { + logs.Warning("get app by id (%d) error. %v", appId, err) + return nil, err + } + + ns, err := NamespaceModel.GetById(app.Namespace.Id) + if err != nil { + logs.Warning("get namespace by id (%d) error. %v", app.Namespace.Id, err) + return nil, err + } + var namespaceMetaData NamespaceMetaData + err = json.Unmarshal(hack.Slice(ns.MetaData), &namespaceMetaData) + if err != nil { + logs.Error("Unmarshal namespace metadata (%s) error. %v", ns.MetaData, err) + return nil, err + } + ns.MetaDataObj = namespaceMetaData + return ns, nil +} From 236bb31833a615a8507454eaa62a4f863e3d0955 Mon Sep 17 00:00:00 2001 From: gaohj Date: Fri, 15 Feb 2019 18:54:37 +0800 Subject: [PATCH 4/5] change package order --- src/backend/controllers/common/ingress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/controllers/common/ingress.go b/src/backend/controllers/common/ingress.go index 9bf16eb1d..ae012aae6 100644 --- a/src/backend/controllers/common/ingress.go +++ b/src/backend/controllers/common/ingress.go @@ -1,9 +1,9 @@ package common import ( - "github.com/Qihoo360/wayne/src/backend/models" - kapiv1beta1 "k8s.io/api/extensions/v1beta1" + + "github.com/Qihoo360/wayne/src/backend/models" ) func IngressPreDeploy(kubeIngress *kapiv1beta1.Ingress, cluster *models.Cluster, namespace *models.Namespace) { From 212bca8072fc4ce9640cb577aa9dc3732df2d3cf Mon Sep 17 00:00:00 2001 From: gaohj Date: Mon, 18 Feb 2019 14:48:18 +0800 Subject: [PATCH 5/5] fix ingress service bug --- src/backend/controllers/common/ingress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/controllers/common/ingress.go b/src/backend/controllers/common/ingress.go index ae012aae6..a4095f7c9 100644 --- a/src/backend/controllers/common/ingress.go +++ b/src/backend/controllers/common/ingress.go @@ -15,11 +15,11 @@ func IngressPreDeploy(kubeIngress *kapiv1beta1.Ingress, cluster *models.Cluster, preDefinedAnnotationMap[k] = v } // cluster defined, overwrite user defined - for k, v := range cluster.MetaDataObj.ServiceAnnotations { + for k, v := range cluster.MetaDataObj.IngressAnnotations { preDefinedAnnotationMap[k] = v } // namespace defined, overwrite cluster and user defined - for k, v := range namespace.MetaDataObj.ServiceAnnotations { + for k, v := range namespace.MetaDataObj.IngressAnnotations { preDefinedAnnotationMap[k] = v } for k, v := range preDefinedAnnotationMap {