From a5f5cf03c84ca976ec98fc6e7526684871e0665b Mon Sep 17 00:00:00 2001 From: anthonyhui Date: Tue, 8 Aug 2023 11:04:15 +0800 Subject: [PATCH 1/6] reduce workloadMap data;only send data of the machine Signed-off-by: anthonyhui --- .../pkg/metadata/kubernetes/pod_delete.go | 2 +- .../pkg/metadata/kubernetes/pod_watch.go | 32 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/collector/pkg/metadata/kubernetes/pod_delete.go b/collector/pkg/metadata/kubernetes/pod_delete.go index 2b62fcf90..ad370cb8b 100644 --- a/collector/pkg/metadata/kubernetes/pod_delete.go +++ b/collector/pkg/metadata/kubernetes/pod_delete.go @@ -64,7 +64,7 @@ func deletePodInfo(podInfo *deletedPodInfo) { if podInfo.name != "" { deletePodInfo, ok := globalPodInfo.delete(podInfo.namespace, podInfo.name) if ok { - globalWorkload.delete(deletePodInfo.Namespace, deletePodInfo.WorkloadName) + workloadMap.delete(deletePodInfo.Namespace, deletePodInfo.WorkloadName) } } if len(podInfo.containerIds) != 0 { diff --git a/collector/pkg/metadata/kubernetes/pod_watch.go b/collector/pkg/metadata/kubernetes/pod_watch.go index 1566d7edf..d0e752eab 100644 --- a/collector/pkg/metadata/kubernetes/pod_watch.go +++ b/collector/pkg/metadata/kubernetes/pod_watch.go @@ -2,6 +2,7 @@ package kubernetes import ( "fmt" + "os" _ "path/filepath" "regexp" "strings" @@ -30,7 +31,7 @@ type podMap struct { mutex sync.RWMutex } -type workloadMap struct { +type WorkloadMap struct { Info map[string]map[string]*WorkloadInfo mutex sync.RWMutex } @@ -42,12 +43,12 @@ type WorkloadInfo struct { } var globalPodInfo = newPodMap() -var globalWorkload = newWorkloadMap() +var workloadMap = newWorkloadMap() func GetWorkloadDataGroup() []*model.DataGroup { - globalWorkload.mutex.RLock() + workloadMap.mutex.RLock() dataGroups := make([]*model.DataGroup, 0) - for _, workloadInfoMap := range globalWorkload.Info { + for _, workloadInfoMap := range workloadMap.Info { for _, workloadInfo := range workloadInfoMap { dataGroups = append(dataGroups, &model.DataGroup{ Name: constnames.K8sWorkloadMetricGroupName, @@ -62,7 +63,7 @@ func GetWorkloadDataGroup() []*model.DataGroup { }) } } - globalWorkload.mutex.RUnlock() + workloadMap.mutex.RUnlock() return dataGroups } @@ -73,8 +74,8 @@ func newPodMap() *podMap { } } -func newWorkloadMap() *workloadMap { - return &workloadMap{ +func newWorkloadMap() *WorkloadMap { + return &WorkloadMap{ Info: make(map[string]map[string]*WorkloadInfo), mutex: sync.RWMutex{}, } @@ -103,7 +104,7 @@ func (m *podMap) delete(namespace string, name string) (*K8sPodInfo, bool) { return podInfo, ok } -func (m *workloadMap) add(info *WorkloadInfo) { +func (m *WorkloadMap) add(info *WorkloadInfo) { m.mutex.Lock() workloadInfoMap, ok := m.Info[info.Namespace] if !ok { @@ -115,7 +116,7 @@ func (m *workloadMap) add(info *WorkloadInfo) { } -func (m *workloadMap) delete(namespace string, name string) { +func (m *WorkloadMap) delete(namespace string, name string) { m.mutex.Lock() workloadInfoMap, ok := m.Info[namespace] if ok { @@ -278,11 +279,14 @@ func onAdd(obj interface{}) { } } globalPodInfo.add(cachePodInfo) - globalWorkload.add(&WorkloadInfo{ - Namespace: cachePodInfo.Namespace, - WorkloadName: cachePodInfo.WorkloadName, - WorkloadKind: cachePodInfo.WorkloadKind, - }) + //workloadMap only restore the workload in this machine + if pod.Spec.NodeName == os.Getenv("NODE_NAME") { + workloadMap.add(&WorkloadInfo{ + Namespace: cachePodInfo.Namespace, + WorkloadName: cachePodInfo.WorkloadName, + WorkloadKind: cachePodInfo.WorkloadKind, + }) + } } func getControllerKindName(pod *corev1.Pod) (workloadKind string, workloadName string) { From cccd413d9d6b0e668df41929864028b00ce59bab Mon Sep 17 00:00:00 2001 From: anthonyhui Date: Tue, 8 Aug 2023 15:47:12 +0800 Subject: [PATCH 2/6] change workloadMap Signed-off-by: anthonyhui --- .../pkg/metadata/kubernetes/pod_delete.go | 2 +- .../pkg/metadata/kubernetes/pod_watch.go | 30 ++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/collector/pkg/metadata/kubernetes/pod_delete.go b/collector/pkg/metadata/kubernetes/pod_delete.go index ad370cb8b..3887038cf 100644 --- a/collector/pkg/metadata/kubernetes/pod_delete.go +++ b/collector/pkg/metadata/kubernetes/pod_delete.go @@ -64,7 +64,7 @@ func deletePodInfo(podInfo *deletedPodInfo) { if podInfo.name != "" { deletePodInfo, ok := globalPodInfo.delete(podInfo.namespace, podInfo.name) if ok { - workloadMap.delete(deletePodInfo.Namespace, deletePodInfo.WorkloadName) + localWorkloadMap.delete(deletePodInfo.Namespace, deletePodInfo.WorkloadName) } } if len(podInfo.containerIds) != 0 { diff --git a/collector/pkg/metadata/kubernetes/pod_watch.go b/collector/pkg/metadata/kubernetes/pod_watch.go index d0e752eab..55e4557a0 100644 --- a/collector/pkg/metadata/kubernetes/pod_watch.go +++ b/collector/pkg/metadata/kubernetes/pod_watch.go @@ -31,24 +31,26 @@ type podMap struct { mutex sync.RWMutex } -type WorkloadMap struct { - Info map[string]map[string]*WorkloadInfo +type workloadMap struct { + Info map[string]map[string]*workloadInfo mutex sync.RWMutex } -type WorkloadInfo struct { +type workloadInfo struct { Namespace string WorkloadName string WorkloadKind string } var globalPodInfo = newPodMap() -var workloadMap = newWorkloadMap() + +// only restore local pods info and will send to promethus.And fasten the filter speed. +var localWorkloadMap = newWorkloadMap() func GetWorkloadDataGroup() []*model.DataGroup { - workloadMap.mutex.RLock() + localWorkloadMap.mutex.RLock() dataGroups := make([]*model.DataGroup, 0) - for _, workloadInfoMap := range workloadMap.Info { + for _, workloadInfoMap := range localWorkloadMap.Info { for _, workloadInfo := range workloadInfoMap { dataGroups = append(dataGroups, &model.DataGroup{ Name: constnames.K8sWorkloadMetricGroupName, @@ -63,7 +65,7 @@ func GetWorkloadDataGroup() []*model.DataGroup { }) } } - workloadMap.mutex.RUnlock() + localWorkloadMap.mutex.RUnlock() return dataGroups } @@ -74,9 +76,9 @@ func newPodMap() *podMap { } } -func newWorkloadMap() *WorkloadMap { - return &WorkloadMap{ - Info: make(map[string]map[string]*WorkloadInfo), +func newWorkloadMap() *workloadMap { + return &workloadMap{ + Info: make(map[string]map[string]*workloadInfo), mutex: sync.RWMutex{}, } } @@ -104,11 +106,11 @@ func (m *podMap) delete(namespace string, name string) (*K8sPodInfo, bool) { return podInfo, ok } -func (m *WorkloadMap) add(info *WorkloadInfo) { +func (m *workloadMap) add(info *workloadInfo) { m.mutex.Lock() workloadInfoMap, ok := m.Info[info.Namespace] if !ok { - workloadInfoMap = make(map[string]*WorkloadInfo) + workloadInfoMap = make(map[string]*workloadInfo) } workloadInfoMap[info.WorkloadName] = info m.Info[info.Namespace] = workloadInfoMap @@ -116,7 +118,7 @@ func (m *WorkloadMap) add(info *WorkloadInfo) { } -func (m *WorkloadMap) delete(namespace string, name string) { +func (m *workloadMap) delete(namespace string, name string) { m.mutex.Lock() workloadInfoMap, ok := m.Info[namespace] if ok { @@ -281,7 +283,7 @@ func onAdd(obj interface{}) { globalPodInfo.add(cachePodInfo) //workloadMap only restore the workload in this machine if pod.Spec.NodeName == os.Getenv("NODE_NAME") { - workloadMap.add(&WorkloadInfo{ + localWorkloadMap.add(&workloadInfo{ Namespace: cachePodInfo.Namespace, WorkloadName: cachePodInfo.WorkloadName, WorkloadKind: cachePodInfo.WorkloadKind, From 8634d48ac1893547c57d4d1432d4e62d861ff95d Mon Sep 17 00:00:00 2001 From: anthonyhui Date: Wed, 9 Aug 2023 15:57:37 +0800 Subject: [PATCH 3/6] localWorkloadMap only stores the workload whose pods are in the local Node Signed-off-by: anthonyhui --- collector/pkg/metadata/kubernetes/pod_watch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/collector/pkg/metadata/kubernetes/pod_watch.go b/collector/pkg/metadata/kubernetes/pod_watch.go index 55e4557a0..c35150e63 100644 --- a/collector/pkg/metadata/kubernetes/pod_watch.go +++ b/collector/pkg/metadata/kubernetes/pod_watch.go @@ -44,7 +44,8 @@ type workloadInfo struct { var globalPodInfo = newPodMap() -// only restore local pods info and will send to promethus.And fasten the filter speed. +// localWorkloadMap only stores the workload whose pods are in the local Node. +// The workload metadata will be sent to prometheus and used to filter metrics. var localWorkloadMap = newWorkloadMap() func GetWorkloadDataGroup() []*model.DataGroup { From 8d89af460c018fdcd7a457a6f175de0f0199c62f Mon Sep 17 00:00:00 2001 From: anthonyhui Date: Fri, 11 Aug 2023 17:50:28 +0800 Subject: [PATCH 4/6] localWorkloadMap only stores the workload whose pods are in the local Node Signed-off-by: anthonyhui --- collector/pkg/metadata/kubernetes/pod_watch.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/collector/pkg/metadata/kubernetes/pod_watch.go b/collector/pkg/metadata/kubernetes/pod_watch.go index c35150e63..bdb3d5e99 100644 --- a/collector/pkg/metadata/kubernetes/pod_watch.go +++ b/collector/pkg/metadata/kubernetes/pod_watch.go @@ -44,7 +44,7 @@ type workloadInfo struct { var globalPodInfo = newPodMap() -// localWorkloadMap only stores the workload whose pods are in the local Node. +// localWorkloadMap only stores the workload whose pods are in the local Node. // The workload metadata will be sent to prometheus and used to filter metrics. var localWorkloadMap = newWorkloadMap() @@ -282,8 +282,9 @@ func onAdd(obj interface{}) { } } globalPodInfo.add(cachePodInfo) + nodeName, _ := os.LookupEnv("MY_NODE_NAME") //workloadMap only restore the workload in this machine - if pod.Spec.NodeName == os.Getenv("NODE_NAME") { + if cachePodInfo.NodeName == nodeName { localWorkloadMap.add(&workloadInfo{ Namespace: cachePodInfo.Namespace, WorkloadName: cachePodInfo.WorkloadName, From 2f69b31142bd1091fbd7125e1a8940cd989a2bc7 Mon Sep 17 00:00:00 2001 From: anthonyhui Date: Tue, 15 Aug 2023 20:52:50 +0800 Subject: [PATCH 5/6] localWorkloadMap only stores the workload whose pods are in the local Node Signed-off-by: anthonyhui --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67013251a..a2ca0a166 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ 1. All notable changes to this project will be documented in this file. 2. Records in this file are not identical to the title of their Pull Requests. A detailed description is necessary for understanding what changes are and why they are made. +## Unreleased +### Bug fixes +- Fix the bug where sending repetitive k8s_info_workload.Now each node only send its own info.（[#554](https://github.com/KindlingProject/kindling/pull/554)） ## v0.8.0 - 2023-06-30 ### New features - Provide a new metric called kindling_k8s_workload_info, which supports workload filtering for k8s, thus preventing frequent crashes of Grafana topology. Please refer to the [doc](http://kindling.harmonycloud.cn/docs/usage/grafana-topology-plugin/) for any limitations.（[#530](https://github.com/KindlingProject/kindling/pull/530)） From 795f858cd04d824de7600b04183ffaeb31a729c2 Mon Sep 17 00:00:00 2001 From: anthonyhui Date: Tue, 15 Aug 2023 20:56:31 +0800 Subject: [PATCH 6/6] localWorkloadMap only stores the workload whose pods are in the local Node Signed-off-by: anthonyhui --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2ca0a166..2ceb17b88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ## Unreleased ### Bug fixes -- Fix the bug where sending repetitive k8s_info_workload.Now each node only send its own info.（[#554](https://github.com/KindlingProject/kindling/pull/554)） +- Fix the bug where sending repetitive k8s_info_workload. Now each node only sends its own info.（[#554](https://github.com/KindlingProject/kindling/pull/554)） ## v0.8.0 - 2023-06-30 ### New features - Provide a new metric called kindling_k8s_workload_info, which supports workload filtering for k8s, thus preventing frequent crashes of Grafana topology. Please refer to the [doc](http://kindling.harmonycloud.cn/docs/usage/grafana-topology-plugin/) for any limitations.（[#530](https://github.com/KindlingProject/kindling/pull/530)）