Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

reduce workloadMap data;only send data of the machine #554

Merged
merged 6 commits into from
Aug 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
1. All notable changes to this project will be documented in this file.
2. Records in this file are not identical to the title of their Pull Requests. A detailed description is necessary for understanding what changes are and why they are made.

## Unreleased
### Bug fixes
- Fix the bug where sending repetitive k8s_info_workload. Now each node only sends its own info.([#554](https://github.com/KindlingProject/kindling/pull/554))
## v0.8.0 - 2023-06-30
### New features
- Provide a new metric called kindling_k8s_workload_info, which supports workload filtering for k8s, thus preventing frequent crashes of Grafana topology. Please refer to the [doc](http://kindling.harmonycloud.cn/docs/usage/grafana-topology-plugin/) for any limitations.([#530](https://github.com/KindlingProject/kindling/pull/530))
Expand Down
2 changes: 1 addition & 1 deletion collector/pkg/metadata/kubernetes/pod_delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func deletePodInfo(podInfo *deletedPodInfo) {
if podInfo.name != "" {
deletePodInfo, ok := globalPodInfo.delete(podInfo.namespace, podInfo.name)
if ok {
globalWorkload.delete(deletePodInfo.Namespace, deletePodInfo.WorkloadName)
localWorkloadMap.delete(deletePodInfo.Namespace, deletePodInfo.WorkloadName)
}
}
if len(podInfo.containerIds) != 0 {
Expand Down
36 changes: 22 additions & 14 deletions collector/pkg/metadata/kubernetes/pod_watch.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package kubernetes

import (
"fmt"
"os"
_ "path/filepath"
"regexp"
"strings"
Expand Down Expand Up @@ -31,23 +32,26 @@ type podMap struct {
}

type workloadMap struct {
Info map[string]map[string]*WorkloadInfo
Info map[string]map[string]*workloadInfo
mutex sync.RWMutex
}

type WorkloadInfo struct {
type workloadInfo struct {
Namespace string
WorkloadName string
WorkloadKind string
}

var globalPodInfo = newPodMap()
var globalWorkload = newWorkloadMap()

// localWorkloadMap only stores the workload whose pods are in the local Node.
// The workload metadata will be sent to prometheus and used to filter metrics.
var localWorkloadMap = newWorkloadMap()

func GetWorkloadDataGroup() []*model.DataGroup {
globalWorkload.mutex.RLock()
localWorkloadMap.mutex.RLock()
dataGroups := make([]*model.DataGroup, 0)
for _, workloadInfoMap := range globalWorkload.Info {
for _, workloadInfoMap := range localWorkloadMap.Info {
for _, workloadInfo := range workloadInfoMap {
dataGroups = append(dataGroups, &model.DataGroup{
Name: constnames.K8sWorkloadMetricGroupName,
Expand All @@ -62,7 +66,7 @@ func GetWorkloadDataGroup() []*model.DataGroup {
})
}
}
globalWorkload.mutex.RUnlock()
localWorkloadMap.mutex.RUnlock()
return dataGroups
}

Expand All @@ -75,7 +79,7 @@ func newPodMap() *podMap {

func newWorkloadMap() *workloadMap {
return &workloadMap{
Info: make(map[string]map[string]*WorkloadInfo),
Info: make(map[string]map[string]*workloadInfo),
mutex: sync.RWMutex{},
}
}
Expand Down Expand Up @@ -103,11 +107,11 @@ func (m *podMap) delete(namespace string, name string) (*K8sPodInfo, bool) {
return podInfo, ok
}

func (m *workloadMap) add(info *WorkloadInfo) {
func (m *workloadMap) add(info *workloadInfo) {
m.mutex.Lock()
workloadInfoMap, ok := m.Info[info.Namespace]
if !ok {
workloadInfoMap = make(map[string]*WorkloadInfo)
workloadInfoMap = make(map[string]*workloadInfo)
}
workloadInfoMap[info.WorkloadName] = info
m.Info[info.Namespace] = workloadInfoMap
Expand Down Expand Up @@ -278,11 +282,15 @@ func onAdd(obj interface{}) {
}
}
globalPodInfo.add(cachePodInfo)
globalWorkload.add(&WorkloadInfo{
Namespace: cachePodInfo.Namespace,
WorkloadName: cachePodInfo.WorkloadName,
WorkloadKind: cachePodInfo.WorkloadKind,
})
nodeName, _ := os.LookupEnv("MY_NODE_NAME")
//workloadMap only restore the workload in this machine
if cachePodInfo.NodeName == nodeName {
localWorkloadMap.add(&workloadInfo{
Namespace: cachePodInfo.Namespace,
WorkloadName: cachePodInfo.WorkloadName,
WorkloadKind: cachePodInfo.WorkloadKind,
})
}
}

func getControllerKindName(pod *corev1.Pod) (workloadKind string, workloadName string) {
Expand Down