Skip to content

Commit

Permalink
refactor: monitoring generator with workspace configuration (#741)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ffforest
ffforest committed Jan 10, 2024
1 parent dbf37e0 commit 5eb12a8
Show file tree
Hide file tree
Showing 9 changed files with 398 additions and 173 deletions.
12 changes: 0 additions & 12 deletions pkg/apis/core/v1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,11 @@ package v1

type (
BuilderType string
MonitorType string
)

const (
KCLBuilder BuilderType = "KCL"
AppConfigurationBuilder BuilderType = "AppConfiguration"
PodMonitorType MonitorType = "Pod"
ServiceMonitorType MonitorType = "Service"
)

// Project is a definition of Kusion Project resource.
Expand All @@ -31,9 +28,6 @@ type Project struct {
// Generator controls how to generate the Intent.
Generator *GeneratorConfig `json:"generator,omitempty" yaml:"generator,omitempty"`

// Prometheus configs
Prometheus *PrometheusConfig `json:"prometheus,omitempty" yaml:"prometheus,omitempty"`

// The set of stacks that are known about this project.
Stacks []*Stack `json:"stacks,omitempty" yaml:"stacks,omitempty"`
}
Expand All @@ -46,12 +40,6 @@ type GeneratorConfig struct {
Configs map[string]interface{} `json:"configs,omitempty" yaml:"configs,omitempty"`
}

// PrometheusConfig represent Prometheus configs saved in project.yaml
type PrometheusConfig struct {
OperatorMode bool `yaml:"operatorMode,omitempty" json:"operatorMode,omitempty"`
MonitorType MonitorType `yaml:"monitorType,omitempty" json:"monitorType,omitempty"`
}

// Stack is a definition of Kusion Stack resource.
//
// Stack provides a mechanism to isolate multiple deploys of same application,
Expand Down
2 changes: 1 addition & 1 deletion pkg/modules/generators/app_configurations_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func (g *appConfigurationGenerator) Generate(i *apiv1.Intent) error {
// Patcher logic patches generated resources
pfs := []modules.NewPatcherFunc{
pattrait.NewOpsRulePatcherFunc(g.app, modulesConfig),
patmonitoring.NewMonitoringPatcherFunc(g.appName, g.app, g.project),
patmonitoring.NewMonitoringPatcherFunc(g.app, modulesConfig),
}
if err := modules.CallPatchers(i.Resources.GVKIndex(), pfs...); err != nil {
return err
Expand Down
241 changes: 170 additions & 71 deletions pkg/modules/generators/monitoring/monitoring_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,26 @@ package monitoring

import (
"fmt"
"time"

prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"kusionstack.io/kusion/pkg/modules/inputs"

apiv1 "kusionstack.io/kusion/pkg/apis/core/v1"
"kusionstack.io/kusion/pkg/modules"
"kusionstack.io/kusion/pkg/modules/inputs/monitoring"
"kusionstack.io/kusion/pkg/workspace"
)

type monitoringGenerator struct {
project *apiv1.Project
monitor *monitoring.Monitor
appName string
namespace string
project *apiv1.Project
stack *apiv1.Stack
appName string
app *inputs.AppConfiguration
modulesConfig map[string]apiv1.GenericConfig
namespace string
}

func NewMonitoringGenerator(ctx modules.GeneratorContext) (modules.Generator, error) {
Expand All @@ -27,10 +33,12 @@ func NewMonitoringGenerator(ctx modules.GeneratorContext) (modules.Generator, er
return nil, fmt.Errorf("app name must not be empty")
}
return &monitoringGenerator{
project: ctx.Project,
monitor: ctx.Application.Monitoring,
appName: ctx.Application.Name,
namespace: ctx.Namespace,
project: ctx.Project,
stack: ctx.Stack,
app: ctx.Application,
appName: ctx.Application.Name,
modulesConfig: ctx.ModuleInputs,
namespace: ctx.Namespace,
}, nil
}

Expand All @@ -44,91 +52,182 @@ func (g *monitoringGenerator) Generate(spec *apiv1.Intent) error {
if spec.Resources == nil {
spec.Resources = make(apiv1.Resources, 0)
}
// If AppConfiguration does not contain monitoring config, return
if g.app.Monitoring == nil {
return nil
}

// If Prometheus runs as an operator, it relies on Custom Resources to
// manage the scrape configs. CRs (ServiceMonitors and PodMonitors) rely on
// corresponding resources (Services and Pods) to have labels that can be
// used as part of the label selector for the CR to determine which
// service/pods to scrape from.
// Here we choose the label name kusion_monitoring_appname for two reasons:
// 1. Unlike the label validation in Kubernetes, the label name accepted by
// Prometheus cannot contain non-alphanumeric characters except underscore:
// https://github.com/prometheus/common/blob/main/model/labels.go#L94
// 2. The name should be unique enough that is only created by Kusion and
// used to identify a certain application
monitoringLabels := map[string]string{
"kusion_monitoring_appname": g.appName,
// Patch workspace configurations for monitoring generator.
if err := g.parseWorkspaceConfig(); err != nil {
return err
}

if g.project.Prometheus != nil && g.project.Prometheus.OperatorMode && g.monitor != nil {
if g.project.Prometheus.MonitorType == apiv1.ServiceMonitorType {
serviceEndpoint := prometheusv1.Endpoint{
Interval: g.monitor.Interval,
ScrapeTimeout: g.monitor.Timeout,
Port: g.monitor.Port,
Path: g.monitor.Path,
Scheme: g.monitor.Scheme,
}
serviceEndpointList := []prometheusv1.Endpoint{serviceEndpoint}
serviceMonitor := &prometheusv1.ServiceMonitor{
TypeMeta: metav1.TypeMeta{
Kind: "ServiceMonitor",
APIVersion: prometheusv1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-service-monitor", g.appName), Namespace: g.namespace},
Spec: prometheusv1.ServiceMonitorSpec{
Selector: metav1.LabelSelector{
MatchLabels: monitoringLabels,
},
Endpoints: serviceEndpointList,
},
if g.app.Monitoring != nil && g.app.Monitoring.OperatorMode {
if g.app.Monitoring.MonitorType == monitoring.ServiceMonitorType {
serviceMonitor, err := g.buildMonitorObject(g.app.Monitoring.MonitorType)
if err != nil {
return err
}
err := modules.AppendToIntent(
err = modules.AppendToIntent(
apiv1.Kubernetes,
modules.KubernetesResourceID(serviceMonitor.TypeMeta, serviceMonitor.ObjectMeta),
modules.KubernetesResourceID(
serviceMonitor.(*prometheusv1.ServiceMonitor).TypeMeta,
serviceMonitor.(*prometheusv1.ServiceMonitor).ObjectMeta,
),
spec,
serviceMonitor,
)
if err != nil {
return err
}
} else if g.project.Prometheus.MonitorType == apiv1.PodMonitorType {
podMetricsEndpoint := prometheusv1.PodMetricsEndpoint{
Interval: g.monitor.Interval,
ScrapeTimeout: g.monitor.Timeout,
Port: g.monitor.Port,
Path: g.monitor.Path,
Scheme: g.monitor.Scheme,
}
podMetricsEndpointList := []prometheusv1.PodMetricsEndpoint{podMetricsEndpoint}

podMonitor := &prometheusv1.PodMonitor{
TypeMeta: metav1.TypeMeta{
Kind: "PodMonitor",
APIVersion: prometheusv1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-pod-monitor", g.appName), Namespace: g.namespace},
Spec: prometheusv1.PodMonitorSpec{
Selector: metav1.LabelSelector{
MatchLabels: monitoringLabels,
},
PodMetricsEndpoints: podMetricsEndpointList,
},
} else if g.app.Monitoring.MonitorType == monitoring.PodMonitorType {
podMonitor, err := g.buildMonitorObject(g.app.Monitoring.MonitorType)
if err != nil {
return err
}

err := modules.AppendToIntent(
err = modules.AppendToIntent(
apiv1.Kubernetes,
modules.KubernetesResourceID(podMonitor.TypeMeta, podMonitor.ObjectMeta),
modules.KubernetesResourceID(
podMonitor.(*prometheusv1.PodMonitor).TypeMeta,
podMonitor.(*prometheusv1.PodMonitor).ObjectMeta,
),
spec,
podMonitor,
)
if err != nil {
return err
}
} else {
return fmt.Errorf("MonitorType should either be service or pod %s", g.project.Prometheus.MonitorType)
return fmt.Errorf("MonitorType should either be service or pod %s", g.app.Monitoring.MonitorType)
}
}

return nil
}

// parseWorkspaceConfig parses the config items for monitoring generator in workspace configurations.
func (g *monitoringGenerator) parseWorkspaceConfig() error {
wsConfig, ok := g.modulesConfig[monitoring.ModuleName]
// If AppConfiguration contains monitoring config but workspace does not,
// respond with the error ErrEmptyModuleConfigBlock
if g.app.Monitoring != nil && !ok {
return workspace.ErrEmptyModuleConfigBlock
}

if operatorMode, ok := wsConfig[monitoring.OperatorModeKey]; ok {
g.app.Monitoring.OperatorMode = operatorMode.(bool)
}

if monitorType, ok := wsConfig[monitoring.MonitorTypeKey]; ok {
g.app.Monitoring.MonitorType = monitoring.MonitorType(monitorType.(string))
} else {
g.app.Monitoring.MonitorType = monitoring.DefaultMonitorType
}

if interval, ok := wsConfig[monitoring.IntervalKey]; ok {
g.app.Monitoring.Interval = prometheusv1.Duration(interval.(string))
} else {
g.app.Monitoring.Interval = monitoring.DefaultInterval
}

if timeout, ok := wsConfig[monitoring.TimeoutKey]; ok {
g.app.Monitoring.Timeout = prometheusv1.Duration(timeout.(string))
} else {
g.app.Monitoring.Timeout = monitoring.DefaultTimeout
}

if scheme, ok := wsConfig[monitoring.SchemeKey]; ok {
g.app.Monitoring.Scheme = scheme.(string)
} else {
g.app.Monitoring.Scheme = monitoring.DefaultScheme
}

parsedTimeout, err := time.ParseDuration(string(g.app.Monitoring.Timeout))
if err != nil {
return err
}
parsedInterval, err := time.ParseDuration(string(g.app.Monitoring.Interval))
if err != nil {
return err
}

if parsedTimeout > parsedInterval {
return monitoring.ErrTimeoutGreaterThanInterval
}

return nil
}

func (g *monitoringGenerator) buildMonitorObject(monitorType monitoring.MonitorType) (runtime.Object, error) {
// If Prometheus runs as an operator, it relies on Custom Resources to
// manage the scrape configs. CRs (ServiceMonitors and PodMonitors) rely on
// corresponding resources (Services and Pods) to have labels that can be
// used as part of the label selector for the CR to determine which
// service/pods to scrape from.
// Here we choose the label name kusion_monitoring_appname for two reasons:
// 1. Unlike the label validation in Kubernetes, the label name accepted by
// Prometheus cannot contain non-alphanumeric characters except underscore:
// https://github.com/prometheus/common/blob/main/model/labels.go#L94
// 2. The name should be unique enough that is only created by Kusion and
// used to identify a certain application
monitoringLabels := map[string]string{
"kusion_monitoring_appname": g.appName,
}

if monitorType == monitoring.ServiceMonitorType {
serviceEndpoint := prometheusv1.Endpoint{
Interval: g.app.Monitoring.Interval,
ScrapeTimeout: g.app.Monitoring.Timeout,
Port: g.app.Monitoring.Port,
Path: g.app.Monitoring.Path,
Scheme: g.app.Monitoring.Scheme,
}
serviceEndpointList := []prometheusv1.Endpoint{serviceEndpoint}
serviceMonitor := &prometheusv1.ServiceMonitor{
TypeMeta: metav1.TypeMeta{
Kind: "ServiceMonitor",
APIVersion: prometheusv1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-service-monitor", modules.UniqueAppName(g.project.Name, g.stack.Name, g.appName)),
Namespace: g.namespace,
},
Spec: prometheusv1.ServiceMonitorSpec{
Selector: metav1.LabelSelector{
MatchLabels: monitoringLabels,
},
Endpoints: serviceEndpointList,
},
}
return serviceMonitor, nil
} else if monitorType == monitoring.PodMonitorType {
podMetricsEndpoint := prometheusv1.PodMetricsEndpoint{
Interval: g.app.Monitoring.Interval,
ScrapeTimeout: g.app.Monitoring.Timeout,
Port: g.app.Monitoring.Port,
Path: g.app.Monitoring.Path,
Scheme: g.app.Monitoring.Scheme,
}
podMetricsEndpointList := []prometheusv1.PodMetricsEndpoint{podMetricsEndpoint}

podMonitor := &prometheusv1.PodMonitor{
TypeMeta: metav1.TypeMeta{
Kind: "PodMonitor",
APIVersion: prometheusv1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-pod-monitor", modules.UniqueAppName(g.project.Name, g.stack.Name, g.appName)),
Namespace: g.namespace,
},
Spec: prometheusv1.PodMonitorSpec{
Selector: metav1.LabelSelector{
MatchLabels: monitoringLabels,
},
PodMetricsEndpoints: podMetricsEndpointList,
},
}
return podMonitor, nil
}

return nil, fmt.Errorf("MonitorType should either be service or pod %s", monitorType)
}

0 comments on commit 5eb12a8

Please sign in to comment.