Skip to content

Commit

Permalink
Merge pull request gocrane#859 from whitebear009/add-container-resour…
Browse files Browse the repository at this point in the history 
…ce-type

add ContainerResource source type for ehpa prediction
  • Loading branch information
@qmhu
qmhu committed Sep 13, 2023
2 parents f436d75 + b5475b7 commit 94616ca
Show file tree
Hide file tree
Showing 6 changed files with 121 additions and 64 deletions.
35 changes: 26 additions & 9 deletions pkg/controller/ehpa/hpa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,10 +208,30 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
var metricIdentifier string
var averageValue *resource.Quantity
switch metric.Type {
case autoscalingv2.ResourceMetricSourceType:
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String())
case autoscalingv2.ResourceMetricSourceType, autoscalingv2.ContainerResourceMetricSourceType:
var averageUtilization *int32
var containerName string
if metric.Resource != nil {
if metric.Resource.Target.AverageUtilization != nil {
averageUtilization = metric.Resource.Target.AverageUtilization
}
if metric.Resource.Target.AverageValue != nil {
averageValue = metric.Resource.Target.AverageValue
}
}
if metric.ContainerResource != nil {
containerName = metric.ContainerResource.Container
if metric.ContainerResource.Target.AverageUtilization != nil {
averageUtilization = metric.ContainerResource.Target.AverageUtilization
}
if metric.ContainerResource.Target.AverageValue != nil {
averageValue = metric.ContainerResource.Target.AverageValue
}
}

// When use AverageUtilization in EffectiveHorizontalPodAutoscaler's metricSpec, convert to AverageValue
if metric.Resource.Target.AverageUtilization != nil {
if averageUtilization != nil {
metricName := utils.GetMetricName(metric)
scale, _, err := utils.GetScale(ctx, c.RestMapper, c.ScaleClient, ehpa.Namespace, ehpa.Spec.ScaleTargetRef)
if err != nil {
return nil, err
Expand All @@ -231,24 +251,21 @@ func (c *EffectiveHPAController) GetHPAMetrics(ctx context.Context, ehpa *autosc
return nil, fmt.Errorf("failed to get available pods. ")
}

requests, err := utils.CalculatePodRequests(availablePods, metric.Resource.Name)
requests, err := utils.CalculatePodRequests(availablePods, v1.ResourceName(metricName), containerName)
if err != nil {
return nil, err
}

value := int64((float64(requests) * float64(*metric.Resource.Target.AverageUtilization) / 100) / float64(len(availablePods)))
value := int64((float64(requests) * float64(*averageUtilization) / 100) / float64(len(availablePods)))
averageValue = resource.NewMilliQuantity(value, resource.DecimalSI)
} else {
averageValue = metric.Resource.Target.AverageValue
}
case autoscalingv2.ExternalMetricSourceType:
metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name)
averageValue = metric.External.Target.AverageValue
case autoscalingv2.PodsMetricSourceType:
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name)
averageValue = metric.Pods.Target.AverageValue
}

metricIdentifier = utils.GetPredictionMetricIdentifier(metric)
if metricIdentifier == "" {
continue
}
Expand Down
52 changes: 25 additions & 27 deletions pkg/controller/ehpa/predict.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,33 +133,20 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect

var predictionMetrics []predictionapi.PredictionMetric
for _, metric := range ehpa.Spec.Metrics {
var metricName string
//get metricIdentifier by metric.Type and metricName
var metricIdentifier string
switch metric.Type {
case autoscalingv2.ResourceMetricSourceType:
metricName = metric.Resource.Name.String()
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Resource.Name.String())
case autoscalingv2.ExternalMetricSourceType:
metricName = metric.External.Metric.Name
metricIdentifier = utils.GetMetricIdentifier(metric, metric.External.Metric.Name)
case autoscalingv2.PodsMetricSourceType:
metricName = metric.Pods.Metric.Name
metricIdentifier = utils.GetMetricIdentifier(metric, metric.Pods.Metric.Name)
}

metricIdentifier := utils.GetPredictionMetricIdentifier(metric)
if metricIdentifier == "" {
continue
}

metricName := utils.GetMetricName(metric)
//get matchLabels
var matchLabels []string
var metricRule *prometheus_adapter.MetricRule

// Supreme priority: annotation
expressionQuery := utils.GetExpressionQueryAnnotation(metricIdentifier, ehpa.Annotations)
if expressionQuery == "" {
var nameReg string
var podNameReg string
// get metricRule from prometheus-adapter
switch metric.Type {
case autoscalingv2.ResourceMetricSourceType:
Expand All @@ -169,7 +156,23 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName)
} else {
klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
}
}
case autoscalingv2.ContainerResourceMetricSourceType:
if len(mrs.MetricRulesResource) > 0 {
metricRule = prometheus_adapter.MatchMetricRule(mrs.MetricRulesResource, metricName)
if metricRule == nil {
klog.Errorf("Got MetricRulesResource prometheus-adapter-resource Failed MetricName[%s]", metricName)
} else {
klog.V(4).Infof("Got MetricRulesResource prometheus-adapter-resource MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
// Compared to ResourceMetricSourceType, there is an additional container-name field
containerLabel := "container"
if metricRule.ContainerLabel != "" {
containerLabel = metricRule.ContainerLabel
}
matchLabels = append(matchLabels, utils.MapSortToArray(map[string]string{containerLabel: metric.ContainerResource.Container})...)
}
}
case autoscalingv2.PodsMetricSourceType:
Expand All @@ -179,12 +182,9 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
klog.Errorf("Got MetricRulesCustomer prometheus-adapter-customer Failed MetricName[%s]", metricName)
} else {
klog.V(4).Infof("Got MetricRulesCustomer prometheus-adapter-customer MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
nameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)

podNameReg = utils.GetPodNameReg(ehpa.Spec.ScaleTargetRef.Name, ehpa.Spec.ScaleTargetRef.Kind)
if metric.Pods.Metric.Selector != nil {
for _, i := range utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels) {
matchLabels = append(matchLabels, i)
}
matchLabels = append(matchLabels, utils.MapSortToArray(metric.Pods.Metric.Selector.MatchLabels)...)
}
}
}
Expand All @@ -196,18 +196,16 @@ func (c *EffectiveHPAController) NewPredictionObject(ehpa *autoscalingapi.Effect
} else {
klog.V(4).Infof("Got MetricRulesExternal prometheus-adapter-external MetricMatches[%s] SeriesName[%s]", metricRule.MetricMatches, metricRule.SeriesName)
if metric.External.Metric.Selector != nil {
for _, i := range utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels) {
matchLabels = append(matchLabels, i)
}
matchLabels = append(matchLabels, utils.MapSortToArray(metric.External.Metric.Selector.MatchLabels)...)
}
}
}
}

if metricRule != nil {
// Second priority: get default expressionQuery
// Second priority: get prometheus-adapter expressionQuery
var err error
expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, nameReg, append(mrs.ExtensionLabels, matchLabels...))
expressionQuery, err = metricRule.QueryForSeries(ehpa.Namespace, podNameReg, append(mrs.ExtensionLabels, matchLabels...))
if err != nil {
klog.Errorf("Got promSelector prometheus-adapter %v %v", metricRule, err)
} else {
Expand Down
33 changes: 18 additions & 15 deletions pkg/prometheus-adapter/expression.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,13 @@ type MetricRules struct {
}

type MetricRule struct {
MetricMatches string
SeriesName string
ResConverter naming.ResourceConverter
Template *template.Template
Namespaced bool
LabelMatchers []string
MetricMatches string
SeriesName string
ResConverter naming.ResourceConverter
Template *template.Template
Namespaced bool
LabelMatchers []string
ContainerLabel string
}

type QueryTemplateArgs struct {
Expand Down Expand Up @@ -129,7 +130,7 @@ func SetExtensionLabels(extensionLabels string) {
}
}

// GetMetricRuleResourceFromRules produces a MetricNamer for each rule in the given config.
// GetMetricRulesFromResourceRules produces a MetricNamer for each rule in the given config.
func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTMapper) (metricRules []MetricRule, metricRulesError []string, err error) {
// get cpu MetricsQuery
if cfg.CPU.ContainerQuery != "" {
Expand All @@ -154,10 +155,11 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM
klog.Errorf("unable to parse metrics query template %q: %v", cfg.CPU.ContainerQuery, err)
} else {
metricRules = append(metricRules, MetricRule{
MetricMatches: "cpu",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
MetricMatches: "cpu",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
ContainerLabel: cfg.CPU.ContainerLabel,
})
}
}
Expand Down Expand Up @@ -190,10 +192,11 @@ func GetMetricRulesFromResourceRules(cfg config.ResourceRules, mapper meta.RESTM
klog.Errorf("unable to parse metrics query template %q: %v", cfg.Memory.ContainerQuery, err)
} else {
metricRules = append(metricRules, MetricRule{
MetricMatches: "memory",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
MetricMatches: "memory",
ResConverter: resConverter,
Template: templ,
Namespaced: true,
ContainerLabel: cfg.Memory.ContainerLabel,
})
}
}
Expand Down
37 changes: 30 additions & 7 deletions pkg/utils/ehpa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@ import (
"strings"

autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1"
autoscalingv2 "k8s.io/api/autoscaling/v2beta2"

"github.com/gocrane/crane/pkg/known"
autoscalingv2 "k8s.io/api/autoscaling/v2beta2"
)

func IsEHPAPredictionEnabled(ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) bool {
Expand Down Expand Up @@ -39,7 +38,7 @@ func IsEHPACronEnabled(ehpa *autoscalingapi.EffectiveHorizontalPodAutoscaler) bo
// GetPredictionMetricName return metric name used by prediction
func GetPredictionMetricName(sourceType autoscalingv2.MetricSourceType) (metricName string) {
switch sourceType {
case autoscalingv2.ResourceMetricSourceType, autoscalingv2.PodsMetricSourceType, autoscalingv2.ExternalMetricSourceType:
case autoscalingv2.ResourceMetricSourceType, autoscalingv2.ContainerResourceMetricSourceType, autoscalingv2.PodsMetricSourceType, autoscalingv2.ExternalMetricSourceType:
metricName = known.MetricNamePrediction
}

Expand All @@ -51,19 +50,36 @@ func GetCronMetricName() string {
return known.MetricNameCron
}

// GetGeneralPredictionMetricName return metric name used by prediction
func GetMetricIdentifier(metric autoscalingv2.MetricSpec, name string) string {
func GetMetricName(metric autoscalingv2.MetricSpec) string {
switch metric.Type {
case autoscalingv2.PodsMetricSourceType:
return metric.Pods.Metric.Name
case autoscalingv2.ResourceMetricSourceType:
return metric.Resource.Name.String()
case autoscalingv2.ContainerResourceMetricSourceType:
return metric.ContainerResource.Name.String()
case autoscalingv2.ExternalMetricSourceType:
return metric.External.Metric.Name
default:
return ""
}
}

// GetPredictionMetricIdentifier return metric name used by prediction
func GetPredictionMetricIdentifier(metric autoscalingv2.MetricSpec) string {
var prefix string
switch metric.Type {
case autoscalingv2.PodsMetricSourceType:
prefix = "pods"
case autoscalingv2.ResourceMetricSourceType:
prefix = "resource"
case autoscalingv2.ContainerResourceMetricSourceType:
prefix = "container-resource"
case autoscalingv2.ExternalMetricSourceType:
prefix = "external"
}

return fmt.Sprintf("%s.%s", prefix, name)
return fmt.Sprintf("%s.%s", prefix, GetMetricName(metric))
}

// GetExpressionQueryAnnotation return metric query from annotation by metricName
Expand Down Expand Up @@ -95,7 +111,7 @@ func IsExpressionQueryAnnotationEnabled(metricIdentifier string, annotations map
return false
}

// GetExpressionQuery return metric query
// GetExpressionQueryDefault return default metric query
func GetExpressionQueryDefault(metric autoscalingv2.MetricSpec, namespace string, name string, kind string) string {
var expressionQuery string
switch metric.Type {
Expand All @@ -106,6 +122,13 @@ func GetExpressionQueryDefault(metric autoscalingv2.MetricSpec, namespace string
case "memory":
expressionQuery = GetWorkloadMemUsageExpression(namespace, name, kind)
}
case autoscalingv2.ContainerResourceMetricSourceType:
switch metric.ContainerResource.Name {
case "cpu":
expressionQuery = GetContainerCpuUsageExpression(namespace, name, kind, metric.ContainerResource.Container)
case "memory":
expressionQuery = GetContainerMemUsageExpression(namespace, name, kind, metric.ContainerResource.Container)
}
case autoscalingv2.PodsMetricSourceType:
var labels []string
if metric.Pods.Metric.Selector != nil {
Expand Down
7 changes: 5 additions & 2 deletions pkg/utils/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,14 @@ func EvictPodWithGracePeriod(client clientset.Interface, pod *v1.Pod, gracePerio
return client.CoreV1().Pods(pod.Namespace).EvictV1beta1(context.Background(), e)
}

// CalculatePodRequests sum request total from pods
func CalculatePodRequests(pods []v1.Pod, resource v1.ResourceName) (int64, error) {
// CalculatePodRequests sum request total from pods. If the containerName is specified, the total amount of requests for that container will be calculated.
func CalculatePodRequests(pods []v1.Pod, resource v1.ResourceName, containerName string) (int64, error) {
var requests int64
for _, pod := range pods {
for _, c := range pod.Spec.Containers {
if containerName != "" && c.Name != containerName {
continue
}
if containerRequest, ok := c.Resources.Requests[resource]; ok {
requests += containerRequest.MilliValue()
} else {
Expand Down
21 changes: 17 additions & 4 deletions pkg/utils/pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,10 @@ func TestCalculatePodRequests(t *testing.T) {
}

tests := []struct {
description string
resource v1.ResourceName
expect int64
description string
resource v1.ResourceName
containerName string
expect int64
}{
{
description: "calculate cpu request total",
Expand All @@ -75,10 +76,22 @@ func TestCalculatePodRequests(t *testing.T) {
resource: v1.ResourceMemory,
expect: 60000,
},
{
description: "calculate cpu request total of container1",
resource: v1.ResourceCPU,
containerName: "container1",
expect: 3000,
},
{
description: "calculate memory request total of container1",
resource: v1.ResourceMemory,
containerName: "container1",
expect: 30000,
},
}

for _, test := range tests {
requests, err := CalculatePodRequests(pods, test.resource)
requests, err := CalculatePodRequests(pods, test.resource, test.containerName)
if err != nil {
t.Errorf("Failed to calculatePodRequests: %v", err)
}
Expand Down

0 comments on commit 94616ca

Please sign in to comment.