Skip to content

Commit

Permalink
RHIROS-1326 Kruize 0.0.20_rm Integration
Browse files Browse the repository at this point in the history
  • Loading branch information
@saltgen
saltgen committed Oct 6, 2023
1 parent 6f54036 commit 8e53cd2
Show file tree
Hide file tree
Showing 8 changed files with 285 additions and 199 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,9 @@ upload-msg-to-rosocp:
get-recommendations:
ifdef env
$(eval APIPOD=$(shell oc get pods -o custom-columns=POD:.metadata.name --no-headers -n ${env} | grep ros-ocp-backend-api))
oc exec ${APIPOD} -c ros-ocp-backend-api -n ${env} -- /bin/bash -c 'curl -s -H "X-Rh-Identity: ${b64_identity}" -H "x-rh-request_id: testtesttest" http://localhost:8000/api/cost-management/v1/recommendations/openshift' | python -m json.tool
oc exec ${APIPOD} -c ros-ocp-backend-api -n ${env} -- /bin/bash -c 'curl -s -H "X-Rh-Identity: ${b64_identity}" -H "x-rh-request_id: testtesttest" http://localhost:8000/api/cost-management/v1/recommendations/openshift?start_date=1992-12-26' | python -m json.tool
else
curl -s -H "x-rh-identity: ${b64_identity}" \
-H "x-rh-request_id: testtesttest" \
http://localhost:8000/api/cost-management/v1/recommendations/openshift | python -m json.tool
http://localhost:8000/api/cost-management/v1/recommendations/openshift?start_date=1992-12-26 | python -m json.tool
endif
62 changes: 35 additions & 27 deletions internal/api/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,11 +223,6 @@ func TransformComponentUnits(jsonData datatypes.JSON) map[string]interface{} {
return nil
}

durationBased, ok := data["duration_based"].(map[string]interface{})
if !ok {
fmt.Printf("duration_based not found in JSON")
}

convertMemory := func(memory map[string]interface{}) error {
amount, ok := memory["amount"].(float64)
if ok {
Expand Down Expand Up @@ -276,45 +271,58 @@ func TransformComponentUnits(jsonData datatypes.JSON) map[string]interface{} {

/*
Recommendation data is available for three periods
for cost and performance
For each of these actual values will be present in
below mentioned dataBlocks > request and limits
*/

for _, period := range []string{"long_term", "medium_term", "short_term"} {
intervalData, ok := durationBased[period].(map[string]interface{})
recommendation_terms, ok := data["recommendation_terms"].(map[string]interface{})
if !ok {
fmt.Printf("recommendation_terms not found in JSON")
}

for _, period := range []string{"short_term", "medium_term", "long_term"} {
intervalData, ok := recommendation_terms[period].(map[string]interface{})
if !ok {
continue
}

for _, dataBlock := range []string{"current", "config", "variation"} {
recommendationSection, ok := intervalData[dataBlock].(map[string]interface{})
for _, recommendationType := range []string{"cost", "performance"} {
engineData, ok := intervalData["recommendation_engines"].(map[string]interface{})[recommendationType].(map[string]interface{})
if !ok {
continue
}

for _, section := range []string{"limits", "requests"} {
for _, dataBlock := range []string{"current", "config", "variation"} {
recommendationSection, ok := engineData[dataBlock].(map[string]interface{})
if !ok {
continue
}

for _, section := range []string{"limits", "requests"} {

sectionObject, ok := recommendationSection[section].(map[string]interface{})
if ok {
memory, ok := sectionObject["memory"].(map[string]interface{})
sectionObject, ok := recommendationSection[section].(map[string]interface{})
if ok {
err := convertMemory(memory)
if err != nil {
fmt.Printf("error converting memory in %s: %v\n", period, err)
continue
memory, ok := sectionObject["memory"].(map[string]interface{})
if ok {
err := convertMemory(memory)
if err != nil {
fmt.Printf("error converting memory in %s: %v\n", period, err)
continue
}
}
}
cpu, ok := sectionObject["cpu"].(map[string]interface{})
if ok {
err := convertCPU(cpu)
if err != nil {
fmt.Printf("error converting cpu in %s: %v\n", period, err)
continue
cpu, ok := sectionObject["cpu"].(map[string]interface{})
if ok {
err := convertCPU(cpu)
if err != nil {
fmt.Printf("error converting cpu in %s: %v\n", period, err)
continue
}
}
}
}
}

}

}
}
}

Expand Down
14 changes: 8 additions & 6 deletions internal/services/report_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,9 @@ func ProcessReport(msg *kafka.Message) {
k8s_object_name,
)

container_names, err := kruize.Create_kruize_experiments(experiment_name, k8s_object)
cluster_identifier := kafkaMsg.Metadata.Cluster_alias + "; " + kafkaMsg.Metadata.Cluster_uuid

container_names, err := kruize.Create_kruize_experiments(experiment_name, cluster_identifier, k8s_object)
if err != nil {
log.Error(err)
continue
Expand Down Expand Up @@ -199,7 +201,7 @@ func ProcessReport(msg *kafka.Message) {
continue
}

if kruize.Is_valid_recommendation(recommendation) {
if kruize.Is_valid_recommendation(recommendation, experiment_name, maxEndTime.String()) {
containers := recommendation[0].Kubernetes_objects[0].Containers
for _, container := range containers {
for _, v := range container.Recommendations.Data {
Expand All @@ -212,8 +214,8 @@ func ProcessReport(msg *kafka.Message) {
recommendationSet := model.RecommendationSet{
WorkloadID: workload.ID,
ContainerName: container.Container_name,
MonitoringStartTime: v.Duration_based.Short_term.Monitoring_start_time,
MonitoringEndTime: v.Duration_based.Short_term.Monitoring_end_time,
MonitoringStartTime: v.RecommendationTerms.Short_term.MonitoringStartTime,
MonitoringEndTime: v.MonitoringEndTime,
Recommendations: marshalData,
}
if err := recommendationSet.CreateRecommendationSet(); err != nil {
Expand All @@ -227,8 +229,8 @@ func ProcessReport(msg *kafka.Message) {
historicalRecommendationSet := model.HistoricalRecommendationSet{
WorkloadID: workload.ID,
ContainerName: container.Container_name,
MonitoringStartTime: v.Duration_based.Short_term.Monitoring_start_time,
MonitoringEndTime: v.Duration_based.Short_term.Monitoring_end_time,
MonitoringStartTime: v.RecommendationTerms.Short_term.MonitoringStartTime,
MonitoringEndTime: v.MonitoringEndTime,
Recommendations: marshalData,
}
if err := historicalRecommendationSet.CreateHistoricalRecommendationSet(); err != nil {
Expand Down
49 changes: 30 additions & 19 deletions internal/types/kruizePayload/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,36 +38,47 @@ type aggregation_info struct {
}

type recommendation struct {
Data map[string]recommendationType `json:"data,omitempty"`
Notifications map[string]notification `json:"notifications,omitempty"`
Version string `json:"version,omitempty"`
Data map[string]RecommendationData `json:"data,omitempty"`
Notifications map[string]Notification `json:"notifications,omitempty"`
}

type notification struct {

type Notification struct {
NotifyType string `json:"type,omitempty"`
Message string `json:"message,omitempty"`
Code int `json:"code,omitempty"`
}

type recommendationType struct {
Duration_based termbased `json:"duration_based,omitempty"`
type RecommendationEngineObject struct {
PodsCount int `json:"pods_count,omitempty"`
ConfidenceLevel float64 `json:"confidence_level,omitempty"`
Config ConfigObject `json:"config,omitempty"`
Variation ConfigObject `json:"variation,omitempty"`
Notifications map[string]Notification `json:"notifications,omitempty"`
}

type RecommendationData struct {
Notifications map[string]Notification `json:"notifications"`
MonitoringEndTime time.Time `json:"monitoring_end_time"`
Current ConfigObject `json:"current"`
RecommendationTerms TermBased `json:"recommendation_terms"`
}

type termbased struct {
Short_term recommendationObject `json:"short_term,omitempty"`
Medium_term recommendationObject `json:"medium_term,omitempty"`
Long_term recommendationObject `json:"long_term,omitempty"`
type RecommendationTerm struct {
DurationInHours float64 `json:"duration_in_hours"`
Notifications map[string]Notification `json:"notifications"`
MonitoringStartTime time.Time
RecommendationEngines struct {
Cost RecommendationEngineObject `json:"cost"`
Performance RecommendationEngineObject `json:"performance"`
} `json:"recommendation_engines"`
}

type recommendationObject struct {
Monitoring_start_time time.Time `json:"monitoring_start_time,omitempty"`
Monitoring_end_time time.Time `json:"monitoring_end_time,omitempty"`
Duration_in_hours float64 `json:"duration_in_hours,omitempty"`
Pods_count int `json:"pods_count,omitempty"`
Confidence_level float64 `json:"confidence_level,omitempty"`
Current ConfigObject `json:"current,omitempty"`
Config ConfigObject `json:"config,omitempty"`
Variation ConfigObject `json:"variation,omitempty"`
Notifications map[string]notification `json:"notifications,omitempty"`
type TermBased struct {
Short_term RecommendationTerm `json:"short_term,omitempty"`
Medium_term RecommendationTerm `json:"medium_term,omitempty"`
Long_term RecommendationTerm `json:"long_term,omitempty"`
}

type ConfigObject struct {
Expand Down
4 changes: 3 additions & 1 deletion internal/types/kruizePayload/createExperiment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
type createExperiment struct {
Version string `json:"version"`
Experiment_name string `json:"experiment_name"`
Cluster_name string `json:"cluster_name"`
Performance_profile string `json:"performance_profile"`
Mode string `json:"mode"`
Target_cluster string `json:"target_cluster"`
Expand All @@ -23,7 +24,7 @@ type recommendation_settings struct {
Threshold string `json:"threshold"`
}

func GetCreateExperimentPayload(experiment_name string, containers []map[string]string, data map[string]string) ([]byte, error) {
func GetCreateExperimentPayload(experiment_name string, cluster_identifier string, containers []map[string]string, data map[string]string) ([]byte, error) {
container_array := []container{}
for _, c := range containers {
container_array = append(container_array, container{
Expand All @@ -35,6 +36,7 @@ func GetCreateExperimentPayload(experiment_name string, containers []map[string]
{
Version: "1.0",
Experiment_name: experiment_name,
Cluster_name: cluster_identifier,
Performance_profile: "resource-optimization-openshift",
Mode: "monitor",
Target_cluster: "remote",
Expand Down
76 changes: 66 additions & 10 deletions internal/utils/kruize/kruize_api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ var log *logrus.Entry = logging.GetLogger()
var cfg *config.Config = config.GetConfig()
var experimentCreateAttempt bool = true

func Create_kruize_experiments(experiment_name string, k8s_object []map[string]interface{}) ([]string, error) {
func Create_kruize_experiments(experiment_name string, cluster_identifier string, k8s_object []map[string]interface{}) ([]string, error) {
// k8s_object (can) contain multiple containers of same k8s object type.
data := map[string]string{
"namespace": k8s_object[0]["namespace"].(string),
Expand All @@ -39,7 +39,7 @@ func Create_kruize_experiments(experiment_name string, k8s_object []map[string]i
})
}
}
payload, err := kruizePayload.GetCreateExperimentPayload(experiment_name, containers, data)
payload, err := kruizePayload.GetCreateExperimentPayload(experiment_name, cluster_identifier, containers, data)
if err != nil {
return nil, fmt.Errorf("unable to create payload: %v", err)
}
Expand Down Expand Up @@ -69,7 +69,7 @@ func Create_kruize_experiments(experiment_name string, k8s_object []map[string]i
log.Info("Tring to create resource_optimization_openshift performance profile")
utils.Setup_kruize_performance_profile()
experimentCreateAttempt = false // Attempting only once
container_names, err := Create_kruize_experiments(experiment_name, k8s_object)
container_names, err := Create_kruize_experiments(experiment_name, cluster_identifier, k8s_object)
experimentCreateAttempt = true
if err != nil {
return nil, err
Expand Down Expand Up @@ -176,15 +176,71 @@ func Update_recommendations(experiment_name string, interval_end_time time.Time)

}

func Is_valid_recommendation(d []kruizePayload.ListRecommendations) bool {
func Is_valid_recommendation(d []kruizePayload.ListRecommendations, experiment_name string, maxEndTime string) bool {
if len(d) > 0 {

// To maintain a local reference the following map has been created from
// https://github.com/kruize/autotune/blob/master/design/NotificationCodes.md#detailed-codes
notificationCodeValidities := map[string]string{
"111000": "INFO",
"120001": "INFO",
"221001": "ERROR",
"221002": "ERROR",
"221003": "ERROR",
"221004": "ERROR",
"223001": "ERROR",
"223002": "ERROR",
"223003": "ERROR",
"223004": "ERROR",
"224001": "ERROR",
"224002": "ERROR",
"224003": "ERROR",
"224004": "ERROR",
}

notifications := d[0].Kubernetes_objects[0].Containers[0].Recommendations.Notifications
// 112101 is notification code for "Duration Based Recommendations Available".
if _, ok := notifications["112101"]; ok {
return true
} else {
return false

for key := range notifications{
notificationType, keyExists := notificationCodeValidities[key]
if !keyExists {
return false
}

if key == "111000" && notificationType == "INFO"{
notificationsLevelTwo := d[0].Kubernetes_objects[0].Containers[0].Recommendations.Data[maxEndTime].Notifications

for key := range notificationsLevelTwo{
if notificationCodeValidities[key] == "ERROR"{
kruizeInvalidRecommendationDetail.WithLabelValues(key, experiment_name).Set(1)

notificationsLevelThreeShortTerm := d[0].Kubernetes_objects[0].Containers[0].Recommendations.Data[maxEndTime].RecommendationTerms.Short_term.Notifications
notificationsLevelThreeMediumTerm := d[0].Kubernetes_objects[0].Containers[0].Recommendations.Data[maxEndTime].RecommendationTerms.Medium_term.Notifications
notificationsLevelThreeLongTerm := d[0].Kubernetes_objects[0].Containers[0].Recommendations.Data[maxEndTime].RecommendationTerms.Long_term.Notifications

notificationSections := []map[string]kruizePayload.Notification{
notificationsLevelTwo,
notificationsLevelThreeShortTerm,
notificationsLevelThreeMediumTerm,
notificationsLevelThreeLongTerm,
}

for _, notificationBody := range notificationSections{
for key := range notificationBody{
if notificationCodeValidities[key] == "ERROR"{
kruizeInvalidRecommendationDetail.WithLabelValues(key, experiment_name).Set(1)
}

}
}
return true
} else {
// Setting the metric counter to 1 as we expect a single metric
// for a combination of notification_code and experiment_name
kruizeInvalidRecommendationDetail.WithLabelValues(key, experiment_name).Set(1)
return false
}
}
}
}}}

return false
}
11 changes: 11 additions & 0 deletions internal/utils/kruize/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,14 @@ var (
[]string{"path"},
)
)


var (
kruizeInvalidRecommendationDetail = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "rosocp_kruize_invalid_recommendation_detail",
Help: "List of INFO/ERROR type recommendations from Kruize",
},
[]string{"notification_code", "experiment_name"},
)
)
Loading

0 comments on commit 8e53cd2

Please sign in to comment.