THREESCALE-7961: Use sum_irate instead of sum_rate

Makefile

@@ -228,6 +228,7 @@ download:
 	@$(GO) mod download
 
 ## licenses.xml: Generate licenses.xml file
+.PHONY: licenses.xml
 licenses.xml: $(DEPENDENCY_DECISION_FILE)
 ifndef LICENSEFINDERBINARY
 	$(error "license-finder is not available please install: gem install license_finder --version 5.7.1")

controllers/apps/apimanager_suite_test.go

@@ -41,6 +41,7 @@ import (
 	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	grafanav1alpha1 "github.com/integr8ly/grafana-operator/v3/pkg/apis/integreatly/v1alpha1"
 	appsv1 "github.com/openshift/api/apps/v1"
+	configv1 "github.com/openshift/api/config/v1"
 	consolev1 "github.com/openshift/api/console/v1"
 	imagev1 "github.com/openshift/api/image/v1"
 	routev1 "github.com/openshift/api/route/v1"
@@ -113,6 +114,9 @@ var _ = BeforeSuite(func(done Done) {
 	err = grafanav1alpha1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = configv1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
 	// +kubebuilder:scaffold:scheme
 
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{

doc/dependency_decisions.yml

@@ -336,3 +336,10 @@
     :why: Apache 2.0 License https://github.com/prometheus/client_golang/blob/master/LICENSE
     :versions: []
     :when: 2020-11-05 10:19:33.685918301 Z
+- - :license
+  - golang.org/x/mod
+  - 3-clause BSD License
+  - :who: 
+    :why: 3-clause BSD License https://cs.opensource.google/go/x/mod/+/master:LICENSE
+    :versions: []
+    :when: 2022-04-29 12:39:16.976400839 Z

doc/prometheusrules/README.md

@@ -8,6 +8,7 @@
 * [System App](system-app.yaml)
 * [System Sidekiq](system-sidekiq.yaml)
 * [3scale Kube State Metrics](threescale-kube-state-metrics.yaml)
+* [3scale Kube State Metrics (Openshift <4.9)](threescale-kube-state-metrics-pre49.yaml)
 * [Zync](zync.yaml)
 * [Zync QUE](zync-que.yaml)
 

doc/prometheusrules/threescale-kube-state-metrics-pre49.yaml

@@ -0,0 +1,60 @@
+metadata:
+  creationTimestamp: null
+  labels:
+    app: 3scale-api-management
+    prometheus: application-monitoring
+    role: alert-rules
+  name: threescale-kube-state-metrics
+spec:
+  groups:
+  - name: __NAMESPACE__/threescale-kube-state-metrics.rules
+    rules:
+    - alert: ThreescalePodCrashLooping
+      annotations:
+        message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes.
+      expr: rate(kube_pod_container_status_restarts_total{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}[15m]) * 60 * 5 > 0
+      for: 5m
+      labels:
+        severity: critical
+    - alert: ThreescalePodNotReady
+      annotations:
+        message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 5 minutes.
+      expr: sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)", phase=~"Pending|Unknown"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{namespace="__NAMESPACE__",owner_kind!="Job"})) > 0
+      for: 5m
+      labels:
+        severity: critical
+    - alert: ThreescaleReplicationControllerReplicasMismatch
+      annotations:
+        message: ReplicationController {{ $labels.namespace }}/{{ $labels.replicationcontroller }} has not matched the expected number of replicas for longer than 5 minutes.
+      expr: kube_replicationcontroller_spec_replicas {namespace="__NAMESPACE__",replicationcontroller=~"(apicast-.*|backend-.*|system-.*|zync-.*)"} != kube_replicationcontroller_status_ready_replicas {namespace="__NAMESPACE__",replicationcontroller=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}
+      for: 5m
+      labels:
+        severity: critical
+    - alert: ThreescaleContainerWaiting
+      annotations:
+        message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container }} has been in waiting state for longer than 1 hour.
+      expr: sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) > 0
+      for: 1h
+      labels:
+        severity: warning
+    - alert: ThreescaleContainerCPUHigh
+      annotations:
+        message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container }} has High CPU usage for longer than 15 minutes.
+      expr: sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) * 100 > 90
+      for: 15m
+      labels:
+        severity: warning
+    - alert: ThreescaleContainerMemoryHigh
+      annotations:
+        message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container }} has High Memory usage for longer than 15 minutes.
+      expr: sum(container_memory_usage_bytes{namespace="__NAMESPACE__",container!="",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by(namespace, container, pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by(namespace, container, pod) * 100 > 90
+      for: 15m
+      labels:
+        severity: warning
+    - alert: ThreescaleContainerCPUThrottlingHigh
+      annotations:
+        message: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.'
+      expr: sum(increase(container_cpu_cfs_throttled_periods_total{namespace="__NAMESPACE__",container!="",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)" }[5m])) by (container, pod, namespace) / sum(increase(container_cpu_cfs_periods_total{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}[5m])) by (container, pod, namespace) > ( 25 / 100 )
+      for: 15m
+      labels:
+        severity: warning

doc/prometheusrules/threescale-kube-state-metrics.yaml

@@ -40,7 +40,7 @@ spec:
     - alert: ThreescaleContainerCPUHigh
       annotations:
         message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container }} has High CPU usage for longer than 15 minutes.
-      expr: sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) * 100 > 90
+      expr: sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace="__NAMESPACE__",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) * 100 > 90
       for: 15m
       labels:
         severity: warning

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.5.1
+	golang.org/x/mod v0.5.1
 	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.18.6
 	k8s.io/apimachinery v0.18.6

go.sum

@@ -983,8 +983,9 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

licenses.xml

@@ -1,9 +1,10 @@
+LicenseFinder::GoModules: is active
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <licenseSummary>
   <dependencies>
           <dependency>
         <packageName>github.com/3scale/3scale-porta-go-client</packageName>
-        <version>v0.0.4</version>
+        <version>v0.6.0</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -33,7 +34,7 @@
       </dependency>
           <dependency>
         <packageName>github.com/getkin/kin-openapi</packageName>
-        <version>v0.22.1</version>
+        <version>v0.94.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -156,8 +157,8 @@
         <version>v1.5.1</version>
         <licenses>
                       <license>
-              <name>unknown</name>
-              <url></url>
+              <name>Apache 2.0</name>
+              <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
             </license>
                   </licenses>
       </dependency>
@@ -190,6 +191,16 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
+      </dependency>
+          <dependency>
+        <packageName>golang.org/x/mod</packageName>
+        <version>v0.5.1</version>
+        <licenses>
+                      <license>
+              <name>3-clause BSD License</name>
+              <url></url>
+            </license>
+                  </licenses>
       </dependency>
           <dependency>
         <packageName>gopkg.in/yaml.v2</packageName>
@@ -233,7 +244,7 @@
       </dependency>
           <dependency>
         <packageName>sigs.k8s.io/controller-runtime</packageName>
-        <version>v0.6.2</version>
+        <version>v0.6.3</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>

main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/getkin/kin-openapi/openapi3"
 	grafanav1alpha1 "github.com/integr8ly/grafana-operator/v3/pkg/apis/integreatly/v1alpha1"
 	appsv1 "github.com/openshift/api/apps/v1"
+	configv1 "github.com/openshift/api/config/v1"
 	consolev1 "github.com/openshift/api/console/v1"
 	imagev1 "github.com/openshift/api/image/v1"
 	routev1 "github.com/openshift/api/route/v1"
@@ -74,6 +75,7 @@ func init() {
 	utilruntime.Must(appsv1.AddToScheme(scheme))
 	utilruntime.Must(monitoringv1.AddToScheme(scheme))
 	utilruntime.Must(grafanav1alpha1.AddToScheme(scheme))
+	utilruntime.Must(configv1.AddToScheme(scheme))
 
 	// +kubebuilder:scaffold:scheme
 }

pkg/3scale/amp/cmd/prometheusrules.go

@@ -24,6 +24,9 @@ import (
 
 var prometheusRulesNamespace string
 
+// Compatibility with Openshift <4.9
+var compatPre49 bool
+
 // templateCmd represents the template command
 var prometheusRulesCmd = &cobra.Command{
 	Use:   getPrometheusRulesUsage(),
@@ -62,7 +65,7 @@ func runPrometheusRulesCommand(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Factory %s not found", prName)
 	}
 
-	prometheusRulesObj := factory.PrometheusRule(prometheusRulesNamespace)
+	prometheusRulesObj := factory.PrometheusRule(compatPre49, prometheusRulesNamespace)
 
 	serializer := json.NewSerializerWithOptions(json.DefaultMetaFactory, nil, nil,
 		json.SerializerOptions{Yaml: true, Pretty: true, Strict: true})
@@ -71,6 +74,7 @@ func runPrometheusRulesCommand(cmd *cobra.Command, args []string) error {
 
 func init() {
 	prometheusRulesCmd.PersistentFlags().StringVar(&prometheusRulesNamespace, "namespace", "", "Namespace to be used when generating the prometheus rules")
+	prometheusRulesCmd.PersistentFlags().BoolVar(&compatPre49, "compat", false, "Generate rules compatible with Openshift releases prior to 4.9")
 	prometheusRulesCmd.MarkFlagRequired("namespace")
 	rootCmd.AddCommand(prometheusRulesCmd)
 }

pkg/3scale/amp/component/apicast_monitoring.go

@@ -51,11 +51,11 @@ func (apicast *Apicast) ApicastStagingPodMonitor() *monitoringv1.PodMonitor {
 	}
 }
 
-func (apicast *Apicast) ApicastMainAppGrafanaDashboard() *grafanav1alpha1.GrafanaDashboard {
+func (apicast *Apicast) ApicastMainAppGrafanaDashboard(sumRate string) *grafanav1alpha1.GrafanaDashboard {
 	data := &struct {
-		Namespace string
+		Namespace, SumRate string
 	}{
-		apicast.Options.Namespace,
+		apicast.Options.Namespace, sumRate,
 	}
 
 	return &grafanav1alpha1.GrafanaDashboard{
@@ -70,11 +70,11 @@ func (apicast *Apicast) ApicastMainAppGrafanaDashboard() *grafanav1alpha1.Grafan
 	}
 }
 
-func (apicast *Apicast) ApicastServicesGrafanaDashboard() *grafanav1alpha1.GrafanaDashboard {
+func (apicast *Apicast) ApicastServicesGrafanaDashboard(sumRate string) *grafanav1alpha1.GrafanaDashboard {
 	data := &struct {
-		Namespace string
+		Namespace, SumRate string
 	}{
-		apicast.Options.Namespace,
+		apicast.Options.Namespace, sumRate,
 	}
 	return &grafanav1alpha1.GrafanaDashboard{
 		ObjectMeta: metav1.ObjectMeta{

pkg/3scale/amp/component/backend_monitoring.go

@@ -50,11 +50,11 @@ func (backend *Backend) BackendWorkerPodMonitor() *monitoringv1.PodMonitor {
 	}
 }
 
-func (backend *Backend) BackendGrafanaDashboard() *grafanav1alpha1.GrafanaDashboard {
+func (backend *Backend) BackendGrafanaDashboard(sumRate string) *grafanav1alpha1.GrafanaDashboard {
 	data := &struct {
-		Namespace string
+		Namespace, SumRate string
 	}{
-		backend.Options.Namespace,
+		backend.Options.Namespace, sumRate,
 	}
 	return &grafanav1alpha1.GrafanaDashboard{
 		ObjectMeta: metav1.ObjectMeta{

pkg/3scale/amp/component/generic_monitoring.go

@@ -11,11 +11,11 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func KubernetesResourcesByNamespaceGrafanaDashboard(ns string, appLabel string) *grafanav1alpha1.GrafanaDashboard {
+func KubernetesResourcesByNamespaceGrafanaDashboard(sumRate, ns, appLabel string) *grafanav1alpha1.GrafanaDashboard {
 	data := &struct {
-		Namespace string
+		Namespace, SumRate string
 	}{
-		ns,
+		ns, sumRate,
 	}
 	return &grafanav1alpha1.GrafanaDashboard{
 		ObjectMeta: metav1.ObjectMeta{
@@ -32,11 +32,11 @@ func KubernetesResourcesByNamespaceGrafanaDashboard(ns string, appLabel string)
 	}
 }
 
-func KubernetesResourcesByPodGrafanaDashboard(ns string, appLabel string) *grafanav1alpha1.GrafanaDashboard {
+func KubernetesResourcesByPodGrafanaDashboard(sumRate, ns, appLabel string) *grafanav1alpha1.GrafanaDashboard {
 	data := &struct {
-		Namespace string
+		Namespace, SumRate string
 	}{
-		ns,
+		ns, sumRate,
 	}
 	return &grafanav1alpha1.GrafanaDashboard{
 		ObjectMeta: metav1.ObjectMeta{
@@ -53,7 +53,7 @@ func KubernetesResourcesByPodGrafanaDashboard(ns string, appLabel string) *grafa
 	}
 }
 
-func KubeStateMetricsPrometheusRules(ns string, appLabel string) *monitoringv1.PrometheusRule {
+func KubeStateMetricsPrometheusRules(sumRate, ns, appLabel string) *monitoringv1.PrometheusRule {
 	return &monitoringv1.PrometheusRule{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "threescale-kube-state-metrics",
@@ -117,7 +117,7 @@ func KubeStateMetricsPrometheusRules(ns string, appLabel string) *monitoringv1.P
 							Annotations: map[string]string{
 								"message": `Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container }} has High CPU usage for longer than 15 minutes.`,
 							},
-							Expr: intstr.FromString(fmt.Sprintf(`sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace="%s",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace="%s",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) * 100 > 90`, ns, ns)),
+							Expr: intstr.FromString(fmt.Sprintf(`sum(node_namespace_pod_container:container_cpu_usage_seconds_total:%s{namespace="%s",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace="%s",pod=~"(apicast-.*|backend-.*|system-.*|zync-.*)"}) by (namespace, container, pod) * 100 > 90`, sumRate, ns, ns)),
 							For:  "15m",
 							Labels: map[string]string{
 								"severity": "warning",

pkg/3scale/amp/component/system_monitoring.go

@@ -78,11 +78,11 @@ func (system *System) SystemAppPodMonitor() *monitoringv1.PodMonitor {
 	}
 }
 
-func (system *System) SystemGrafanaDashboard() *grafanav1alpha1.GrafanaDashboard {
+func (system *System) SystemGrafanaDashboard(sumRate string) *grafanav1alpha1.GrafanaDashboard {
 	data := &struct {
-		Namespace string
+		Namespace, SumRate string
 	}{
-		system.Options.Namespace,
+		system.Options.Namespace, sumRate,
 	}
 	return &grafanav1alpha1.GrafanaDashboard{
 		ObjectMeta: metav1.ObjectMeta{

pkg/3scale/amp/component/zync_monitoring.go

@@ -51,11 +51,11 @@ func (zync *Zync) ZyncQuePodMonitor() *monitoringv1.PodMonitor {
 	}
 }
 
-func (zync *Zync) ZyncGrafanaDashboard() *grafanav1alpha1.GrafanaDashboard {
+func (zync *Zync) ZyncGrafanaDashboard(sumRate string) *grafanav1alpha1.GrafanaDashboard {
 	data := &struct {
-		Namespace string
+		Namespace, SumRate string
 	}{
-		zync.Options.Namespace,
+		zync.Options.Namespace, sumRate,
 	}
 	return &grafanav1alpha1.GrafanaDashboard{
 		ObjectMeta: metav1.ObjectMeta{

pkg/3scale/amp/operator/apicast_reconciler.go

@@ -139,12 +139,17 @@ func (r *ApicastReconciler) Reconcile() (reconcile.Result, error) {
 		return reconcile.Result{}, err
 	}
 
-	err = r.ReconcileGrafanaDashboard(apicast.ApicastMainAppGrafanaDashboard(), reconcilers.GenericGrafanaDashboardsMutator)
+	sumRate, err := helper.SumRateForOpenshiftVersion(r.Context(), r.Client())
 	if err != nil {
 		return reconcile.Result{}, err
 	}
 
-	err = r.ReconcileGrafanaDashboard(apicast.ApicastServicesGrafanaDashboard(), reconcilers.GenericGrafanaDashboardsMutator)
+	err = r.ReconcileGrafanaDashboard(apicast.ApicastMainAppGrafanaDashboard(sumRate), reconcilers.GenericGrafanaDashboardsMutator)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	err = r.ReconcileGrafanaDashboard(apicast.ApicastServicesGrafanaDashboard(sumRate), reconcilers.GenericGrafanaDashboardsMutator)
 	if err != nil {
 		return reconcile.Result{}, err
 	}