Monitoring resources

Makefile

@@ -23,7 +23,7 @@ help: Makefile
 
 ## vendor: Populate vendor directory
 vendor:
-	@GO111MODULE=on $(GO) mod vendor
+	$(GO) mod vendor
 
 IMAGE ?= quay.io/3scale/3scale-operator
 SOURCE_VERSION ?= master
@@ -39,6 +39,16 @@ download:
 	@echo Download go.mod dependencies
 	@go mod download
 
+## install-tools: Installing tools from tools.go
+install-tools: download
+	@echo Installing tools from tools.go
+	@cat tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install %
+
+## assets: Generate embedded assets
+assets: install-tools
+	@echo Generate Go embedded assets files by processing source
+	$(GO) generate github.com/3scale/3scale-operator/pkg/assets
+
 ## build: Build operator
 build:
 	$(OPERATOR_SDK) build $(IMAGE):$(VERSION)
@@ -111,7 +121,7 @@ endif
 	cd $(PROJECT_PATH)/deploy/olm-catalog && operator-courier verify --ui_validate_io 3scale-operator-master/
 
 ## licenses.xml: Generate licenses.xml file
-licenses.xml:
+licenses.xml: $(DEPENDENCY_DECISION_FILE)
 ifndef LICENSEFINDERBINARY
 	$(error "license-finder is not available please install: gem install license_finder --version 5.7.1")
 endif

cmd/manager/main.go

@@ -14,10 +14,13 @@ import (
 
 	"github.com/3scale/3scale-operator/pkg/3scale/amp/product"
 	"github.com/3scale/3scale-operator/pkg/apis"
+	appsv1alpha1 "github.com/3scale/3scale-operator/pkg/apis/apps/v1alpha1"
+	"github.com/3scale/3scale-operator/pkg/common"
 	"github.com/3scale/3scale-operator/pkg/controller"
 	"github.com/3scale/3scale-operator/version"
-	"github.com/prometheus/client_golang/prometheus"
 
+	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+	grafanav1alpha1 "github.com/integr8ly/grafana-operator/v3/pkg/apis/integreatly/v1alpha1"
 	appsv1 "github.com/openshift/api/apps/v1"
 	imagev1 "github.com/openshift/api/image/v1"
 	routev1 "github.com/openshift/api/route/v1"
@@ -27,10 +30,12 @@ import (
 	"github.com/operator-framework/operator-sdk/pkg/log/zap"
 	"github.com/operator-framework/operator-sdk/pkg/metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/pflag"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -127,6 +132,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Setup Scheme for all monitoring resources
+	if err := monitoringv1.AddToScheme(mgr.GetScheme()); err != nil {
+		log.Error(err, "")
+		os.Exit(1)
+	}
+
+	// Setup Scheme for all grafana resources
+	if err := grafanav1alpha1.AddToScheme(mgr.GetScheme()); err != nil {
+		log.Error(err, "")
+		os.Exit(1)
+	}
+
 	// Setup Scheme for all resources
 	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
 		log.Error(err, "")
@@ -191,6 +208,12 @@ func addMetrics(ctx context.Context, cfg *rest.Config, namespace string) {
 		log.Info("Could not create metrics Service", "error", err.Error())
 	}
 
+	// Adding the monitoring-key:middleware to the operator service which will get propagated to the serviceMonitor
+	err = addMonitoringKeyLabelToOperatorService(ctx, cfg, service)
+	if err != nil {
+		log.Error(err, "Could not add monitoring-key label to operator metrics Service")
+	}
+
 	// CreateServiceMonitors will automatically create the prometheus-operator ServiceMonitor resources
 	// necessary to configure Prometheus to scrape metrics from this operator.
 	services := []*v1.Service{service}
@@ -298,3 +321,30 @@ func filterGKVsFromAddToScheme(gvks []schema.GroupVersionKind) []schema.GroupVer
 
 	return ownGVKs
 }
+
+func addMonitoringKeyLabelToOperatorService(ctx context.Context, cfg *rest.Config, service *v1.Service) error {
+	if service == nil {
+		return fmt.Errorf("service doesn't exist")
+	}
+
+	kclient, err := client.New(cfg, client.Options{})
+	if err != nil {
+		return err
+	}
+
+	updatedLabels := map[string]string{
+		"monitoring-key": common.MonitoringKey,
+		"app":            appsv1alpha1.Default3scaleAppLabel,
+	}
+	for k, v := range service.ObjectMeta.Labels {
+		updatedLabels[k] = v
+	}
+	service.ObjectMeta.Labels = updatedLabels
+
+	err = kclient.Update(ctx, service)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

deploy/crds/apps.3scale.net_apimanagers_crd.yaml

@@ -4068,6 +4068,11 @@ spec:
               type: object
             imageStreamTagImportInsecure:
               type: boolean
+            monitoring:
+              properties:
+                enabled:
+                  type: boolean
+              type: object
             podDisruptionBudget:
               properties:
                 enabled:

deploy/crds/apps.3scale.net_v1alpha1_apimanager_cr_monitoring.yaml

@@ -0,0 +1,8 @@
+apiVersion: apps.3scale.net/v1alpha1
+kind: APIManager
+metadata:
+  name: example-apimanager-monitoring
+spec:
+  wildcardDomain: example.com
+  monitoring:
+    enabled: true

deploy/olm-catalog/3scale-operator-master/0.0.0/3scale-operator-master.v0.0.0.clusterserviceversion.yaml

@@ -62,6 +62,19 @@ metadata:
             "wildcardDomain": "example.com"
           }
         },
+        {
+          "apiVersion": "apps.3scale.net/v1alpha1",
+          "kind": "APIManager",
+          "metadata": {
+            "name": "example-apimanager-monitoring"
+          },
+          "spec": {
+            "monitoring": {
+              "enabled": true
+            },
+            "wildcardDomain": "example.com"
+          }
+        },
         {
           "apiVersion": "apps.3scale.net/v1alpha1",
           "kind": "APIManager",
@@ -520,13 +533,6 @@ spec:
           - patch
           - update
           - watch
-        - apiGroups:
-          - monitoring.coreos.com
-          resources:
-          - servicemonitors
-          verbs:
-          - get
-          - create
         - apiGroups:
           - apps
           resourceNames:
@@ -662,6 +668,27 @@ spec:
           - update
           - watch
           - delete
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          - prometheusrules
+          verbs:
+          - list
+          - get
+          - create
+          - update
+          - watch
+        - apiGroups:
+          - integreatly.org
+          resources:
+          - grafanadashboards
+          verbs:
+          - get
+          - list
+          - create
+          - update
+          - watch
         serviceAccountName: 3scale-operator
     strategy: deployment
   installModes:

deploy/role.yaml

@@ -41,13 +41,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - monitoring.coreos.com
-  resources:
-  - servicemonitors
-  verbs:
-  - get
-  - create
 - apiGroups:
   - apps
   resourceNames:
@@ -183,3 +176,24 @@ rules:
   - update
   - watch
   - delete
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  - prometheusrules
+  verbs:
+  - list
+  - get
+  - create
+  - update
+  - watch
+- apiGroups:
+  - integreatly.org
+  resources:
+  - grafanadashboards
+  verbs:
+  - get
+  - list
+  - create
+  - update
+  - watch

doc/apimanager-reference.md

@@ -32,6 +32,7 @@ This resource is the resource used to deploy a 3scale API Management solution.
 | ZyncSpec    | `zync`    | \*ZyncSpec    | No | See [ZyncSpec](#ZyncSpec) reference | Spec of the Zync part    |
 | HighAvailabilitySpec | `highAvailability` | \*HighAvailabilitySpec | No | See [HighAvailabilitySpec](#HighAvailabilitySpec) reference | Spec of the HighAvailability part |
 | PodDisruptionBudgetSpec | `podDisruptionBudget` | \*PodDisruptionBudgetSpec | No | See [PodDisruptionBudgetSpec](#PodDisruptionBudgetSpec) reference | Spec of the PodDisruptionBudgetSpec part |
+| MonitoringSpec | `monitoring` | \*MonitoringSpec | No | Disabled | [MonitoringSpec](#MonitoringSpec) reference |
 
 #### ApicastSpec
 
@@ -279,6 +280,11 @@ pre-created by the user:
 | --- | --- | --- | --- | --- | --- |
 | Enabled | `enabled` | bool | No | `false` | Enable to automatically create [PodDisruptionBudgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) for components that can scale. Not including any of the databases or redis services.|
 
+#### MonitoringSpec
+
+| **Field** | **json/yaml field**| **Type** | **Required** | **Default value** | **Description** |
+| --- | --- | --- | --- | --- | --- |
+| Enabled | `enabled` | bool | No | `false` | [Enable to automatically create monitoring resources](operator-monitoring-resources.md) |
 
 #### APIManagerStatus
 

doc/dependency_decisions.yml

@@ -179,7 +179,20 @@
 - - :license
   - github.com/evanphx/json-patch
   - 3-clause BSD License
-  - :who:
+  - :who: 
     :why: 3-clause BSD License https://github.com/evanphx/json-patch/blob/master/LICENSE
     :versions: []
     :when: 2019-10-24 09:56:58.733603332 Z
+- - :whitelist
+  - CC0 1.0 Universal
+  - :who: Jeff Kaufmann and Richard Fontana (Red Hat Legal)
+    :why: 
+    :versions: []
+    :when: 2019-11-26 10:15:34.001520959 Z
+- - :license
+  - github.com/go-bindata/go-bindata
+  - CC0 1.0 Universal
+  - :who: 
+    :why: https://github.com/go-bindata/go-bindata/blob/master/LICENSE
+    :versions: []
+    :when: 2020-06-15 09:58:31.161619176 Z

doc/operator-monitoring-resources.md

@@ -0,0 +1,51 @@
+# 3scale Monitoring Resources
+
+The 3scale monitoring resources are (optionally) installed when 3scale is installed on Openshift using the 3scale Operator.
+
+## Prerequirements
+
+* [prometheus-operator](https://github.com/coreos/prometheus-operator/tree/master/contrib/kube-prometheus#quickstart) needs to be deployed in the cluster.
+
+The prometheus operator is an operator that creates, configures, and manages Prometheus clusters atop Kubernetes. It provides `ServiceMonitor` and `PrometheusRule` custom resources required by 3scale monitoring.
+
+* [grafana-operator](https://github.com/coreos/prometheus-operator/tree/master/contrib/kube-prometheus#quickstart) needs to be deployed in the cluster.
+
+The grafana operator is an operator for creating and managing Grafana instances. It provides `GrafanaDashboard` custom resources required by 3scale monitoring.
+
+## Enabling 3scale monitoring
+
+3scale monitoring is disabled by default. It can be enabled by setting monitoring to `true` in the [APIManager CR](apimanager-reference.md).
+
+```
+apiVersion: apps.3scale.net/v1alpha1
+kind: APIManager
+metadata:
+  name: apimanager1
+spec:
+  wildcardDomain: example.com
+  monitoring:
+    enabled: true
+```
+
+## Monitored components
+
+* Kubernetes resources at namespace level where 3scale is installed
+* Apicast Staging
+* Apicast Production
+* 3scale Backend worker
+* 3scale Backend listener
+* System sidekiq
+* Zync
+* Zync-que
+
+## Exposing monitoring resources
+
+3scale operator created monitoring resources, i.e. `ServiceMonitor`, `PrometheusRule` and `ServiceMonitor`, will all be labeled with
+
+```
+monitoring-key: middleware
+```
+
+Make sure the prometheus services and grafana services created by respective operators are configured to monitor resources with that label.
+
+Depending on the prometheus and grafana service configuration, the namespace where 3scale is installed might require labels too. Check your monitoring provider configuration like grafana and prometheus servers.

doc/operator-user-guide.md

@@ -12,6 +12,7 @@
     * [PostgreSQL Installation](#postgresql-installation)
     * [Enabling Pod Disruption Budgets](#enabling-pod-disruption-budgets)
     * [Setting custom affinity and tolerations](#setting-custom-affinity-and-tolerations)
+    * [Enabling monitoring resources](operator-monitoring-resources.md)
 * [Reconciliation](#reconciliation)
 * [Upgrading 3scale](#upgrading-3scale)
 * [Feature Operator (in *TechPreview*)](operator-capabilities.md)

go.mod

@@ -4,12 +4,16 @@ go 1.13
 
 require (
 	github.com/3scale/3scale-porta-go-client v0.0.3
+	github.com/Azure/go-autorest v12.2.0+incompatible
 	github.com/RHsyseng/operator-utils v0.0.0-20200204194854-c5b0d8533458
+	github.com/coreos/prometheus-operator v0.34.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
+	github.com/go-bindata/go-bindata v3.1.1+incompatible
 	github.com/go-logr/logr v0.1.0
 	github.com/go-openapi/spec v0.19.4
 	github.com/go-playground/validator/v10 v10.2.0
 	github.com/google/go-cmp v0.3.1
+	github.com/integr8ly/grafana-operator/v3 v3.1.0
 	github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/openshift/api v3.9.1-0.20190924102528-32369d4db2ad+incompatible