Add Stackdriver Exporter

Add Stackdriver helm config variable as well as change reporting period
to 1 minute.

Gopkg.toml

@@ -95,3 +95,7 @@
 [[constraint]]
   name = "github.com/evanphx/json-patch"
   version = "4.1.0"
+
+[[constraint]]
+  name = "contrib.go.opencensus.io/exporter/stackdriver"
+  version = "v0.8.0"

cmd/controller/main.go

@@ -49,18 +49,20 @@ import (
 )
 
 const (
-	enableMetricsFlag     = "metrics"
-	sidecarImageFlag      = "sidecar-image"
-	sidecarCPURequestFlag = "sidecar-cpu-request"
-	sidecarCPULimitFlag   = "sidecar-cpu-limit"
-	pullSidecarFlag       = "always-pull-sidecar"
-	minPortFlag           = "min-port"
-	maxPortFlag           = "max-port"
-	certFileFlag          = "cert-file"
-	keyFileFlag           = "key-file"
-	kubeconfigFlag        = "kubeconfig"
-	workers               = 2
-	defaultResync         = 30 * time.Second
+	enableStackdriverMetricsFlag = "stackdriver-exporter"
+	enablePrometheusMetricsFlag  = "prometheus-exporter"
+	projectIDFlag                = "gcp-project-id"
+	sidecarImageFlag             = "sidecar-image"
+	sidecarCPURequestFlag        = "sidecar-cpu-request"
+	sidecarCPULimitFlag          = "sidecar-cpu-limit"
+	pullSidecarFlag              = "always-pull-sidecar"
+	minPortFlag                  = "min-port"
+	maxPortFlag                  = "max-port"
+	certFileFlag                 = "cert-file"
+	keyFileFlag                  = "key-file"
+	kubeconfigFlag               = "kubeconfig"
+	workers                      = 2
+	defaultResync                = 30 * time.Second
 )
 
 var (
@@ -106,19 +108,39 @@ func main() {
 	var rs []runner
 	var health healthcheck.Handler
 
-	if ctlConf.Metrics {
+	// Stackdriver metrics
+	if ctlConf.Stackdriver {
+		sd, err := metrics.RegisterStackdriverExporter(ctlConf.GCPProjectID)
+		if err != nil {
+			logger.WithError(err).Fatal("Could not register stackdriver exporter")
+		}
+		// It is imperative to invoke flush before your main function exits
+		defer sd.Flush()
+	}
+
+	// Prometheus metrics
+	if ctlConf.PrometheusMetrics {
 		registry := prom.NewRegistry()
 		metricHandler, err := metrics.RegisterPrometheusExporter(registry)
 		if err != nil {
-			logger.WithError(err).Fatal("Could not create register prometheus exporter")
+			logger.WithError(err).Fatal("Could not register prometheus exporter")
 		}
 		server.Handle("/metrics", metricHandler)
 		health = healthcheck.NewMetricsHandler(registry, "agones")
-		rs = append(rs, metrics.NewController(kubeClient, agonesClient, agonesInformerFactory))
 	} else {
 		health = healthcheck.NewHandler()
 	}
 
+	// If we are using Prometheus only exporter we can make reporting more often,
+	// every 1 seconds, if we are using Stackdriver we would use 60 seconds reporting period,
+	// which is a requirements of Stackdriver, otherwise most of time series would be invalid for Stackdriver
+	metrics.SetReportingPeriod(ctlConf.PrometheusMetrics, ctlConf.Stackdriver)
+
+	// Add metrics controller only if we configure one of metrics exporters
+	if ctlConf.PrometheusMetrics || ctlConf.Stackdriver {
+		rs = append(rs, metrics.NewController(kubeClient, agonesClient, agonesInformerFactory))
+	}
+
 	server.Handle("/", health)
 
 	allocationMutex := &sync.Mutex{}
@@ -170,7 +192,9 @@ func parseEnvFlags() config {
 	viper.SetDefault(pullSidecarFlag, false)
 	viper.SetDefault(certFileFlag, filepath.Join(base, "certs/server.crt"))
 	viper.SetDefault(keyFileFlag, filepath.Join(base, "certs/server.key"))
-	viper.SetDefault(enableMetricsFlag, true)
+	viper.SetDefault(enablePrometheusMetricsFlag, true)
+	viper.SetDefault(enableStackdriverMetricsFlag, false)
+	viper.SetDefault(projectIDFlag, "")
 
 	pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable")
 	pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable")
@@ -181,7 +205,9 @@ func parseEnvFlags() config {
 	pflag.String(keyFileFlag, viper.GetString(keyFileFlag), "Optional. Path to the key file")
 	pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file")
 	pflag.String(kubeconfigFlag, viper.GetString(kubeconfigFlag), "Optional. kubeconfig to run the controller out of the cluster. Only use it for debugging as webhook won't works.")
-	pflag.Bool(enableMetricsFlag, viper.GetBool(enableMetricsFlag), "Flag to activate metrics of Agones. Can also use METRICS env variable.")
+	pflag.Bool(enablePrometheusMetricsFlag, viper.GetBool(enablePrometheusMetricsFlag), "Flag to activate metrics of Agones. Can also use PROMETHEUS_EXPORTER env variable.")
+	pflag.Bool(enableStackdriverMetricsFlag, viper.GetBool(enableStackdriverMetricsFlag), "Flag to activate stackdriver monitoring metrics for Agones. Can also use STACKDRIVER_EXPORTER env variable.")
+	pflag.String(projectIDFlag, viper.GetString(projectIDFlag), "GCP ProjectID used for Stackdriver, if not specified ProjectID from Application Default Credentials would be used. Can also use GCP_PROJECT_ID env variable.")
 	pflag.Parse()
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -194,7 +220,9 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(keyFileFlag))
 	runtime.Must(viper.BindEnv(certFileFlag))
 	runtime.Must(viper.BindEnv(kubeconfigFlag))
-	runtime.Must(viper.BindEnv(enableMetricsFlag))
+	runtime.Must(viper.BindEnv(enablePrometheusMetricsFlag))
+	runtime.Must(viper.BindEnv(enableStackdriverMetricsFlag))
+	runtime.Must(viper.BindEnv(projectIDFlag))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 
 	request, err := resource.ParseQuantity(viper.GetString(sidecarCPURequestFlag))
@@ -217,7 +245,9 @@ func parseEnvFlags() config {
 		KeyFile:           viper.GetString(keyFileFlag),
 		CertFile:          viper.GetString(certFileFlag),
 		KubeConfig:        viper.GetString(kubeconfigFlag),
-		Metrics:           viper.GetBool(enableMetricsFlag),
+		PrometheusMetrics: viper.GetBool(enablePrometheusMetricsFlag),
+		Stackdriver:       viper.GetBool(enableStackdriverMetricsFlag),
+		GCPProjectID:      viper.GetString(projectIDFlag),
 	}
 }
 
@@ -229,10 +259,12 @@ type config struct {
 	SidecarCPURequest resource.Quantity
 	SidecarCPULimit   resource.Quantity
 	AlwaysPullSidecar bool
-	Metrics           bool
+	PrometheusMetrics bool
+	Stackdriver       bool
 	KeyFile           string
 	CertFile          string
 	KubeConfig        string
+	GCPProjectID      string
 }
 
 // validate ensures the ctlConfig data is valid.

docs/metrics.md

@@ -19,6 +19,7 @@ Table of Contents
   - [Installation](#installation)
     - [Prometheus installation](#prometheus-installation)
     - [Grafana installation](#grafana-installation)
+    - [Stackdriver installation](#stackdriver-installation)
   - [Adding more metrics](#adding-more-metrics)
 
 ## Backend integrations
@@ -50,7 +51,10 @@ Finally include that `ServiceMonitor` in your [Prometheus instance CRD](https://
 
 ### Stackdriver
 
-We don't yet support the [OpenCensus Stackdriver exporter](https://opencensus.io/exporters/supported-exporters/go/stackdriver/) but you can still use the Prometheus Stackdriver integration by following these [instructions](https://cloud.google.com/monitoring/kubernetes-engine/prometheus).
+We support the [OpenCensus Stackdriver exporter](https://opencensus.io/exporters/supported-exporters/go/stackdriver/). 
+In order to configure it you should enable Stackdriver Monitoring API in Google Cloud Console.
+
+Also you can use the Prometheus Stackdriver integration by following these [instructions](https://cloud.google.com/monitoring/kubernetes-engine/prometheus).
 Annotations required by this integration can be activated by setting the `agones.metrics.prometheusServiceDiscovery` to true (default) via the [helm chart value](../install/helm/agones/README.md#configuration).
 
 ## Metrics available
@@ -164,6 +168,45 @@ Open a web browser to [http://127.0.0.1:3000](http://127.0.0.1:3000), you should
 
 > Makefile targets `make grafana-portforward`,`make kind-grafana-portforward` and `make minikube-grafana-portforward`.
 
+### Stackdriver installation
+
+In order to use [Stackdriver monitoring](https://app.google.stackdriver.com) you should enable Stackdriver Monitoring API on Google Cloud Console. You need to grant all necessary permissions to the users (see [Access Control Guide](https://cloud.google.com/monitoring/access-control)). Stackdriver exporter uses a strategy called Application Default Credentials (ADC) to find your application's credentials. Details could be found here [Setting Up Authentication for Server to Server Production Applications](https://cloud.google.com/docs/authentication/production).
+
+Default metrics exporter is Prometheus. In order to change it to Stackdriver update following helm parameters in [values file](../install/helm/agones/values.yaml):
+```
+agones:
+  metrics:
+    prometheusEnabled: false
+    prometheusServiceDiscovery: false
+    stackdriverEnabled: true
+    stackdriverProjectID: ""
+```
+
+With this configuration only Stackdriver exporter would be used instead of Prometheus exporter.
+
+Run the following command to install Agones to a cluster with an updated configuration:
+```
+make install
+```
+
+Create a Fleet or a Gameserver in order to check that connection with stackdriver API is configured properly and so that you will be able to see the metrics data.
+
+Visit [Stackdriver monitoring](https://app.google.stackdriver.com) website, select your project, or choose `Create a new Workspace` and select GCP project where your cluster resides. In [Stackdriver metrics explorer](https://cloud.google.com/monitoring/charts/metrics-explorer) you should be able to find new metrics with prefix `agones/` (resource type is `Global`) after a couple of minutes. Choose the metrics you are interested in and add to a single or separate graphs. You can create multiple graphs, save them into your dashboard and use various aggregation parameters and reducers for each graph.
+
+Example of the dashboard appearance is provided below:
+
+![](stackdriver-metrics-dashboard.png)
+
+Currently there exists only manual way of configuring Stackdriver Dashboard. So it is up to you to set Alignment Period (minimal is 1 minute), group by parameter where you choose Metric label, filter parameter, etc.
+
+#### Troubleshooting
+If you could not see agones metrics please check the controller logs for connection errors. Check also that Stackdriver Monitoring API is enabled and user has all necessary permissions to access Stackdriver Monitoring, configure `stackdriverProjectID` manually, if default is not working.
+For example if wrong ProjectID is configured you would see such an error, same error will occure with Permissions is not configured in GCP:
+```
+Failed to export to Stackdriver: rpc error: code = PermissionDenied desc = Permission monitoring.metricDescriptors.create denied (or the resource may not exist).
+```
+
+
 ## Adding more metrics
 
 If you want to contribute and add more metrics we recommend to use shared informers (cache) as it is currently implemented in the [metrics controller](../pkg/metrics/controller.go). Using shared informers allows to keep metrics code in one place and doesn't overload the Kubernetes API.

install/helm/agones/README.md

@@ -86,8 +86,10 @@ The following tables lists the configurable parameters of the Agones chart and t
 | `agones.rbacEnabled`                                | Creates RBAC resources. Must be set for any cluster configured with RBAC                        | `true`                 |
 | `agones.crds.install`                               | Install the CRDs with this chart. Useful to disable if you want to subchart (since crd-install hook is broken), so you can copy the CRDs into your own chart. | `true` |
 | `agones.crds.cleanupOnDelete`                       | Run the pre-delete hook to delete all GameServers and their backing Pods when deleting the helm chart, so that all CRDs can be removed on chart deletion | `true`          |  
-| `agones.metrics.enabled`                            | Enables controller metrics on port `8080` and path `/metrics`                                   | `true`                 |
+| `agones.metrics.prometheusEnabled`                  | Enables controller metrics on port `8080` and path `/metrics`                                   | `true`                 |
 | `agones.metrics.prometheusServiceDiscovery`         | Adds annotations for Prometheus ServiceDiscovery (and also Strackdriver)                        | `true`                 |
+| `agones.metrics.stackdriverEnabled`                 | (⚠️ **development feature**⚠️) Enables Stackdriver exporter of controller metrics              | `false`                 |
+| `agones.metrics.stackdriverProjectID`               | (⚠️ **development feature**⚠️) Project ID where current GKE cluster is deployed, if empty string project id from Application Default Credentials would be used. Used for Stackdriver.            | ``                 |
 | `agones.serviceaccount.controller`                  | Service account name for the controller                                                         | `agones-controller`    |
 | `agones.serviceaccount.sdk`                         | Service account name for the sdk                                                                | `agones-sdk`           |
 | `agones.image.registry`                             | Global image registry for all images                                                            | `gcr.io/agones-images` |

install/helm/agones/templates/controller.yaml

@@ -40,7 +40,7 @@ spec:
 {{- if .Values.agones.controller.generateTLS }}
         revision/tls-cert: {{ .Release.Revision | quote }}
 {{- end }}
-{{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.enabled) }}
+{{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.prometheusEnabled) }}
         prometheus.io/scrape: "true"
         prometheus.io/port: {{ .Values.agones.controller.http.port | quote }}
         prometheus.io/path: "/metrics"
@@ -81,8 +81,12 @@ spec:
           value: {{ .Values.agones.image.sdk.alwaysPull | quote }}
         - name: SIDECAR_CPU_REQUEST
           value: {{ .Values.agones.image.sdk.cpuRequest | quote }}
-        - name: METRICS
-          value: {{ .Values.agones.metrics.enabled | quote }}
+        - name: PROMETHEUS_EXPORTER
+          value: {{ .Values.agones.metrics.prometheusEnabled | quote }}
+        - name: STACKDRIVER_EXPORTER
+          value: {{ .Values.agones.metrics.stackdriverEnabled | quote }}
+        - name: GCP_PROJECT_ID
+          value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
         - name: SIDECAR_CPU_LIMIT
           value: {{ .Values.agones.image.sdk.cpuLimit | quote }}
         livenessProbe:

install/helm/agones/values.yaml

@@ -16,8 +16,10 @@
 
 agones:
   metrics:
-    enabled: true
+    prometheusEnabled: true
     prometheusServiceDiscovery: true
+    stackdriverEnabled: false
+    stackdriverProjectID: ""
   rbacEnabled: true
   crds:
     install: true

install/yaml/install.yaml

@@ -1031,8 +1031,12 @@ spec:
           value: "false"
         - name: SIDECAR_CPU_REQUEST
           value: "30m"
-        - name: METRICS
+        - name: PROMETHEUS_EXPORTER
           value: "true"
+        - name: STACKDRIVER_EXPORTER
+          value: "false"
+        - name: GCP_PROJECT_ID
+          value: ""
         - name: SIDECAR_CPU_LIMIT
           value: "0"
         livenessProbe:

pkg/metrics/exporter.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"time"
 
+	"contrib.go.opencensus.io/exporter/stackdriver"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"go.opencensus.io/exporter/prometheus"
 	"go.opencensus.io/stats/view"
@@ -41,7 +42,40 @@ func RegisterPrometheusExporter(registry *prom.Registry) (http.Handler, error) {
 		return nil, err
 	}
 	view.RegisterExporter(pe)
-	// since we're using prometheus we can report faster as we're only exposing metrics in memory
-	view.SetReportingPeriod(1 * time.Second)
+
 	return pe, nil
 }
+
+// RegisterStackdriverExporter register a Stackdriver exporter to OpenCensus.
+// It will add Agones metrics into Stackdriver on Google Cloud.
+func RegisterStackdriverExporter(projectID string) (sd *stackdriver.Exporter, err error) {
+	// Default project will be used
+	sd, err = stackdriver.NewExporter(stackdriver.Options{
+		ProjectID: projectID,
+		// MetricPrefix helps uniquely identify your metrics.
+		MetricPrefix: "agones",
+	})
+	if err != nil {
+		return
+	}
+
+	// Register it as a metrics exporter
+	view.RegisterExporter(sd)
+	return
+}
+
+// SetReportingPeriod set appropriate reporting period which depends on exporters
+// we are going to use
+func SetReportingPeriod(prometheus, stackdriver bool) {
+	// if we're using only prometheus we can report faster as we're only exposing metrics in memory
+	reportingPeriod := 1 * time.Second
+	if stackdriver {
+		// There is a limitation on Stackdriver that reporting should
+		// be equal or more than 1 minute
+		reportingPeriod = 60 * time.Second
+	}
+
+	if stackdriver || prometheus {
+		view.SetReportingPeriod(reportingPeriod)
+	}
+}