Document and fix recorded metric forwarding (#266)

This should address most of #104

- fix target cache lookup for recorded metrics
- update documentation with examples

Also: allow overriding docker FROM target with DOCKER_IMAGE_BASE env var
(helpful for testing and runtime scenarios where /bin/sh is needed)

Dockerfile

@@ -1,4 +1,5 @@
-FROM  gcr.io/distroless/static:latest
+ARG DOCKER_IMAGE_BASE=gcr.io/distroless/static:latest
+FROM  $DOCKER_IMAGE_BASE
 LABEL maintainer "Stackdriver Engineering <engineering@stackdriver.com>"
 
 COPY stackdriver-prometheus-sidecar         /bin/stackdriver-prometheus-sidecar

Makefile

@@ -43,6 +43,7 @@ PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_
 PREFIX                  ?= $(shell pwd)
 BIN_DIR                 ?= $(shell pwd)
 # Private repo.
+DOCKER_IMAGE_BASE       ?= gcr.io/distroless/static:latest
 DOCKER_IMAGE_NAME       ?= gcr.io/stackdriver-prometheus/stackdriver-prometheus-sidecar
 DOCKER_IMAGE_TAG        ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
 
@@ -110,7 +111,7 @@ tarball: promu
 
 docker: build-linux-amd64
 	@echo ">> building docker image"
-	docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
+	docker build --build-arg "DOCKER_IMAGE_BASE=$(DOCKER_IMAGE_BASE)" -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
 
 push: test docker
 	@echo ">> pushing docker image"

README.md

@@ -70,7 +70,7 @@ stackdriver-prometheus-sidecar --include='{__name__!~"cadvisor_.+",job="k8s"}' .
 
 This drops all series which do not have a `job` label `k8s` and all metrics that have a name starting with `cadvisor_`.
 
-For equality filter on metric name you can use the simpler notation, e.g. `--include='metric_name{label="foo"}'`.
+For equality filter on metric name you can use the simpler notation, e.g., `--include='metric_name{label="foo"}'`.
 
 The flag may be repeated to provide several sets of filters, in which case the metric will be forwarded if it matches at least one of them. Please note that inclusion filters only apply to Prometheus metrics proxied directly, and do not apply to [aggregated counters](#counter-aggregator).
 
@@ -95,6 +95,28 @@ static_metadata:
   * All `static_metadata` entries must have `type` specified. This specifies the Stackdriver metric type and overrides the metric type chosen by the Prometheus client.
   * If `value_type` is specified, it will override the default value type for counters and gauges. All Prometheus metrics have a default type of double.
 
+#### Dealing with recording rules
+
+The default Prometheus naming format for [recording rules](https://prometheus.io/docs/practices/rules/) is `level:metric:operations`, e.g., `instance:requests_total:sum`, but colons are not allowed in Stackdriver metric descriptor names.  To forward a recorded Prometheus metric to Stackdriver, you must use the `metric_renames` feature to replace the colon characters:
+
+```yaml
+metric_renames:
+  - from: instance:requests_total:sum
+    to: recorded_instance_requests_total_sum
+```
+
+Additionally, the sidecar assumes that any recorded metrics are gauges.  If this is not the case (e.g., for a Prometheus counter metric) you will need to specify that in `static_metadata`:
+
+```yaml
+static_metadata:
+  - metric: recorded_instance_requests_total_sum
+    type: counter
+    value_type: int64
+    help: an arbitrary help string
+```
+
+*Warning:* recorded metrics _must_ have minimally an "instance" and "job" label, otherwise they will not be forwarded.
+
 #### Counter Aggregator
 
 Counter Aggregator is an advanced feature of the sidecar that can be used to export a sum of multiple Prometheus counters to Stackdriver as a single CUMULATIVE metric.

targets/cache.go

@@ -179,7 +179,7 @@ func targetMatch(targets []*Target, lset labels.Labels) (*Target, bool) {
 Outer:
 	for _, t := range targets {
 		for _, tl := range t.Labels {
-			if lset.Get(tl.Name) != tl.Value {
+			if v := lset.Get(tl.Name); v != "" && v != tl.Value {
 				continue Outer
 			}
 		}