Merge branch 'master' into linasn/bootstrap-out-of-retention-blocks-fix

* master: [dtest] endpoint to fetch tagged (#3138) Refactor FetchTagged to return an Iterator of results (#3141) [dbnode] Add aggregate term limit regression test (#3135) [DOCS] Adding Prometheus steps to quickstart (#3043) [dbnode] Revert AggregateQuery changes (#3133) Fix TestSessionFetchIDs flaky test (#3132) [dbnode] Alter multi-segments builder to order by size before processing (#3128) [dbnode] Emit aggregate usage metrics (#3123) [dbnode] Add Shard.OpenStreamingReader method (#3119)
m3db · Feb 1, 2021 · 8e87325 · 8e87325
2 parents 9b9102c + 54474fd
commit 8e87325
Show file tree

Hide file tree

Showing 41 changed files with 1,976 additions and 756 deletions.
diff --git a/scripts/docker-integration-tests/prometheus/test.sh b/scripts/docker-integration-tests/prometheus/test.sh
@@ -233,7 +233,7 @@ function test_query_limits_applied {
   ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
     '[[ $(curl -s -o /dev/null -w "%{http_code}" -H "M3-Limit-Max-Series: 3" -H "M3-Limit-Require-Exhaustive: true" 0.0.0.0:7201/api/v1/query?query=database_write_tagged_success) = "400" ]]'
 
-  # Test the docs limit applied when directly querying
+  # Test the default docs limit applied when directly querying
   # coordinator (docs limit set by header)
   echo "Test query docs limit with coordinator limit header"
   ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
@@ -391,46 +391,6 @@ function test_series {
     '[[ $(curl -s "0.0.0.0:7201/api/v1/series?match[]=prometheus_remote_storage_samples_total&start=-292273086-05-16T16:47:06Z&end=292277025-08-18T07:12:54.999999999Z" | jq -r ".data | length") -eq 1 ]]'
 }
 
-function test_label_query_limits_applied {
-  # Test that require exhaustive does nothing if limits are not hit
-  echo "Test label limits with require-exhaustive headers true (below limit therefore no error)"
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ $(curl -s -o /dev/null -w "%{http_code}" -H "M3-Limit-Max-Series: 10000" -H "M3-Limit-Max-Series: 10000" -H "M3-Limit-Require-Exhaustive: true" 0.0.0.0:7201/api/v1/label/__name__/values) = "200" ]]'
-
-  # the header takes precedence over the configured default series limit
-  echo "Test label series limit with coordinator limit header (default requires exhaustive so error)"
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ -n $(curl -s -H "M3-Limit-Max-Series: 1" 0.0.0.0:7201/api/v1/label/__name__/values | jq ."error" | grep "query exceeded limit") ]]'
-
-  echo "Test label series limit with require-exhaustive headers false"
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ $(curl -s -H "M3-Limit-Max-Series: 2" -H "M3-Limit-Require-Exhaustive: false" 0.0.0.0:7201/api/v1/label/__name__/values | jq -r ".data | length") -eq 1 ]]'
-
-  echo "Test label series limit with require-exhaustive headers true (above limit therefore error)"
-  # Test that require exhaustive error is returned
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ -n $(curl -s -H "M3-Limit-Max-Series: 2" -H "M3-Limit-Require-Exhaustive: true" 0.0.0.0:7201/api/v1/label/__name__/values | jq ."error" | grep "query exceeded limit") ]]'
-  # Test that require exhaustive error is 4xx
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ $(curl -s -o /dev/null -w "%{http_code}" -H "M3-Limit-Max-Series: 2" -H "M3-Limit-Require-Exhaustive: true" 0.0.0.0:7201/api/v1/label/__name__/values) = "400" ]]'
-
-  echo "Test label docs limit with coordinator limit header (default requires exhaustive so error)"
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ -n $(curl -s -H "M3-Limit-Max-Docs: 1" 0.0.0.0:7201/api/v1/label/__name__/values | jq ."error" | grep "query exceeded limit") ]]'
-
-  echo "Test label docs limit with require-exhaustive headers false"
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ $(curl -s -H "M3-Limit-Max-Docs: 2" -H "M3-Limit-Require-Exhaustive: false" 0.0.0.0:7201/api/v1/label/__name__/values | jq -r ".data | length") -eq 1 ]]'
-
- echo "Test label docs limit with require-exhaustive headers true (above limit therefore error)"
-  # Test that require exhaustive error is returned
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ -n $(curl -s -H "M3-Limit-Max-Docs: 1" -H "M3-Limit-Require-Exhaustive: true" 0.0.0.0:7201/api/v1/label/__name__/values | jq ."error" | grep "query exceeded limit") ]]'
-  # Test that require exhaustive error is 4xx
-  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff  \
-    '[[ $(curl -s -o /dev/null -w "%{http_code}" -H "M3-Limit-Max-Docs: 1" -H "M3-Limit-Require-Exhaustive: true" 0.0.0.0:7201/api/v1/label/__name__/values) = "400" ]]'
-}
-
 echo "Running readiness test"
 test_readiness
 
@@ -449,7 +409,6 @@ test_prometheus_query_native_timeout
 test_query_restrict_tags
 test_prometheus_remote_write_map_tags
 test_series
-test_label_query_limits_applied
 
 echo "Running function correctness tests"
 test_correctness

diff --git a/scripts/docker-integration-tests/simple/test.sh b/scripts/docker-integration-tests/simple/test.sh
@@ -143,24 +143,18 @@ curl -vvvsS -X POST 0.0.0.0:9003/writetagged -d '{
 }'
 
 echo "Read data"
-queryResult=$(curl -sSf -X POST 0.0.0.0:9003/query -d '{
-  "namespace": "unagg",
-  "query": {
-    "regexp": {
-      "field": "city",
-      "regexp": ".*"
-    }
-  },
-  "rangeStart": 0,
-  "rangeEnd":'"$(date +"%s")"'
-}' | jq '.results | length')
-
-if [ "$queryResult" -lt 1 ]; then
-  echo "Result not found"
-  exit 1
-else
-  echo "Result found"
-fi
+ATTEMPTS=3 TIMEOUT=1 retry_with_backoff  \
+  '[ "$(curl -sSf -X POST 0.0.0.0:9003/query -d "{
+    \"namespace\": \"unagg\",
+    \"query\": {
+      \"regexp\": {
+        \"field\": \"city\",
+        \"regexp\": \".*\"
+      }
+    },
+    \"rangeStart\": 0,
+    \"rangeEnd\":'\"$(date +\"%s\")\"'
+  }" | jq ".results | length")" == "1" ]'
 
 echo "Deleting placement"
 curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/services/m3db/placement

diff --git a/site/content/includes/prometheus.yml b/site/content/includes/prometheus.yml
@@ -0,0 +1,32 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: 'codelab-monitor'
+
+remote_read:
+  - url: "http://localhost:7201/api/v1/prom/remote/read"
+    # To test reading even when local Prometheus has the data
+    read_recent: true
+remote_write:
+  - url: "http://localhost:7201/api/v1/prom/remote/write"
+
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:8080', 'localhost:8081']
+        labels:
+          group: 'production'
+      - targets: ['localhost:8082']
+        labels:
+          group: 'canary'
+
+  - job_name: 'm3' 
+    static_configs: 
+      - targets: ['localhost:7203']
diff --git a/site/content/includes/quickstart-common-steps.md b/site/content/includes/quickstart-common-steps.md
@@ -276,34 +276,15 @@ You can write metrics using one of two endpoints:
 -   _[{{% apiendpoint %}}prom/remote/write](/docs/m3coordinator/api/remote/)_ - Write a Prometheus remote write query to M3DB with a binary snappy compressed Prometheus WriteRequest protobuf message.
 -   _{{% apiendpoint %}}json/write_ - Write a JSON payload of metrics data. This endpoint is quick for testing purposes but is not as performant for production usage.
 
-For this quickstart, use the _{{% apiendpoint %}}json/write_ endpoint to write a tagged metric to M3 with the following data in the request body, all fields are required:
+{{< tabs name="prom_http_write" >}}
+{{< tab name="Prometheus" >}}
 
--   `tags`: An object of at least one `name`/`value` pairs
--   `timestamp`: The UNIX timestamp for the data
--   `value`: The value for the data, can be of any type
-
-{{% notice tip %}}
-The examples below use `__name__` as the name for one of the tags, which is a Prometheus reserved tag that allows you to query metrics using the value of the tag to filter results.
-{{% /notice %}}
-
-{{% notice tip %}}
-Label names may contain ASCII letters, numbers, underscores, and Unicode characters. They must match the regex `[a-zA-Z_][a-zA-Z0-9_]*`. Label names beginning with `__` are reserved for internal use. [Read more in the Prometheus documentation](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels).
-{{% /notice %}}
-
-{{< tabs name="write_metrics" >}}
-{{< tab name="Command 1" >}}
-
-{{< codeinclude file="docs/includes/write-metrics-1.sh" language="shell" >}}
-
-{{< /tab >}}
-{{< tab name="Command 2" >}}
-
-{{< codeinclude file="docs/includes/write-metrics-2.sh" language="shell" >}}
+{{< fileinclude file="quickstart-prometheus-steps.md" >}}
 
 {{< /tab >}}
-{{< tab name="Command 3" >}}
+{{< tab name="HTTP API" >}}
 
-{{< codeinclude file="docs/includes/write-metrics-3.sh" language="shell" >}}
+{{< fileinclude file="quickstart-http-steps.md" >}}
 
 {{< /tab >}}
 {{< /tabs >}}
@@ -443,6 +424,95 @@ curl -X "POST" -G "{{% apiendpoint %}}query_range" \
 {{% /tab %}}
 {{< /tabs >}}
 
+#### Values collected from Prometheus
+
+If you followed the steps above for collecting metrics from Prometheus, the examples above work, but don't return any results. To query those results, use the following commands to return a sum of the values.
+
+{{< tabs name="example_promql_sum" >}}
+{{% tab name="Linux" %}}
+
+<!-- TODO: Check Linux command -->
+
+```shell
+curl -X "POST" -G "{{% apiendpoint %}}query_range" \
+  -d "query=third_avenue_sum" \
+  -d "start=$(date "+%s" -d "45 seconds ago")" \
+  -d "end=$( date +%s )" \
+  -d "step=500s" | jq .
+```
+
+{{% /tab %}}
+{{% tab name="macOS/BSD" %}}
+
+```shell
+curl -X "POST" -G "{{% apiendpoint %}}query_range" \
+  -d "query=third_avenue_sum" \
+  -d "start=$( date -v -45S +%s )" \
+  -d "end=$( date +%s )" \
+  -d "step=500s" | jq .
+```
+
+{{% /tab %}}
+{{% tab name="Output" %}}
+
+```json
+{
+  "status": "success",
+  "data": {
+    "resultType": "matrix",
+    "result": [
+      {
+        "metric": {
+          "__name__": "third_avenue_sum",
+          "group": "canary",
+          "instance": "localhost:8082",
+          "job": "node",
+          "monitor": "codelab-monitor"
+        },
+        "values": [
+          [
+            1608737991,
+            "5801.45"
+          ]
+        ]
+      },
+      {
+        "metric": {
+          "__name__": "third_avenue_sum",
+          "group": "production",
+          "instance": "localhost:8080",
+          "job": "node",
+          "monitor": "codelab-monitor"
+        },
+        "values": [
+          [
+            1608737991,
+            "5501.45"
+          ]
+        ]
+      },
+      {
+        "metric": {
+          "__name__": "third_avenue_sum",
+          "group": "production",
+          "instance": "localhost:8081",
+          "job": "node",
+          "monitor": "codelab-monitor"
+        },
+        "values": [
+          [
+            1608737991,
+            "13480.27"
+          ]
+        ]
+      }
+    ]
+  }
+}
+```
+
+{{% /tab %}}
+{{< /tabs >}}
 <!-- ## Next Steps
 
 This quickstart covered getting a single-node M3DB cluster running, and writing and querying metrics to the cluster. Some next steps are:

diff --git a/site/content/includes/quickstart-http-query-steps.md b/site/content/includes/quickstart-http-query-steps.md
@@ -0,0 +1,128 @@
+To query metrics, use the _{{% apiendpoint %}}query_range_ endpoint with the following data in the request body, all fields are required:
+
+-   `query`: A PromQL query
+-   `start`: Timestamp in `RFC3339Nano` of start range for results
+-   `end`: Timestamp in `RFC3339Nano` of end range for results
+-   `step`: A duration or float of the query resolution, the interval between results in the timespan between `start` and `end`.
+
+Below are some examples using the metrics written above.
+
+#### Return results in past 45 seconds
+
+{{< tabs name="example_promql_regex" >}}
+{{% tab name="Linux" %}}
+
+<!-- TODO: Check this on Linux -->
+
+```shell
+curl -X "POST" -G "{{% apiendpoint %}}query_range" \
+  -d "query=third_avenue" \
+  -d "start=$(date "+%s" -d "45 seconds ago")" \
+  -d "end=$( date +%s )" \
+  -d "step=5s" | jq .  
+```
+
+{{% /tab %}}
+{{% tab name="macOS/BSD" %}}
+
+```shell
+curl -X "POST" -G "{{% apiendpoint %}}query_range" \
+  -d "query=third_avenue" \
+  -d "start=$( date -v -45S +%s )" \
+  -d "end=$( date +%s )" \
+  -d "step=5s" | jq .
+```
+
+{{% /tab %}}
+{{% tab name="Output" %}}
+
+```json
+{
+  "status": "success",
+  "data": {
+    "resultType": "matrix",
+    "result": [
+      {
+        "metric": {
+          "__name__": "third_avenue",
+          "checkout": "1",
+          "city": "new_york"
+        },
+        "values": [
+          [
+            {{% now %}},
+            "3347.26"
+          ],
+          [
+            {{% now %}},
+            "5347.26"
+          ],
+          [
+            {{% now %}},
+            "7347.26"
+          ]
+        ]
+      }
+    ]
+  }
+}
+```
+
+{{% /tab %}}
+{{< /tabs >}}
+
+#### Values above a certain number
+
+{{< tabs name="example_promql_range" >}}
+{{% tab name="Linux" %}}
+
+<!-- TODO: Check Linux command -->
+
+```shell
+curl -X "POST" -G "{{% apiendpoint %}}query_range" \
+  -d "query=third_avenue > 6000" \
+  -d "start=$(date "+%s" -d "45 seconds ago")" \
+  -d "end=$( date +%s )" \
+  -d "step=5s" | jq .
+```
+
+{{% /tab %}}
+{{% tab name="macOS/BSD" %}}
+
+```shell
+curl -X "POST" -G "{{% apiendpoint %}}query_range" \
+  -d "query=third_avenue > 6000" \
+  -d "start=$(date -v -45S "+%s")" \
+  -d "end=$( date +%s )" \
+  -d "step=5s" | jq .
+```
+
+{{% /tab %}}
+{{% tab name="Output" %}}
+
+```json
+{
+  "status": "success",
+  "data": {
+    "resultType": "matrix",
+    "result": [
+      {
+        "metric": {
+          "__name__": "third_avenue",
+          "checkout": "1",
+          "city": "new_york"
+        },
+        "values": [
+          [
+            {{% now %}},
+            "7347.26"
+          ]
+        ]
+      }
+    ]
+  }
+}
+```
+
+{{% /tab %}}
+{{< /tabs >}}