rule: add --query.http-method

CHANGELOG.md

@@ -20,6 +20,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#3346](https://github.com/thanos-io/thanos/pull/3346) Ruler UI: Fix a bug preventing the /rules endpoint from loading.
 - [#3115](https://github.com/thanos-io/thanos/pull/3115) compact: now deletes partially uploaded and blocks with deletion marks concurrently. It does that at the beginning and then every `--compact.cleanup-interval` time period. By default it is 5 minutes.
 - [#3312](https://github.com/thanos-io/thanos/pull/3312) s3: add list_objects_version config option for compatibility.
+- [#3378](https://github.com/thanos-io/thanos/pull/3378) Ruler: learned to send queries via POST. Helps when alerting/recording rules are extra long. It now uses POST unless `--query.http-method` is `GET`.
 
 ### Fixed
 - [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors.

cmd/thanos/rule.go

@@ -116,6 +116,9 @@ func registerRule(app *extkingpin.App) {
 	dnsSDInterval := extkingpin.ModelDuration(cmd.Flag("query.sd-dns-interval", "Interval between DNS resolutions.").
 		Default("30s"))
 
+	httpMethod := cmd.Flag("query.http-method", "HTTP method to use when sending queries. Possible options: [GET, POST]").
+		Default("POST").Enum("GET", "POST")
+
 	dnsSDResolver := cmd.Flag("query.sd-dns-resolver", "Resolver to use. Possible options: [golang, miekgdns]").
 		Default("golang").Hidden().String()
 
@@ -210,6 +213,7 @@ func registerRule(app *extkingpin.App) {
 			*dnsSDResolver,
 			comp,
 			*allowOutOfOrderUpload,
+			*httpMethod,
 			getFlagsMap(cmd.Flags()),
 		)
 	})
@@ -299,6 +303,7 @@ func runRule(
 	dnsSDResolver string,
 	comp component.Component,
 	allowOutOfOrderUpload bool,
+	httpMethod string,
 	flagsMap map[string]string,
 ) error {
 	metrics := newRuleMetrics(reg)
@@ -463,7 +468,7 @@ func runRule(
 				Queryable:   db,
 				ResendDelay: resendDelay,
 			},
-			queryFuncCreator(logger, queryClients, metrics.duplicatedQuery, metrics.ruleEvalWarnings),
+			queryFuncCreator(logger, queryClients, metrics.duplicatedQuery, metrics.ruleEvalWarnings, httpMethod),
 			lset,
 		)
 
@@ -729,6 +734,7 @@ func queryFuncCreator(
 	queriers []*http_util.Client,
 	duplicatedQuery prometheus.Counter,
 	ruleEvalWarnings *prometheus.CounterVec,
+	httpMethod string,
 ) func(partialResponseStrategy storepb.PartialResponseStrategy) rules.QueryFunc {
 
 	// queryFunc returns query function that hits the HTTP query API of query peers in randomized order until we get a result
@@ -760,6 +766,7 @@ func queryFuncCreator(
 					v, warns, err := promClient.PromqlQueryInstant(ctx, endpoints[i], q, t, promclient.QueryOptions{
 						Deduplicate:             true,
 						PartialResponseStrategy: partialResponseStrategy,
+						Method:                  httpMethod,
 					})
 					span.Finish()
 

docs/components/rule.md

@@ -392,6 +392,8 @@ Flags:
                                  (used as a fallback)
       --query.sd-dns-interval=30s
                                  Interval between DNS resolutions.
+      --query.http-method=POST   HTTP method to use when sending queries.
+                                 Possible options: [GET, POST]
 
 ```
 

pkg/promclient/promclient.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -97,14 +98,26 @@ func NewWithTracingClient(logger log.Logger, userAgent string) *Client {
 	)
 }
 
-func (c *Client) get2xx(ctx context.Context, u *url.URL) (_ []byte, _ int, err error) {
-	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+// req2xx sends a request to the given url.URL. If method is http.MethodPost then
+// the raw query is encoded in the body and the appropriate Content-Type is set.
+func (c *Client) req2xx(ctx context.Context, u *url.URL, method string) (_ []byte, _ int, err error) {
+	var b io.Reader
+	if method == http.MethodPost {
+		rq := u.RawQuery
+		b = strings.NewReader(rq)
+		u.RawQuery = ""
+	}
+
+	req, err := http.NewRequest(method, u.String(), b)
 	if err != nil {
 		return nil, 0, errors.Wrap(err, "create GET request")
 	}
 	if c.userAgent != "" {
 		req.Header.Set("User-Agent", c.userAgent)
 	}
+	if method == http.MethodPost {
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	}
 
 	resp, err := c.Do(req.WithContext(ctx))
 	if err != nil {
@@ -148,7 +161,7 @@ func (c *Client) ExternalLabels(ctx context.Context, base *url.URL) (labels.Labe
 	span, ctx := tracing.StartSpan(ctx, "/prom_config HTTP[client]")
 	defer span.Finish()
 
-	body, _, err := c.get2xx(ctx, &u)
+	body, _, err := c.req2xx(ctx, &u, http.MethodGet)
 	if err != nil {
 		return nil, err
 	}
@@ -339,6 +352,7 @@ func (c *Client) Snapshot(ctx context.Context, base *url.URL, skipHead bool) (st
 type QueryOptions struct {
 	Deduplicate             bool
 	PartialResponseStrategy storepb.PartialResponseStrategy
+	Method                  string
 }
 
 func (p *QueryOptions) AddTo(values url.Values) error {
@@ -381,7 +395,12 @@ func (c *Client) QueryInstant(ctx context.Context, base *url.URL, query string,
 	span, ctx := tracing.StartSpan(ctx, "/prom_query_instant HTTP[client]")
 	defer span.Finish()
 
-	body, _, err := c.get2xx(ctx, &u)
+	method := opts.Method
+	if method == "" {
+		method = http.MethodGet
+	}
+
+	body, _, err := c.req2xx(ctx, &u, method)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "read query instant response")
 	}
@@ -483,7 +502,7 @@ func (c *Client) QueryRange(ctx context.Context, base *url.URL, query string, st
 	span, ctx := tracing.StartSpan(ctx, "/prom_query_range HTTP[client]")
 	defer span.Finish()
 
-	body, _, err := c.get2xx(ctx, &u)
+	body, _, err := c.req2xx(ctx, &u, http.MethodGet)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "read query range response")
 	}
@@ -565,7 +584,7 @@ func (c *Client) AlertmanagerAlerts(ctx context.Context, base *url.URL) ([]*mode
 	span, ctx := tracing.StartSpan(ctx, "/alertmanager_alerts HTTP[client]")
 	defer span.Finish()
 
-	body, _, err := c.get2xx(ctx, &u)
+	body, _, err := c.req2xx(ctx, &u, http.MethodGet)
 	if err != nil {
 		return nil, err
 	}
@@ -592,7 +611,7 @@ func (c *Client) get2xxResultWithGRPCErrors(ctx context.Context, spanName string
 	span, ctx := tracing.StartSpan(ctx, spanName)
 	defer span.Finish()
 
-	body, code, err := c.get2xx(ctx, u)
+	body, code, err := c.req2xx(ctx, u, http.MethodGet)
 	if err != nil {
 		if code, exists := statusToCode[code]; exists && code != 0 {
 			return status.Error(code, err.Error())