From fc8829c9a7e4855ffe0fb16340f64ba2a6cd9ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Thu, 29 Oct 2020 14:43:28 +0200 Subject: [PATCH 1/2] rule: add --query.http-method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new parameter which let's you specify what kind of HTTP method to use when sending queries. Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 1 + cmd/thanos/rule.go | 9 ++++++++- docs/components/rule.md | 2 ++ pkg/promclient/promclient.go | 33 ++++++++++++++++++++++++++------- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45a2c90e53..4db6ee9dba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#3346](https://github.com/thanos-io/thanos/pull/3346) Ruler UI: Fix a bug preventing the /rules endpoint from loading. - [#3115](https://github.com/thanos-io/thanos/pull/3115) compact: now deletes partially uploaded and blocks with deletion marks concurrently. It does that at the beginning and then every `--compact.cleanup-interval` time period. By default it is 5 minutes. - [#3312](https://github.com/thanos-io/thanos/pull/3312) s3: add list_objects_version config option for compatibility. +- [#3378](https://github.com/thanos-io/thanos/pull/3378) Ruler: learned to send queries via POST. Helps when alerting/recording rules are extra long. It now uses POST unless `--query.http-method` is `GET`. ### Fixed - [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors. diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index fd42daf6f9..224bd28724 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -116,6 +116,9 @@ func registerRule(app *extkingpin.App) { dnsSDInterval := extkingpin.ModelDuration(cmd.Flag("query.sd-dns-interval", "Interval between DNS resolutions."). Default("30s")) + httpMethod := cmd.Flag("query.http-method", "HTTP method to use when sending queries. Possible options: [GET, POST]"). + Default("POST").Enum("GET", "POST") + dnsSDResolver := cmd.Flag("query.sd-dns-resolver", "Resolver to use. Possible options: [golang, miekgdns]"). Default("golang").Hidden().String() @@ -210,6 +213,7 @@ func registerRule(app *extkingpin.App) { *dnsSDResolver, comp, *allowOutOfOrderUpload, + *httpMethod, getFlagsMap(cmd.Flags()), ) }) @@ -299,6 +303,7 @@ func runRule( dnsSDResolver string, comp component.Component, allowOutOfOrderUpload bool, + httpMethod string, flagsMap map[string]string, ) error { metrics := newRuleMetrics(reg) @@ -463,7 +468,7 @@ func runRule( Queryable: db, ResendDelay: resendDelay, }, - queryFuncCreator(logger, queryClients, metrics.duplicatedQuery, metrics.ruleEvalWarnings), + queryFuncCreator(logger, queryClients, metrics.duplicatedQuery, metrics.ruleEvalWarnings, httpMethod), lset, ) @@ -729,6 +734,7 @@ func queryFuncCreator( queriers []*http_util.Client, duplicatedQuery prometheus.Counter, ruleEvalWarnings *prometheus.CounterVec, + httpMethod string, ) func(partialResponseStrategy storepb.PartialResponseStrategy) rules.QueryFunc { // queryFunc returns query function that hits the HTTP query API of query peers in randomized order until we get a result @@ -760,6 +766,7 @@ func queryFuncCreator( v, warns, err := promClient.PromqlQueryInstant(ctx, endpoints[i], q, t, promclient.QueryOptions{ Deduplicate: true, PartialResponseStrategy: partialResponseStrategy, + Method: httpMethod, }) span.Finish() diff --git a/docs/components/rule.md b/docs/components/rule.md index 93449a1135..62fda3978c 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -392,6 +392,8 @@ Flags: (used as a fallback) --query.sd-dns-interval=30s Interval between DNS resolutions. + --query.http-method=POST HTTP method to use when sending queries. + Possible options: [GET, POST] ``` diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index ab6c15c6f8..c20e5b162b 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "net/url" @@ -97,14 +98,26 @@ func NewWithTracingClient(logger log.Logger, userAgent string) *Client { ) } -func (c *Client) get2xx(ctx context.Context, u *url.URL) (_ []byte, _ int, err error) { - req, err := http.NewRequest(http.MethodGet, u.String(), nil) +// req2xx sends a request to the given url.URL. If method is http.MethodPost then +// the raw query is encoded in the body and the appropriate Content-Type is set. +func (c *Client) req2xx(ctx context.Context, u *url.URL, method string) (_ []byte, _ int, err error) { + var b io.Reader + if method == http.MethodPost { + rq := u.RawQuery + b = strings.NewReader(rq) + u.RawQuery = "" + } + + req, err := http.NewRequest(method, u.String(), b) if err != nil { return nil, 0, errors.Wrap(err, "create GET request") } if c.userAgent != "" { req.Header.Set("User-Agent", c.userAgent) } + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } resp, err := c.Do(req.WithContext(ctx)) if err != nil { @@ -148,7 +161,7 @@ func (c *Client) ExternalLabels(ctx context.Context, base *url.URL) (labels.Labe span, ctx := tracing.StartSpan(ctx, "/prom_config HTTP[client]") defer span.Finish() - body, _, err := c.get2xx(ctx, &u) + body, _, err := c.req2xx(ctx, &u, http.MethodGet) if err != nil { return nil, err } @@ -339,6 +352,7 @@ func (c *Client) Snapshot(ctx context.Context, base *url.URL, skipHead bool) (st type QueryOptions struct { Deduplicate bool PartialResponseStrategy storepb.PartialResponseStrategy + Method string } func (p *QueryOptions) AddTo(values url.Values) error { @@ -381,7 +395,12 @@ func (c *Client) QueryInstant(ctx context.Context, base *url.URL, query string, span, ctx := tracing.StartSpan(ctx, "/prom_query_instant HTTP[client]") defer span.Finish() - body, _, err := c.get2xx(ctx, &u) + method := opts.Method + if method == "" { + method = http.MethodGet + } + + body, _, err := c.req2xx(ctx, &u, method) if err != nil { return nil, nil, errors.Wrap(err, "read query instant response") } @@ -483,7 +502,7 @@ func (c *Client) QueryRange(ctx context.Context, base *url.URL, query string, st span, ctx := tracing.StartSpan(ctx, "/prom_query_range HTTP[client]") defer span.Finish() - body, _, err := c.get2xx(ctx, &u) + body, _, err := c.req2xx(ctx, &u, http.MethodGet) if err != nil { return nil, nil, errors.Wrap(err, "read query range response") } @@ -565,7 +584,7 @@ func (c *Client) AlertmanagerAlerts(ctx context.Context, base *url.URL) ([]*mode span, ctx := tracing.StartSpan(ctx, "/alertmanager_alerts HTTP[client]") defer span.Finish() - body, _, err := c.get2xx(ctx, &u) + body, _, err := c.req2xx(ctx, &u, http.MethodGet) if err != nil { return nil, err } @@ -592,7 +611,7 @@ func (c *Client) get2xxResultWithGRPCErrors(ctx context.Context, spanName string span, ctx := tracing.StartSpan(ctx, spanName) defer span.Finish() - body, code, err := c.get2xx(ctx, u) + body, code, err := c.req2xx(ctx, u, http.MethodGet) if err != nil { if code, exists := statusToCode[code]; exists && code != 0 { return status.Error(code, err.Error()) From a727a48f562fb311984bb6001e8e10508fc454a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 30 Oct 2020 13:46:53 +0200 Subject: [PATCH 2/2] CHANGELOG: rework the line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make it more technical :-P Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4db6ee9dba..1f56204ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#3346](https://github.com/thanos-io/thanos/pull/3346) Ruler UI: Fix a bug preventing the /rules endpoint from loading. - [#3115](https://github.com/thanos-io/thanos/pull/3115) compact: now deletes partially uploaded and blocks with deletion marks concurrently. It does that at the beginning and then every `--compact.cleanup-interval` time period. By default it is 5 minutes. - [#3312](https://github.com/thanos-io/thanos/pull/3312) s3: add list_objects_version config option for compatibility. -- [#3378](https://github.com/thanos-io/thanos/pull/3378) Ruler: learned to send queries via POST. Helps when alerting/recording rules are extra long. It now uses POST unless `--query.http-method` is `GET`. +- [#3378](https://github.com/thanos-io/thanos/pull/3378) Ruler: added the ability to send queries via the HTTP method POST. Helps when alerting/recording rules are extra long because it encodes the actual parameters inside of the body instead of the URI. Thanos Ruler now uses POST by default unless `--query.http-method` is set `GET`. ### Fixed - [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors.