-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Ruler: remote rule evaluation (#8744)
**What this PR does / why we need it**: Adds the ability to evaluate recording & alerting rules against a given `query-frontend`, allowing these queries to be executed with all the parallelisation & optimisation that regular adhoc queries have. This is important because with `local` evaluation all queries are single-threaded, and rules that evaluate a large range/volume of data may timeout or OOM the `ruler` itself, leading to missed metrics or alerts. When `remote` evaluation mode is enabled, the `ruler` effectively just becomes a gRPC client for the `query-frontend`, which will dramatically improve the reliability of the `ruler` and also drastically reduce its resource requirements. **Which issue(s) this PR fixes**: This PR implements the feature discussed in #8129 (**LID 0002: Remote Rule Evaluation**).
- Loading branch information
Danny Kopping
committed
Mar 13, 2023
1 parent
540380f
commit 33e44ed
Showing
22 changed files
with
996 additions
and
125 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
package cluster | ||
|
||
import ( | ||
"fmt" | ||
"os" | ||
"path/filepath" | ||
"strings" | ||
) | ||
|
||
func (c *Component) WithRulerRemoteWrite(name, url string) { | ||
|
||
// ensure remote-write is enabled | ||
c.WithExtraConfig(` | ||
ruler: | ||
remote_write: | ||
enabled: true | ||
`) | ||
|
||
c.WithExtraConfig(fmt.Sprintf(` | ||
ruler: | ||
remote_write: | ||
clients: | ||
%s: | ||
url: %s/api/v1/write | ||
queue_config: | ||
# send immediately as soon as a sample is generated | ||
capacity: 1 | ||
batch_send_deadline: 0s | ||
`, name, url)) | ||
} | ||
|
||
func (c *Component) WithTenantRules(tenantFilesMap map[string]map[string]string) error { | ||
sharedPath := c.ClusterSharedPath() | ||
rulesPath := filepath.Join(sharedPath, "rules") | ||
|
||
if err := os.Mkdir(rulesPath, 0755); err != nil { | ||
return fmt.Errorf("error creating rules path: %w", err) | ||
} | ||
|
||
for tenant, files := range tenantFilesMap { | ||
for filename, file := range files { | ||
path := filepath.Join(rulesPath, tenant) | ||
if err := os.Mkdir(path, 0755); err != nil { | ||
return fmt.Errorf("error creating tenant %s rules path: %w", tenant, err) | ||
} | ||
if err := os.WriteFile(filepath.Join(path, filename), []byte(strings.TrimSpace(file)), 0644); err != nil { | ||
return fmt.Errorf("error creating rule file at path %s: %w", path, err) | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} |
Oops, something went wrong.