diff --git a/.changelog/12034.txt b/.changelog/12034.txt new file mode 100644 index 000000000000..2e0069d26b0c --- /dev/null +++ b/.changelog/12034.txt @@ -0,0 +1,3 @@ +```release-note:improvement +api: filter values of evaluation and deployment list api endpoints +``` diff --git a/api/api.go b/api/api.go index 1dfc19205c27..a92df23fa057 100644 --- a/api/api.go +++ b/api/api.go @@ -65,6 +65,10 @@ type QueryOptions struct { // AuthToken is the secret ID of an ACL token AuthToken string + // Filter specifies the go-bexpr filter expression to be used for + // filtering the data prior to returning a response + Filter string + // PerPage is the number of entries to be returned in queries that support // paginated lists. PerPage int32 @@ -586,6 +590,9 @@ func (r *request) setQueryOptions(q *QueryOptions) { if q.Prefix != "" { r.params.Set("prefix", q.Prefix) } + if q.Filter != "" { + r.params.Set("filter", q.Filter) + } if q.PerPage != 0 { r.params.Set("per_page", fmt.Sprint(q.PerPage)) } diff --git a/api/evaluations_test.go b/api/evaluations_test.go index a734e57053f3..2b5b561348b6 100644 --- a/api/evaluations_test.go +++ b/api/evaluations_test.go @@ -77,6 +77,14 @@ func TestEvaluations_List(t *testing.T) { if len(result) != 1 { t.Fatalf("expected no evals after last one but got %v", result[0]) } + + // Query evaluations using a filter. + results, _, err = e.List(&QueryOptions{ + Filter: `TriggeredBy == "job-register"`, + }) + if len(result) != 1 { + t.Fatalf("expected 1 eval, got %d", len(result)) + } } func TestEvaluations_PrefixList(t *testing.T) { diff --git a/command/agent/http.go b/command/agent/http.go index 706a7c303b01..d2ac5cdf5d38 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -537,6 +537,9 @@ func (s *HTTPServer) wrap(handler func(resp http.ResponseWriter, req *http.Reque } else if strings.HasSuffix(errMsg, structs.ErrJobRegistrationDisabled.Error()) { errMsg = structs.ErrJobRegistrationDisabled.Error() code = 403 + } else if strings.HasSuffix(errMsg, structs.ErrIncompatibleFiltering.Error()) { + errMsg = structs.ErrIncompatibleFiltering.Error() + code = 400 } } @@ -784,6 +787,7 @@ func (s *HTTPServer) parse(resp http.ResponseWriter, req *http.Request, r *strin parsePrefix(req, b) parseNamespace(req, &b.Namespace) parsePagination(req, b) + parseFilter(req, b) return parseWait(resp, req, b) } @@ -801,6 +805,14 @@ func parsePagination(req *http.Request, b *structs.QueryOptions) { b.NextToken = query.Get("next_token") } +// parseFilter parses the filter query parameter for QueryOptions +func parseFilter(req *http.Request, b *structs.QueryOptions) { + query := req.URL.Query() + if filter := query.Get("filter"); filter != "" { + b.Filter = filter + } +} + // parseWriteRequest is a convenience method for endpoints that need to parse a // write request. func (s *HTTPServer) parseWriteRequest(req *http.Request, w *structs.WriteRequest) { diff --git a/command/deployment_list.go b/command/deployment_list.go index 58e3da3a2f1f..95ea0b2ec3fe 100644 --- a/command/deployment_list.go +++ b/command/deployment_list.go @@ -30,6 +30,9 @@ List Options: -json Output the deployments in a JSON format. + -filter + Specifies an expression used to filter query results. + -t Format and display the deployments using a Go template. @@ -43,6 +46,7 @@ func (c *DeploymentListCommand) AutocompleteFlags() complete.Flags { return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ "-json": complete.PredictNothing, + "-filter": complete.PredictAnything, "-t": complete.PredictAnything, "-verbose": complete.PredictNothing, }) @@ -60,12 +64,13 @@ func (c *DeploymentListCommand) Name() string { return "deployment list" } func (c *DeploymentListCommand) Run(args []string) int { var json, verbose bool - var tmpl string + var filter, tmpl string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } flags.BoolVar(&verbose, "verbose", false, "") flags.BoolVar(&json, "json", false, "") + flags.StringVar(&filter, "filter", "", "") flags.StringVar(&tmpl, "t", "", "") if err := flags.Parse(args); err != nil { @@ -93,7 +98,10 @@ func (c *DeploymentListCommand) Run(args []string) int { return 1 } - deploys, _, err := client.Deployments().List(nil) + opts := &api.QueryOptions{ + Filter: filter, + } + deploys, _, err := client.Deployments().List(opts) if err != nil { c.Ui.Error(fmt.Sprintf("Error retrieving deployments: %s", err)) return 1 diff --git a/command/eval_list.go b/command/eval_list.go index 20ea04477236..86d2b361fff8 100644 --- a/command/eval_list.go +++ b/command/eval_list.go @@ -35,6 +35,9 @@ Eval List Options: -page-token Where to start pagination. + -filter + Specifies an expression used to filter query results. + -job Only show evaluations for this job ID. @@ -61,6 +64,7 @@ func (c *EvalListCommand) AutocompleteFlags() complete.Flags { "-json": complete.PredictNothing, "-t": complete.PredictAnything, "-verbose": complete.PredictNothing, + "-filter": complete.PredictAnything, "-job": complete.PredictAnything, "-status": complete.PredictAnything, "-per-page": complete.PredictAnything, @@ -88,7 +92,7 @@ func (c *EvalListCommand) Name() string { return "eval list" } func (c *EvalListCommand) Run(args []string) int { var monitor, verbose, json bool var perPage int - var tmpl, pageToken, filterJobID, filterStatus string + var tmpl, pageToken, filter, filterJobID, filterStatus string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } @@ -98,6 +102,7 @@ func (c *EvalListCommand) Run(args []string) int { flags.StringVar(&tmpl, "t", "", "") flags.IntVar(&perPage, "per-page", 0, "") flags.StringVar(&pageToken, "page-token", "", "") + flags.StringVar(&filter, "filter", "", "") flags.StringVar(&filterJobID, "job", "", "") flags.StringVar(&filterStatus, "status", "", "") @@ -120,6 +125,7 @@ func (c *EvalListCommand) Run(args []string) int { } opts := &api.QueryOptions{ + Filter: filter, PerPage: int32(perPage), NextToken: pageToken, Params: map[string]string{}, diff --git a/go.mod b/go.mod index 03bc93ee4c45..3fd28ecf8cb9 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/hashicorp/consul/api v1.9.1 github.com/hashicorp/consul/sdk v0.8.0 github.com/hashicorp/cronexpr v1.1.1 + github.com/hashicorp/go-bexpr v0.1.11 github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-connlimit v0.3.0 @@ -209,6 +210,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/pointerstructure v1.2.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mrunalp/fileutils v0.5.0 // indirect diff --git a/go.sum b/go.sum index 68d65e1eaf83..21f2fcd1110f 100644 --- a/go.sum +++ b/go.sum @@ -667,6 +667,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.2/go.mod h1:ANbpTX1oAql27TZkKVeW8p1w8NTdnyzPe/0qqPCKohU= +github.com/hashicorp/go-bexpr v0.1.11 h1:6DqdA/KBjurGby9yTY0bmkathya0lfwF2SeuubCI7dY= +github.com/hashicorp/go-bexpr v0.1.11/go.mod h1:f03lAo0duBlDIUMGCuad8oLcgejw4m7U+N8T+6Kz1AE= github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de h1:XDCSythtg8aWSRSO29uwhgh7b127fWr+m5SemqjSUL8= github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de/go.mod h1:xIwEieBHERyEvaeKF/TcHh1Hu+lxPM+n2vT1+g9I4m4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -938,9 +940,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw= +github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= diff --git a/nomad/deployment_endpoint.go b/nomad/deployment_endpoint.go index ce8c82e0b668..04ccacf1669e 100644 --- a/nomad/deployment_endpoint.go +++ b/nomad/deployment_endpoint.go @@ -2,6 +2,7 @@ package nomad import ( "fmt" + "net/http" "time" metrics "github.com/armon/go-metrics" @@ -421,13 +422,22 @@ func (d *Deployment) List(args *structs.DeploymentListRequest, reply *structs.De } var deploys []*structs.Deployment - paginator := state.NewPaginator(iter, args.QueryOptions, + paginator, err := state.NewPaginator(iter, args.QueryOptions, func(raw interface{}) { deploy := raw.(*structs.Deployment) deploys = append(deploys, deploy) }) + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to create result paginator: %v", err) + } + + nextToken, err := paginator.Page() + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to read result page: %v", err) + } - nextToken := paginator.Page() reply.QueryMeta.NextToken = nextToken reply.Deployments = deploys diff --git a/nomad/deployment_endpoint_test.go b/nomad/deployment_endpoint_test.go index 867b3e86671c..42300062355f 100644 --- a/nomad/deployment_endpoint_test.go +++ b/nomad/deployment_endpoint_test.go @@ -1288,17 +1288,19 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { } aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read", - mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)). + mock.NamespacePolicy("*", "read", nil)). SecretID cases := []struct { name string namespace string prefix string + filter string nextToken string pageSize int32 expectedNextToken string expectedIDs []string + expectedError string }{ { name: "test01 size-2 page-1 default NS", @@ -1341,12 +1343,57 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { }, }, { - name: "test5 no valid results with filters and prefix", + name: "test05 no valid results with filters and prefix", prefix: "cccc", pageSize: 2, nextToken: "", expectedIDs: []string{}, }, + { + name: "test06 go-bexpr filter", + namespace: "*", + filter: `ID matches "^a+[123]"`, + expectedIDs: []string{ + "aaaa1111-3350-4b4b-d185-0e1992ed43e9", + "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", + "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test07 go-bexpr filter with pagination", + namespace: "*", + filter: `ID matches "^a+[123]"`, + pageSize: 2, + expectedNextToken: "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", + expectedIDs: []string{ + "aaaa1111-3350-4b4b-d185-0e1992ed43e9", + "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test08 go-bexpr filter in namespace", + namespace: "non-default", + filter: `Status == "cancelled"`, + expectedIDs: []string{ + "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test09 go-bexpr wrong namespace", + namespace: "default", + filter: `Namespace == "non-default"`, + expectedIDs: []string{}, + }, + { + name: "test10 go-bexpr invalid expression", + filter: `NotValid`, + expectedError: "failed to read filter expression", + }, + { + name: "test11 go-bexpr invalid field", + filter: `InvalidField == "value"`, + expectedError: "error finding value in datum", + }, } for _, tc := range cases { @@ -1357,13 +1404,22 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { Region: "global", Namespace: tc.namespace, Prefix: tc.prefix, + Filter: tc.filter, PerPage: tc.pageSize, NextToken: tc.nextToken, }, } req.AuthToken = aclToken var resp structs.DeploymentListResponse - require.NoError(t, msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp)) + err := msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp) + if tc.expectedError == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedError) + return + } + gotIDs := []string{} for _, deployment := range resp.Deployments { gotIDs = append(gotIDs, deployment.ID) diff --git a/nomad/eval_endpoint.go b/nomad/eval_endpoint.go index 1c6408c52f69..9094258d1c14 100644 --- a/nomad/eval_endpoint.go +++ b/nomad/eval_endpoint.go @@ -2,6 +2,7 @@ package nomad import ( "fmt" + "net/http" "time" metrics "github.com/armon/go-metrics" @@ -397,6 +398,14 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon return structs.ErrPermissionDenied } + if args.Filter != "" { + // Check for incompatible filtering. + hasLegacyFilter := args.FilterJobID != "" || args.FilterEvalStatus != "" + if hasLegacyFilter { + return structs.ErrIncompatibleFiltering + } + } + // Setup the blocking query opts := blockingOptions{ queryOpts: &args.QueryOptions, @@ -425,13 +434,22 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon }) var evals []*structs.Evaluation - paginator := state.NewPaginator(iter, args.QueryOptions, + paginator, err := state.NewPaginator(iter, args.QueryOptions, func(raw interface{}) { eval := raw.(*structs.Evaluation) evals = append(evals, eval) }) + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to create result paginator: %v", err) + } + + nextToken, err := paginator.Page() + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to read result page: %v", err) + } - nextToken := paginator.Page() reply.QueryMeta.NextToken = nextToken reply.Evaluations = evals diff --git a/nomad/eval_endpoint_test.go b/nomad/eval_endpoint_test.go index e2cef6904f37..ed9c2e590187 100644 --- a/nomad/eval_endpoint_test.go +++ b/nomad/eval_endpoint_test.go @@ -1050,7 +1050,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { } aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read", - mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)). + mock.NamespacePolicy("*", "read", nil)). SecretID cases := []struct { @@ -1060,9 +1060,11 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { nextToken string filterJobID string filterStatus string + filter string pageSize int32 expectedNextToken string expectedIDs []string + expectedError string }{ { name: "test01 size-2 page-1 default NS", @@ -1194,6 +1196,52 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { nextToken: "aaaaaa11-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{}, }, + { + name: "test14 go-bexpr filter", + filter: `Status == "blocked"`, + nextToken: "", + expectedIDs: []string{"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"}, + }, + { + name: "test15 go-bexpr filter with pagination", + filter: `JobID == "example"`, + pageSize: 2, + expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedIDs: []string{ + "aaaa1111-3350-4b4b-d185-0e1992ed43e9", + "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test16 go-bexpr filter namespace", + namespace: "non-default", + filter: `ID contains "aaa"`, + expectedIDs: []string{ + "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test17 go-bexpr wrong namespace", + namespace: "default", + filter: `Namespace == "non-default"`, + expectedIDs: []string{}, + }, + { + name: "test18 incompatible filtering", + filter: `JobID == "example"`, + filterStatus: "complete", + expectedError: structs.ErrIncompatibleFiltering.Error(), + }, + { + name: "test19 go-bexpr invalid expression", + filter: `NotValid`, + expectedError: "failed to read filter expression", + }, + { + name: "test20 go-bexpr invalid field", + filter: `InvalidField == "value"`, + expectedError: "error finding value in datum", + }, } for _, tc := range cases { @@ -1208,11 +1256,20 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { Prefix: tc.prefix, PerPage: tc.pageSize, NextToken: tc.nextToken, + Filter: tc.filter, }, } req.AuthToken = aclToken var resp structs.EvalListResponse - require.NoError(t, msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp)) + err := msgpackrpc.CallWithCodec(codec, "Eval.List", req, &resp) + if tc.expectedError == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedError) + return + } + gotIDs := []string{} for _, eval := range resp.Evaluations { gotIDs = append(gotIDs, eval.ID) diff --git a/nomad/state/filter_test.go b/nomad/state/filter_test.go new file mode 100644 index 000000000000..73f6754caa3e --- /dev/null +++ b/nomad/state/filter_test.go @@ -0,0 +1,228 @@ +package state + +import ( + "testing" + "time" + + "github.com/hashicorp/go-bexpr" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" +) + +func BenchmarkEvalListFilter(b *testing.B) { + const evalCount = 100_000 + + b.Run("filter with index", func(b *testing.B) { + state := setupPopulatedState(b, evalCount) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace) + var lastSeen string + var countSeen int + for { + raw := iter.Next() + if raw == nil { + break + } + eval := raw.(*structs.Evaluation) + lastSeen = eval.ID + countSeen++ + } + if countSeen < evalCount/2 { + b.Fatalf("failed: %d evals seen, lastSeen=%s", countSeen, lastSeen) + } + } + }) + + b.Run("filter with go-bexpr", func(b *testing.B) { + state := setupPopulatedState(b, evalCount) + evaluator, err := bexpr.CreateEvaluator(`Namespace == "default"`) + if err != nil { + b.Fatalf("failed: %v", err) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter, _ := state.Evals(nil, false) + var lastSeen string + var countSeen int + for { + raw := iter.Next() + if raw == nil { + break + } + match, err := evaluator.Evaluate(raw) + if !match || err != nil { + continue + } + eval := raw.(*structs.Evaluation) + lastSeen = eval.ID + countSeen++ + } + if countSeen < evalCount/2 { + b.Fatalf("failed: %d evals seen, lastSeen=%s", countSeen, lastSeen) + } + } + }) + + b.Run("paginated filter with index", func(b *testing.B) { + state := setupPopulatedState(b, evalCount) + opts := structs.QueryOptions{ + PerPage: 100, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace) + var evals []*structs.Evaluation + paginator, err := NewPaginator(iter, opts, func(raw interface{}) { + eval := raw.(*structs.Evaluation) + evals = append(evals, eval) + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + paginator.Page() + } + }) + + b.Run("paginated filter with go-bexpr", func(b *testing.B) { + state := setupPopulatedState(b, evalCount) + opts := structs.QueryOptions{ + PerPage: 100, + Filter: `Namespace == "default"`, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter, _ := state.Evals(nil, false) + var evals []*structs.Evaluation + paginator, err := NewPaginator(iter, opts, func(raw interface{}) { + eval := raw.(*structs.Evaluation) + evals = append(evals, eval) + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + paginator.Page() + } + }) + + b.Run("paginated filter with index last page", func(b *testing.B) { + state := setupPopulatedState(b, evalCount) + + // Find the last eval ID. + iter, _ := state.Evals(nil, false) + var lastSeen string + for { + raw := iter.Next() + if raw == nil { + break + } + eval := raw.(*structs.Evaluation) + lastSeen = eval.ID + } + + opts := structs.QueryOptions{ + PerPage: 100, + NextToken: lastSeen, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace) + var evals []*structs.Evaluation + paginator, err := NewPaginator(iter, opts, func(raw interface{}) { + eval := raw.(*structs.Evaluation) + evals = append(evals, eval) + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + paginator.Page() + } + }) + + b.Run("paginated filter with go-bexpr last page", func(b *testing.B) { + state := setupPopulatedState(b, evalCount) + + // Find the last eval ID. + iter, _ := state.Evals(nil, false) + var lastSeen string + for { + raw := iter.Next() + if raw == nil { + break + } + eval := raw.(*structs.Evaluation) + lastSeen = eval.ID + } + + opts := structs.QueryOptions{ + PerPage: 100, + NextToken: lastSeen, + Filter: `Namespace == "default"`, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter, _ := state.Evals(nil, false) + var evals []*structs.Evaluation + paginator, err := NewPaginator(iter, opts, func(raw interface{}) { + eval := raw.(*structs.Evaluation) + evals = append(evals, eval) + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + paginator.Page() + } + }) +} + +// ----------------- +// BENCHMARK HELPER FUNCTIONS + +func setupPopulatedState(b *testing.B, evalCount int) *StateStore { + evals := generateEvals(evalCount) + + index := uint64(0) + var err error + state := TestStateStore(b) + for _, eval := range evals { + index++ + err = state.UpsertEvals( + structs.MsgTypeTestSetup, index, []*structs.Evaluation{eval}) + } + if err != nil { + b.Fatalf("failed: %v", err) + } + return state +} + +func generateEvals(count int) []*structs.Evaluation { + evals := []*structs.Evaluation{} + ns := structs.DefaultNamespace + for i := 0; i < count; i++ { + if i > count/2 { + ns = "other" + } + evals = append(evals, generateEval(i, ns)) + } + return evals +} + +func generateEval(i int, ns string) *structs.Evaluation { + now := time.Now().UTC().UnixNano() + return &structs.Evaluation{ + ID: uuid.Generate(), + Namespace: ns, + Priority: 50, + Type: structs.JobTypeService, + JobID: uuid.Generate(), + Status: structs.EvalStatusPending, + CreateTime: now, + ModifyTime: now, + } +} diff --git a/nomad/state/paginator.go b/nomad/state/paginator.go index ecf131b52afd..138398908d13 100644 --- a/nomad/state/paginator.go +++ b/nomad/state/paginator.go @@ -1,19 +1,33 @@ package state import ( - memdb "github.com/hashicorp/go-memdb" + "fmt" + + "github.com/hashicorp/go-bexpr" "github.com/hashicorp/nomad/nomad/structs" ) +// Iterator is the interface that must be implemented to use the Paginator. +type Iterator interface { + // Next returns the next element to be considered for pagination. + // The page will end if nil is returned. + Next() interface{} +} + // Paginator is an iterator over a memdb.ResultIterator that returns // only the expected number of pages. type Paginator struct { - iter memdb.ResultIterator + iter Iterator perPage int32 itemCount int32 seekingToken string nextToken string nextTokenFound bool + pageErr error + + // filterEvaluator is used to filter results using go-bexpr. It's nil if + // no filter expression is defined. + filterEvaluator *bexpr.Evaluator // appendFunc is the function the caller should use to append raw // entries to the results set. The object is guaranteed to be @@ -21,19 +35,30 @@ type Paginator struct { appendFunc func(interface{}) } -func NewPaginator(iter memdb.ResultIterator, opts structs.QueryOptions, appendFunc func(interface{})) *Paginator { - return &Paginator{ - iter: iter, - perPage: opts.PerPage, - seekingToken: opts.NextToken, - nextTokenFound: opts.NextToken == "", - appendFunc: appendFunc, +func NewPaginator(iter Iterator, opts structs.QueryOptions, appendFunc func(interface{})) (*Paginator, error) { + var evaluator *bexpr.Evaluator + var err error + + if opts.Filter != "" { + evaluator, err = bexpr.CreateEvaluator(opts.Filter) + if err != nil { + return nil, fmt.Errorf("failed to read filter expression: %v", err) + } } + + return &Paginator{ + iter: iter, + perPage: opts.PerPage, + seekingToken: opts.NextToken, + nextTokenFound: opts.NextToken == "", + filterEvaluator: evaluator, + appendFunc: appendFunc, + }, nil } // Page populates a page by running the append function // over all results. Returns the next token -func (p *Paginator) Page() string { +func (p *Paginator) Page() (string, error) { DONE: for { raw, andThen := p.next() @@ -46,7 +71,7 @@ DONE: break DONE } } - return p.nextToken + return p.nextToken, p.pageErr } func (p *Paginator) next() (interface{}, paginatorState) { @@ -62,6 +87,19 @@ func (p *Paginator) next() (interface{}, paginatorState) { if !p.nextTokenFound && id < p.seekingToken { return nil, paginatorSkip } + + // apply filter if defined + if p.filterEvaluator != nil { + match, err := p.filterEvaluator.Evaluate(raw) + if err != nil { + p.pageErr = err + return nil, paginatorComplete + } + if !match { + return nil, paginatorSkip + } + } + p.nextTokenFound = true // have we produced enough results for this page? diff --git a/nomad/state/paginator_test.go b/nomad/state/paginator_test.go index cae606783300..27d34d8f86d7 100644 --- a/nomad/state/paginator_test.go +++ b/nomad/state/paginator_test.go @@ -55,7 +55,7 @@ func TestPaginator(t *testing.T) { iter := newTestIterator(ids) results := []string{} - paginator := NewPaginator(iter, + paginator, err := NewPaginator(iter, structs.QueryOptions{ PerPage: tc.perPage, NextToken: tc.nextToken, }, @@ -64,8 +64,10 @@ func TestPaginator(t *testing.T) { results = append(results, result.GetID()) }, ) + require.NoError(t, err) - nextToken := paginator.Page() + nextToken, err := paginator.Page() + require.NoError(t, err) require.Equal(t, tc.expected, results) require.Equal(t, tc.expectedNextToken, nextToken) }) diff --git a/nomad/structs/errors.go b/nomad/structs/errors.go index 10f8fac1cd22..4063303df8ca 100644 --- a/nomad/structs/errors.go +++ b/nomad/structs/errors.go @@ -19,6 +19,7 @@ const ( errUnknownNomadVersion = "Unable to determine Nomad version" errNodeLacksRpc = "Node does not support RPC; requires 0.8 or later" errMissingAllocID = "Missing allocation ID" + errIncompatibleFiltering = "Filter expression cannot be used with other filter parameters" // Prefix based errors that are used to check if the error is of a given // type. These errors should be created with the associated constructor. @@ -53,6 +54,7 @@ var ( ErrUnknownNomadVersion = errors.New(errUnknownNomadVersion) ErrNodeLacksRpc = errors.New(errNodeLacksRpc) ErrMissingAllocID = errors.New(errMissingAllocID) + ErrIncompatibleFiltering = errors.New(errIncompatibleFiltering) ErrUnknownNode = errors.New(ErrUnknownNodePrefix) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 65d9a8c6e36f..9d6331a7528e 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -274,6 +274,10 @@ type QueryOptions struct { // AuthToken is secret portion of the ACL token used for the request AuthToken string + // Filter specifies the go-bexpr filter expression to be used for + // filtering the data prior to returning a response + Filter string + // PerPage is the number of entries to be returned in queries that support // paginated lists. PerPage int32 diff --git a/website/content/api-docs/deployments.mdx b/website/content/api-docs/deployments.mdx index 484b5003c887..d9cc238d4659 100644 --- a/website/content/api-docs/deployments.mdx +++ b/website/content/api-docs/deployments.mdx @@ -29,11 +29,11 @@ The table below shows this endpoint's support for - `prefix` `(string: "")`- Specifies a string to filter deployments based on an ID prefix. Because the value is decoded to bytes, the prefix must have an even number of hexadecimal characters (0-9a-f) .This is specified as a query - string parameter. + string parameter and is used before any `filter` expression is applied. -- `namespace` `(string: "default")` - Specifies the target - namespace. Specifying `*` will return all evaluations across all - authorized namespaces. +- `namespace` `(string: "default")` - Specifies the target namespace. + Specifying `*` will return all evaluations across all authorized namespaces. + This parameter is used before any `filter` expression is applied. - `next_token` `(string: "")` - This endpoint supports paging. The `next_token` parameter accepts a string which is the `ID` field of @@ -46,6 +46,10 @@ The table below shows this endpoint's support for used as the `last_token` of the next request to fetch additional pages. +- `filter` `(string: "")` - Specifies the expression used to filter the query + results. Consider using pagination or a query parameter to reduce resource + used to serve the request. + - `ascending` `(bool: false)` - Specifies the list of returned deployments should be sorted in chronological order (oldest evaluations first). By default deployments are returned sorted in reverse chronological order (newest deployments first). diff --git a/website/content/api-docs/evaluations.mdx b/website/content/api-docs/evaluations.mdx index 835eacfeffff..37a49228a573 100644 --- a/website/content/api-docs/evaluations.mdx +++ b/website/content/api-docs/evaluations.mdx @@ -29,7 +29,7 @@ The table below shows this endpoint's support for - `prefix` `(string: "")`- Specifies a string to filter evaluations based on an ID prefix. Because the value is decoded to bytes, the prefix must have an even number of hexadecimal characters (0-9a-f). This is specified as a query - string parameter. + string parameter and and is used before any `filter` expression is applied. - `next_token` `(string: "")` - This endpoint supports paging. The `next_token` parameter accepts a string which is the `ID` field of @@ -42,6 +42,10 @@ The table below shows this endpoint's support for used as the `last_token` of the next request to fetch additional pages. +- `filter` `(string: "")` - Specifies the expression used to filter the query + results. Consider using pagination or a query parameter to reduce resource + used to serve the request. + - `job` `(string: "")` - Filter the list of evaluations to a specific job ID. @@ -49,9 +53,9 @@ The table below shows this endpoint's support for specific evaluation status (one of `blocked`, `pending`, `complete`, `failed`, or `canceled`). -- `namespace` `(string: "default")` - Specifies the target - namespace. Specifying `*` will return all evaluations across all - authorized namespaces. +- `namespace` `(string: "default")` - Specifies the target namespace. + Specifying `*` will return all evaluations across all authorized namespaces. + This parameter is used before any `filter` expression is applied. - `ascending` `(bool: false)` - Specifies the list of returned evaluations should be sorted in chronological order (oldest evaluations first). By default evaluations diff --git a/website/content/docs/commands/deployment/list.mdx b/website/content/docs/commands/deployment/list.mdx index 5d6bfc6cfc06..b6e2d35e633b 100644 --- a/website/content/docs/commands/deployment/list.mdx +++ b/website/content/docs/commands/deployment/list.mdx @@ -27,6 +27,7 @@ capability for the deployment's namespace. ## List Options - `-json` : Output the deployments in their JSON format. +- `-filter`: Specifies an expression used to filter query results. - `-t` : Format and display the deployments using a Go template. - `-verbose`: Show full information. diff --git a/website/content/docs/commands/eval/list.mdx b/website/content/docs/commands/eval/list.mdx index 467fbb90b9c4..e3dc5ac86fb1 100644 --- a/website/content/docs/commands/eval/list.mdx +++ b/website/content/docs/commands/eval/list.mdx @@ -29,6 +29,7 @@ capability for the requested namespace. - `-verbose`: Show full information. - `-per-page`: How many results to show per page. - `-page-token`: Where to start pagination. +- `-filter`: Specifies an expression used to filter query results. - `-job`: Only show evaluations for this job ID. - `-status`: Only show evaluations with this status. - `-json`: Output the evaluation in its JSON format.