Add pagination, filtering and sort to more API endpoints (#12186)

.changelog/12186.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+api: Add support for filtering, sorting, and pagination to the ACL tokens and allocations list endpoint
+```
+
+```release-note:improvement
+api: Add support for filtering and pagination to the jobs and volumes list endpoint
+```

api/api.go

@@ -82,10 +82,10 @@ type QueryOptions struct {
 	// previous response.
 	NextToken string
 
-	// Ascending is used to have results sorted in ascending chronological order.
+	// Reverse is used to reverse the default order of list results.
 	//
-	// Currently only supported by evaluations.List and deployments.list endpoints.
-	Ascending bool
+	// Currently only supported by specific endpoints.
+	Reverse bool
 
 	// ctx is an optional context pass through to the underlying HTTP
 	// request layer. Use Context() and WithContext() to manage this.
@@ -605,8 +605,8 @@ func (r *request) setQueryOptions(q *QueryOptions) {
 	if q.NextToken != "" {
 		r.params.Set("next_token", q.NextToken)
 	}
-	if q.Ascending {
-		r.params.Set("ascending", "true")
+	if q.Reverse {
+		r.params.Set("reverse", "true")
 	}
 	for k, v := range q.Params {
 		r.params.Set(k, v)

api/api_test.go

@@ -181,7 +181,7 @@ func TestSetQueryOptions(t *testing.T) {
 		WaitIndex:  1000,
 		WaitTime:   100 * time.Second,
 		AuthToken:  "foobar",
-		Ascending:  true,
+		Reverse:    true,
 	}
 	r.setQueryOptions(q)
 
@@ -199,7 +199,7 @@ func TestSetQueryOptions(t *testing.T) {
 	try("stale", "") // should not be present
 	try("index", "1000")
 	try("wait", "100000ms")
-	try("ascending", "true")
+	try("reverse", "true")
 }
 
 func TestQueryOptionsContext(t *testing.T) {

command/agent/http.go

@@ -788,7 +788,7 @@ func (s *HTTPServer) parse(resp http.ResponseWriter, req *http.Request, r *strin
 	parseNamespace(req, &b.Namespace)
 	parsePagination(req, b)
 	parseFilter(req, b)
-	parseAscending(req, b)
+	parseReverse(req, b)
 	return parseWait(resp, req, b)
 }
 
@@ -814,10 +814,10 @@ func parseFilter(req *http.Request, b *structs.QueryOptions) {
 	}
 }
 
-// parseAscending parses the ascending query parameter for QueryOptions
-func parseAscending(req *http.Request, b *structs.QueryOptions) {
+// parseReverse parses the reverse query parameter for QueryOptions
+func parseReverse(req *http.Request, b *structs.QueryOptions) {
 	query := req.URL.Query()
-	b.Ascending = query.Get("ascending") == "true"
+	b.Reverse = query.Get("reverse") == "true"
 }
 
 // parseWriteRequest is a convenience method for endpoints that need to parse a

helper/raftutil/fsm.go

@@ -185,28 +185,28 @@ func (f *FSMHelper) StateAsMap() map[string][]interface{} {
 }
 
 // StateAsMap returns a json-able representation of the state
-func StateAsMap(state *state.StateStore) map[string][]interface{} {
+func StateAsMap(store *state.StateStore) map[string][]interface{} {
 	result := map[string][]interface{}{
-		"ACLPolicies":      toArray(state.ACLPolicies(nil)),
-		"ACLTokens":        toArray(state.ACLTokens(nil)),
-		"Allocs":           toArray(state.Allocs(nil)),
-		"CSIPlugins":       toArray(state.CSIPlugins(nil)),
-		"CSIVolumes":       toArray(state.CSIVolumes(nil)),
-		"Deployments":      toArray(state.Deployments(nil, false)),
-		"Evals":            toArray(state.Evals(nil, false)),
-		"Indexes":          toArray(state.Indexes()),
-		"JobSummaries":     toArray(state.JobSummaries(nil)),
-		"JobVersions":      toArray(state.JobVersions(nil)),
-		"Jobs":             toArray(state.Jobs(nil)),
-		"Nodes":            toArray(state.Nodes(nil)),
-		"PeriodicLaunches": toArray(state.PeriodicLaunches(nil)),
-		"SITokenAccessors": toArray(state.SITokenAccessors(nil)),
-		"ScalingEvents":    toArray(state.ScalingEvents(nil)),
-		"ScalingPolicies":  toArray(state.ScalingPolicies(nil)),
-		"VaultAccessors":   toArray(state.VaultAccessors(nil)),
+		"ACLPolicies":      toArray(store.ACLPolicies(nil)),
+		"ACLTokens":        toArray(store.ACLTokens(nil, state.SortDefault)),
+		"Allocs":           toArray(store.Allocs(nil, state.SortDefault)),
+		"CSIPlugins":       toArray(store.CSIPlugins(nil)),
+		"CSIVolumes":       toArray(store.CSIVolumes(nil)),
+		"Deployments":      toArray(store.Deployments(nil, state.SortDefault)),
+		"Evals":            toArray(store.Evals(nil, state.SortDefault)),
+		"Indexes":          toArray(store.Indexes()),
+		"JobSummaries":     toArray(store.JobSummaries(nil)),
+		"JobVersions":      toArray(store.JobVersions(nil)),
+		"Jobs":             toArray(store.Jobs(nil)),
+		"Nodes":            toArray(store.Nodes(nil)),
+		"PeriodicLaunches": toArray(store.PeriodicLaunches(nil)),
+		"SITokenAccessors": toArray(store.SITokenAccessors(nil)),
+		"ScalingEvents":    toArray(store.ScalingEvents(nil)),
+		"ScalingPolicies":  toArray(store.ScalingPolicies(nil)),
+		"VaultAccessors":   toArray(store.VaultAccessors(nil)),
 	}
 
-	insertEnterpriseState(result, state)
+	insertEnterpriseState(result, store)
 
 	return result
 

nomad/acl_endpoint.go

@@ -3,6 +3,7 @@ package nomad
 import (
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -14,6 +15,7 @@ import (
 	policy "github.com/hashicorp/nomad/acl"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/nomad/state"
+	"github.com/hashicorp/nomad/nomad/state/paginator"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -652,41 +654,67 @@ func (a *ACL) ListTokens(args *structs.ACLTokenListRequest, reply *structs.ACLTo
 	}
 
 	// Setup the blocking query
+	sort := state.SortOption(args.Reverse)
 	opts := blockingOptions{
 		queryOpts: &args.QueryOptions,
 		queryMeta: &reply.QueryMeta,
 		run: func(ws memdb.WatchSet, state *state.StateStore) error {
 			// Iterate over all the tokens
 			var err error
 			var iter memdb.ResultIterator
+			var opts paginator.StructsTokenizerOptions
+
 			if prefix := args.QueryOptions.Prefix; prefix != "" {
 				iter, err = state.ACLTokenByAccessorIDPrefix(ws, prefix)
+				opts = paginator.StructsTokenizerOptions{
+					WithID: true,
+				}
 			} else if args.GlobalOnly {
 				iter, err = state.ACLTokensByGlobal(ws, true)
+				opts = paginator.StructsTokenizerOptions{
+					WithID: true,
+				}
 			} else {
-				iter, err = state.ACLTokens(ws)
+				iter, err = state.ACLTokens(ws, sort)
+				opts = paginator.StructsTokenizerOptions{
+					WithCreateIndex: true,
+					WithID:          true,
+				}
 			}
 			if err != nil {
 				return err
 			}
 
-			// Convert all the tokens to a list stub
-			reply.Tokens = nil
-			for {
-				raw := iter.Next()
-				if raw == nil {
-					break
-				}
-				token := raw.(*structs.ACLToken)
-				reply.Tokens = append(reply.Tokens, token.Stub())
+			tokenizer := paginator.NewStructsTokenizer(iter, opts)
+
+			var tokens []*structs.ACLTokenListStub
+			paginator, err := paginator.NewPaginator(iter, tokenizer, nil, args.QueryOptions,
+				func(raw interface{}) error {
+					token := raw.(*structs.ACLToken)
+					tokens = append(tokens, token.Stub())
+					return nil
+				})
+			if err != nil {
+				return structs.NewErrRPCCodedf(
+					http.StatusBadRequest, "failed to create result paginator: %v", err)
+			}
+
+			nextToken, err := paginator.Page()
+			if err != nil {
+				return structs.NewErrRPCCodedf(
+					http.StatusBadRequest, "failed to read result page: %v", err)
 			}
 
+			reply.QueryMeta.NextToken = nextToken
+			reply.Tokens = tokens
+
 			// Use the last index that affected the token table
 			index, err := state.Index("acl_token")
 			if err != nil {
 				return err
 			}
 			reply.Index = index
+
 			return nil
 		}}
 	return a.srv.blockingRPC(&opts)