From 88884843386cf6b33fba298da318ad67e25b6915 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 23 Dec 2021 13:45:14 -0500 Subject: [PATCH] api: paginate deployment list and accept wildcard namespace Add `per_page` and `next_token` handling to `Deployment.List` RPC, and allow the use of a wildcard namespace for namespace filtering. --- .changelog/11743.txt | 7 ++ nomad/deployment_endpoint.go | 26 ++-- nomad/deployment_endpoint_test.go | 148 +++++++++++++++++++++++ website/content/api-docs/deployments.mdx | 15 +++ 4 files changed, 184 insertions(+), 12 deletions(-) create mode 100644 .changelog/11743.txt diff --git a/.changelog/11743.txt b/.changelog/11743.txt new file mode 100644 index 000000000000..a4f1114b715a --- /dev/null +++ b/.changelog/11743.txt @@ -0,0 +1,7 @@ +```release-note:improvement +api: Updated the deployments list API to respect wildcard namespaces +``` + +```release-note:improvement +api: Added pagination to deployments list API +``` diff --git a/nomad/deployment_endpoint.go b/nomad/deployment_endpoint.go index b7073eb8f881..d6e06cc933c7 100644 --- a/nomad/deployment_endpoint.go +++ b/nomad/deployment_endpoint.go @@ -400,32 +400,34 @@ func (d *Deployment) List(args *structs.DeploymentListRequest, reply *structs.De opts := blockingOptions{ queryOpts: &args.QueryOptions, queryMeta: &reply.QueryMeta, - run: func(ws memdb.WatchSet, state *state.StateStore) error { + run: func(ws memdb.WatchSet, store *state.StateStore) error { // Capture all the deployments var err error var iter memdb.ResultIterator if prefix := args.QueryOptions.Prefix; prefix != "" { - iter, err = state.DeploymentsByIDPrefix(ws, args.RequestNamespace(), prefix) + iter, err = store.DeploymentsByIDPrefix(ws, args.RequestNamespace(), prefix) + } else if args.RequestNamespace() == structs.AllNamespacesSentinel { + iter, err = store.Deployments(ws) } else { - iter, err = state.DeploymentsByNamespace(ws, args.RequestNamespace()) + iter, err = store.DeploymentsByNamespace(ws, args.RequestNamespace()) } if err != nil { return err } var deploys []*structs.Deployment - for { - raw := iter.Next() - if raw == nil { - break - } - deploy := raw.(*structs.Deployment) - deploys = append(deploys, deploy) - } + paginator := state.NewPaginator(iter, args.QueryOptions, + func(raw interface{}) { + deploy := raw.(*structs.Deployment) + deploys = append(deploys, deploy) + }) + + nextToken := paginator.Page() + reply.QueryMeta.NextToken = nextToken reply.Deployments = deploys // Use the last index that affected the deployment table - index, err := state.Index("deployment") + index, err := store.Index("deployment") if err != nil { return err } diff --git a/nomad/deployment_endpoint_test.go b/nomad/deployment_endpoint_test.go index 300d7790cc4f..9868fb021bd6 100644 --- a/nomad/deployment_endpoint_test.go +++ b/nomad/deployment_endpoint_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDeploymentEndpoint_GetDeployment(t *testing.T) { @@ -1006,6 +1007,28 @@ func TestDeploymentEndpoint_List(t *testing.T) { assert.EqualValues(resp.Index, 1000, "Wrong Index") assert.Len(resp2.Deployments, 1, "Deployments") assert.Equal(resp2.Deployments[0].ID, d.ID, "Deployment ID") + + // add another deployment in another namespace + + j2 := mock.Job() + d2 := mock.Deployment() + j2.Namespace = "prod" + d2.Namespace = "prod" + d2.JobID = j2.ID + assert.Nil(state.UpsertNamespaces(1001, []*structs.Namespace{{Name: "prod"}})) + assert.Nil(state.UpsertJob(structs.MsgTypeTestSetup, 1002, j2), "UpsertJob") + assert.Nil(state.UpsertDeployment(1003, d2), "UpsertDeployment") + + // Lookup the deployments with wildcard namespace + get = &structs.DeploymentListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: structs.AllNamespacesSentinel, + }, + } + assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.List", get, &resp), "RPC") + assert.EqualValues(resp.Index, 1003, "Wrong Index") + assert.Len(resp.Deployments, 2, "Deployments") } func TestDeploymentEndpoint_List_ACL(t *testing.T) { @@ -1135,6 +1158,131 @@ func TestDeploymentEndpoint_List_Blocking(t *testing.T) { } } +func TestDeploymentEndpoint_List_Pagination(t *testing.T) { + t.Parallel() + s1, _, cleanupS1 := TestACLServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // create a set of deployments. these are in the order that the + // state store will return them from the iterator (sorted by key), + // for ease of writing tests + mocks := []struct { + id string + namespace string + jobID string + status string + }{ + {id: "aaaa1111-3350-4b4b-d185-0e1992ed43e9"}, + {id: "aaaaaa22-3350-4b4b-d185-0e1992ed43e9"}, + {id: "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", namespace: "non-default"}, + {id: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"}, + {id: "aaaaaabb-3350-4b4b-d185-0e1992ed43e9"}, + {id: "aaaaaacc-3350-4b4b-d185-0e1992ed43e9"}, + {id: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9"}, + } + + state := s1.fsm.State() + index := uint64(1000) + + for _, m := range mocks { + index++ + deployment := mock.Deployment() + deployment.Status = structs.DeploymentStatusCancelled + deployment.ID = m.id + if m.namespace != "" { // defaults to "default" + deployment.Namespace = m.namespace + } + require.NoError(t, state.UpsertDeployment(index, deployment)) + } + + aclToken := mock.CreatePolicyAndToken(t, state, 1100, "test-valid-read", + mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)). + SecretID + + cases := []struct { + name string + namespace string + prefix string + nextToken string + pageSize int32 + expectedNextToken string + expectedIDs []string + }{ + { + name: "test01 size-2 page-1 default NS", + pageSize: 2, + expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedIDs: []string{ + "aaaa1111-3350-4b4b-d185-0e1992ed43e9", + "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test02 size-2 page-1 default NS with prefix", + prefix: "aaaa", + pageSize: 2, + expectedNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedIDs: []string{ + "aaaa1111-3350-4b4b-d185-0e1992ed43e9", + "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test03 size-2 page-2 default NS", + pageSize: 2, + nextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "aaaaaacc-3350-4b4b-d185-0e1992ed43e9", + expectedIDs: []string{ + "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test04 size-2 page-2 default NS with prefix", + prefix: "aaaa", + pageSize: 2, + nextToken: "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9", + expectedIDs: []string{ + "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", + "aaaaaacc-3350-4b4b-d185-0e1992ed43e9", + }, + }, + { + name: "test5 no valid results with filters and prefix", + prefix: "cccc", + pageSize: 2, + nextToken: "", + expectedIDs: []string{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + req := &structs.DeploymentListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: tc.namespace, + Prefix: tc.prefix, + PerPage: tc.pageSize, + NextToken: tc.nextToken, + }, + } + req.AuthToken = aclToken + var resp structs.DeploymentListResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp)) + gotIDs := []string{} + for _, deployment := range resp.Deployments { + gotIDs = append(gotIDs, deployment.ID) + } + require.Equal(t, tc.expectedIDs, gotIDs, "unexpected page of deployments") + require.Equal(t, tc.expectedNextToken, resp.QueryMeta.NextToken, "unexpected NextToken") + }) + } +} + func TestDeploymentEndpoint_Allocations(t *testing.T) { t.Parallel() diff --git a/website/content/api-docs/deployments.mdx b/website/content/api-docs/deployments.mdx index c9db087636c3..a608ab357d8d 100644 --- a/website/content/api-docs/deployments.mdx +++ b/website/content/api-docs/deployments.mdx @@ -31,6 +31,21 @@ The table below shows this endpoint's support for even number of hexadecimal characters (0-9a-f) .This is specified as a query string parameter. +- `namespace` `(string: "default")` - Specifies the target + namespace. Specifying `*` will return all evaluations across all + authorized namespaces. + +- `next_token` `(string: "")` - This endpoint supports paging. The + `next_token` parameter accepts a string which is the `ID` field of + the next expected deployment. This value can be obtained from the + `X-Nomad-NextToken` header from the previous response. + +- `per_page` `(int: 0)` - Specifies a maximum number of deployments to + return for this request. If omitted, the response is not + paginated. The `ID` of the last deployment in the response can be + used as the `last_token` of the next request to fetch additional + pages. + ### Sample Request ```shell-session