Merge pull request #9055 from hashicorp/f-9017-resources

api: add field filters to /v1/{allocations,nodes}

CHANGELOG.md

@@ -8,6 +8,8 @@ IMPROVEMENTS:
  * core: Improved job deregistration error logging. [[GH-8745](https://github.com/hashicorp/nomad/issues/8745)]
  * api: Added support for cancellation contexts to HTTP API. [[GH-8836](https://github.com/hashicorp/nomad/issues/8836)]
  * api: Job Register API now permits non-zero initial Version to accommodate multi-region deployments. [[GH-9071](https://github.com/hashicorp/nomad/issues/9071)]
+ * api: Added ?resources=true query parameter to /v1/nodes and /v1/allocations to include resource allocations in listings. [[GH-9055](https://github.com/hashicorp/nomad/issues/9055)]
+ * api: Added ?task_states=false query parameter to /v1/allocations to remove TaskStates from listings. Defaults to being included as before. [[GH-9055](https://github.com/hashicorp/nomad/issues/9055)]
  * cli: Added `scale` and `scaling-events` subcommands to the `job` command. [[GH-9023](https://github.com/hashicorp/nomad/pull/9023)]
  * cli: Added `scaling` command for interaction with the scaling API endpoint. [[GH-9025](https://github.com/hashicorp/nomad/pull/9025)]
  * client: Use ec2 CPU perf data from AWS API [[GH-7830](https://github.com/hashicorp/nomad/issues/7830)]

api/allocations.go

@@ -432,6 +432,7 @@ type AllocationListStub struct {
 	JobType               string
 	JobVersion            uint64
 	TaskGroup             string
+	AllocatedResources    *AllocatedResources `json:",omitempty"`
 	DesiredStatus         string
 	DesiredDescription    string
 	ClientStatus          string

api/allocations_test.go

@@ -7,15 +7,17 @@ import (
 	"reflect"
 	"sort"
 	"testing"
-
 	"time"
 
+	"github.com/hashicorp/nomad/api/internal/testutil"
 	"github.com/stretchr/testify/require"
 )
 
 func TestAllocations_List(t *testing.T) {
 	t.Parallel()
-	c, s := makeClient(t, nil, nil)
+	c, s := makeClient(t, nil, func(c *testutil.TestServerConfig) {
+		c.DevMode = true
+	})
 	defer s.Stop()
 	a := c.Allocations()
 
@@ -31,33 +33,28 @@ func TestAllocations_List(t *testing.T) {
 		t.Fatalf("expected 0 allocs, got: %d", n)
 	}
 
-	// TODO: do something that causes an allocation to actually happen
-	// so we can query for them.
-	return
+	// Create a job and attempt to register it
+	job := testJob()
+	resp, wm, err := c.Jobs().Register(job, nil)
+	require.NoError(t, err)
+	require.NotNil(t, resp)
+	require.NotEmpty(t, resp.EvalID)
+	assertWriteMeta(t, wm)
 
-	//job := &Job{
-	//ID:   stringToPtr("job1"),
-	//Name: stringToPtr("Job #1"),
-	//Type: stringToPtr(JobTypeService),
-	//}
-	//eval, _, err := c.Jobs().Register(job, nil)
-	//if err != nil {
-	//t.Fatalf("err: %s", err)
-	//}
+	// List the allocations again
+	qo := &QueryOptions{
+		WaitIndex: wm.LastIndex,
+	}
+	allocs, qm, err = a.List(qo)
+	require.NoError(t, err)
+	require.NotZero(t, qm.LastIndex)
 
-	//// List the allocations again
-	//allocs, qm, err = a.List(nil)
-	//if err != nil {
-	//t.Fatalf("err: %s", err)
-	//}
-	//if qm.LastIndex == 0 {
-	//t.Fatalf("bad index: %d", qm.LastIndex)
-	//}
+	// Check that we got the allocation back
+	require.Len(t, allocs, 1)
+	require.Equal(t, resp.EvalID, allocs[0].EvalID)
 
-	//// Check that we got the allocation back
-	//if len(allocs) == 0 || allocs[0].EvalID != eval {
-	//t.Fatalf("bad: %#v", allocs)
-	//}
+	// Resources should be unset by default
+	require.Nil(t, allocs[0].AllocatedResources)
 }
 
 func TestAllocations_PrefixList(t *testing.T) {
@@ -108,6 +105,37 @@ func TestAllocations_PrefixList(t *testing.T) {
 	//}
 }
 
+func TestAllocations_List_Resources(t *testing.T) {
+	t.Parallel()
+	c, s := makeClient(t, nil, func(c *testutil.TestServerConfig) {
+		c.DevMode = true
+	})
+	defer s.Stop()
+	a := c.Allocations()
+
+	// Create a job and register it
+	job := testJob()
+	resp, wm, err := c.Jobs().Register(job, nil)
+	require.NoError(t, err)
+	require.NotNil(t, resp)
+	require.NotEmpty(t, resp.EvalID)
+	assertWriteMeta(t, wm)
+
+	// List the allocations
+	qo := &QueryOptions{
+		Params:    map[string]string{"resources": "true"},
+		WaitIndex: wm.LastIndex,
+	}
+	allocs, qm, err := a.List(qo)
+	require.NoError(t, err)
+	require.NotZero(t, qm.LastIndex)
+
+	// Check that we got the allocation back with resources
+	require.Len(t, allocs, 1)
+	require.Equal(t, resp.EvalID, allocs[0].EvalID)
+	require.NotNil(t, allocs[0].AllocatedResources)
+}
+
 func TestAllocations_CreateIndexSort(t *testing.T) {
 	t.Parallel()
 	allocs := []*AllocationListStub{

api/nodes.go

@@ -786,6 +786,8 @@ type NodeListStub struct {
 	Status                string
 	StatusDescription     string
 	Drivers               map[string]*DriverInfo
+	NodeResources         *NodeResources         `json:",omitempty"`
+	ReservedResources     *NodeReservedResources `json:",omitempty"`
 	CreateIndex           uint64
 	ModifyIndex           uint64
 }

api/nodes_test.go

@@ -83,6 +83,45 @@ func TestNodes_PrefixList(t *testing.T) {
 	assertQueryMeta(t, qm)
 }
 
+// TestNodes_List_Resources asserts that ?resources=true includes allocated and
+// reserved resources in the response.
+func TestNodes_List_Resources(t *testing.T) {
+	t.Parallel()
+	c, s := makeClient(t, nil, func(c *testutil.TestServerConfig) {
+		c.DevMode = true
+	})
+	defer s.Stop()
+	nodes := c.Nodes()
+
+	var out []*NodeListStub
+	var err error
+
+	testutil.WaitForResult(func() (bool, error) {
+		out, _, err = nodes.List(nil)
+		if err != nil {
+			return false, err
+		}
+		if n := len(out); n != 1 {
+			return false, fmt.Errorf("expected 1 node, got: %d", n)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatalf("err: %s", err)
+	})
+
+	// By default resources should *not* be included
+	require.Nil(t, out[0].NodeResources)
+	require.Nil(t, out[0].ReservedResources)
+
+	qo := &QueryOptions{
+		Params: map[string]string{"resources": "true"},
+	}
+	out, _, err = nodes.List(qo)
+	require.NoError(t, err)
+	require.NotNil(t, out[0].NodeResources)
+	require.NotNil(t, out[0].ReservedResources)
+}
+
 func TestNodes_Info(t *testing.T) {
 	t.Parallel()
 	startTime := time.Now().Unix()

api/util_test.go

@@ -24,7 +24,7 @@ func assertWriteMeta(t *testing.T, wm *WriteMeta) {
 }
 
 func testJob() *Job {
-	task := NewTask("task1", "exec").
+	task := NewTask("task1", "raw_exec").
 		SetConfig("command", "/bin/sleep").
 		Require(&Resources{
 			CPU:      intToPtr(100),

command/agent/alloc_endpoint.go

@@ -33,6 +33,26 @@ func (s *HTTPServer) AllocsRequest(resp http.ResponseWriter, req *http.Request)
 		return nil, nil
 	}
 
+	// Parse resources and task_states field selection
+	resources, err := parseBool(req, "resources")
+	if err != nil {
+		return nil, err
+	}
+	taskStates, err := parseBool(req, "task_states")
+	if err != nil {
+		return nil, err
+	}
+
+	if resources != nil || taskStates != nil {
+		args.Fields = structs.NewAllocStubFields()
+		if resources != nil {
+			args.Fields.Resources = *resources
+		}
+		if taskStates != nil {
+			args.Fields.TaskStates = *taskStates
+		}
+	}
+
 	var out structs.AllocListResponse
 	if err := s.agent.RPC("Alloc.List", &args, &out); err != nil {
 		return nil, err

command/agent/csi_endpoint.go

@@ -344,13 +344,13 @@ func structsCSIVolumeToApi(vol *structs.CSIVolume) *api.CSIVolume {
 
 	for _, a := range vol.WriteAllocs {
 		if a != nil {
-			out.Allocations = append(out.Allocations, structsAllocListStubToApi(a.Stub()))
+			out.Allocations = append(out.Allocations, structsAllocListStubToApi(a.Stub(nil)))
 		}
 	}
 
 	for _, a := range vol.ReadAllocs {
 		if a != nil {
-			out.Allocations = append(out.Allocations, structsAllocListStubToApi(a.Stub()))
+			out.Allocations = append(out.Allocations, structsAllocListStubToApi(a.Stub(nil)))
 		}
 	}
 

command/agent/http.go

@@ -655,6 +655,20 @@ func parseNamespace(req *http.Request, n *string) {
 	}
 }
 
+// parseBool parses a query parameter to a boolean or returns (nil, nil) if the
+// parameter is not present.
+func parseBool(req *http.Request, field string) (*bool, error) {
+	if str := req.URL.Query().Get(field); str != "" {
+		param, err := strconv.ParseBool(str)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to parse value of %q (%v) as a bool: %v", field, str, err)
+		}
+		return &param, nil
+	}
+
+	return nil, nil
+}
+
 // parseToken is used to parse the X-Nomad-Token param
 func (s *HTTPServer) parseToken(req *http.Request, token *string) {
 	if other := req.Header.Get("X-Nomad-Token"); other != "" {

command/agent/http_test.go

@@ -515,6 +515,53 @@ func TestParseToken(t *testing.T) {
 	}
 }
 
+func TestParseBool(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		Input    string
+		Expected *bool
+		Err      bool // true if an error should be expected
+	}{
+		{
+			Input:    "",
+			Expected: nil,
+		},
+		{
+			Input:    "true",
+			Expected: helper.BoolToPtr(true),
+		},
+		{
+			Input:    "false",
+			Expected: helper.BoolToPtr(false),
+		},
+		{
+			Input: "1234",
+			Err:   true,
+		},
+	}
+
+	for i := range cases {
+		tc := cases[i]
+		t.Run("Input-"+tc.Input, func(t *testing.T) {
+			testURL, err := url.Parse("http://localhost/foo?resources=" + tc.Input)
+			require.NoError(t, err)
+			req := &http.Request{
+				URL: testURL,
+			}
+
+			result, err := parseBool(req, "resources")
+			if tc.Err {
+				require.Error(t, err)
+				require.Nil(t, result)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.Expected, result)
+			}
+		})
+	}
+}
+
 // TestHTTP_VerifyHTTPSClient asserts that a client certificate signed by the
 // appropriate CA is required when VerifyHTTPSClient=true.
 func TestHTTP_VerifyHTTPSClient(t *testing.T) {

command/agent/node_endpoint.go

@@ -20,6 +20,17 @@ func (s *HTTPServer) NodesRequest(resp http.ResponseWriter, req *http.Request) (
 		return nil, nil
 	}
 
+	// Parse resources field selection
+	resources, err := parseBool(req, "resources")
+	if err != nil {
+		return nil, err
+	}
+	if resources != nil {
+		args.Fields = &structs.NodeStubFields{
+			Resources: *resources,
+		}
+	}
+
 	var out structs.NodeListResponse
 	if err := s.agent.RPC("Node.List", &args, &out); err != nil {
 		return nil, err

nomad/alloc_endpoint.go

@@ -72,7 +72,7 @@ func (a *Alloc) List(args *structs.AllocListRequest, reply *structs.AllocListRes
 					break
 				}
 				alloc := raw.(*structs.Allocation)
-				allocs = append(allocs, alloc.Stub())
+				allocs = append(allocs, alloc.Stub(args.Fields))
 			}
 			reply.Allocations = allocs