api: implement fuzzy search API

This PR introduces the /v1/search/fuzzy API endpoint, used for fuzzy
searching objects in Nomad. The fuzzy search endpoint routes requests
to the Nomad Server leader, which implements the Search.FuzzySearch RPC
method.

Requests to the fuzzy search API are based on the api.FuzzySearchRequest
object, e.g.

{
  "Text": "ed",
  "Context": "all"
}

Responses from the fuzzy search API are based on the api.FuzzySearchResponse
object, e.g.

{
  "Index": 27,
  "KnownLeader": true,
  "LastContact": 0,
  "Matches": {
    "tasks": [
      {
        "ID": "redis",
        "Scope": [
          "default",
          "example",
          "cache"
        ]
      }
    ],
    "evals": [],
    "deployment": [],
    "volumes": [],
    "scaling_policy": [],
    "images": [
      {
        "ID": "redis:3.2",
        "Scope": [
          "default",
          "example",
          "cache",
          "redis"
        ]
      }
    ]
  },
  "Truncations": {
    "volumes": false,
    "scaling_policy": false,
    "evals": false,
    "deployment": false
  }
}

The API is tunable using the new server.search stanza, e.g.

server {
  search {
    fuzzy_enabled   = true
    limit_query     = 200
    limit_results   = 1000
    min_term_length = 5
  }
}

These values can be increased or decreased, so as to provide more
search results or to reduce load on the Nomad Server. The fuzzy search
API can be disabled entirely by setting `fuzzy_enabled` to `false`.

CHANGELOG.md

@@ -10,6 +10,7 @@ __BACKWARDS INCOMPATIBILITIES:__
  * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)]
 
 IMPROVEMENTS:
+ * api: Added an API endpoint for fuzzy search queries [[GH-10184](https://github.com/hashicorp/nomad/pull/10184)]
  * api: Removed unimplemented `CSIVolumes.PluginList` API. [[GH-10158](https://github.com/hashicorp/nomad/issues/10158)]
  * cli: Update defaults for `nomad operator debug` flags `-interval` and `-server-id` to match common usage. [[GH-10121](https://github.com/hashicorp/nomad/issues/10121)]
  * consul/connect: Enable setting `local_bind_address` field on connect upstreams [[GH-6248](https://github.com/hashicorp/nomad/issues/6248)]

api/contexts/contexts.go

@@ -1,9 +1,12 @@
+// Package contexts provides constants used with the Nomad Search API.
 package contexts
 
-// Context defines the scope in which a search for Nomad object operates
+// Context defines the scope in which a search for Nomad object operates.
 type Context string
 
 const (
+	// These Context types are used to reference the high level Nomad object
+	// types than can be searched.
 	Allocs          Context = "allocs"
 	Deployments     Context = "deployment"
 	Evals           Context = "evals"
@@ -15,5 +18,16 @@ const (
 	ScalingPolicies Context = "scaling_policy"
 	Plugins         Context = "plugins"
 	Volumes         Context = "volumes"
-	All             Context = "all"
+
+	// These Context types are used to associate a search result from a lower
+	// level Nomad object with one of the higher level Context types above.
+	Groups   Context = "groups"
+	Services Context = "services"
+	Tasks    Context = "tasks"
+	Images   Context = "images"
+	Commands Context = "commands"
+	Classes  Context = "classes"
+
+	// Context used to represent the set of all the higher level Context types.
+	All Context = "all"
 )

api/search.go

@@ -13,7 +13,7 @@ func (c *Client) Search() *Search {
 	return &Search{client: c}
 }
 
-// PrefixSearch returns a list of matches for a particular context and prefix.
+// PrefixSearch returns a set of matches for a particular context and prefix.
 func (s *Search) PrefixSearch(prefix string, context contexts.Context, q *QueryOptions) (*SearchResponse, *QueryMeta, error) {
 	var resp SearchResponse
 	req := &SearchRequest{Prefix: prefix, Context: context}
@@ -26,14 +26,72 @@ func (s *Search) PrefixSearch(prefix string, context contexts.Context, q *QueryO
 	return &resp, qm, nil
 }
 
+type SearchResponse struct {
+	Matches     map[contexts.Context][]string
+	Truncations map[contexts.Context]bool
+	QueryMeta
+}
+
 type SearchRequest struct {
 	Prefix  string
 	Context contexts.Context
 	QueryOptions
 }
 
-type SearchResponse struct {
-	Matches     map[contexts.Context][]string
+// FuzzySearch returns a set of matches for a given context and string.
+func (s *Search) FuzzySearch(text string, context contexts.Context, q *QueryOptions) (*FuzzySearchResponse, *QueryMeta, error) {
+	var resp FuzzySearchResponse
+
+	req := &FuzzySearchRequest{
+		Context: context,
+		Text:    text,
+	}
+
+	qm, err := s.client.putQuery("/v1/search/fuzzy", req, &resp, q)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &resp, qm, nil
+}
+
+// FuzzyMatch is used to describe the ID of an object which may be a machine
+// readable UUID or a human readable Name. If the object is a component of a Job,
+// the Scope is a list of IDs starting from Namespace down to the parent object of
+// ID.
+//
+// e.g. A Task-level service would have scope like,
+//   ["<namespace>", "<job>", "<group>", "<task>"]
+type FuzzyMatch struct {
+	ID    string   // ID is UUID or Name of object
+	Scope []string `json:",omitempty"` // IDs of parent objects
+}
+
+// FuzzySearchResponse is used to return fuzzy matches and information about
+// whether the match list is truncated specific to each type of searchable Context.
+type FuzzySearchResponse struct {
+	// Matches is a map of Context types to IDs which fuzzy match a specified query.
+	Matches map[contexts.Context][]FuzzyMatch
+
+	// Truncations indicates whether the matches for a particular Context have
+	// been truncated.
 	Truncations map[contexts.Context]bool
+
 	QueryMeta
 }
+
+// FuzzySearchRequest is used to parameterize a fuzzy search request, and returns
+// a list of matches made up of jobs, allocations, evaluations, and/or nodes,
+// along with whether or not the information returned is truncated.
+type FuzzySearchRequest struct {
+	// Text is what names are fuzzy-matched to. E.g. if the given text were
+	// "py", potential matches might be "python", "mypy", etc. of jobs, nodes,
+	// allocs, groups, services, commands, images, classes.
+	Text string
+
+	// Context is the type that can be matched against. A Context of "all" indicates
+	// all Contexts types are queried for matching.
+	Context contexts.Context
+
+	QueryOptions
+}

api/search_test.go

@@ -7,26 +7,46 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestSearch_List(t *testing.T) {
-	require := require.New(t)
+func TestSearch_PrefixSearch(t *testing.T) {
 	t.Parallel()
 
 	c, s := makeClient(t, nil, nil)
 	defer s.Stop()
 
 	job := testJob()
 	_, _, err := c.Jobs().Register(job, nil)
-	require.Nil(err)
+	require.NoError(t, err)
 
 	id := *job.ID
 	prefix := id[:len(id)-2]
 	resp, qm, err := c.Search().PrefixSearch(prefix, contexts.Jobs, nil)
-
-	require.Nil(err)
-	require.NotNil(qm)
-	require.NotNil(qm)
+	require.NoError(t, err)
+	require.NotNil(t, qm)
+	require.NotNil(t, resp)
 
 	jobMatches := resp.Matches[contexts.Jobs]
-	require.Equal(1, len(jobMatches))
-	require.Equal(id, jobMatches[0])
+	require.Len(t, jobMatches, 1)
+	require.Equal(t, id, jobMatches[0])
+}
+
+func TestSearch_FuzzySearch(t *testing.T) {
+	t.Parallel()
+	c, s := makeClient(t, nil, nil)
+	defer s.Stop()
+
+	job := testJob()
+	_, _, err := c.Jobs().Register(job, nil)
+	require.NoError(t, err)
+
+	resp, qm, err := c.Search().FuzzySearch("bin", contexts.All, nil)
+	require.NoError(t, err)
+	require.NotNil(t, qm)
+	require.NotNil(t, resp)
+
+	commandMatches := resp.Matches[contexts.Commands]
+	require.Len(t, commandMatches, 1)
+	require.Equal(t, "/bin/sleep", commandMatches[0].ID)
+	require.Equal(t, []string{
+		"default", *job.ID, "group1", "task1",
+	}, commandMatches[0].Scope)
 }

client/taskenv/env_test.go

@@ -217,123 +217,6 @@ func TestEnvironment_AsList(t *testing.T) {
 	require.Equal(t, exp, act)
 }
 
-// COMPAT(0.11): Remove in 0.11
-func TestEnvironment_AsList_Old(t *testing.T) {
-	n := mock.Node()
-	n.Meta = map[string]string{
-		"metaKey": "metaVal",
-	}
-	a := mock.Alloc()
-	a.AllocatedResources = nil
-	a.Resources = &structs.Resources{
-		CPU:      500,
-		MemoryMB: 256,
-		DiskMB:   150,
-		Networks: []*structs.NetworkResource{
-			{
-				Device: "eth0",
-				IP:     "192.168.0.100",
-				ReservedPorts: []structs.Port{
-					{Label: "ssh", Value: 22},
-					{Label: "other", Value: 1234},
-				},
-				MBits:        50,
-				DynamicPorts: []structs.Port{{Label: "http", Value: 2000}},
-			},
-		},
-	}
-	a.TaskResources = map[string]*structs.Resources{
-		"web": {
-			CPU:      500,
-			MemoryMB: 256,
-			Networks: []*structs.NetworkResource{
-				{
-					Device:        "eth0",
-					IP:            "127.0.0.1",
-					ReservedPorts: []structs.Port{{Label: "https", Value: 8080}},
-					MBits:         50,
-					DynamicPorts:  []structs.Port{{Label: "http", Value: 80}},
-				},
-			},
-		},
-	}
-	a.TaskResources["ssh"] = &structs.Resources{
-		Networks: []*structs.NetworkResource{
-			{
-				Device: "eth0",
-				IP:     "192.168.0.100",
-				MBits:  50,
-				ReservedPorts: []structs.Port{
-					{Label: "ssh", Value: 22},
-					{Label: "other", Value: 1234},
-				},
-			},
-		},
-	}
-
-	// simulate canonicalization on restore or fetch
-	a.Canonicalize()
-
-	task := a.Job.TaskGroups[0].Tasks[0]
-	task.Env = map[string]string{
-		"taskEnvKey": "taskEnvVal",
-	}
-	task.Resources.Networks = []*structs.NetworkResource{
-		// Nomad 0.8 didn't fully populate the fields in task Resource Networks
-		{
-			IP:            "",
-			ReservedPorts: []structs.Port{{Label: "https"}},
-			DynamicPorts:  []structs.Port{{Label: "http"}},
-		},
-	}
-	env := NewBuilder(n, a, task, "global").SetDriverNetwork(
-		&drivers.DriverNetwork{PortMap: map[string]int{"https": 443}},
-	)
-
-	act := env.Build().List()
-	exp := []string{
-		"taskEnvKey=taskEnvVal",
-		"NOMAD_ADDR_http=127.0.0.1:80",
-		"NOMAD_PORT_http=80",
-		"NOMAD_IP_http=127.0.0.1",
-		"NOMAD_ADDR_https=127.0.0.1:8080",
-		"NOMAD_PORT_https=443",
-		"NOMAD_IP_https=127.0.0.1",
-		"NOMAD_HOST_PORT_http=80",
-		"NOMAD_HOST_PORT_https=8080",
-		"NOMAD_TASK_NAME=web",
-		"NOMAD_GROUP_NAME=web",
-		"NOMAD_ADDR_ssh_other=192.168.0.100:1234",
-		"NOMAD_ADDR_ssh_ssh=192.168.0.100:22",
-		"NOMAD_IP_ssh_other=192.168.0.100",
-		"NOMAD_IP_ssh_ssh=192.168.0.100",
-		"NOMAD_PORT_ssh_other=1234",
-		"NOMAD_PORT_ssh_ssh=22",
-		"NOMAD_CPU_LIMIT=500",
-		"NOMAD_DC=dc1",
-		"NOMAD_NAMESPACE=default",
-		"NOMAD_REGION=global",
-		"NOMAD_MEMORY_LIMIT=256",
-		"NOMAD_META_ELB_CHECK_INTERVAL=30s",
-		"NOMAD_META_ELB_CHECK_MIN=3",
-		"NOMAD_META_ELB_CHECK_TYPE=http",
-		"NOMAD_META_FOO=bar",
-		"NOMAD_META_OWNER=armon",
-		"NOMAD_META_elb_check_interval=30s",
-		"NOMAD_META_elb_check_min=3",
-		"NOMAD_META_elb_check_type=http",
-		"NOMAD_META_foo=bar",
-		"NOMAD_META_owner=armon",
-		fmt.Sprintf("NOMAD_JOB_ID=%s", a.Job.ID),
-		"NOMAD_JOB_NAME=my-job",
-		fmt.Sprintf("NOMAD_ALLOC_ID=%s", a.ID),
-		"NOMAD_ALLOC_INDEX=0",
-	}
-	sort.Strings(act)
-	sort.Strings(exp)
-	require.Equal(t, exp, act)
-}
-
 func TestEnvironment_AllValues(t *testing.T) {
 	t.Parallel()
 

command/agent/agent.go

@@ -422,6 +422,16 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 	conf.LicenseEnv = agentConfig.Server.LicenseEnv
 	conf.LicensePath = agentConfig.Server.LicensePath
 
+	// Add the search configuration
+	if search := agentConfig.Server.Search; search != nil {
+		conf.SearchConfig = &structs.SearchConfig{
+			FuzzyEnabled:  search.FuzzyEnabled,
+			LimitQuery:    search.LimitQuery,
+			LimitResults:  search.LimitResults,
+			MinTermLength: search.MinTermLength,
+		}
+	}
+
 	return conf, nil
 }
 

command/agent/config.go

@@ -503,6 +503,44 @@ type ServerConfig struct {
 
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
+
+	Search *Search `hcl:"search"`
+}
+
+// Search is used in servers to configure search API options.
+type Search struct {
+	// FuzzyEnabled toggles whether the FuzzySearch API is enabled. If not
+	// enabled, requests to /v1/search/fuzzy will reply with a 404 response code.
+	//
+	// Default: enabled.
+	FuzzyEnabled bool `hcl:"fuzzy_enabled"`
+
+	// LimitQuery limits the number of objects searched in the FuzzySearch API.
+	// The results are indicated as truncated if the limit is reached.
+	//
+	// Lowering this value can reduce resource consumption of Nomad server when
+	// the FuzzySearch API is enabled.
+	//
+	// Default value: 20.
+	LimitQuery int `hcl:"limit_query"`
+
+	// LimitResults limits the number of results provided by the FuzzySearch API.
+	// The results are indicated as truncate if the limit is reached.
+	//
+	// Lowering this value can reduce resource consumption of Nomad server per
+	// fuzzy search request when the FuzzySearch API is enabled.
+	//
+	// Default value: 100.
+	LimitResults int `hcl:"limit_results"`
+
+	// MinTermLength is the minimum length of Text required before the FuzzySearch
+	// API will return results.
+	//
+	// Increasing this value can avoid resource consumption on Nomad server by
+	// reducing searches with less meaningful results.
+	//
+	// Default value: 2.
+	MinTermLength int `hcl:"min_term_length"`
 }
 
 // ServerJoin is used in both clients and servers to bootstrap connections to
@@ -900,6 +938,12 @@ func DefaultConfig() *Config {
 				RetryInterval:    30 * time.Second,
 				RetryMaxAttempts: 0,
 			},
+			Search: &Search{
+				FuzzyEnabled:  true,
+				LimitQuery:    20,
+				LimitResults:  100,
+				MinTermLength: 2,
+			},
 		},
 		ACL: &ACLConfig{
 			Enabled:   false,
@@ -1434,6 +1478,19 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		result.DefaultSchedulerConfig = &c
 	}
 
+	if b.Search != nil {
+		result.Search = &Search{FuzzyEnabled: b.Search.FuzzyEnabled}
+		if b.Search.LimitQuery > 0 {
+			result.Search.LimitQuery = b.Search.LimitQuery
+		}
+		if b.Search.LimitResults > 0 {
+			result.Search.LimitResults = b.Search.LimitResults
+		}
+		if b.Search.MinTermLength > 0 {
+			result.Search.MinTermLength = b.Search.MinTermLength
+		}
+	}
+
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)