Merge pull request #10184 from hashicorp/f-fuzzy-search

api: implement fuzzy search API

CHANGELOG.md

@@ -10,6 +10,7 @@ __BACKWARDS INCOMPATIBILITIES:__
  * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)]
 
 IMPROVEMENTS:
+ * api: Added an API endpoint for fuzzy search queries [[GH-10184](https://github.com/hashicorp/nomad/pull/10184)]
  * api: Removed unimplemented `CSIVolumes.PluginList` API. [[GH-10158](https://github.com/hashicorp/nomad/issues/10158)]
  * cli: Update defaults for `nomad operator debug` flags `-interval` and `-server-id` to match common usage. [[GH-10121](https://github.com/hashicorp/nomad/issues/10121)]
  * cli: Added `nomad ui -authenticate` flag to generate a one-time token for authenticating to the web UI when ACLs are enabled. [[GH-10097](https://github.com/hashicorp/nomad/issues/10097)]

api/contexts/contexts.go

@@ -1,9 +1,12 @@
+// Package contexts provides constants used with the Nomad Search API.
 package contexts
 
-// Context defines the scope in which a search for Nomad object operates
+// Context defines the scope in which a search for Nomad object operates.
 type Context string
 
 const (
+	// These Context types are used to reference the high level Nomad object
+	// types than can be searched.
 	Allocs          Context = "allocs"
 	Deployments     Context = "deployment"
 	Evals           Context = "evals"
@@ -15,5 +18,16 @@ const (
 	ScalingPolicies Context = "scaling_policy"
 	Plugins         Context = "plugins"
 	Volumes         Context = "volumes"
-	All             Context = "all"
+
+	// These Context types are used to associate a search result from a lower
+	// level Nomad object with one of the higher level Context types above.
+	Groups   Context = "groups"
+	Services Context = "services"
+	Tasks    Context = "tasks"
+	Images   Context = "images"
+	Commands Context = "commands"
+	Classes  Context = "classes"
+
+	// Context used to represent the set of all the higher level Context types.
+	All Context = "all"
 )

api/search.go

@@ -13,7 +13,7 @@ func (c *Client) Search() *Search {
 	return &Search{client: c}
 }
 
-// PrefixSearch returns a list of matches for a particular context and prefix.
+// PrefixSearch returns a set of matches for a particular context and prefix.
 func (s *Search) PrefixSearch(prefix string, context contexts.Context, q *QueryOptions) (*SearchResponse, *QueryMeta, error) {
 	var resp SearchResponse
 	req := &SearchRequest{Prefix: prefix, Context: context}
@@ -26,14 +26,72 @@ func (s *Search) PrefixSearch(prefix string, context contexts.Context, q *QueryO
 	return &resp, qm, nil
 }
 
+type SearchResponse struct {
+	Matches     map[contexts.Context][]string
+	Truncations map[contexts.Context]bool
+	QueryMeta
+}
+
 type SearchRequest struct {
 	Prefix  string
 	Context contexts.Context
 	QueryOptions
 }
 
-type SearchResponse struct {
-	Matches     map[contexts.Context][]string
+// FuzzySearch returns a set of matches for a given context and string.
+func (s *Search) FuzzySearch(text string, context contexts.Context, q *QueryOptions) (*FuzzySearchResponse, *QueryMeta, error) {
+	var resp FuzzySearchResponse
+
+	req := &FuzzySearchRequest{
+		Context: context,
+		Text:    text,
+	}
+
+	qm, err := s.client.putQuery("/v1/search/fuzzy", req, &resp, q)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &resp, qm, nil
+}
+
+// FuzzyMatch is used to describe the ID of an object which may be a machine
+// readable UUID or a human readable Name. If the object is a component of a Job,
+// the Scope is a list of IDs starting from Namespace down to the parent object of
+// ID.
+//
+// e.g. A Task-level service would have scope like,
+//   ["<namespace>", "<job>", "<group>", "<task>"]
+type FuzzyMatch struct {
+	ID    string   // ID is UUID or Name of object
+	Scope []string `json:",omitempty"` // IDs of parent objects
+}
+
+// FuzzySearchResponse is used to return fuzzy matches and information about
+// whether the match list is truncated specific to each type of searchable Context.
+type FuzzySearchResponse struct {
+	// Matches is a map of Context types to IDs which fuzzy match a specified query.
+	Matches map[contexts.Context][]FuzzyMatch
+
+	// Truncations indicates whether the matches for a particular Context have
+	// been truncated.
 	Truncations map[contexts.Context]bool
+
 	QueryMeta
 }
+
+// FuzzySearchRequest is used to parameterize a fuzzy search request, and returns
+// a list of matches made up of jobs, allocations, evaluations, and/or nodes,
+// along with whether or not the information returned is truncated.
+type FuzzySearchRequest struct {
+	// Text is what names are fuzzy-matched to. E.g. if the given text were
+	// "py", potential matches might be "python", "mypy", etc. of jobs, nodes,
+	// allocs, groups, services, commands, images, classes.
+	Text string
+
+	// Context is the type that can be matched against. A Context of "all" indicates
+	// all Contexts types are queried for matching.
+	Context contexts.Context
+
+	QueryOptions
+}

api/search_test.go

@@ -7,26 +7,46 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestSearch_List(t *testing.T) {
-	require := require.New(t)
+func TestSearch_PrefixSearch(t *testing.T) {
 	t.Parallel()
 
 	c, s := makeClient(t, nil, nil)
 	defer s.Stop()
 
 	job := testJob()
 	_, _, err := c.Jobs().Register(job, nil)
-	require.Nil(err)
+	require.NoError(t, err)
 
 	id := *job.ID
 	prefix := id[:len(id)-2]
 	resp, qm, err := c.Search().PrefixSearch(prefix, contexts.Jobs, nil)
-
-	require.Nil(err)
-	require.NotNil(qm)
-	require.NotNil(qm)
+	require.NoError(t, err)
+	require.NotNil(t, qm)
+	require.NotNil(t, resp)
 
 	jobMatches := resp.Matches[contexts.Jobs]
-	require.Equal(1, len(jobMatches))
-	require.Equal(id, jobMatches[0])
+	require.Len(t, jobMatches, 1)
+	require.Equal(t, id, jobMatches[0])
+}
+
+func TestSearch_FuzzySearch(t *testing.T) {
+	t.Parallel()
+	c, s := makeClient(t, nil, nil)
+	defer s.Stop()
+
+	job := testJob()
+	_, _, err := c.Jobs().Register(job, nil)
+	require.NoError(t, err)
+
+	resp, qm, err := c.Search().FuzzySearch("bin", contexts.All, nil)
+	require.NoError(t, err)
+	require.NotNil(t, qm)
+	require.NotNil(t, resp)
+
+	commandMatches := resp.Matches[contexts.Commands]
+	require.Len(t, commandMatches, 1)
+	require.Equal(t, "/bin/sleep", commandMatches[0].ID)
+	require.Equal(t, []string{
+		"default", *job.ID, "group1", "task1",
+	}, commandMatches[0].Scope)
 }

client/taskenv/env_test.go

@@ -217,123 +217,6 @@ func TestEnvironment_AsList(t *testing.T) {
 	require.Equal(t, exp, act)
 }
 
-// COMPAT(0.11): Remove in 0.11
-func TestEnvironment_AsList_Old(t *testing.T) {
-	n := mock.Node()
-	n.Meta = map[string]string{
-		"metaKey": "metaVal",
-	}
-	a := mock.Alloc()
-	a.AllocatedResources = nil
-	a.Resources = &structs.Resources{
-		CPU:      500,
-		MemoryMB: 256,
-		DiskMB:   150,
-		Networks: []*structs.NetworkResource{
-			{
-				Device: "eth0",
-				IP:     "192.168.0.100",
-				ReservedPorts: []structs.Port{
-					{Label: "ssh", Value: 22},
-					{Label: "other", Value: 1234},
-				},
-				MBits:        50,
-				DynamicPorts: []structs.Port{{Label: "http", Value: 2000}},
-			},
-		},
-	}
-	a.TaskResources = map[string]*structs.Resources{
-		"web": {
-			CPU:      500,
-			MemoryMB: 256,
-			Networks: []*structs.NetworkResource{
-				{
-					Device:        "eth0",
-					IP:            "127.0.0.1",
-					ReservedPorts: []structs.Port{{Label: "https", Value: 8080}},
-					MBits:         50,
-					DynamicPorts:  []structs.Port{{Label: "http", Value: 80}},
-				},
-			},
-		},
-	}
-	a.TaskResources["ssh"] = &structs.Resources{
-		Networks: []*structs.NetworkResource{
-			{
-				Device: "eth0",
-				IP:     "192.168.0.100",
-				MBits:  50,
-				ReservedPorts: []structs.Port{
-					{Label: "ssh", Value: 22},
-					{Label: "other", Value: 1234},
-				},
-			},
-		},
-	}
-
-	// simulate canonicalization on restore or fetch
-	a.Canonicalize()
-
-	task := a.Job.TaskGroups[0].Tasks[0]
-	task.Env = map[string]string{
-		"taskEnvKey": "taskEnvVal",
-	}
-	task.Resources.Networks = []*structs.NetworkResource{
-		// Nomad 0.8 didn't fully populate the fields in task Resource Networks
-		{
-			IP:            "",
-			ReservedPorts: []structs.Port{{Label: "https"}},
-			DynamicPorts:  []structs.Port{{Label: "http"}},
-		},
-	}
-	env := NewBuilder(n, a, task, "global").SetDriverNetwork(
-		&drivers.DriverNetwork{PortMap: map[string]int{"https": 443}},
-	)
-
-	act := env.Build().List()
-	exp := []string{
-		"taskEnvKey=taskEnvVal",
-		"NOMAD_ADDR_http=127.0.0.1:80",
-		"NOMAD_PORT_http=80",
-		"NOMAD_IP_http=127.0.0.1",
-		"NOMAD_ADDR_https=127.0.0.1:8080",
-		"NOMAD_PORT_https=443",
-		"NOMAD_IP_https=127.0.0.1",
-		"NOMAD_HOST_PORT_http=80",
-		"NOMAD_HOST_PORT_https=8080",
-		"NOMAD_TASK_NAME=web",
-		"NOMAD_GROUP_NAME=web",
-		"NOMAD_ADDR_ssh_other=192.168.0.100:1234",
-		"NOMAD_ADDR_ssh_ssh=192.168.0.100:22",
-		"NOMAD_IP_ssh_other=192.168.0.100",
-		"NOMAD_IP_ssh_ssh=192.168.0.100",
-		"NOMAD_PORT_ssh_other=1234",
-		"NOMAD_PORT_ssh_ssh=22",
-		"NOMAD_CPU_LIMIT=500",
-		"NOMAD_DC=dc1",
-		"NOMAD_NAMESPACE=default",
-		"NOMAD_REGION=global",
-		"NOMAD_MEMORY_LIMIT=256",
-		"NOMAD_META_ELB_CHECK_INTERVAL=30s",
-		"NOMAD_META_ELB_CHECK_MIN=3",
-		"NOMAD_META_ELB_CHECK_TYPE=http",
-		"NOMAD_META_FOO=bar",
-		"NOMAD_META_OWNER=armon",
-		"NOMAD_META_elb_check_interval=30s",
-		"NOMAD_META_elb_check_min=3",
-		"NOMAD_META_elb_check_type=http",
-		"NOMAD_META_foo=bar",
-		"NOMAD_META_owner=armon",
-		fmt.Sprintf("NOMAD_JOB_ID=%s", a.Job.ID),
-		"NOMAD_JOB_NAME=my-job",
-		fmt.Sprintf("NOMAD_ALLOC_ID=%s", a.ID),
-		"NOMAD_ALLOC_INDEX=0",
-	}
-	sort.Strings(act)
-	sort.Strings(exp)
-	require.Equal(t, exp, act)
-}
-
 func TestEnvironment_AllValues(t *testing.T) {
 	t.Parallel()
 

command/agent/agent.go

@@ -424,6 +424,16 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 	conf.LicenseEnv = agentConfig.Server.LicenseEnv
 	conf.LicensePath = agentConfig.Server.LicensePath
 
+	// Add the search configuration
+	if search := agentConfig.Server.Search; search != nil {
+		conf.SearchConfig = &structs.SearchConfig{
+			FuzzyEnabled:  search.FuzzyEnabled,
+			LimitQuery:    search.LimitQuery,
+			LimitResults:  search.LimitResults,
+			MinTermLength: search.MinTermLength,
+		}
+	}
+
 	return conf, nil
 }
 

command/agent/config.go

@@ -510,6 +510,44 @@ type ServerConfig struct {
 
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
+
+	Search *Search `hcl:"search"`
+}
+
+// Search is used in servers to configure search API options.
+type Search struct {
+	// FuzzyEnabled toggles whether the FuzzySearch API is enabled. If not
+	// enabled, requests to /v1/search/fuzzy will reply with a 404 response code.
+	//
+	// Default: enabled.
+	FuzzyEnabled bool `hcl:"fuzzy_enabled"`
+
+	// LimitQuery limits the number of objects searched in the FuzzySearch API.
+	// The results are indicated as truncated if the limit is reached.
+	//
+	// Lowering this value can reduce resource consumption of Nomad server when
+	// the FuzzySearch API is enabled.
+	//
+	// Default value: 20.
+	LimitQuery int `hcl:"limit_query"`
+
+	// LimitResults limits the number of results provided by the FuzzySearch API.
+	// The results are indicated as truncate if the limit is reached.
+	//
+	// Lowering this value can reduce resource consumption of Nomad server per
+	// fuzzy search request when the FuzzySearch API is enabled.
+	//
+	// Default value: 100.
+	LimitResults int `hcl:"limit_results"`
+
+	// MinTermLength is the minimum length of Text required before the FuzzySearch
+	// API will return results.
+	//
+	// Increasing this value can avoid resource consumption on Nomad server by
+	// reducing searches with less meaningful results.
+	//
+	// Default value: 2.
+	MinTermLength int `hcl:"min_term_length"`
 }
 
 // ServerJoin is used in both clients and servers to bootstrap connections to
@@ -900,6 +938,12 @@ func DefaultConfig() *Config {
 				RetryInterval:    30 * time.Second,
 				RetryMaxAttempts: 0,
 			},
+			Search: &Search{
+				FuzzyEnabled:  true,
+				LimitQuery:    20,
+				LimitResults:  100,
+				MinTermLength: 2,
+			},
 		},
 		ACL: &ACLConfig{
 			Enabled:   false,
@@ -1434,6 +1478,19 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		result.DefaultSchedulerConfig = &c
 	}
 
+	if b.Search != nil {
+		result.Search = &Search{FuzzyEnabled: b.Search.FuzzyEnabled}
+		if b.Search.LimitQuery > 0 {
+			result.Search.LimitQuery = b.Search.LimitQuery
+		}
+		if b.Search.LimitResults > 0 {
+			result.Search.LimitResults = b.Search.LimitResults
+		}
+		if b.Search.MinTermLength > 0 {
+			result.Search.MinTermLength = b.Search.MinTermLength
+		}
+	}
+
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)