Mailroom contact search API

.gitignore

@@ -10,6 +10,7 @@ dist
 .vscode
 .envrc
 docs/*
+docs
 
 # Test binary, build with `go test -c`
 *.test

README.md

@@ -71,7 +71,7 @@ To run the tests you need to create the test database:
 
 ```
 $ createdb mailroom_test
-$ createuser -P -E temba (set no password)
+$ createuser -P -E -s mailroom_test (set no password)
 ```
 
 To run all of the tests:

cmd/mailroom/main.go

@@ -24,6 +24,7 @@ import (
 	_ "github.com/nyaruka/mailroom/tasks/stats"
 	_ "github.com/nyaruka/mailroom/tasks/timeouts"
 
+	_ "github.com/nyaruka/mailroom/web/contact"
 	_ "github.com/nyaruka/mailroom/web/docs"
 	_ "github.com/nyaruka/mailroom/web/expression"
 	_ "github.com/nyaruka/mailroom/web/flow"

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/gorilla/schema v1.0.2
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect
 	github.com/lib/pq v1.0.0
 	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
@@ -36,6 +37,7 @@ require (
 	golang.org/x/net v0.0.0-20181217023233-e147a9138326 // indirect
 	google.golang.org/appengine v1.4.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/go-playground/validator.v9 v9.21.0
 	gopkg.in/mail.v2 v2.3.1
 )

go.sum

@@ -53,6 +53,11 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
@@ -123,6 +128,8 @@ gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gG
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v9 v9.12.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=

mailroom.go

@@ -211,7 +211,7 @@ func (mr *Mailroom) Start() error {
 	mr.handlerForeman.Start()
 
 	// start our web server
-	mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.S3Client, mr.WaitGroup)
+	mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.S3Client, mr.ElasticClient, mr.WaitGroup)
 	mr.webserver.Start()
 
 	logrus.Info("mailroom started")

models/contacts.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/nyaruka/gocommon/urns"
 	"github.com/nyaruka/goflow/assets"
+	"github.com/nyaruka/goflow/contactql"
 	"github.com/nyaruka/goflow/envs"
 	"github.com/nyaruka/goflow/excellent/types"
 	"github.com/nyaruka/goflow/flows"
@@ -148,34 +149,108 @@ func ContactIDsFromReferences(ctx context.Context, tx Queryer, org *OrgAssets, r
 	return ids, nil
 }
 
-// ContactIDsForQuery returns the ids of all the contacts that match the passed in query
-func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAssets, query string) ([]ContactID, error) {
-	start := time.Now()
-
-	if client == nil {
-		return nil, errors.Errorf("no elastic client available, check your configuration")
-	}
-
-	// our field resolver
-	resolver := func(key string) assets.Field {
+// buildFieldResolver builds a field resolver function for the passed in Org
+func buildFieldResolver(org *OrgAssets) contactql.FieldResolverFunc {
+	return func(key string) assets.Field {
 		f := org.FieldByKey(key)
 		if f == nil {
 			return nil
 		}
 		return f
 	}
+}
 
+// ParseQuery parses the passed in query for the passed in org, returning the parsed query and the resulting elastic query
+func ParseQuery(org *OrgAssets, resolver contactql.FieldResolverFunc, query string) (string, elastic.Query, error) {
 	// turn into elastic query
-	eq, err := search.ToElasticQuery(org.Env(), resolver, query)
+	parsed, eq, err := search.ToElasticQuery(org.Env(), resolver, query)
 	if err != nil {
-		return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
+		return "", nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
 	}
 
-	// filter by org, active, blocked and stopped
+	// additionally filter by org and active contacts
 	eq = elastic.NewBoolQuery().Must(
 		eq,
 		elastic.NewTermQuery("org_id", org.OrgID()),
 		elastic.NewTermQuery("is_active", true),
+	)
+
+	return parsed, eq, nil
+}
+
+// ContactIDsForQueryPage returns the ids of the contacts for the passed in query page
+func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *OrgAssets, group assets.GroupUUID, query string, sort string, offset int, pageSize int) (string, []ContactID, int64, error) {
+	start := time.Now()
+
+	if client == nil {
+		return "", nil, 0, errors.Errorf("no elastic client available, check your configuration")
+	}
+
+	resolver := buildFieldResolver(org)
+	parsed, eq, err := ParseQuery(org, resolver, query)
+	if err != nil {
+		return "", nil, 0, errors.Wrapf(err, "error parsing query: %s", query)
+	}
+
+	fieldSort, err := search.ToElasticFieldSort(resolver, sort)
+	if err != nil {
+		return "", nil, 0, errors.Wrapf(err, "error parsing sort")
+	}
+
+	// filter by our base group
+	eq = elastic.NewBoolQuery().Must(
+		eq,
+		elastic.NewTermQuery("groups", group),
+	)
+
+	s := client.Search("contacts").Routing(strconv.FormatInt(int64(org.OrgID()), 10))
+	s = s.Size(pageSize).From(offset).Query(eq).SortBy(fieldSort).FetchSource(false)
+
+	results, err := s.Do(ctx)
+	if err != nil {
+		return "", nil, 0, errors.Wrapf(err, "error performing query")
+	}
+
+	ids := make([]ContactID, 0, pageSize)
+	for _, hit := range results.Hits.Hits {
+		id, err := strconv.Atoi(hit.Id)
+		if err != nil {
+			return "", nil, 0, errors.Wrapf(err, "unexpected non-integer contact id: %s for search: %s", hit.Id, query)
+		}
+		ids = append(ids, ContactID(id))
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"org_id":      org.OrgID(),
+		"parsed":      parsed,
+		"group_uuid":  group,
+		"query":       query,
+		"elapsed":     time.Since(start),
+		"page_count":  len(ids),
+		"total_count": results.Hits.TotalHits,
+	}).Debug("paged contact query complete")
+
+	return parsed, ids, results.Hits.TotalHits, nil
+}
+
+// ContactIDsForQuery returns the ids of all the contacts that match the passed in query
+func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAssets, query string) ([]ContactID, error) {
+	start := time.Now()
+
+	if client == nil {
+		return nil, errors.Errorf("no elastic client available, check your configuration")
+	}
+
+	// turn into elastic query
+	resolver := buildFieldResolver(org)
+	_, eq, err := ParseQuery(org, resolver, query)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
+	}
+
+	// only include unblocked and unstopped contacts
+	eq = elastic.NewBoolQuery().Must(
+		eq,
 		elastic.NewTermQuery("is_blocked", false),
 		elastic.NewTermQuery("is_stopped", false),
 	)

models/contacts_test.go

@@ -46,11 +46,19 @@ func TestElasticContacts(t *testing.T) {
 				"query":{
 					"bool":{
 						"must":[
-							{"match":{"name":{"query":"george"}}},
-							{"term":{"org_id":1}},
-							{"term":{"is_active":true}},
-							{"term":{"is_blocked":false}},
-							{"term":{"is_stopped":false}}
+							{ "bool":{
+								"must":[
+									{"match":{"name":{"query":"george"}}},
+									{"term":{"org_id":1}},
+									{"term":{"is_active":true}}
+								]
+							}},
+							{ "term":{
+								"is_blocked":false
+							}},
+							{"term":
+								{"is_stopped":false
+							}}
 						]
 					}
 				},
@@ -91,9 +99,13 @@ func TestElasticContacts(t *testing.T) {
 				"query":{
 					"bool":{
 						"must":[
-							{"match":{"name":{"query":"nobody"}}},
-							{"term":{"org_id":1}},
-							{"term":{"is_active":true}},
+							{"bool":
+								{"must":[
+									{"match":{"name":{"query":"nobody"}}},
+									{"term":{"org_id":1}},
+									{"term":{"is_active":true}}
+								]}
+							},
 							{"term":{"is_blocked":false}},
 							{"term":{"is_stopped":false}}
 						]
@@ -133,7 +145,7 @@ func TestElasticContacts(t *testing.T) {
 			assert.Error(t, err)
 		} else {
 			assert.NoError(t, err, "%d: error encountered performing query", i)
-			assert.JSONEq(t, tc.Request, es.LastBody, "%d: request mismatch", i)
+			assert.JSONEq(t, tc.Request, es.LastBody, "%d: request mismatch, got: %s", i, es.LastBody)
 			assert.Equal(t, tc.Contacts, ids, "%d: ids mismatch", i)
 		}
 	}

models/test_constants.go

@@ -80,6 +80,9 @@ var RemindersEvent2ID = CampaignEventID(10001)
 var DoctorsGroupID = GroupID(10000)
 var DoctorsGroupUUID = assets.GroupUUID("c153e265-f7c9-4539-9dbc-9b358714b638")
 
+var AllContactsGroupID = GroupID(1)
+var AllContactsGroupUUID = assets.GroupUUID("bc268217-9ffa-49e0-883e-e4e09c252a5a")
+
 var TestersGroupID = GroupID(10001)
 var TestersGroupUUID = assets.GroupUUID("5e9d8fab-5e7e-4f51-b533-261af5dea70d")