sow for searching API, add unit test

README.md

@@ -71,7 +71,7 @@ To run the tests you need to create the test database:
 
 ```
 $ createdb mailroom_test
-$ createuser -P -E temba (set no password)
+$ createuser -P -E -s mailroom_test (set no password)
 ```
 
 To run all of the tests:

models/contacts.go

@@ -148,29 +148,45 @@ func ContactIDsFromReferences(ctx context.Context, tx Queryer, org *OrgAssets, r
 	return ids, nil
 }
 
-// ContactIDsForQueryPage returns the ids of the contacts for the passed in query page
-func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *OrgAssets, group assets.GroupUUID, query string, offset int, pageSize int) ([]ContactID, int64, error) {
-	start := time.Now()
-
-	if client == nil {
-		return nil, 0, errors.Errorf("no elastic client available, check your configuration")
-	}
-
+// ParseQuery parses the passed in query for the passed in org, returning the parsed query and the resulting elastic query
+func ParseQuery(org *OrgAssets, query string) (string, elastic.Query, error) {
 	// our field resolver
 	resolver := func(key string) assets.Field {
 		return org.FieldByKey(key)
 	}
 
 	// turn into elastic query
-	eq, err := search.ToElasticQuery(org.Env(), resolver, query)
+	parsed, eq, err := search.ToElasticQuery(org.Env(), resolver, query)
 	if err != nil {
-		return nil, 0, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
+		return "", nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
 	}
 
+	// additionally filter by org and active contacts
 	eq = elastic.NewBoolQuery().Must(
 		eq,
 		elastic.NewTermQuery("org_id", org.OrgID()),
-		elastic.NewTermQuery("is_active", true), // technically this ought to be redundant with the group, but better to be safe
+		elastic.NewTermQuery("is_active", true),
+	)
+
+	return parsed, eq, nil
+}
+
+// ContactIDsForQueryPage returns the ids of the contacts for the passed in query page
+func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *OrgAssets, group assets.GroupUUID, query string, offset int, pageSize int) (string, []ContactID, int64, error) {
+	start := time.Now()
+
+	if client == nil {
+		return "", nil, 0, errors.Errorf("no elastic client available, check your configuration")
+	}
+
+	parsed, eq, err := ParseQuery(org, query)
+	if err != nil {
+		return "", nil, 0, errors.Wrapf(err, "error parsing query: %s", query)
+	}
+
+	// filter by our base group
+	eq = elastic.NewBoolQuery().Must(
+		eq,
 		elastic.NewTermQuery("groups", group),
 	)
 
@@ -179,28 +195,29 @@ func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *Or
 
 	results, err := s.Do(ctx)
 	if err != nil {
-		return nil, 0, errors.Wrapf(err, "error performing query")
+		return "", nil, 0, errors.Wrapf(err, "error performing query")
 	}
 
 	ids := make([]ContactID, 0, pageSize)
 	for _, hit := range results.Hits.Hits {
 		id, err := strconv.Atoi(hit.Id)
 		if err != nil {
-			return nil, 0, errors.Wrapf(err, "unexpected non-integer contact id: %s for search: %s", hit.Id, query)
+			return "", nil, 0, errors.Wrapf(err, "unexpected non-integer contact id: %s for search: %s", hit.Id, query)
 		}
 
 		ids = append(ids, ContactID(id))
 	}
 
 	logrus.WithFields(logrus.Fields{
 		"org_id":      org.OrgID(),
+		"parsed":      parsed,
 		"group_uuid":  group,
 		"query":       query,
 		"elapsed":     time.Since(start),
 		"match_count": len(ids),
 	}).Debug("paged contact query complete")
 
-	return ids, results.Hits.TotalHits, nil
+	return parsed, ids, results.Hits.TotalHits, nil
 }
 
 // ContactIDsForQuery returns the ids of all the contacts that match the passed in query
@@ -211,22 +228,15 @@ func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAss
 		return nil, errors.Errorf("no elastic client available, check your configuration")
 	}
 
-	// our field resolver
-	resolver := func(key string) assets.Field {
-		return org.FieldByKey(key)
-	}
-
 	// turn into elastic query
-	eq, err := search.ToElasticQuery(org.Env(), resolver, query)
+	_, eq, err := ParseQuery(org, query)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
 	}
 
-	// filter by org, active, blocked and stopped
+	// only include unblocked and unstopped contacts
 	eq = elastic.NewBoolQuery().Must(
 		eq,
-		elastic.NewTermQuery("org_id", org.OrgID()),
-		elastic.NewTermQuery("is_active", true),
 		elastic.NewTermQuery("is_blocked", false),
 		elastic.NewTermQuery("is_stopped", false),
 	)

models/test_constants.go

@@ -80,6 +80,9 @@ var RemindersEvent2ID = CampaignEventID(10001)
 var DoctorsGroupID = GroupID(10000)
 var DoctorsGroupUUID = assets.GroupUUID("c153e265-f7c9-4539-9dbc-9b358714b638")
 
+var AllContactsGroupID = GroupID(1)
+var AllContactsGroupUUID = assets.GroupUUID("bc268217-9ffa-49e0-883e-e4e09c252a5a")
+
 var TestersGroupID = GroupID(10001)
 var TestersGroupUUID = assets.GroupUUID("5e9d8fab-5e7e-4f51-b533-261af5dea70d")
 

search/search.go

@@ -13,14 +13,19 @@ import (
 	"github.com/shopspring/decimal"
 )
 
-// ToElasticQuery converts a contactql query string to an Elastic query
-func ToElasticQuery(env envs.Environment, resolver contactql.FieldResolverFunc, query string) (elastic.Query, error) {
+// ToElasticQuery converts a contactql query string to an Elastic query returning the normalized view as well as the elastic query
+func ToElasticQuery(env envs.Environment, resolver contactql.FieldResolverFunc, query string) (string, elastic.Query, error) {
 	node, err := contactql.ParseQuery(query, env.RedactionPolicy(), resolver)
 	if err != nil {
-		return nil, errors.Wrapf(err, "error parsing query: %s", query)
+		return "", nil, errors.Wrapf(err, "error parsing query: %s", query)
 	}
 
-	return nodeToElasticQuery(env, resolver, node.Root())
+	eq, err := nodeToElasticQuery(env, resolver, node.Root())
+	if err != nil {
+		return "", nil, errors.Wrapf(err, "error parsing query: %s", query)
+	}
+
+	return node.String(), eq, nil
 }
 
 func nodeToElasticQuery(env envs.Environment, resolver contactql.FieldResolverFunc, node contactql.QueryNode) (elastic.Query, error) {

search/search_test.go

@@ -61,7 +61,7 @@ func TestElasticQuery(t *testing.T) {
 		}
 		env := envs.NewBuilder().WithTimezone(ny).WithRedactionPolicy(redactionPolicy).Build()
 
-		query, err := ToElasticQuery(env, resolver, tc.Search)
+		_, query, err := ToElasticQuery(env, resolver, tc.Search)
 
 		if tc.Error != "" {
 			assert.Error(t, err, "%s: error not received converting to elastic: %s", tc.Label, tc.Search)

web/search/search.go → web/contact/contact.go

@@ -1,4 +1,4 @@
-package search
+package contact
 
 import (
 	"context"
@@ -12,7 +12,7 @@ import (
 )
 
 func init() {
-	web.RegisterJSONRoute(http.MethodPost, "/mr/search/search", web.RequireAuthToken(handleSearch))
+	web.RegisterJSONRoute(http.MethodPost, "/mr/contact/search", web.RequireAuthToken(handleSearch))
 }
 
 // Searches the contacts for an org
@@ -33,11 +33,11 @@ type searchRequest struct {
 
 // Response for a contact search
 type searchResponse struct {
-	Parsed  string             `json:"parsed"`
-	Error   string             `json:"error"`
-	Results []models.ContactID `json:"results"`
-	Total   int64              `json:"total"`
-	Offset  int                `json:"offset"`
+	Query      string             `json:"query"`
+	Error      string             `json:"error"`
+	ContactIDs []models.ContactID `json:"contact_ids"`
+	Total      int64              `json:"total"`
+	Offset     int                `json:"offset"`
 }
 
 // handles a a contact search request
@@ -48,22 +48,23 @@ func handleSearch(ctx context.Context, s *web.Server, r *http.Request) (interfac
 	}
 
 	// grab our org
-	org, err := models.NewOrgAssets(s.CTX, s.DB, request.OrgID, nil)
+	org, err := models.GetOrgAssets(s.CTX, s.DB, request.OrgID)
 	if err != nil {
 		return nil, http.StatusBadRequest, errors.Wrapf(err, "unable to load org assets")
 	}
 
 	// Perform our search
-	hits, total, err := models.ContactIDsForQueryPage(ctx, s.ElasticClient, org, request.GroupUUID, request.Query, request.Offset, request.PageSize)
+	parsed, hits, total, err := models.ContactIDsForQueryPage(ctx, s.ElasticClient, org, request.GroupUUID, request.Query, request.Offset, request.PageSize)
 	if err != nil {
 		return nil, http.StatusServiceUnavailable, errors.Wrapf(err, "error performing query")
 	}
 
 	// build our response
 	response := &searchResponse{
-		Results: hits,
-		Total:   total,
-		Offset:  request.Offset,
+		ContactIDs: hits,
+		Total:      total,
+		Offset:     request.Offset,
+		Query:      parsed,
 	}
 
 	return response, http.StatusOK, nil

web/contact/contact_test.go

@@ -0,0 +1,123 @@
+package contact
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nyaruka/mailroom/config"
+	"github.com/nyaruka/mailroom/models"
+	"github.com/nyaruka/mailroom/search"
+	"github.com/nyaruka/mailroom/testsuite"
+	"github.com/nyaruka/mailroom/web"
+	"github.com/olivere/elastic"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServer(t *testing.T) {
+	testsuite.Reset()
+	ctx := testsuite.CTX()
+	db := testsuite.DB()
+	rp := testsuite.RP()
+	wg := &sync.WaitGroup{}
+
+	es := search.NewMockElasticServer()
+	defer es.Close()
+
+	client, err := elastic.NewClient(
+		elastic.SetURL(es.URL()),
+		elastic.SetHealthcheck(false),
+		elastic.SetSniff(false),
+	)
+	assert.NoError(t, err)
+
+	server := web.NewServer(ctx, config.Mailroom, db, rp, nil, client, wg)
+	server.Start()
+
+	// give our server time to start
+	time.Sleep(time.Second)
+
+	defer server.Stop()
+
+	tcs := []struct {
+		URL        string
+		Method     string
+		Body       string
+		Status     int
+		Response   string
+		Hits       []models.ContactID
+		ESResponse string
+	}{
+		{"/mr/contact/search", "GET", "", 405, "illegal", nil, ""},
+		{
+			"/mr/contact/search", "POST",
+			fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s"}`, models.AllContactsGroupUUID),
+			200,
+			`name~Cathy`,
+			[]models.ContactID{models.CathyID},
+			fmt.Sprintf(`{
+				"_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==",
+				"took": 2,
+				"timed_out": false,
+				"_shards": {
+				  "total": 1,
+				  "successful": 1,
+				  "skipped": 0,
+				  "failed": 0
+				},
+				"hits": {
+				  "total": 1,
+				  "max_score": null,
+				  "hits": [
+					{
+					  "_index": "contacts",
+					  "_type": "_doc",
+					  "_id": "%d",
+					  "_score": null,
+					  "_routing": "1",
+					  "sort": [
+						15124352
+					  ]
+					}
+				  ]
+				}
+			}`, models.CathyID),
+		},
+	}
+
+	for i, tc := range tcs {
+		var body io.Reader
+		es.NextResponse = tc.ESResponse
+
+		if tc.Body != "" {
+			body = bytes.NewReader([]byte(tc.Body))
+		}
+
+		req, err := http.NewRequest(tc.Method, "http://localhost:8090"+tc.URL, body)
+		assert.NoError(t, err, "%d: error creating request", i)
+
+		resp, err := http.DefaultClient.Do(req)
+		assert.NoError(t, err, "%d: error making request", i)
+
+		assert.Equal(t, tc.Status, resp.StatusCode, "%d: unexpected status", i)
+
+		content, err := ioutil.ReadAll(resp.Body)
+		assert.NoError(t, err, "%d: error reading body", i)
+		assert.True(t, strings.Contains(string(content), tc.Response), "%d: did not find string: %s in body: %s", i, tc.Response, string(content))
+
+		// on 200 responses parse them
+		if resp.StatusCode == 200 {
+			r := &searchResponse{}
+			err = json.Unmarshal(content, r)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.Hits, r.ContactIDs)
+		}
+	}
+}