Merge pull request #22 from Ilhasoft/update/mailroom

Update mailroom to 5.3.21 version

.gitignore

@@ -10,6 +10,7 @@ dist
 .vscode
 .envrc
 docs/*
+docs
 
 # Test binary, build with `go test -c`
 *.test

CHANGELOG.md

@@ -1,3 +1,25 @@
+v5.3.21
+----------
+ * Return field dependencies with queries on contact search endpoint
+ * Latest goflow, larger webhook bodies, trim expressions
+
+v5.3.20
+----------
+ * Update to latest goflow v0.64.9
+ * Add contact search web endpoint
+
+v5.3.19
+----------
+ * Update to goflow v0.64.8
+
+v5.3.18
+----------
+ * Update to goflow v0.64.7
+
+v5.3.17
+----------
+ * Include evaluation context with simulation requests
+
 v5.3.16
 ----------
  * Update to goflow v0.64.2

README.md

@@ -71,7 +71,7 @@ To run the tests you need to create the test database:
 
 ```
 $ createdb mailroom_test
-$ createuser -P -E temba (set no password)
+$ createuser -P -E -s mailroom_test (set no password)
 ```
 
 To run all of the tests:

cmd/mailroom/main.go

@@ -24,6 +24,7 @@ import (
 	_ "github.com/nyaruka/mailroom/tasks/stats"
 	_ "github.com/nyaruka/mailroom/tasks/timeouts"
 
+	_ "github.com/nyaruka/mailroom/web/contact"
 	_ "github.com/nyaruka/mailroom/web/docs"
 	_ "github.com/nyaruka/mailroom/web/expression"
 	_ "github.com/nyaruka/mailroom/web/flow"

config/config.go

@@ -24,6 +24,7 @@ type Config struct {
 
 	WebhooksTimeout        int     `help:"the timeout in milliseconds for webhook calls from engine"`
 	WebhooksMaxRetries     int     `help:"the number of times to retry a failed webhook call"`
+	WebhooksMaxBodyBytes   int     `help:"the maximum size of bytes to a webhook call response body"`
 	WebhooksInitialBackoff int     `help:"the initial backoff in milliseconds when retrying a failed webhook call"`
 	WebhooksBackoffJitter  float64 `help:"the amount of jitter to apply to backoff times"`
 	SMTPServer             string  `help:"the smtp configuration for sending emails ex: smtp://user%40password@server:port/?from=foo%40gmail.com"`
@@ -66,6 +67,7 @@ func NewMailroomConfig() *Config {
 
 		WebhooksTimeout:        15000,
 		WebhooksMaxRetries:     2,
+		WebhooksMaxBodyBytes:   1024 * 1024, // 1MB
 		WebhooksInitialBackoff: 5000,
 		WebhooksBackoffJitter:  0.5,
 		SMTPServer:             "",

go.mod

@@ -17,13 +17,14 @@ require (
 	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/gorilla/schema v1.0.2
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect
 	github.com/lib/pq v1.0.0
 	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/nyaruka/ezconf v0.2.1
 	github.com/nyaruka/gocommon v1.1.1
-	github.com/nyaruka/goflow v0.64.2
+	github.com/nyaruka/goflow v0.64.11
 	github.com/nyaruka/librato v1.0.0
 	github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d
 	github.com/nyaruka/null v1.2.0
@@ -36,6 +37,7 @@ require (
 	golang.org/x/net v0.0.0-20181217023233-e147a9138326 // indirect
 	google.golang.org/appengine v1.4.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/go-playground/validator.v9 v9.21.0
 	gopkg.in/mail.v2 v2.3.1
 )

go.sum

@@ -53,6 +53,11 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
@@ -70,8 +75,10 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
 github.com/nyaruka/gocommon v1.1.1 h1:RnQ+kMzN1lA+W0NpkBDd0mGU3UqadJygR3SMpITMYTQ=
 github.com/nyaruka/gocommon v1.1.1/go.mod h1:QbdU2J9WBsqBmeZRuwndf2f6O7rD7mkC0bGn5UNnwjI=
-github.com/nyaruka/goflow v0.64.2 h1:QFjEAqwFrzTBwqSWiW1o8NrS9wBRLz7Wt7Gcg5u5us4=
-github.com/nyaruka/goflow v0.64.2/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM=
+github.com/nyaruka/goflow v0.64.10 h1:vBfE8J1Uuw/jo9hZd5C3G1dLIUc/Zz6ioQODRB8oi0I=
+github.com/nyaruka/goflow v0.64.10/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM=
+github.com/nyaruka/goflow v0.64.11 h1:ErJnURsTjossFzs1mzKN8vEfQAUrbc7xFCU1om4RliI=
+github.com/nyaruka/goflow v0.64.11/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM=
 github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
 github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc=
@@ -123,6 +130,8 @@ gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gG
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v9 v9.12.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=

goflow/engine.go

@@ -55,7 +55,7 @@ func Engine() flows.Engine {
 		httpClient, httpRetries := webhooksHTTP()
 
 		eng = engine.NewBuilder().
-			WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, httpRetries, webhookHeaders, 10000)).
+			WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, httpRetries, webhookHeaders, config.Mailroom.WebhooksMaxBodyBytes)).
 			WithEmailServiceFactory(emailFactory).
 			WithClassificationServiceFactory(classificationFactory).
 			WithAirtimeServiceFactory(airtimeFactory).
@@ -77,7 +77,7 @@ func Simulator() flows.Engine {
 		httpClient, _ := webhooksHTTP() // don't do retries in simulator
 
 		simulator = engine.NewBuilder().
-			WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, nil, webhookHeaders, 10000)).
+			WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, nil, webhookHeaders, config.Mailroom.WebhooksMaxBodyBytes)).
 			WithClassificationServiceFactory(classificationFactory).   // simulated sessions do real classification
 			WithEmailServiceFactory(simulatorEmailServiceFactory).     // but faked emails
 			WithAirtimeServiceFactory(simulatorAirtimeServiceFactory). // and faked airtime transfers

mailroom.go

@@ -211,7 +211,7 @@ func (mr *Mailroom) Start() error {
 	mr.handlerForeman.Start()
 
 	// start our web server
-	mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.S3Client, mr.WaitGroup)
+	mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.S3Client, mr.ElasticClient, mr.WaitGroup)
 	mr.webserver.Start()
 
 	logrus.Info("mailroom started")

models/classifiers.go

@@ -29,7 +29,7 @@ const (
 	// Our classifier types
 	ClassifierTypeWit    = "wit"
 	ClassifierTypeLuis   = "luis"
-	ClassifierTypeBothub = "bh"
+	ClassifierTypeBothub = "bothub"
 
 	// Wit.ai config options
 	WitConfigAccessToken = "access_token"

models/contacts.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/nyaruka/gocommon/urns"
 	"github.com/nyaruka/goflow/assets"
+	"github.com/nyaruka/goflow/contactql"
 	"github.com/nyaruka/goflow/envs"
 	"github.com/nyaruka/goflow/excellent/types"
 	"github.com/nyaruka/goflow/flows"
@@ -148,34 +149,119 @@ func ContactIDsFromReferences(ctx context.Context, tx Queryer, org *OrgAssets, r
 	return ids, nil
 }
 
-// ContactIDsForQuery returns the ids of all the contacts that match the passed in query
-func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAssets, query string) ([]ContactID, error) {
-	start := time.Now()
-
-	if client == nil {
-		return nil, errors.Errorf("no elastic client available, check your configuration")
-	}
-
-	// our field resolver
-	resolver := func(key string) assets.Field {
+// buildFieldResolver builds a field resolver function for the passed in Org
+func buildFieldResolver(org *OrgAssets) contactql.FieldResolverFunc {
+	return func(key string) assets.Field {
 		f := org.FieldByKey(key)
 		if f == nil {
 			return nil
 		}
 		return f
 	}
+}
 
+// BuildElasticQuery turns the passed in contact ql query into an elastic query
+func BuildElasticQuery(org *OrgAssets, resolver contactql.FieldResolverFunc, query *contactql.ContactQuery) (elastic.Query, error) {
 	// turn into elastic query
 	eq, err := search.ToElasticQuery(org.Env(), resolver, query)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
 	}
 
-	// filter by org, active, blocked and stopped
+	// additionally filter by org and active contacts
 	eq = elastic.NewBoolQuery().Must(
 		eq,
 		elastic.NewTermQuery("org_id", org.OrgID()),
 		elastic.NewTermQuery("is_active", true),
+	)
+
+	return eq, nil
+}
+
+// ContactIDsForQueryPage returns the ids of the contacts for the passed in query page
+func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *OrgAssets, group assets.GroupUUID, query string, sort string, offset int, pageSize int) (*contactql.ContactQuery, []ContactID, int64, error) {
+	start := time.Now()
+
+	if client == nil {
+		return nil, nil, 0, errors.Errorf("no elastic client available, check your configuration")
+	}
+
+	resolver := buildFieldResolver(org)
+
+	parsed, err := search.ParseQuery(org.Env(), resolver, query)
+	if err != nil {
+		return nil, nil, 0, errors.Wrapf(err, "error parsing query: %s", query)
+	}
+
+	eq, err := BuildElasticQuery(org, resolver, parsed)
+	if err != nil {
+		return nil, nil, 0, errors.Wrapf(err, "error parsing query: %s", query)
+	}
+
+	fieldSort, err := search.ToElasticFieldSort(resolver, sort)
+	if err != nil {
+		return nil, nil, 0, errors.Wrapf(err, "error parsing sort")
+	}
+
+	// filter by our base group
+	eq = elastic.NewBoolQuery().Must(
+		eq,
+		elastic.NewTermQuery("groups", group),
+	)
+
+	s := client.Search("contacts").Routing(strconv.FormatInt(int64(org.OrgID()), 10))
+	s = s.Size(pageSize).From(offset).Query(eq).SortBy(fieldSort).FetchSource(false)
+
+	results, err := s.Do(ctx)
+	if err != nil {
+		return nil, nil, 0, errors.Wrapf(err, "error performing query")
+	}
+
+	ids := make([]ContactID, 0, pageSize)
+	for _, hit := range results.Hits.Hits {
+		id, err := strconv.Atoi(hit.Id)
+		if err != nil {
+			return nil, nil, 0, errors.Wrapf(err, "unexpected non-integer contact id: %s for search: %s", hit.Id, query)
+		}
+		ids = append(ids, ContactID(id))
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"org_id":      org.OrgID(),
+		"parsed":      parsed,
+		"group_uuid":  group,
+		"query":       query,
+		"elapsed":     time.Since(start),
+		"page_count":  len(ids),
+		"total_count": results.Hits.TotalHits,
+	}).Debug("paged contact query complete")
+
+	return parsed, ids, results.Hits.TotalHits, nil
+}
+
+// ContactIDsForQuery returns the ids of all the contacts that match the passed in query
+func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAssets, query string) ([]ContactID, error) {
+	start := time.Now()
+
+	if client == nil {
+		return nil, errors.Errorf("no elastic client available, check your configuration")
+	}
+
+	// turn into elastic query
+	resolver := buildFieldResolver(org)
+	parsed, err := search.ParseQuery(org.Env(), resolver, query)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error parsing query: %s", query)
+	}
+
+	eq, err := BuildElasticQuery(org, resolver, parsed)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query)
+	}
+
+	// only include unblocked and unstopped contacts
+	eq = elastic.NewBoolQuery().Must(
+		eq,
 		elastic.NewTermQuery("is_blocked", false),
 		elastic.NewTermQuery("is_stopped", false),
 	)

models/contacts_test.go

@@ -46,11 +46,19 @@ func TestElasticContacts(t *testing.T) {
 				"query":{
 					"bool":{
 						"must":[
-							{"match":{"name":{"query":"george"}}},
-							{"term":{"org_id":1}},
-							{"term":{"is_active":true}},
-							{"term":{"is_blocked":false}},
-							{"term":{"is_stopped":false}}
+							{ "bool":{
+								"must":[
+									{"match":{"name":{"query":"george"}}},
+									{"term":{"org_id":1}},
+									{"term":{"is_active":true}}
+								]
+							}},
+							{ "term":{
+								"is_blocked":false
+							}},
+							{"term":
+								{"is_stopped":false
+							}}
 						]
 					}
 				},
@@ -91,9 +99,13 @@ func TestElasticContacts(t *testing.T) {
 				"query":{
 					"bool":{
 						"must":[
-							{"match":{"name":{"query":"nobody"}}},
-							{"term":{"org_id":1}},
-							{"term":{"is_active":true}},
+							{"bool":
+								{"must":[
+									{"match":{"name":{"query":"nobody"}}},
+									{"term":{"org_id":1}},
+									{"term":{"is_active":true}}
+								]}
+							},
 							{"term":{"is_blocked":false}},
 							{"term":{"is_stopped":false}}
 						]
@@ -133,7 +145,7 @@ func TestElasticContacts(t *testing.T) {
 			assert.Error(t, err)
 		} else {
 			assert.NoError(t, err, "%d: error encountered performing query", i)
-			assert.JSONEq(t, tc.Request, es.LastBody, "%d: request mismatch", i)
+			assert.JSONEq(t, tc.Request, es.LastBody, "%d: request mismatch, got: %s", i, es.LastBody)
 			assert.Equal(t, tc.Contacts, ids, "%d: ids mismatch", i)
 		}
 	}

models/test_constants.go

@@ -80,6 +80,9 @@ var RemindersEvent2ID = CampaignEventID(10001)
 var DoctorsGroupID = GroupID(10000)
 var DoctorsGroupUUID = assets.GroupUUID("c153e265-f7c9-4539-9dbc-9b358714b638")
 
+var AllContactsGroupID = GroupID(1)
+var AllContactsGroupUUID = assets.GroupUUID("bc268217-9ffa-49e0-883e-e4e09c252a5a")
+
 var TestersGroupID = GroupID(10001)
 var TestersGroupUUID = assets.GroupUUID("5e9d8fab-5e7e-4f51-b533-261af5dea70d")