diff --git a/.gitignore b/.gitignore index 3b14996bb..e1d7979d6 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ dist .vscode .envrc docs/* +docs # Test binary, build with `go test -c` *.test diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ab58991..82c811f1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +v5.3.21 +---------- + * Return field dependencies with queries on contact search endpoint + * Latest goflow, larger webhook bodies, trim expressions + +v5.3.20 +---------- + * Update to latest goflow v0.64.9 + * Add contact search web endpoint + +v5.3.19 +---------- + * Update to goflow v0.64.8 + +v5.3.18 +---------- + * Update to goflow v0.64.7 + +v5.3.17 +---------- + * Include evaluation context with simulation requests + v5.3.16 ---------- * Update to goflow v0.64.2 diff --git a/README.md b/README.md index 88e944c8e..1776d7a08 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ To run the tests you need to create the test database: ``` $ createdb mailroom_test -$ createuser -P -E temba (set no password) +$ createuser -P -E -s mailroom_test (set no password) ``` To run all of the tests: diff --git a/cmd/mailroom/main.go b/cmd/mailroom/main.go index 6db0cd3ce..ae5292de7 100644 --- a/cmd/mailroom/main.go +++ b/cmd/mailroom/main.go @@ -24,6 +24,7 @@ import ( _ "github.com/nyaruka/mailroom/tasks/stats" _ "github.com/nyaruka/mailroom/tasks/timeouts" + _ "github.com/nyaruka/mailroom/web/contact" _ "github.com/nyaruka/mailroom/web/docs" _ "github.com/nyaruka/mailroom/web/expression" _ "github.com/nyaruka/mailroom/web/flow" diff --git a/config/config.go b/config/config.go index b574d8806..68cce57e8 100644 --- a/config/config.go +++ b/config/config.go @@ -24,6 +24,7 @@ type Config struct { WebhooksTimeout int `help:"the timeout in milliseconds for webhook calls from engine"` WebhooksMaxRetries int `help:"the number of times to retry a failed webhook call"` + WebhooksMaxBodyBytes int `help:"the maximum size of bytes to a webhook call response body"` WebhooksInitialBackoff int `help:"the initial backoff in milliseconds when retrying a failed webhook call"` WebhooksBackoffJitter float64 `help:"the amount of jitter to apply to backoff times"` SMTPServer string `help:"the smtp configuration for sending emails ex: smtp://user%40password@server:port/?from=foo%40gmail.com"` @@ -66,6 +67,7 @@ func NewMailroomConfig() *Config { WebhooksTimeout: 15000, WebhooksMaxRetries: 2, + WebhooksMaxBodyBytes: 1024 * 1024, // 1MB WebhooksInitialBackoff: 5000, WebhooksBackoffJitter: 0.5, SMTPServer: "", diff --git a/go.mod b/go.mod index 66ea4f7df..939190006 100644 --- a/go.mod +++ b/go.mod @@ -17,13 +17,14 @@ require ( github.com/google/go-cmp v0.3.0 // indirect github.com/gorilla/schema v1.0.2 github.com/jmoiron/sqlx v1.2.0 + github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.0.0 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 github.com/nyaruka/gocommon v1.1.1 - github.com/nyaruka/goflow v0.64.2 + github.com/nyaruka/goflow v0.64.11 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d github.com/nyaruka/null v1.2.0 @@ -36,6 +37,7 @@ require ( golang.org/x/net v0.0.0-20181217023233-e147a9138326 // indirect google.golang.org/appengine v1.4.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/go-playground/validator.v9 v9.21.0 gopkg.in/mail.v2 v2.3.1 ) diff --git a/go.sum b/go.sum index e92d03697..11d62786b 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,11 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= @@ -70,8 +75,10 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.1.1 h1:RnQ+kMzN1lA+W0NpkBDd0mGU3UqadJygR3SMpITMYTQ= github.com/nyaruka/gocommon v1.1.1/go.mod h1:QbdU2J9WBsqBmeZRuwndf2f6O7rD7mkC0bGn5UNnwjI= -github.com/nyaruka/goflow v0.64.2 h1:QFjEAqwFrzTBwqSWiW1o8NrS9wBRLz7Wt7Gcg5u5us4= -github.com/nyaruka/goflow v0.64.2/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM= +github.com/nyaruka/goflow v0.64.10 h1:vBfE8J1Uuw/jo9hZd5C3G1dLIUc/Zz6ioQODRB8oi0I= +github.com/nyaruka/goflow v0.64.10/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM= +github.com/nyaruka/goflow v0.64.11 h1:ErJnURsTjossFzs1mzKN8vEfQAUrbc7xFCU1om4RliI= +github.com/nyaruka/goflow v0.64.11/go.mod h1:fb6eGAXiTL2hjbzMpXwSIlNx0O4IsR9XBunXBN+8pWM= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc= @@ -123,6 +130,8 @@ gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gG gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.12.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= diff --git a/goflow/engine.go b/goflow/engine.go index 5fe8f8f94..8117af6e5 100644 --- a/goflow/engine.go +++ b/goflow/engine.go @@ -55,7 +55,7 @@ func Engine() flows.Engine { httpClient, httpRetries := webhooksHTTP() eng = engine.NewBuilder(). - WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, httpRetries, webhookHeaders, 10000)). + WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, httpRetries, webhookHeaders, config.Mailroom.WebhooksMaxBodyBytes)). WithEmailServiceFactory(emailFactory). WithClassificationServiceFactory(classificationFactory). WithAirtimeServiceFactory(airtimeFactory). @@ -77,7 +77,7 @@ func Simulator() flows.Engine { httpClient, _ := webhooksHTTP() // don't do retries in simulator simulator = engine.NewBuilder(). - WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, nil, webhookHeaders, 10000)). + WithWebhookServiceFactory(webhooks.NewServiceFactory(httpClient, nil, webhookHeaders, config.Mailroom.WebhooksMaxBodyBytes)). WithClassificationServiceFactory(classificationFactory). // simulated sessions do real classification WithEmailServiceFactory(simulatorEmailServiceFactory). // but faked emails WithAirtimeServiceFactory(simulatorAirtimeServiceFactory). // and faked airtime transfers diff --git a/mailroom.go b/mailroom.go index d0dbac4dc..8986d9f71 100644 --- a/mailroom.go +++ b/mailroom.go @@ -211,7 +211,7 @@ func (mr *Mailroom) Start() error { mr.handlerForeman.Start() // start our web server - mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.S3Client, mr.WaitGroup) + mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.S3Client, mr.ElasticClient, mr.WaitGroup) mr.webserver.Start() logrus.Info("mailroom started") diff --git a/models/classifiers.go b/models/classifiers.go index e2f5f324e..a87306862 100644 --- a/models/classifiers.go +++ b/models/classifiers.go @@ -29,7 +29,7 @@ const ( // Our classifier types ClassifierTypeWit = "wit" ClassifierTypeLuis = "luis" - ClassifierTypeBothub = "bh" + ClassifierTypeBothub = "bothub" // Wit.ai config options WitConfigAccessToken = "access_token" diff --git a/models/contacts.go b/models/contacts.go index 111d45768..28ea21b98 100644 --- a/models/contacts.go +++ b/models/contacts.go @@ -11,6 +11,7 @@ import ( "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/contactql" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" @@ -148,34 +149,119 @@ func ContactIDsFromReferences(ctx context.Context, tx Queryer, org *OrgAssets, r return ids, nil } -// ContactIDsForQuery returns the ids of all the contacts that match the passed in query -func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAssets, query string) ([]ContactID, error) { - start := time.Now() - - if client == nil { - return nil, errors.Errorf("no elastic client available, check your configuration") - } - - // our field resolver - resolver := func(key string) assets.Field { +// buildFieldResolver builds a field resolver function for the passed in Org +func buildFieldResolver(org *OrgAssets) contactql.FieldResolverFunc { + return func(key string) assets.Field { f := org.FieldByKey(key) if f == nil { return nil } return f } +} +// BuildElasticQuery turns the passed in contact ql query into an elastic query +func BuildElasticQuery(org *OrgAssets, resolver contactql.FieldResolverFunc, query *contactql.ContactQuery) (elastic.Query, error) { // turn into elastic query eq, err := search.ToElasticQuery(org.Env(), resolver, query) if err != nil { return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query) } - // filter by org, active, blocked and stopped + // additionally filter by org and active contacts eq = elastic.NewBoolQuery().Must( eq, elastic.NewTermQuery("org_id", org.OrgID()), elastic.NewTermQuery("is_active", true), + ) + + return eq, nil +} + +// ContactIDsForQueryPage returns the ids of the contacts for the passed in query page +func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *OrgAssets, group assets.GroupUUID, query string, sort string, offset int, pageSize int) (*contactql.ContactQuery, []ContactID, int64, error) { + start := time.Now() + + if client == nil { + return nil, nil, 0, errors.Errorf("no elastic client available, check your configuration") + } + + resolver := buildFieldResolver(org) + + parsed, err := search.ParseQuery(org.Env(), resolver, query) + if err != nil { + return nil, nil, 0, errors.Wrapf(err, "error parsing query: %s", query) + } + + eq, err := BuildElasticQuery(org, resolver, parsed) + if err != nil { + return nil, nil, 0, errors.Wrapf(err, "error parsing query: %s", query) + } + + fieldSort, err := search.ToElasticFieldSort(resolver, sort) + if err != nil { + return nil, nil, 0, errors.Wrapf(err, "error parsing sort") + } + + // filter by our base group + eq = elastic.NewBoolQuery().Must( + eq, + elastic.NewTermQuery("groups", group), + ) + + s := client.Search("contacts").Routing(strconv.FormatInt(int64(org.OrgID()), 10)) + s = s.Size(pageSize).From(offset).Query(eq).SortBy(fieldSort).FetchSource(false) + + results, err := s.Do(ctx) + if err != nil { + return nil, nil, 0, errors.Wrapf(err, "error performing query") + } + + ids := make([]ContactID, 0, pageSize) + for _, hit := range results.Hits.Hits { + id, err := strconv.Atoi(hit.Id) + if err != nil { + return nil, nil, 0, errors.Wrapf(err, "unexpected non-integer contact id: %s for search: %s", hit.Id, query) + } + ids = append(ids, ContactID(id)) + } + + logrus.WithFields(logrus.Fields{ + "org_id": org.OrgID(), + "parsed": parsed, + "group_uuid": group, + "query": query, + "elapsed": time.Since(start), + "page_count": len(ids), + "total_count": results.Hits.TotalHits, + }).Debug("paged contact query complete") + + return parsed, ids, results.Hits.TotalHits, nil +} + +// ContactIDsForQuery returns the ids of all the contacts that match the passed in query +func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAssets, query string) ([]ContactID, error) { + start := time.Now() + + if client == nil { + return nil, errors.Errorf("no elastic client available, check your configuration") + } + + // turn into elastic query + resolver := buildFieldResolver(org) + parsed, err := search.ParseQuery(org.Env(), resolver, query) + if err != nil { + return nil, errors.Wrapf(err, "error parsing query: %s", query) + } + + eq, err := BuildElasticQuery(org, resolver, parsed) + if err != nil { + return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query) + } + + // only include unblocked and unstopped contacts + eq = elastic.NewBoolQuery().Must( + eq, elastic.NewTermQuery("is_blocked", false), elastic.NewTermQuery("is_stopped", false), ) diff --git a/models/contacts_test.go b/models/contacts_test.go index dbbac76e7..7e0f43195 100644 --- a/models/contacts_test.go +++ b/models/contacts_test.go @@ -46,11 +46,19 @@ func TestElasticContacts(t *testing.T) { "query":{ "bool":{ "must":[ - {"match":{"name":{"query":"george"}}}, - {"term":{"org_id":1}}, - {"term":{"is_active":true}}, - {"term":{"is_blocked":false}}, - {"term":{"is_stopped":false}} + { "bool":{ + "must":[ + {"match":{"name":{"query":"george"}}}, + {"term":{"org_id":1}}, + {"term":{"is_active":true}} + ] + }}, + { "term":{ + "is_blocked":false + }}, + {"term": + {"is_stopped":false + }} ] } }, @@ -91,9 +99,13 @@ func TestElasticContacts(t *testing.T) { "query":{ "bool":{ "must":[ - {"match":{"name":{"query":"nobody"}}}, - {"term":{"org_id":1}}, - {"term":{"is_active":true}}, + {"bool": + {"must":[ + {"match":{"name":{"query":"nobody"}}}, + {"term":{"org_id":1}}, + {"term":{"is_active":true}} + ]} + }, {"term":{"is_blocked":false}}, {"term":{"is_stopped":false}} ] @@ -133,7 +145,7 @@ func TestElasticContacts(t *testing.T) { assert.Error(t, err) } else { assert.NoError(t, err, "%d: error encountered performing query", i) - assert.JSONEq(t, tc.Request, es.LastBody, "%d: request mismatch", i) + assert.JSONEq(t, tc.Request, es.LastBody, "%d: request mismatch, got: %s", i, es.LastBody) assert.Equal(t, tc.Contacts, ids, "%d: ids mismatch", i) } } diff --git a/models/test_constants.go b/models/test_constants.go index 01a9e2f2e..e29904254 100644 --- a/models/test_constants.go +++ b/models/test_constants.go @@ -80,6 +80,9 @@ var RemindersEvent2ID = CampaignEventID(10001) var DoctorsGroupID = GroupID(10000) var DoctorsGroupUUID = assets.GroupUUID("c153e265-f7c9-4539-9dbc-9b358714b638") +var AllContactsGroupID = GroupID(1) +var AllContactsGroupUUID = assets.GroupUUID("bc268217-9ffa-49e0-883e-e4e09c252a5a") + var TestersGroupID = GroupID(10001) var TestersGroupUUID = assets.GroupUUID("5e9d8fab-5e7e-4f51-b533-261af5dea70d") diff --git a/search/search.go b/search/search.go index 437da80a9..513937575 100644 --- a/search/search.go +++ b/search/search.go @@ -2,6 +2,7 @@ package search import ( "fmt" + "sort" "strings" "github.com/nyaruka/goflow/assets" @@ -13,14 +14,88 @@ import ( "github.com/shopspring/decimal" ) -// ToElasticQuery converts a contactql query string to an Elastic query -func ToElasticQuery(env envs.Environment, resolver contactql.FieldResolverFunc, query string) (elastic.Query, error) { - node, err := contactql.ParseQuery(query, env.RedactionPolicy(), resolver) +// ParseQuery parses the passed in query returning the result +func ParseQuery(env envs.Environment, resolver contactql.FieldResolverFunc, query string) (*contactql.ContactQuery, error) { + parsed, err := contactql.ParseQuery(query, env.RedactionPolicy(), resolver) if err != nil { - return nil, errors.Wrapf(err, "error parsing query: %s", query) + return nil, NewError(err.Error()) + } + return parsed, nil +} + +// ToElasticQuery converts a contactql query to an Elastic query returning the normalized view as well as the elastic query +func ToElasticQuery(env envs.Environment, resolver contactql.FieldResolverFunc, query *contactql.ContactQuery) (elastic.Query, error) { + eq, err := nodeToElasticQuery(env, resolver, query.Root()) + if err != nil { + return nil, NewError(err.Error()) + } + + return eq, nil +} + +// FieldDependencies returns all the field this query is dependent on. This includes attributes such as "id" and "name" +func FieldDependencies(query *contactql.ContactQuery) []string { + seen := make(map[string]bool) + var appendFields func(node contactql.QueryNode, seen map[string]bool) + appendFields = func(node contactql.QueryNode, seen map[string]bool) { + switch n := node.(type) { + case *contactql.BoolCombination: + for _, c := range n.Children() { + appendFields(c, seen) + } + + case *contactql.Condition: + seen[n.PropertyKey()] = true + + default: + panic(fmt.Sprintf("unknown type in contactql query: %v", n)) + } + } + + appendFields(query.Root(), seen) + fields := make([]string, 0, len(seen)) + for k := range seen { + fields = append(fields, k) + } + + // order to make deterministic + sort.Strings(fields) + + return fields +} + +// ToElasticFieldSort returns the FieldSort for the passed in field +func ToElasticFieldSort(resolver contactql.FieldResolverFunc, fieldName string) (*elastic.FieldSort, error) { + // no field name? default to most recent first by id + if fieldName == "" { + return elastic.NewFieldSort("id").Desc(), nil + } + + // figure out if we are ascending or descending (default is ascending, can be changed with leading -) + ascending := true + if strings.HasPrefix(fieldName, "-") { + ascending = false + fieldName = fieldName[1:] + } + + fieldName = strings.ToLower(fieldName) + + // we are sorting by an attribute + if fieldName == contactql.AttributeID || fieldName == contactql.AttributeCreatedOn || + fieldName == contactql.AttributeLanguage || fieldName == contactql.AttributeName { + return elastic.NewFieldSort(fieldName).Order(ascending), nil } - return nodeToElasticQuery(env, resolver, node.Root()) + // we are sorting by a custom field + field := resolver(fieldName) + if field == nil { + return nil, NewError("unable to find field with name: %s", fieldName) + } + + sort := elastic.NewFieldSort(fmt.Sprintf("fields.%s", field.Type())) + sort = sort.Nested(elastic.NewNestedSort("fields").Filter(elastic.NewTermQuery("fields.field", field.UUID()))) + sort = sort.Order(ascending) + return sort, nil } func nodeToElasticQuery(env envs.Environment, resolver contactql.FieldResolverFunc, node contactql.QueryNode) (elastic.Query, error) { @@ -58,7 +133,7 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol if c.PropertyType() == contactql.PropertyTypeField { field := resolver(key) if field == nil { - return nil, errors.Errorf("unable to find field: %s", key) + return nil, NewError("unable to find field: %s", key) } fieldQuery := elastic.NewTermQuery("fields.field", field.UUID()) @@ -90,7 +165,7 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol ) return elastic.NewBoolQuery().MustNot(elastic.NewNestedQuery("fields", query)), nil } else { - return nil, fmt.Errorf("unsupported text comparator: %s", c.Comparator()) + return nil, NewError("unsupported text comparator: %s", c.Comparator()) } return elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(fieldQuery, query)), nil @@ -98,7 +173,7 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol } else if fieldType == assets.FieldTypeNumber { value, err := decimal.NewFromString(c.Value()) if err != nil { - return nil, errors.Errorf("can't convert '%s' to a number", c.Value()) + return nil, NewError("can't convert '%s' to a number", c.Value()) } if c.Comparator() == "=" { @@ -112,7 +187,7 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol } else if c.Comparator() == "<=" { query = elastic.NewRangeQuery("fields.number").Lte(value) } else { - return nil, fmt.Errorf("unsupported number comparator: %s", c.Comparator()) + return nil, NewError("unsupported number comparator: %s", c.Comparator()) } return elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(fieldQuery, query)), nil @@ -120,7 +195,7 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol } else if fieldType == assets.FieldTypeDatetime { value, err := envs.DateTimeFromString(env, c.Value(), false) if err != nil { - return nil, errors.Wrapf(err, "unable to parse datetime: %s", c.Value()) + return nil, NewError("string '%s' couldn't be parsed as a date", c.Value()) } start, end := dates.DayToUTCRange(value, value.Location()) @@ -135,7 +210,7 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol } else if c.Comparator() == "<=" { query = elastic.NewRangeQuery("fields.datetime").Lt(end) } else { - return nil, fmt.Errorf("unsupported datetime comparator: %s", c.Comparator()) + return nil, NewError("unsupported datetime comparator: %s", c.Comparator()) } return elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(fieldQuery, query)), nil @@ -156,12 +231,12 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol ), ), nil } else { - return nil, fmt.Errorf("unsupported location comparator: %s", c.Comparator()) + return nil, NewError("unsupported location comparator: %s", c.Comparator()) } return elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(fieldQuery, query)), nil } else { - return nil, fmt.Errorf("unsupported contact field type: %s", field.Type()) + return nil, NewError("unsupported contact field type: %s", field.Type()) } } else if c.PropertyType() == contactql.PropertyTypeAttribute { value := strings.ToLower(c.Value()) @@ -190,25 +265,25 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol } else if c.Comparator() == "!=" { return elastic.NewBoolQuery().MustNot(elastic.NewTermQuery("name.keyword", c.Value())), nil } else { - return nil, fmt.Errorf("unsupported name query comparator: %s", c.Comparator()) + return nil, NewError("unsupported name query comparator: %s", c.Comparator()) } } else if key == contactql.AttributeID { if c.Comparator() == "=" { return elastic.NewIdsQuery().Ids(value), nil } - return nil, fmt.Errorf("unsupported comparator for id: %s", c.Comparator()) + return nil, NewError("unsupported comparator for id: %s", c.Comparator()) } else if key == contactql.AttributeLanguage { if c.Comparator() == "=" { return elastic.NewTermQuery("language", value), nil } else if c.Comparator() == "!=" { return elastic.NewBoolQuery().MustNot(elastic.NewTermQuery("language", value)), nil } else { - return nil, fmt.Errorf("unsupported language comparator: %s", c.Comparator()) + return nil, NewError("unsupported language comparator: %s", c.Comparator()) } } else if key == contactql.AttributeCreatedOn { value, err := envs.DateTimeFromString(env, c.Value(), false) if err != nil { - return nil, errors.Wrapf(err, "unable to parse datetime: %s", c.Value()) + return nil, NewError("string '%s' couldn't be parsed as a date", c.Value()) } start, end := dates.DayToUTCRange(value, value.Location()) @@ -223,10 +298,10 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol } else if c.Comparator() == "<=" { return elastic.NewRangeQuery("created_on").Lt(end), nil } else { - return nil, fmt.Errorf("unsupported created_on comparator: %s", c.Comparator()) + return nil, NewError("unsupported created_on comparator: %s", c.Comparator()) } } else { - return nil, fmt.Errorf("unsupported contact attribute: %s", key) + return nil, NewError("unsupported contact attribute: %s", key) } } else if c.PropertyType() == contactql.PropertyTypeScheme { value := strings.ToLower(c.Value()) @@ -254,9 +329,22 @@ func conditionToElasticQuery(env envs.Environment, resolver contactql.FieldResol elastic.NewTermQuery("urns.scheme", key)), ), nil } else { - return nil, fmt.Errorf("unsupported scheme comparator: %s", c.Comparator()) + return nil, NewError("unsupported scheme comparator: %s", c.Comparator()) } } - return nil, errors.Errorf("unsupported property type: %s", c.PropertyType()) + return nil, NewError("unsupported property type: %s", c.PropertyType()) +} + +// Error is used when an error is in the parsing of a field or query format +type Error struct { + error string +} + +func (e *Error) Error() string { + return e.error +} + +func NewError(err string, args ...interface{}) *Error { + return &Error{fmt.Sprintf(err, args...)} } diff --git a/search/search_test.go b/search/search_test.go index 94047424c..d65616a8f 100644 --- a/search/search_test.go +++ b/search/search_test.go @@ -3,12 +3,15 @@ package search import ( "bytes" "encoding/json" + "fmt" "io/ioutil" "testing" "time" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/contactql" "github.com/nyaruka/goflow/envs" + "github.com/olivere/elastic" "github.com/stretchr/testify/assert" ) @@ -23,7 +26,7 @@ func (f *MockField) Name() string { return f.fieldKey } func (f *MockField) Type() assets.FieldType { return f.fieldType } func (f *MockField) UUID() assets.FieldUUID { return f.fieldUUID } -func TestElasticQuery(t *testing.T) { +func buildResolver() contactql.FieldResolverFunc { registry := map[string]assets.Field{ "age": &MockField{"age", assets.FieldTypeNumber, "6b6a43fa-a26d-4017-bede-328bcdd5c93b"}, "color": &MockField{"color", assets.FieldTypeText, "ecc7b13b-c698-4f46-8a90-24a8fab6fe34"}, @@ -33,6 +36,81 @@ func TestElasticQuery(t *testing.T) { "ward": &MockField{"ward", assets.FieldTypeWard, "fde8f740-c337-421b-8abb-83b954897c80"}, } + resolver := func(key string) assets.Field { + field, found := registry[key] + if !found { + return nil + } + return field + } + + return resolver +} + +func TestElasticSort(t *testing.T) { + resolver := buildResolver() + + tcs := []struct { + Label string + Sort string + Elastic string + Error error + }{ + {"empty", "", `{"id":{"order":"desc"}}`, nil}, + {"descending created_on", "-created_on", `{"created_on":{"order":"desc"}}`, nil}, + {"ascending name", "name", `{"name":{"order":"asc"}}`, nil}, + {"descending language", "-language", `{"language":{"order":"desc"}}`, nil}, + {"descending numeric", "-AGE", `{"fields.number":{"nested":{"filter":{"term":{"fields.field":"6b6a43fa-a26d-4017-bede-328bcdd5c93b"}},"path":"fields"},"order":"desc"}}`, nil}, + {"ascending text", "color", `{"fields.text":{"nested":{"filter":{"term":{"fields.field":"ecc7b13b-c698-4f46-8a90-24a8fab6fe34"}},"path":"fields"},"order":"asc"}}`, nil}, + {"descending date", "-dob", `{"fields.datetime":{"nested":{"filter":{"term":{"fields.field":"cbd3fc0e-9b74-4207-a8c7-248082bb4572"}},"path":"fields"},"order":"desc"}}`, nil}, + {"descending state", "-state", `{"fields.state":{"nested":{"filter":{"term":{"fields.field":"67663ad1-3abc-42dd-a162-09df2dea66ec"}},"path":"fields"},"order":"desc"}}`, nil}, + {"ascending district", "district", `{"fields.district":{"nested":{"filter":{"term":{"fields.field":"54c72635-d747-4e45-883c-099d57dd998e"}},"path":"fields"},"order":"asc"}}`, nil}, + {"ascending ward", "ward", `{"fields.ward":{"nested":{"filter":{"term":{"fields.field":"fde8f740-c337-421b-8abb-83b954897c80"}},"path":"fields"},"order":"asc"}}`, nil}, + + {"unknown field", "foo", "", fmt.Errorf("unable to find field with name: foo")}, + } + + for _, tc := range tcs { + sort, err := ToElasticFieldSort(resolver, tc.Sort) + + if err != nil { + assert.Equal(t, tc.Error.Error(), err.Error()) + continue + } + + src, _ := sort.Source() + encoded, _ := json.Marshal(src) + assert.Equal(t, tc.Elastic, string(encoded)) + } +} + +func TestQueryTerms(t *testing.T) { + resolver := buildResolver() + + tcs := []struct { + Query string + Fields []string + }{ + {"joe", []string{"name"}}, + {"id = 10", []string{"id"}}, + {"name = joe or AGE > 10", []string{"age", "name"}}, + } + + env := envs.NewBuilder().Build() + + for _, tc := range tcs { + parsed, err := ParseQuery(env, resolver, tc.Query) + assert.NoError(t, err) + + fields := FieldDependencies(parsed) + assert.Equal(t, fields, tc.Fields) + } + +} + +func TestElasticQuery(t *testing.T) { + resolver := buildResolver() + type TestCase struct { Label string `json:"label"` Search string `json:"search"` @@ -40,7 +118,6 @@ func TestElasticQuery(t *testing.T) { Error string `json:"error"` IsAnon bool `json:"is_anon"` } - tcs := make([]TestCase, 0, 20) tcJSON, err := ioutil.ReadFile("testdata/elastic_test.json") assert.NoError(t, err) @@ -50,10 +127,6 @@ func TestElasticQuery(t *testing.T) { ny, _ := time.LoadLocation("America/New_York") - resolver := func(key string) assets.Field { - return registry[key] - } - for _, tc := range tcs { redactionPolicy := envs.RedactionPolicyNone if tc.IsAnon { @@ -61,7 +134,12 @@ func TestElasticQuery(t *testing.T) { } env := envs.NewBuilder().WithTimezone(ny).WithRedactionPolicy(redactionPolicy).Build() - query, err := ToElasticQuery(env, resolver, tc.Search) + qlQuery, err := ParseQuery(env, resolver, tc.Search) + + var query elastic.Query + if err == nil { + query, err = ToElasticQuery(env, resolver, qlQuery) + } if tc.Error != "" { assert.Error(t, err, "%s: error not received converting to elastic: %s", tc.Label, tc.Search) diff --git a/search/testdata/elastic_test.json b/search/testdata/elastic_test.json index 1dd14326f..c3e8a7d3a 100644 --- a/search/testdata/elastic_test.json +++ b/search/testdata/elastic_test.json @@ -1226,7 +1226,7 @@ { "label": "invalid created_on operand", "search": "created_on 10", +// "sort": "-age" +// } +// +type searchRequest struct { + OrgID models.OrgID `json:"org_id" validate:"required"` + GroupUUID assets.GroupUUID `json:"group_uuid" validate:"required"` + Query string `json:"query" validate:"required"` + PageSize int `json:"page_size"` + Offset int `json:"offset"` + Sort string `json:"sort"` +} + +// Response for a contact search +// +// { +// "query": "age > 10", +// "contact_ids": [5,10,15], +// "total": 3, +// "offset": 0 +// } +type searchResponse struct { + Query string `json:"query"` + ContactIDs []models.ContactID `json:"contact_ids"` + Fields []string `json:"fields"` + Total int64 `json:"total"` + Offset int `json:"offset"` + Sort string `json:"sort"` +} + +// handles a a contact search request +func handleSearch(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) { + request := &searchRequest{ + Offset: 0, + PageSize: 50, + Sort: "-created_on", + } + if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil { + return errors.Wrapf(err, "request failed validation"), http.StatusBadRequest, nil + } + + // grab our org + org, err := models.GetOrgAssets(s.CTX, s.DB, request.OrgID) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") + } + + // Perform our search + parsed, hits, total, err := models.ContactIDsForQueryPage(ctx, s.ElasticClient, org, + request.GroupUUID, request.Query, request.Sort, request.Offset, request.PageSize) + + if err != nil { + switch cause := errors.Cause(err).(type) { + case *search.Error: + return cause, http.StatusBadRequest, nil + default: + return nil, http.StatusInternalServerError, err + } + } + + // build our response + response := &searchResponse{ + Query: parsed.String(), + ContactIDs: hits, + Fields: search.FieldDependencies(parsed), + Total: total, + Offset: request.Offset, + Sort: request.Sort, + } + + return response, http.StatusOK, nil +} diff --git a/web/contact/contact_test.go b/web/contact/contact_test.go new file mode 100644 index 000000000..87f656228 --- /dev/null +++ b/web/contact/contact_test.go @@ -0,0 +1,156 @@ +package contact + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "sync" + "testing" + "time" + + "github.com/nyaruka/mailroom/config" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/search" + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/web" + "github.com/olivere/elastic" + "github.com/stretchr/testify/assert" +) + +func TestServer(t *testing.T) { + testsuite.Reset() + ctx := testsuite.CTX() + db := testsuite.DB() + rp := testsuite.RP() + wg := &sync.WaitGroup{} + + es := search.NewMockElasticServer() + defer es.Close() + + client, err := elastic.NewClient( + elastic.SetURL(es.URL()), + elastic.SetHealthcheck(false), + elastic.SetSniff(false), + ) + assert.NoError(t, err) + + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, client, wg) + server.Start() + + // give our server time to start + time.Sleep(time.Second) + + defer server.Stop() + + singleESResponse := fmt.Sprintf(`{ + "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": null, + "hits": [ + { + "_index": "contacts", + "_type": "_doc", + "_id": "%d", + "_score": null, + "_routing": "1", + "sort": [ + 15124352 + ] + } + ] + } + }`, models.CathyID) + + tcs := []struct { + URL string + Method string + Body string + Status int + Error string + Hits []models.ContactID + Query string + Fields []string + ESResponse string + }{ + {"/mr/contact/search", "GET", "", 405, "illegal method: GET", nil, "", nil, ""}, + { + "/mr/contact/search", "POST", + fmt.Sprintf(`{"org_id": 1, "query": "birthday = tomorrow", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + 400, "can't resolve 'birthday' to attribute, scheme or field", + nil, "", nil, "", + }, + { + "/mr/contact/search", "POST", + fmt.Sprintf(`{"org_id": 1, "query": "age > tomorrow", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + 400, "can't convert 'tomorrow' to a number", + nil, "", nil, "", + }, + { + "/mr/contact/search", "POST", + fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + 200, + "", + []models.ContactID{models.CathyID}, + `name ~ "Cathy"`, + []string{"name"}, + singleESResponse, + }, + { + "/mr/contact/search", "POST", + fmt.Sprintf(`{"org_id": 1, "query": "AGE = 10 and gender = M", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + 200, + "", + []models.ContactID{models.CathyID}, + `age = 10 AND gender = "M"`, + []string{"age", "gender"}, + singleESResponse, + }, + } + + for i, tc := range tcs { + var body io.Reader + es.NextResponse = tc.ESResponse + + if tc.Body != "" { + body = bytes.NewReader([]byte(tc.Body)) + } + + req, err := http.NewRequest(tc.Method, "http://localhost:8090"+tc.URL, body) + assert.NoError(t, err, "%d: error creating request", i) + + resp, err := http.DefaultClient.Do(req) + assert.NoError(t, err, "%d: error making request", i) + + assert.Equal(t, tc.Status, resp.StatusCode, "%d: unexpected status", i) + + content, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err, "%d: error reading body", i) + + // on 200 responses parse them + if resp.StatusCode == 200 { + r := &searchResponse{} + err = json.Unmarshal(content, r) + assert.NoError(t, err) + assert.Equal(t, tc.Hits, r.ContactIDs) + assert.Equal(t, tc.Query, r.Query) + assert.Equal(t, tc.Fields, r.Fields) + } else { + r := &web.ErrorResponse{} + err = json.Unmarshal(content, r) + assert.NoError(t, err) + assert.Equal(t, tc.Error, r.Error) + } + } +} diff --git a/web/expression/expression_test.go b/web/expression/expression_test.go index 00b1a93fd..0465112dd 100644 --- a/web/expression/expression_test.go +++ b/web/expression/expression_test.go @@ -26,7 +26,7 @@ func TestServer(t *testing.T) { rp := testsuite.RP() wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) server.Start() // give our server time to start diff --git a/web/flow/flow_test.go b/web/flow/flow_test.go index 7e4bb384d..ea3f7b1bd 100644 --- a/web/flow/flow_test.go +++ b/web/flow/flow_test.go @@ -27,7 +27,7 @@ func TestServer(t *testing.T) { rp := testsuite.RP() wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) server.Start() // give our server time to start diff --git a/web/ivr/ivr_test.go b/web/ivr/ivr_test.go index c7d535694..db0bc0f98 100644 --- a/web/ivr/ivr_test.go +++ b/web/ivr/ivr_test.go @@ -60,7 +60,7 @@ func TestTwilioIVR(t *testing.T) { twiml.IgnoreSignatures = true wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) server.Start() defer server.Stop() @@ -364,7 +364,7 @@ func TestNexmoIVR(t *testing.T) { defer ts.Close() wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) server.Start() defer server.Stop() diff --git a/web/server.go b/web/server.go index d0dadfead..6530ed5d0 100644 --- a/web/server.go +++ b/web/server.go @@ -11,14 +11,15 @@ import ( "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/models" + "github.com/olivere/elastic" - "github.com/pkg/errors" + "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" - "github.com/sirupsen/logrus" - "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) const ( @@ -60,13 +61,14 @@ func RegisterRoute(method string, pattern string, handler Handler) { } // NewServer creates a new web server, it will need to be started after being created -func NewServer(ctx context.Context, config *config.Config, db *sqlx.DB, rp *redis.Pool, s3Client s3iface.S3API, wg *sync.WaitGroup) *Server { +func NewServer(ctx context.Context, config *config.Config, db *sqlx.DB, rp *redis.Pool, s3Client s3iface.S3API, elasticClient *elastic.Client, wg *sync.WaitGroup) *Server { s := &Server{ - CTX: ctx, - RP: rp, - DB: db, - S3Client: s3Client, - Config: config, + CTX: ctx, + RP: rp, + DB: db, + S3Client: s3Client, + ElasticClient: elasticClient, + Config: config, wg: wg, } @@ -180,12 +182,12 @@ func (s *Server) WrapJSONHandler(handler JSONHandler) http.HandlerFunc { // handler errored (a hard error) if err != nil { - value = errorAsResponse(err) + value = NewErrorResponse(err) } else { // handler returned an error to use as a the response asError, isError := value.(error) if isError { - value = errorAsResponse(asError) + value = NewErrorResponse(asError) } } @@ -219,7 +221,7 @@ func (s *Server) WrapHandler(handler Handler) http.HandlerFunc { logrus.WithError(err).WithField("http_request", r).Error("error handling request") w.WriteHeader(http.StatusInternalServerError) - serialized, _ := json.Marshal(errorAsResponse(err)) + serialized, _ := json.Marshal(NewErrorResponse(err)) w.Write(serialized) return } @@ -271,19 +273,24 @@ func handle405(ctx context.Context, s *Server, r *http.Request) (interface{}, in } type Server struct { - CTX context.Context - RP *redis.Pool - DB *sqlx.DB - S3Client s3iface.S3API - Config *config.Config + CTX context.Context + RP *redis.Pool + DB *sqlx.DB + S3Client s3iface.S3API + Config *config.Config + ElasticClient *elastic.Client wg *sync.WaitGroup httpServer *http.Server } -func errorAsResponse(err error) interface{} { - return map[string]string{ - "error": err.Error(), - } +// ErrorResponse is the type for our error responses, it just contains a single error field +type ErrorResponse struct { + Error string `json:"error"` +} + +// NewErrorResponse creates a new error response from the passed in errro +func NewErrorResponse(err error) *ErrorResponse { + return &ErrorResponse{err.Error()} } diff --git a/web/server_test.go b/web/server_test.go index 723b66764..c75df80e3 100644 --- a/web/server_test.go +++ b/web/server_test.go @@ -23,7 +23,7 @@ func TestServer(t *testing.T) { rp := testsuite.RP() wg := &sync.WaitGroup{} - server := NewServer(ctx, config.Mailroom, db, rp, nil, wg) + server := NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) server.Start() // give our server time to start diff --git a/web/simulation/simulation.go b/web/simulation/simulation.go index 38eb82526..e9f852cd5 100644 --- a/web/simulation/simulation.go +++ b/web/simulation/simulation.go @@ -8,6 +8,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/assets/static/types" + "github.com/nyaruka/goflow/excellent/tools" + xtypes "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/flows/resumes" @@ -38,9 +40,25 @@ type sessionRequest struct { } `json:"assets"` } -type sessionResponse struct { - Session flows.Session `json:"session"` - Events []flows.Event `json:"events"` +type simulationResponse struct { + Session flows.Session `json:"session"` + Events []flows.Event `json:"events"` + Context *xtypes.XObject `json:"context,omitempty"` +} + +func newSimulationResponse(session flows.Session, sprint flows.Sprint) *simulationResponse { + var context *xtypes.XObject + if session != nil { + context = session.CurrentContext() + + // include object defaults which are not marshaled by default + if context != nil { + tools.ContextWalkObjects(context, func(o *xtypes.XObject) { + o.SetMarshalDefault(true) + }) + } + } + return &simulationResponse{Session: session, Events: sprint.Events(), Context: context} } // Starts a new engine session @@ -136,7 +154,7 @@ func triggerFlow(ctx context.Context, db *sqlx.DB, org *models.OrgAssets, sa flo return nil, http.StatusInternalServerError, errors.Wrapf(err, "error handling simulation events") } - return &sessionResponse{Session: session, Events: sprint.Events()}, http.StatusOK, nil + return newSimulationResponse(session, sprint), http.StatusOK, nil } // Resumes an existing engine session @@ -236,7 +254,7 @@ func handleResume(ctx context.Context, s *web.Server, r *http.Request) (interfac // if our session is already complete, then this is a no-op, return the session unchanged if session.Status() != flows.SessionStatusWaiting { - return &sessionResponse{Session: session, Events: nil}, http.StatusOK, nil + return &simulationResponse{Session: session, Events: nil}, http.StatusOK, nil } // resume our session @@ -250,7 +268,7 @@ func handleResume(ctx context.Context, s *web.Server, r *http.Request) (interfac return nil, http.StatusInternalServerError, errors.Wrapf(err, "error handling simulation events") } - return &sessionResponse{Session: session, Events: sprint.Events()}, http.StatusOK, nil + return newSimulationResponse(session, sprint), http.StatusOK, nil } // populateFlow takes care of setting the definition for the flow with the passed in UUID according to the passed in definitions diff --git a/web/simulation/simulation_test.go b/web/simulation/simulation_test.go index 43caf3f39..b2aa48de6 100644 --- a/web/simulation/simulation_test.go +++ b/web/simulation/simulation_test.go @@ -256,7 +256,7 @@ func TestServer(t *testing.T) { rp := testsuite.RP() wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) server.Start() // give our server time to start @@ -329,6 +329,12 @@ func TestServer(t *testing.T) { sessionJSON, _ := json.Marshal(parsed["session"]) session = string(sessionJSON) fmt.Println(session) + + context, hasContext := parsed["context"] + if hasContext { + assert.Contains(t, context, "contact") + assert.Contains(t, context, "globals") + } } assert.True(t, strings.Contains(string(content), tc.Response), "%d: did not find string: %s in body: %s", i, tc.Response, string(content)) diff --git a/web/surveyor/surveyor_test.go b/web/surveyor/surveyor_test.go index 5648657d8..f7558c1b4 100644 --- a/web/surveyor/surveyor_test.go +++ b/web/surveyor/surveyor_test.go @@ -9,8 +9,8 @@ import ( "sync" "testing" - "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/flows" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" @@ -27,7 +27,7 @@ func TestSurveyor(t *testing.T) { defer rc.Close() wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) server.Start() defer server.Stop()