diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4839a0f2f..8dc90630e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: "1.17.x" + go-version: "1.18.x" postgis-version: "3.1" redis-version: "5.0.6" jobs: @@ -50,9 +50,9 @@ jobs: - name: Upload coverage if: success() - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 with: - fail_ci_if_error: false + fail_ci_if_error: true release: name: Release diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f9c96a71..b5bd8d844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,114 @@ +v7.4.1 +---------- + * Update to latest goflow + +v7.4.0 +---------- + * Update README + +v7.3.20 +---------- + * Use proper query construction for preview_start endpoint and return search errors for invalid user queries + +v7.3.19 +---------- + * Update dependencies + * Log version at startup + +v7.3.18 +---------- + * Use new orgmembership table to load users + * Update to latest goflow + +v7.3.17 +---------- + * Update to latest goflow and simplify code for exiting session runs + * Add support for excluding contacts already in a flow in start_session actions + * Don't blow up in msg_created handler if flow has been deleted + * Use analytics package from gocommon instead of librato directly + +v7.3.16 +---------- + * Update to latest goflow + +v7.3.15 +---------- + * Simplify BroadcastBatch + * Record first-reply timings for tickets + * Add arm64 as a build target + +v7.3.14 +---------- + * Update to latest goflow which fixes contact query bug + +v7.3.13 +---------- + * Update to latest goflow which fixes contact query simplification + * Record ticket daily counts when opening, assigning and replying to tickets + * Update to latest gocommon, phonenumbers, jsonparser + +v7.3.12 +---------- + * Update to go 1.18 and use some generics + +v7.3.11 +---------- + * Rework flow/preview_start endpoint to take a number of days since last seen on + * Update to latest goflow that has fix for whatsapp template selection + +v7.3.10 +---------- + * Changes to preview_start endpoint - 1) rename count to total to match other search endpoints, 2) add +query inspection metadata to preview_start endpoint response 3) switch to UUIDs for contacts and groups + +v7.3.9 +---------- + * Move search into its own package and add more tests + * Add endpoint to generate a flow start preview + +v7.3.8 +---------- + * Use new contactfield.name and is_system fields + +v7.3.7 +---------- + * Update to latest goflow and start using httpx.DetectContentType + +v7.3.6 +---------- + * Update modified_on for flow history changes by handling flow entered and sprint ended +events + +v7.3.5 +---------- + * Update to latest goflow which requires mapping groups and flows to ids for ES queries + +v7.3.4 +---------- + * Fix unstopping of contacts who message in + +v7.3.3 +---------- + * ContactGroup.group_type can no longer be 'U' + * Clear session timeout if timeout resume rejected by wait + * Update golang.org/x/sys + +v7.3.2 +---------- + * Add is_system to contact groups, filter groups by group_type = M|Q|U + +v7.3.1 +---------- + * Simplify cron jobs and add them to the main mailroom waitgroup + * Allow expirations and timeouts to resume sessions for stopped, blocked and archived contacts + * Messages to stopped, blocked or archived contacts should immediately fail + +v7.3.0 +---------- + * Update to latest goflow + * Replace last usages of old locker code + * Cleanup some SQL variables + v7.2.6 ---------- * Batch calls to delete event fires diff --git a/README.md b/README.md index 40abc7132..de7d000c9 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,28 @@ -# mailroom +# Mailroom [![Build Status](https://github.com/nyaruka/mailroom/workflows/CI/badge.svg)](https://github.com/nyaruka/mailroom/actions?query=workflow%3ACI) [![codecov](https://codecov.io/gh/nyaruka/mailroom/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/mailroom) -# About +Service for RapidPro/TextIt which does the heavy lifting of running flow starts, campaigns etc. +flows. It interacts directly with the database and sends and receives messages with [Courier](https://github.com/nyaruka/courier) +for handling via Redis. -Mailroom is the [RapidPro](https://github.com/rapidpro/rapidpro) components which does the heavy lifting of running flow starts, campaigns etc. -flows. It interacts directly with the RapidPro database and sends and receives messages with [Courier](https://github.com/nyaruka/courier) for handling via Redis. +## Deploying -# Deploying - -As Mailroom is a Go application, it compiles to a binary and that binary along with the config file is all +As a Go application, it compiles to a binary and that binary along with the config file is all you need to run it on your server. You can find bundles for each platform in the -[releases directory](https://github.com/nyaruka/mailroom/releases). We recommend running Mailroom +[releases directory](https://github.com/nyaruka/mailroom/releases). We recommend running it behind a reverse proxy such as nginx or Elastic Load Balancer that provides HTTPs encryption. -# Configuration +## Configuration -Mailroom uses a tiered configuration system, each option takes precendence over the ones above it: +The service uses a tiered configuration system, each option takes precendence over the ones above it: 1. The configuration file 2. Environment variables starting with `MAILROOM_` 3. Command line parameters -We recommend running Mailroom with no changes to the configuration and no parameters, using only +We recommend running it with no changes to the configuration and no parameters, using only environment variables to configure it. You can use `% mailroom --help` to see a list of the environment variables and parameters and for more details on each option. @@ -67,9 +66,9 @@ Recommended settings for error and performance monitoring: - `MAILROOM_SENTRY_DSN`: The DSN to use when logging errors to Sentry - `MAILROOM_LOG_LEVEL`: the logging level mailroom should use (default "error", use "debug" for more) -# Development +## Development -Once you've checked out the code, you can build Mailroom with: +Once you've checked out the code, you can build the service with: ``` go build github.com/nyaruka/mailroom/cmd/mailroom diff --git a/cmd/mailroom/main.go b/cmd/mailroom/main.go index d5fa4d0c8..21c0216b4 100644 --- a/cmd/mailroom/main.go +++ b/cmd/mailroom/main.go @@ -49,10 +49,15 @@ import ( "github.com/sirupsen/logrus" ) -var version = "Dev" +var ( + // https://goreleaser.com/cookbooks/using-main.version + version = "dev" + date = "unknown" +) func main() { config := runtime.NewDefaultConfig() + config.Version = version loader := ezconf.NewLoader( config, "mailroom", "Mailroom - flow event handler for RapidPro", @@ -65,18 +70,15 @@ func main() { logrus.Fatalf("invalid config: %s", err) } - // if we have a custom version, use it - if version != "Dev" { - config.Version = version - } - - // configure our logger - logrus.SetOutput(os.Stdout) level, err := logrus.ParseLevel(config.LogLevel) if err != nil { logrus.Fatalf("invalid log level '%s'", level) } + logrus.SetLevel(level) + logrus.SetOutput(os.Stdout) + logrus.SetFormatter(&logrus.TextFormatter{}) + logrus.WithField("version", version).WithField("released", date).Info("starting mailroom") // if we have a DSN entry, try to initialize it if config.SentryDSN != "" { @@ -109,7 +111,7 @@ func main() { // handleSignals takes care of trapping quit, interrupt or terminate signals and doing the right thing func handleSignals(mr *mailroom.Mailroom) { - sigs := make(chan os.Signal) + sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) for { diff --git a/core/handlers/base_test.go b/core/handlers/base_test.go index 3af2ed672..b486e5abd 100644 --- a/core/handlers/base_test.go +++ b/core/handlers/base_test.go @@ -145,6 +145,7 @@ func createTestFlow(t *testing.T, uuid assets.FlowUUID, tc TestCase) flows.Flow definition.NewLocalization(), nodes, nil, + nil, ) require.NoError(t, err) diff --git a/core/handlers/contact_flow_changed.go b/core/handlers/contact_flow_changed.go deleted file mode 100644 index 3a5ef9667..000000000 --- a/core/handlers/contact_flow_changed.go +++ /dev/null @@ -1,28 +0,0 @@ -package handlers - -import ( - "context" - - "github.com/jmoiron/sqlx" - "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/mailroom/core/hooks" - "github.com/nyaruka/mailroom/core/models" - "github.com/nyaruka/mailroom/runtime" - "github.com/sirupsen/logrus" -) - -func init() { - models.RegisterEventHandler(models.TypeContactFlowChanged, handleContactFlowChanged) -} - -// handleContactFlowChanged handles contact_flow_changed events which the engine doesn't produce but we append to update a contact's current flow -func handleContactFlowChanged(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *models.OrgAssets, scene *models.Scene, e flows.Event) error { - event := e.(*models.ContactFlowChangedEvent) - - logrus.WithFields(logrus.Fields{"contact_uuid": scene.ContactUUID(), "session_id": scene.SessionID(), "flow_id": event.FlowID}).Debug("contact flow changed") - - scene.AppendToEventPreCommitHook(hooks.CommitFlowChangesHook, event) - scene.AppendToEventPostCommitHook(hooks.ContactModifiedHook, event) - - return nil -} diff --git a/core/handlers/contact_status_changed_test.go b/core/handlers/contact_status_changed_test.go index 93a77b600..05c7aad89 100644 --- a/core/handlers/contact_status_changed_test.go +++ b/core/handlers/contact_status_changed_test.go @@ -11,12 +11,9 @@ import ( ) func TestContactStatusChanged(t *testing.T) { - ctx, rt, db, _ := testsuite.Get() + ctx, rt, _, _ := testsuite.Get() - defer testsuite.Reset(testsuite.ResetAll) - - // make sure cathyID contact is active - db.Exec(`UPDATE contacts_contact SET status = 'A' WHERE id = $1`, testdata.Cathy.ID) + defer testsuite.Reset(testsuite.ResetData) tcs := []handlers.TestCase{ { diff --git a/core/handlers/contact_urns_changed_test.go b/core/handlers/contact_urns_changed_test.go index 6ccfc4054..3bd8bd021 100644 --- a/core/handlers/contact_urns_changed_test.go +++ b/core/handlers/contact_urns_changed_test.go @@ -2,7 +2,6 @@ package handlers_test import ( "testing" - "time" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/flows" @@ -20,8 +19,6 @@ func TestContactURNsChanged(t *testing.T) { // add a URN to george that cathy will steal testdata.InsertContactURN(db, testdata.Org1, testdata.George, urns.URN("tel:+12065551212"), 100) - now := time.Now() - tcs := []handlers.TestCase{ { Actions: handlers.ContactActionMap{ @@ -55,12 +52,6 @@ func TestContactURNsChanged(t *testing.T) { Args: []interface{}{testdata.George.ID}, Count: 1, }, - // two contacts updated, both cathy and evan since their URNs changed - { - SQL: "select count(*) from contacts_contact where modified_on > $1", - Args: []interface{}{now}, - Count: 2, - }, }, }, } diff --git a/core/handlers/flow_entered.go b/core/handlers/flow_entered.go new file mode 100644 index 000000000..aae420089 --- /dev/null +++ b/core/handlers/flow_entered.go @@ -0,0 +1,35 @@ +package handlers + +import ( + "context" + + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/flows/events" + "github.com/nyaruka/mailroom/core/hooks" + "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/runtime" + + "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" +) + +func init() { + models.RegisterEventHandler(events.TypeFlowEntered, handleFlowEntered) +} + +func handleFlowEntered(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *models.OrgAssets, scene *models.Scene, e flows.Event) error { + event := e.(*events.FlowEnteredEvent) + + logrus.WithFields(logrus.Fields{ + "contact_uuid": scene.ContactUUID(), + "session_id": scene.SessionID(), + "flow_name": event.Flow.Name, + "flow_uuid": event.Flow.UUID, + }).Debug("flow entered") + + // we've potentially changed contact flow history.. only way to be sure would be loading contacts with their + // flow history, but not sure that is worth it given how likely we are to be updating modified_on anyway + scene.AppendToEventPreCommitHook(hooks.ContactModifiedHook, event) + + return nil +} diff --git a/core/handlers/flow_entered_test.go b/core/handlers/flow_entered_test.go new file mode 100644 index 000000000..f30f2fc42 --- /dev/null +++ b/core/handlers/flow_entered_test.go @@ -0,0 +1,42 @@ +package handlers_test + +import ( + "testing" + + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/flows/actions" + "github.com/nyaruka/mailroom/core/handlers" + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/testsuite/testdata" + "github.com/stretchr/testify/assert" +) + +func TestFlowEntered(t *testing.T) { + ctx, rt, _, _ := testsuite.Get() + + defer testsuite.Reset(testsuite.ResetAll) + + oa := testdata.Org1.Load(rt) + + flow, err := oa.FlowByID(testdata.PickANumber.ID) + assert.NoError(t, err) + + tcs := []handlers.TestCase{ + { + Actions: handlers.ContactActionMap{ + testdata.Cathy: []flows.Action{ + actions.NewEnterFlow(handlers.NewActionUUID(), flow.Reference(), false), + }, + }, + SQLAssertions: []handlers.SQLAssertion{ + { + SQL: `select count(*) from contacts_contact where current_flow_id = $1`, + Args: []interface{}{flow.ID()}, + Count: 1, + }, + }, + }, + } + + handlers.RunTestCases(t, ctx, rt, tcs) +} diff --git a/core/handlers/msg_created.go b/core/handlers/msg_created.go index 9317e8fe8..5666c93df 100644 --- a/core/handlers/msg_created.go +++ b/core/handlers/msg_created.go @@ -50,7 +50,7 @@ func handlePreMsgCreated(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, } // everybody else gets their timeout cleared, will be set by courier - scene.Session().ClearTimeoutOn() + scene.Session().ClearWaitTimeout(ctx, nil) return nil } @@ -93,10 +93,15 @@ func handleMsgCreated(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa } } + // and the flow + var flow *models.Flow run, _ := scene.Session().FindStep(e.StepUUID()) - flow, _ := oa.FlowByUUID(run.FlowReference().UUID) + flowAsset, _ := oa.FlowByUUID(run.FlowReference().UUID) + if flowAsset != nil { + flow = flowAsset.(*models.Flow) + } - msg, err := models.NewOutgoingFlowMsg(rt, oa.Org(), channel, scene.Session(), flow.(*models.Flow), event.Msg, event.CreatedOn()) + msg, err := models.NewOutgoingFlowMsg(rt, oa.Org(), channel, scene.Session(), flow, event.Msg, event.CreatedOn()) if err != nil { return errors.Wrapf(err, "error creating outgoing message to %s", event.Msg.URN()) } diff --git a/core/handlers/noop.go b/core/handlers/noop.go index a2f46c6fd..28a1471a0 100644 --- a/core/handlers/noop.go +++ b/core/handlers/noop.go @@ -16,7 +16,6 @@ func init() { models.RegisterEventHandler(events.TypeEnvironmentRefreshed, NoopHandler) models.RegisterEventHandler(events.TypeError, NoopHandler) models.RegisterEventHandler(events.TypeFailure, NoopHandler) - models.RegisterEventHandler(events.TypeFlowEntered, NoopHandler) models.RegisterEventHandler(events.TypeMsgWait, NoopHandler) models.RegisterEventHandler(events.TypeRunExpired, NoopHandler) models.RegisterEventHandler(events.TypeRunResultChanged, NoopHandler) diff --git a/core/handlers/session_triggered.go b/core/handlers/session_triggered.go index f8fdce670..50dadb317 100644 --- a/core/handlers/session_triggered.go +++ b/core/handlers/session_triggered.go @@ -17,16 +17,15 @@ func init() { models.RegisterEventHandler(events.TypeSessionTriggered, handleSessionTriggered) } -// handleSessionTriggered queues this event for being started after our scene are committed func handleSessionTriggered(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *models.OrgAssets, scene *models.Scene, e flows.Event) error { event := e.(*events.SessionTriggeredEvent) logrus.WithFields(logrus.Fields{ "contact_uuid": scene.ContactUUID(), "session_id": scene.SessionID(), - "flow": event.Flow.Name, + "flow_name": event.Flow.Name, "flow_uuid": event.Flow.UUID, - }).Debug("scene triggered") + }).Debug("session triggered") scene.AppendToEventPreCommitHook(hooks.InsertStartHook, event) diff --git a/core/handlers/sprint_ended.go b/core/handlers/sprint_ended.go new file mode 100644 index 000000000..34ec40844 --- /dev/null +++ b/core/handlers/sprint_ended.go @@ -0,0 +1,35 @@ +package handlers + +import ( + "context" + + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom/core/hooks" + "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/runtime" + + "github.com/jmoiron/sqlx" +) + +func init() { + models.RegisterEventHandler(models.TypeSprintEnded, handleSprintEnded) +} + +func handleSprintEnded(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *models.OrgAssets, scene *models.Scene, e flows.Event) error { + event := e.(*models.SprintEndedEvent) + + // if we're in a flow type that can wait then contact current flow has potentially changed + currentFlowChanged := scene.Session().SessionType().Interrupts() && event.Contact.CurrentFlowID() != scene.Session().CurrentFlowID() + + if currentFlowChanged { + scene.AppendToEventPreCommitHook(hooks.CommitFlowChangesHook, scene.Session().CurrentFlowID()) + } + + // if current flow has changed then we need to update modified_on, but also if this is a new session + // then flow history may have changed too in a way that won't be captured by a flow_entered event + if currentFlowChanged || !event.Resumed { + scene.AppendToEventPostCommitHook(hooks.ContactModifiedHook, event) + } + + return nil +} diff --git a/core/handlers/ticket_opened_test.go b/core/handlers/ticket_opened_test.go index 376c79f93..da57129c9 100644 --- a/core/handlers/ticket_opened_test.go +++ b/core/handlers/ticket_opened_test.go @@ -44,9 +44,11 @@ func TestTicketOpened(t *testing.T) { }, })) + oa := testdata.Org1.Load(rt) + // an existing ticket cathyTicket := models.NewTicket(flows.TicketUUID(uuids.New()), testdata.Org1.ID, testdata.Cathy.ID, testdata.Mailgun.ID, "748363", testdata.DefaultTopic.ID, "Who?", models.NilUserID, nil) - err := models.InsertTickets(ctx, db, []*models.Ticket{cathyTicket}) + err := models.InsertTickets(ctx, db, oa, []*models.Ticket{cathyTicket}) require.NoError(t, err) tcs := []handlers.TestCase{ diff --git a/core/hooks/commit_field_changes.go b/core/hooks/commit_field_changes.go index 8bb345fef..732345c14 100644 --- a/core/hooks/commit_field_changes.go +++ b/core/hooks/commit_field_changes.go @@ -68,7 +68,7 @@ func (h *commitFieldChangesHook) Apply(ctx context.Context, rt *runtime.Runtime, // first apply our deletes // in pg9.6 we need to do this as one query per field type, in pg10 we can rewrite this to be a single query for _, fds := range fieldDeletes { - err := models.BulkQuery(ctx, "deleting contact field values", tx, deleteContactFieldsSQL, fds) + err := models.BulkQuery(ctx, "deleting contact field values", tx, sqlDeleteContactFields, fds) if err != nil { return errors.Wrapf(err, "error deleting contact fields") } @@ -76,7 +76,7 @@ func (h *commitFieldChangesHook) Apply(ctx context.Context, rt *runtime.Runtime, // then our updates if len(fieldUpdates) > 0 { - err := models.BulkQuery(ctx, "updating contact field values", tx, updateContactFieldsSQL, fieldUpdates) + err := models.BulkQuery(ctx, "updating contact field values", tx, sqlUpdateContactFields, fieldUpdates) if err != nil { return errors.Wrapf(err, "error updating contact fields") } @@ -99,28 +99,14 @@ type FieldValue struct { Text string `json:"text"` } -const updateContactFieldsSQL = ` -UPDATE - contacts_contact c -SET - fields = COALESCE(fields,'{}'::jsonb) || r.updates::jsonb -FROM ( - VALUES(:contact_id, :updates) -) AS - r(contact_id, updates) -WHERE - c.id = r.contact_id::int -` - -const deleteContactFieldsSQL = ` -UPDATE - contacts_contact c -SET - fields = fields - r.field_uuid -FROM ( - VALUES(:contact_id, :field_uuid) -) AS - r(contact_id, field_uuid) -WHERE - c.id = r.contact_id::int -` +const sqlUpdateContactFields = ` +UPDATE contacts_contact c + SET fields = COALESCE(fields,'{}'::jsonb) || r.updates::jsonb + FROM (VALUES(:contact_id, :updates)) AS r(contact_id, updates) + WHERE c.id = r.contact_id::int` + +const sqlDeleteContactFields = ` +UPDATE contacts_contact c + SET fields = fields - r.field_uuid + FROM (VALUES(:contact_id, :field_uuid)) AS r(contact_id, field_uuid) + WHERE c.id = r.contact_id::int` diff --git a/core/hooks/commit_flow_changes.go b/core/hooks/commit_flow_changes.go index 488c11428..5bf12ea28 100644 --- a/core/hooks/commit_flow_changes.go +++ b/core/hooks/commit_flow_changes.go @@ -16,11 +16,11 @@ type commitFlowChangesHook struct{} // Apply commits our contact current_flow changes as a bulk update for the passed in map of scene func (h *commitFlowChangesHook) Apply(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error { // build up our list of pairs of contact id and current flow id - updates := make([]interface{}, 0, len(scenes)) + updates := make([]*currentFlowUpdate, 0, len(scenes)) for s, evts := range scenes { // there is only ever one of these events per scene - event := evts[len(evts)-1].(*models.ContactFlowChangedEvent) - updates = append(updates, ¤tFlowUpdate{s.ContactID(), event.FlowID}) + flowID := evts[len(evts)-1].(models.FlowID) + updates = append(updates, ¤tFlowUpdate{s.ContactID(), flowID}) } // do our update diff --git a/core/hooks/commit_language_changes.go b/core/hooks/commit_language_changes.go index bfd43b263..23c8ab820 100644 --- a/core/hooks/commit_language_changes.go +++ b/core/hooks/commit_language_changes.go @@ -18,7 +18,7 @@ type commitLanguageChangesHook struct{} // Apply applies our contact language change before our commit func (h *commitLanguageChangesHook) Apply(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error { // build up our list of pairs of contact id and language name - updates := make([]interface{}, 0, len(scenes)) + updates := make([]*languageUpdate, 0, len(scenes)) for s, e := range scenes { // we only care about the last name change event := e[len(e)-1].(*events.ContactLanguageChangedEvent) @@ -26,7 +26,7 @@ func (h *commitLanguageChangesHook) Apply(ctx context.Context, rt *runtime.Runti } // do our update - return models.BulkQuery(ctx, "updating contact language", tx, updateContactLanguageSQL, updates) + return models.BulkQuery(ctx, "updating contact language", tx, sqlUpdateContactLanguage, updates) } // struct used for our bulk update @@ -35,15 +35,8 @@ type languageUpdate struct { Language null.String `db:"language"` } -const updateContactLanguageSQL = ` - UPDATE - contacts_contact c - SET - language = r.language - FROM ( - VALUES(:id, :language) - ) AS - r(id, language) - WHERE - c.id = r.id::int -` +const sqlUpdateContactLanguage = ` +UPDATE contacts_contact c + SET language = r.language + FROM (VALUES(:id, :language)) AS r(id, language) + WHERE c.id = r.id::int` diff --git a/core/hooks/commit_name_changes.go b/core/hooks/commit_name_changes.go index d627f8c92..dafc0cba9 100644 --- a/core/hooks/commit_name_changes.go +++ b/core/hooks/commit_name_changes.go @@ -19,7 +19,7 @@ type commitNameChangesHook struct{} // Apply commits our contact name changes as a bulk update for the passed in map of scene func (h *commitNameChangesHook) Apply(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error { // build up our list of pairs of contact id and contact name - updates := make([]interface{}, 0, len(scenes)) + updates := make([]*nameUpdate, 0, len(scenes)) for s, e := range scenes { // we only care about the last name change event := e[len(e)-1].(*events.ContactNameChangedEvent) @@ -27,7 +27,7 @@ func (h *commitNameChangesHook) Apply(ctx context.Context, rt *runtime.Runtime, } // do our update - return models.BulkQuery(ctx, "updating contact name", tx, updateContactNameSQL, updates) + return models.BulkQuery(ctx, "updating contact name", tx, sqlUpdateContactName, updates) } // struct used for our bulk insert @@ -36,15 +36,8 @@ type nameUpdate struct { Name null.String `db:"name"` } -const updateContactNameSQL = ` - UPDATE - contacts_contact c - SET - name = r.name - FROM ( - VALUES(:id, :name) - ) AS - r(id, name) - WHERE - c.id = r.id::int -` +const sqlUpdateContactName = ` +UPDATE contacts_contact c + SET name = r.name + FROM (VALUES(:id, :name)) AS r(id, name) + WHERE c.id = r.id::int` diff --git a/core/hooks/insert_start.go b/core/hooks/insert_start.go index 6a5d63a0d..43bc4eef8 100644 --- a/core/hooks/insert_start.go +++ b/core/hooks/insert_start.go @@ -57,11 +57,12 @@ func (h *insertStartHook) Apply(ctx context.Context, rt *runtime.Runtime, tx *sq } // create our start - start := models.NewFlowStart(oa.OrgID(), models.StartTypeFlowAction, flow.FlowType(), flow.ID(), true, true). + start := models.NewFlowStart(oa.OrgID(), models.StartTypeFlowAction, flow.FlowType(), flow.ID()). WithGroupIDs(groupIDs). WithContactIDs(contactIDs). WithURNs(event.URNs). WithQuery(event.ContactQuery). + WithExcludeInAFlow(event.Exclusions.InAFlow). WithCreateContact(event.CreateContact). WithParentSummary(event.RunSummary). WithSessionHistory(historyJSON) diff --git a/core/hooks/insert_tickets.go b/core/hooks/insert_tickets.go index 3acf2606b..ee93baf5e 100644 --- a/core/hooks/insert_tickets.go +++ b/core/hooks/insert_tickets.go @@ -27,7 +27,7 @@ func (h *insertTicketsHook) Apply(ctx context.Context, rt *runtime.Runtime, tx * } // insert the tickets - err := models.InsertTickets(ctx, tx, tickets) + err := models.InsertTickets(ctx, tx, oa, tickets) if err != nil { return errors.Wrapf(err, "error inserting tickets") } diff --git a/core/ivr/ivr.go b/core/ivr/ivr.go index 7b5d36147..0ef5ab086 100644 --- a/core/ivr/ivr.go +++ b/core/ivr/ivr.go @@ -450,7 +450,7 @@ func ResumeIVRFlow( if body != nil { // guess our content type and set it - contentType := http.DetectContentType(body) + contentType := httpx.DetectContentType(body) w.Header().Set("Content-Type", contentType) _, err := w.Write(body) return err diff --git a/core/models/assets_test.go b/core/models/assets_test.go index 9d3321734..6d6d95d3e 100644 --- a/core/models/assets_test.go +++ b/core/models/assets_test.go @@ -15,15 +15,9 @@ import ( ) func TestAssets(t *testing.T) { - ctx, rt, db, _ := testsuite.Get() - - defer testsuite.Reset(testsuite.ResetData) + ctx, rt, _, _ := testsuite.Get() - // create new flow with same name as an existing flow - testdata.InsertFlow(db, testdata.Org1, []byte(`{ - "uuid": "fd7d16dd-3a38-4351-aea6-7a80acb41dd9", - "name": "Pick a Number" - }`)) + defer models.FlushCache() oa, err := models.GetOrgAssets(ctx, rt, testdata.Org1.ID) require.NoError(t, err) @@ -42,7 +36,6 @@ func TestAssets(t *testing.T) { flow, err = oa.FlowByName("PICK A NUMBER") // from db assert.NoError(t, err) - assert.Equal(t, assets.FlowUUID("fd7d16dd-3a38-4351-aea6-7a80acb41dd9"), flow.UUID()) // new flow as newer saved_on assert.Equal(t, "Pick a Number", flow.Name()) flow, err = oa.FlowByName("pick a number") // from cache diff --git a/core/models/campaigns.go b/core/models/campaigns.go index 339e43dd5..8ed0bb05b 100644 --- a/core/models/campaigns.go +++ b/core/models/campaigns.go @@ -626,41 +626,23 @@ type eligibleContact struct { RelToValue *time.Time `db:"rel_to_value"` } -const eligibleContactsForCreatedOnSQL = ` -SELECT - c.id AS contact_id, - c.created_on AS rel_to_value -FROM - contacts_contact c -INNER JOIN - contacts_contactgroup_contacts gc ON gc.contact_id = c.id -WHERE - gc.contactgroup_id = $1 AND c.is_active = TRUE -` - -const eligibleContactsForLastSeenOnSQL = ` -SELECT - c.id AS contact_id, - c.last_seen_on AS rel_to_value -FROM - contacts_contact c -INNER JOIN - contacts_contactgroup_contacts gc ON gc.contact_id = c.id -WHERE - gc.contactgroup_id = $1 AND c.is_active = TRUE AND c.last_seen_on IS NOT NULL -` - -const eligibleContactsForFieldSQL = ` -SELECT - c.id AS contact_id, - (c.fields->$2->>'datetime')::timestamptz AS rel_to_value -FROM - contacts_contact c -INNER JOIN - contacts_contactgroup_contacts gc ON gc.contact_id = c.id -WHERE - gc.contactgroup_id = $1 AND c.is_active = TRUE AND ARRAY[$2]::text[] <@ (extract_jsonb_keys(c.fields)) IS NOT NULL -` +const sqlEligibleContactsForCreatedOn = ` + SELECT c.id AS contact_id, c.created_on AS rel_to_value + FROM contacts_contact c +INNER JOIN contacts_contactgroup_contacts gc ON gc.contact_id = c.id + WHERE gc.contactgroup_id = $1 AND c.is_active = TRUE` + +const sqlEligibleContactsForLastSeenOn = ` + SELECT c.id AS contact_id, c.last_seen_on AS rel_to_value + FROM contacts_contact c +INNER JOIN contacts_contactgroup_contacts gc ON gc.contact_id = c.id + WHERE gc.contactgroup_id = $1 AND c.is_active = TRUE AND c.last_seen_on IS NOT NULL` + +const sqlEligibleContactsForField = ` + SELECT c.id AS contact_id, (c.fields->$2->>'datetime')::timestamptz AS rel_to_value + FROM contacts_contact c +INNER JOIN contacts_contactgroup_contacts gc ON gc.contact_id = c.id + WHERE gc.contactgroup_id = $1 AND c.is_active = TRUE AND ARRAY[$2]::text[] <@ (extract_jsonb_keys(c.fields)) IS NOT NULL` func campaignEventEligibleContacts(ctx context.Context, db Queryer, groupID GroupID, field *Field) ([]*eligibleContact, error) { var query string @@ -668,13 +650,13 @@ func campaignEventEligibleContacts(ctx context.Context, db Queryer, groupID Grou switch field.Key() { case CreatedOnKey: - query = eligibleContactsForCreatedOnSQL + query = sqlEligibleContactsForCreatedOn params = []interface{}{groupID} case LastSeenOnKey: - query = eligibleContactsForLastSeenOnSQL + query = sqlEligibleContactsForLastSeenOn params = []interface{}{groupID} default: - query = eligibleContactsForFieldSQL + query = sqlEligibleContactsForField params = []interface{}{groupID, field.UUID()} } diff --git a/core/models/channel_connection.go b/core/models/channel_connection.go index d02beb1f2..2d6d97037 100644 --- a/core/models/channel_connection.go +++ b/core/models/channel_connection.go @@ -105,9 +105,8 @@ func (c *ChannelConnection) ErrorReason() ConnectionError { return ConnectionErr func (c *ChannelConnection) ErrorCount() int { return c.c.ErrorCount } func (c *ChannelConnection) NextAttempt() *time.Time { return c.c.NextAttempt } -const insertConnectionSQL = ` -INSERT INTO - channels_channelconnection +const sqlInsertConnection = ` +INSERT INTO channels_channelconnection ( created_on, modified_on, @@ -122,7 +121,6 @@ INSERT INTO contact_urn_id, error_count ) - VALUES( NOW(), NOW(), @@ -137,10 +135,7 @@ VALUES( :contact_urn_id, 0 ) -RETURNING - id, - NOW(); -` +RETURNING id, NOW();` // InsertIVRConnection creates a new IVR session for the passed in org, channel and contact, inserting it func InsertIVRConnection(ctx context.Context, db *sqlx.DB, orgID OrgID, channelID ChannelID, startID StartID, contactID ContactID, urnID URNID, @@ -159,7 +154,7 @@ func InsertIVRConnection(ctx context.Context, db *sqlx.DB, orgID OrgID, channelI c.ExternalID = externalID c.StartID = startID - rows, err := db.NamedQueryContext(ctx, insertConnectionSQL, c) + rows, err := db.NamedQueryContext(ctx, sqlInsertConnection, c) if err != nil { return nil, errors.Wrapf(err, "error inserting new channel connection") } diff --git a/core/models/channel_event.go b/core/models/channel_event.go index 12164dd9f..2c1281266 100644 --- a/core/models/channel_event.go +++ b/core/models/channel_event.go @@ -78,18 +78,15 @@ func (e *ChannelEvent) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &e.e) } -const insertChannelEventSQL = ` -INSERT INTO - channels_channelevent(event_type, extra, occurred_on, created_on, channel_id, contact_id, contact_urn_id, org_id) - VALUES(:event_type, :extra, :occurred_on, :created_on, :channel_id, :contact_id, :contact_urn_id, :org_id) -RETURNING - id -` +const sqlInsertChannelEvent = ` +INSERT INTO channels_channelevent(event_type, extra, occurred_on, created_on, channel_id, contact_id, contact_urn_id, org_id) + VALUES(:event_type, :extra, :occurred_on, :created_on, :channel_id, :contact_id, :contact_urn_id, :org_id) + RETURNING id` // Insert inserts this channel event to our DB. The ID of the channel event will be // set if no error is returned func (e *ChannelEvent) Insert(ctx context.Context, db Queryer) error { - return BulkQuery(ctx, "insert channel event", db, insertChannelEventSQL, []interface{}{&e.e}) + return BulkQuery(ctx, "insert channel event", db, sqlInsertChannelEvent, []interface{}{&e.e}) } // NewChannelEvent creates a new channel event for the passed in parameters, returning it diff --git a/core/models/channel_logs.go b/core/models/channel_logs.go index a0ff14b9c..385992d6c 100644 --- a/core/models/channel_logs.go +++ b/core/models/channel_logs.go @@ -35,13 +35,10 @@ type ChannelLog struct { // ID returns the id of this channel log func (l *ChannelLog) ID() ChannelLogID { return l.l.ID } -const insertChannelLogSQL = ` -INSERT INTO - channels_channellog( description, is_error, url, method, request, response, response_status, created_on, request_time, channel_id, connection_id) - VALUES(:description, :is_error, :url, :method, :request, :response, :response_status, :created_on, :request_time, :channel_id, :connection_id) -RETURNING - id as id -` +const sqlInsertChannelLog = ` +INSERT INTO channels_channellog( description, is_error, url, method, request, response, response_status, created_on, request_time, channel_id, connection_id) + VALUES(:description, :is_error, :url, :method, :request, :response, :response_status, :created_on, :request_time, :channel_id, :connection_id) + RETURNING id as id` // NewChannelLog creates a new channel log func NewChannelLog(trace *httpx.Trace, isError bool, desc string, channel *Channel, conn *ChannelConnection) *ChannelLog { @@ -87,7 +84,7 @@ func InsertChannelLogs(ctx context.Context, db Queryer, logs []*ChannelLog) erro ls[i] = &logs[i].l } - err := BulkQuery(ctx, "insert channel log", db, insertChannelLogSQL, ls) + err := BulkQuery(ctx, "insert channel log", db, sqlInsertChannelLog, ls) if err != nil { return errors.Wrapf(err, "error inserting channel log") } @@ -117,7 +114,7 @@ func InsertChannelLog(ctx context.Context, db Queryer, l.ConnectionID = conn.ID() } - err := BulkQuery(ctx, "insert channel log", db, insertChannelLogSQL, []interface{}{l}) + err := BulkQuery(ctx, "insert channel log", db, sqlInsertChannelLog, []interface{}{l}) if err != nil { return nil, errors.Wrapf(err, "error inserting channel log") } diff --git a/core/models/contacts.go b/core/models/contacts.go index ade4e38e9..f29507e47 100644 --- a/core/models/contacts.go +++ b/core/models/contacts.go @@ -19,6 +19,7 @@ import ( "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/null" + "github.com/nyaruka/redisx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -195,9 +196,12 @@ func (c *Contact) UpdatePreferredURN(ctx context.Context, db Queryer, oa *OrgAss // FlowContact converts our mailroom contact into a flow contact for use in the engine func (c *Contact) FlowContact(oa *OrgAssets) (*flows.Contact, error) { // convert our groups to a list of references - groups := make([]*assets.GroupReference, len(c.groups)) - for i, g := range c.groups { - groups[i] = assets.NewGroupReference(g.UUID(), g.Name()) + groups := make([]*assets.GroupReference, 0, len(c.groups)) + for _, g := range c.groups { + // exclude the db-trigger based status groups for now + if g.Type() == GroupTypeManual || g.Type() == GroupTypeSmart { + groups = append(groups, assets.NewGroupReference(g.UUID(), g.Name())) + } } // convert our tickets to flow tickets @@ -1039,7 +1043,9 @@ func StopContact(ctx context.Context, db Queryer, orgID OrgID, contactID Contact const sqlDeleteAllContactGroups = ` DELETE FROM contacts_contactgroup_contacts - WHERE contact_id = $2 AND contactgroup_id = ANY(SELECT id from contacts_contactgroup WHERE org_id = $1 and group_type = 'U')` + WHERE contact_id = $2 AND contactgroup_id = ANY( + SELECT id from contacts_contactgroup WHERE org_id = $1 and group_type IN ('M', 'Q') + )` const sqlDeleteAllContactTriggers = ` DELETE FROM triggers_trigger_contacts @@ -1130,7 +1136,7 @@ func updateURNChannelPriority(urn urns.URN, channel *Channel, priority int) (urn // UpdateContactModifiedOn updates modified_on the passed in contacts func UpdateContactModifiedOn(ctx context.Context, db Queryer, contactIDs []ContactID) error { - for _, idBatch := range chunkContactIDs(contactIDs, 100) { + for _, idBatch := range chunkSlice(contactIDs, 100) { _, err := db.ExecContext(ctx, `UPDATE contacts_contact SET modified_on = NOW() WHERE id = ANY($1)`, pq.Array(idBatch)) if err != nil { return errors.Wrap(err, "error updating modified_on for contact batch") @@ -1341,18 +1347,10 @@ func (i *ContactID) Scan(value interface{}) error { return null.ScanInt(value, (*null.Int)(i)) } -// ContactLock returns the lock key for a particular contact, used with locker -func ContactLock(orgID OrgID, contactID ContactID) string { - return fmt.Sprintf("c:%d:%d", orgID, contactID) -} - -// UpdateContactModifiedBy updates modified by the passed user id on the passed in contacts -func UpdateContactModifiedBy(ctx context.Context, db Queryer, contactIDs []ContactID, userID UserID) error { - if userID == NilUserID || len(contactIDs) == 0 { - return nil - } - _, err := db.ExecContext(ctx, `UPDATE contacts_contact SET modified_on = NOW(), modified_by_id = $2 WHERE id = ANY($1)`, pq.Array(contactIDs), userID) - return err +// GetContactLocker returns the locker for a particular contact +func GetContactLocker(orgID OrgID, contactID ContactID) *redisx.Locker { + key := fmt.Sprintf("lock:c:%d:%d", orgID, contactID) + return redisx.NewLocker(key, time.Minute*5) } // ContactStatusChange struct used for our contact status change diff --git a/core/models/contacts_test.go b/core/models/contacts_test.go index 8a34a0aca..764069862 100644 --- a/core/models/contacts_test.go +++ b/core/models/contacts_test.go @@ -26,9 +26,9 @@ func TestContacts(t *testing.T) { defer testsuite.Reset(testsuite.ResetAll) testdata.InsertContactURN(db, testdata.Org1, testdata.Bob, "whatsapp:250788373373", 999) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.SupportTopic, "Where are my shoes?", "1234", testdata.Agent) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.SalesTopic, "Where are my pants?", "2345", nil) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Bob, testdata.Mailgun, testdata.DefaultTopic, "His name is Bob", "", testdata.Editor) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.SupportTopic, "Where are my shoes?", "1234", time.Now(), testdata.Agent) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.SalesTopic, "Where are my pants?", "2345", time.Now(), nil) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Bob, testdata.Mailgun, testdata.DefaultTopic, "His name is Bob", "", time.Now(), testdata.Editor) // delete mailgun ticketer db.MustExec(`UPDATE tickets_ticketer SET is_active = false WHERE id = $1`, testdata.Mailgun.ID) @@ -509,27 +509,6 @@ func TestUpdateContactLastSeenAndModifiedOn(t *testing.T) { assert.True(t, cathy.ModifiedOn().After(t2)) } -func TestUpdateContactModifiedBy(t *testing.T) { - ctx, _, db, _ := testsuite.Get() - - defer testsuite.Reset(testsuite.ResetAll) - - err := models.UpdateContactModifiedBy(ctx, db, []models.ContactID{}, models.UserID(0)) - assert.NoError(t, err) - - assertdb.Query(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND modified_by_id = NULL`, testdata.Cathy.ID).Returns(0) - - err = models.UpdateContactModifiedBy(ctx, db, []models.ContactID{testdata.Cathy.ID}, models.UserID(0)) - assert.NoError(t, err) - - assertdb.Query(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND modified_by_id = NULL`, testdata.Cathy.ID).Returns(0) - - err = models.UpdateContactModifiedBy(ctx, db, []models.ContactID{testdata.Cathy.ID}, models.UserID(1)) - assert.NoError(t, err) - - assertdb.Query(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND modified_by_id = 1`, testdata.Cathy.ID).Returns(1) -} - func TestUpdateContactStatus(t *testing.T) { ctx, _, db, _ := testsuite.Get() diff --git a/core/models/counts.go b/core/models/counts.go new file mode 100644 index 000000000..238e89eb0 --- /dev/null +++ b/core/models/counts.go @@ -0,0 +1,79 @@ +package models + +import ( + "context" + "fmt" + "time" + + "github.com/nyaruka/gocommon/dates" +) + +type scopedCount struct { + CountType string `db:"count_type"` + Scope string `db:"scope"` + Count int `db:"count"` +} + +type dailyCount struct { + scopedCount + + Day dates.Date `db:"day"` +} + +const sqlInsertDailyCount = `INSERT INTO %s(count_type, scope, day, count, is_squashed) VALUES(:count_type, :scope, :day, :count, FALSE)` + +func insertDailyCounts(ctx context.Context, tx Queryer, table string, countType TicketDailyCountType, tz *time.Location, scopeCounts map[string]int) error { + day := dates.ExtractDate(dates.Now().In(tz)) + + counts := make([]*dailyCount, 0, len(scopeCounts)) + for scope, count := range scopeCounts { + counts = append(counts, &dailyCount{ + scopedCount: scopedCount{ + CountType: string(countType), + Scope: scope, + Count: count, + }, + Day: day, + }) + } + + return BulkQuery(ctx, "inserted daily counts", tx, fmt.Sprintf(sqlInsertDailyCount, table), counts) +} + +type dailyTiming struct { + dailyCount + + Seconds int64 `db:"seconds"` +} + +const sqlInsertDailyTiming = `INSERT INTO %s(count_type, scope, day, count, seconds, is_squashed) VALUES(:count_type, :scope, :day, :count, :seconds, FALSE)` + +func insertDailyTiming(ctx context.Context, tx Queryer, table string, countType TicketDailyTimingType, tz *time.Location, scope string, duration time.Duration) error { + day := dates.ExtractDate(dates.Now().In(tz)) + timing := &dailyTiming{ + dailyCount: dailyCount{ + scopedCount: scopedCount{ + CountType: string(countType), + Scope: scope, + Count: 1, + }, + Day: day, + }, + Seconds: int64(duration / time.Second), + } + + _, err := tx.NamedExecContext(ctx, fmt.Sprintf(sqlInsertDailyTiming, table), timing) + return err +} + +func scopeOrg(oa *OrgAssets) string { + return fmt.Sprintf("o:%d", oa.OrgID()) +} + +func scopeTeam(t *Team) string { + return fmt.Sprintf("t:%d", t.ID) +} + +func scopeUser(oa *OrgAssets, u *User) string { + return fmt.Sprintf("o:%d:u:%d", oa.OrgID(), u.ID()) +} diff --git a/core/models/events.go b/core/models/events.go index e92990d75..4a82bad7f 100644 --- a/core/models/events.go +++ b/core/models/events.go @@ -260,18 +260,21 @@ func ApplyModifiers(ctx context.Context, rt *runtime.Runtime, oa *OrgAssets, mod return eventsByContact, nil } -// TypeContactFlowChanged is the type of our event that the contact flow changed -const TypeContactFlowChanged string = "contact_flow_changed" +// TypeSprintEnded is a pseudo event that lets add hooks for changes to a contacts current flow or flow history +const TypeSprintEnded string = "sprint_ended" -type ContactFlowChangedEvent struct { +type SprintEndedEvent struct { events.BaseEvent - FlowID FlowID + Contact *Contact // model contact so we can access current flow + Resumed bool // whether this was a resume } -func NewContactFlowChangedEvent(flowID FlowID) *ContactFlowChangedEvent { - return &ContactFlowChangedEvent{ - BaseEvent: events.NewBaseEvent(TypeContactFlowChanged), - FlowID: flowID, +// NewSprintEndedEvent creates a new sprint ended event +func NewSprintEndedEvent(c *Contact, resumed bool) *SprintEndedEvent { + return &SprintEndedEvent{ + BaseEvent: events.NewBaseEvent(TypeSprintEnded), + Contact: c, + Resumed: resumed, } } diff --git a/core/models/fields.go b/core/models/fields.go index 2a5658e87..f634a51a2 100644 --- a/core/models/fields.go +++ b/core/models/fields.go @@ -17,12 +17,12 @@ type FieldID int // Field is our mailroom type for contact field types type Field struct { f struct { - ID FieldID `json:"id"` - UUID assets.FieldUUID `json:"uuid"` - Key string `json:"key"` - Name string `json:"name"` - FieldType assets.FieldType `json:"field_type"` - System bool `json:"is_system"` + ID FieldID `json:"id"` + UUID assets.FieldUUID `json:"uuid"` + Key string `json:"key"` + Name string `json:"name"` + Type assets.FieldType `json:"field_type"` + System bool `json:"is_system"` } } @@ -39,7 +39,7 @@ func (f *Field) Key() string { return f.f.Key } func (f *Field) Name() string { return f.f.Name } // Type returns the value type for this field -func (f *Field) Type() assets.FieldType { return f.f.FieldType } +func (f *Field) Type() assets.FieldType { return f.f.Type } // System returns whether this is a system field func (f *Field) System() bool { return f.f.System } @@ -77,26 +77,17 @@ func loadFields(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Fie } const sqlSelectFields = ` -SELECT ROW_TO_JSON(f) FROM (SELECT - id, - uuid, - key, - label as name, - (SELECT CASE value_type - WHEN 'T' THEN 'text' - WHEN 'N' THEN 'number' - WHEN 'D' THEN 'datetime' - WHEN 'S' THEN 'state' - WHEN 'I' THEN 'district' - WHEN 'W' THEN 'ward' - END) as field_type, - field_type = 'S' as is_system -FROM - contacts_contactfield -WHERE - org_id = $1 AND - is_active = TRUE -ORDER BY - key ASC -) f; -` +SELECT ROW_TO_JSON(f) FROM ( + SELECT id, uuid, key, name, is_system, + (SELECT CASE value_type + WHEN 'T' THEN 'text' + WHEN 'N' THEN 'number' + WHEN 'D' THEN 'datetime' + WHEN 'S' THEN 'state' + WHEN 'I' THEN 'district' + WHEN 'W' THEN 'ward' + END) as field_type + FROM contacts_contactfield + WHERE org_id = $1 AND is_active = TRUE + ORDER BY key ASC +) f;` diff --git a/core/models/globals_test.go b/core/models/globals_test.go index 656987c18..6c61be52f 100644 --- a/core/models/globals_test.go +++ b/core/models/globals_test.go @@ -13,6 +13,10 @@ import ( func TestLoadGlobals(t *testing.T) { ctx, rt, db, _ := testsuite.Get() + defer func() { + db.MustExec(`UPDATE globals_global SET value = 'Nyaruka' WHERE org_id = $1 AND key = $2`, testdata.Org1.ID, "org_name") + }() + // set one of our global values to empty db.MustExec(`UPDATE globals_global SET value = '' WHERE org_id = $1 AND key = $2`, testdata.Org1.ID, "org_name") diff --git a/core/models/groups.go b/core/models/groups.go index ad4d42aa1..dfbdf7fb2 100644 --- a/core/models/groups.go +++ b/core/models/groups.go @@ -10,11 +10,13 @@ import ( "github.com/jmoiron/sqlx" "github.com/lib/pq" - "github.com/olivere/elastic/v7" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +// GroupID is our type for group ids +type GroupID int + // GroupStatus is the current status of the passed in group type GroupStatus string @@ -24,16 +26,23 @@ const ( GroupStatusReady = GroupStatus("R") ) -// GroupID is our type for group ids -type GroupID int +// GroupType is the the type of a group +type GroupType string + +const ( + GroupTypeManual = GroupType("M") + GroupTypeSmart = GroupType("Q") +) // Group is our mailroom type for contact groups type Group struct { g struct { - ID GroupID `json:"id"` - UUID assets.GroupUUID `json:"uuid"` - Name string `json:"name"` - Query string `json:"query"` + ID GroupID `json:"id"` + UUID assets.GroupUUID `json:"uuid"` + Name string `json:"name"` + Query string `json:"query"` + Status GroupStatus `json:"status"` + Type GroupType `json:"group_type"` } } @@ -49,6 +58,12 @@ func (g *Group) Name() string { return g.g.Name } // Query returns the query string (if any) for this group func (g *Group) Query() string { return g.g.Query } +// Status returns the status of this group +func (g *Group) Status() GroupStatus { return g.g.Status } + +// Type returns the type of this group +func (g *Group) Type() GroupType { return g.g.Type } + // LoadGroups loads the groups for the passed in org func LoadGroups(ctx context.Context, db Queryer, orgID OrgID) ([]assets.Group, error) { start := time.Now() @@ -76,34 +91,16 @@ func LoadGroups(ctx context.Context, db Queryer, orgID OrgID) ([]assets.Group, e } const selectGroupsSQL = ` -SELECT ROW_TO_JSON(r) FROM (SELECT - id, - uuid, - name, - query -FROM - contacts_contactgroup -WHERE - org_id = $1 AND - is_active = TRUE AND - group_type = 'U' -ORDER BY - name ASC -) r; -` +SELECT ROW_TO_JSON(r) FROM ( + SELECT id, uuid, name, query, status, group_type + FROM contacts_contactgroup + WHERE org_id = $1 AND is_active = TRUE + ORDER BY name ASC +) r;` // RemoveContactsFromGroups fires a bulk SQL query to remove all the contacts in the passed in groups func RemoveContactsFromGroups(ctx context.Context, tx Queryer, removals []*GroupRemove) error { - if len(removals) == 0 { - return nil - } - - // convert to list of interfaces - is := make([]interface{}, len(removals)) - for i := range removals { - is[i] = removals[i] - } - return BulkQuery(ctx, "removing contacts from groups", tx, removeContactsFromGroupsSQL, is) + return BulkQuery(ctx, "removing contacts from groups", tx, removeContactsFromGroupsSQL, removals) } // GroupRemove is our struct to track group removals @@ -130,16 +127,7 @@ IN ( // AddContactsToGroups fires a bulk SQL query to remove all the contacts in the passed in groups func AddContactsToGroups(ctx context.Context, tx Queryer, adds []*GroupAdd) error { - if len(adds) == 0 { - return nil - } - - // convert to list of interfaces - is := make([]interface{}, len(adds)) - for i := range adds { - is[i] = adds[i] - } - return BulkQuery(ctx, "adding contacts to groups", tx, addContactsToGroupsSQL, is) + return BulkQuery(ctx, "adding contacts to groups", tx, addContactsToGroupsSQL, adds) } // GroupAdd is our struct to track a final group additions @@ -331,93 +319,3 @@ func AddContactsToGroupAndCampaigns(ctx context.Context, db *sqlx.DB, oa *OrgAss return nil } - -// PopulateDynamicGroup calculates which members should be part of a group and populates the contacts -// for that group by performing the minimum number of inserts / deletes. -func PopulateDynamicGroup(ctx context.Context, db *sqlx.DB, es *elastic.Client, oa *OrgAssets, groupID GroupID, query string) (int, error) { - err := UpdateGroupStatus(ctx, db, groupID, GroupStatusEvaluating) - if err != nil { - return 0, errors.Wrapf(err, "error marking dynamic group as evaluating") - } - - start := time.Now() - - // we have a bit of a race with the indexer process.. we want to make sure that any contacts that changed - // before this group was updated but after the last index are included, so if a contact was modified - // more recently than 10 seconds ago, we wait that long before starting in populating our group - newest, err := GetNewestContactModifiedOn(ctx, db, oa) - if err != nil { - return 0, errors.Wrapf(err, "error getting most recent contact modified_on for org: %d", oa.OrgID()) - } - if newest != nil { - n := *newest - - // if it was more recent than 10 seconds ago, sleep until it has been 10 seconds - if n.Add(time.Second * 10).After(start) { - sleep := n.Add(time.Second * 10).Sub(start) - logrus.WithField("sleep", sleep).Info("sleeping before evaluating dynamic group") - time.Sleep(sleep) - } - } - - // get current set of contacts in our group - ids, err := ContactIDsForGroupIDs(ctx, db, []GroupID{groupID}) - if err != nil { - return 0, errors.Wrapf(err, "unable to look up contact ids for group: %d", groupID) - } - present := make(map[ContactID]bool, len(ids)) - for _, i := range ids { - present[i] = true - } - - // calculate new set of ids - new, err := GetContactIDsForQuery(ctx, es, oa, query, -1) - if err != nil { - return 0, errors.Wrapf(err, "error performing query: %s for group: %d", query, groupID) - } - - // find which contacts need to be added or removed - adds := make([]ContactID, 0, 100) - for _, id := range new { - if !present[id] { - adds = append(adds, id) - } - delete(present, id) - } - - // build our list of removals - removals := make([]ContactID, 0, len(present)) - for id := range present { - removals = append(removals, id) - } - - // first remove all the contacts - err = RemoveContactsFromGroupAndCampaigns(ctx, db, oa, groupID, removals) - if err != nil { - return 0, errors.Wrapf(err, "error removing contacts from group: %d", groupID) - } - - // then add them all - err = AddContactsToGroupAndCampaigns(ctx, db, oa, groupID, adds) - if err != nil { - return 0, errors.Wrapf(err, "error adding contacts to group: %d", groupID) - } - - // mark our group as no longer evaluating - err = UpdateGroupStatus(ctx, db, groupID, GroupStatusReady) - if err != nil { - return 0, errors.Wrapf(err, "error marking dynamic group as ready") - } - - // finally update modified_on for all affected contacts to ensure these changes are seen by rp-indexer - changed := make([]ContactID, 0, len(adds)) - changed = append(changed, adds...) - changed = append(changed, removals...) - - err = UpdateContactModifiedOn(ctx, db, changed) - if err != nil { - return 0, errors.Wrapf(err, "error updating contact modified_on after group population") - } - - return len(new), nil -} diff --git a/core/models/groups_test.go b/core/models/groups_test.go index b4339dc73..fee836534 100644 --- a/core/models/groups_test.go +++ b/core/models/groups_test.go @@ -2,17 +2,13 @@ package models_test import ( "errors" - "fmt" "testing" - "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/testsuite/testdata" - "github.com/lib/pq" - "github.com/olivere/elastic/v7" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -35,136 +31,27 @@ func TestLoadGroups(t *testing.T) { require.NoError(t, err) tcs := []struct { - ID models.GroupID - UUID assets.GroupUUID - Name string - Query string + id models.GroupID + uuid assets.GroupUUID + name string + query string }{ + {testdata.ActiveGroup.ID, testdata.ActiveGroup.UUID, "Active", ""}, + {testdata.ArchivedGroup.ID, testdata.ArchivedGroup.UUID, "Archived", ""}, + {testdata.BlockedGroup.ID, testdata.BlockedGroup.UUID, "Blocked", ""}, {testdata.DoctorsGroup.ID, testdata.DoctorsGroup.UUID, "Doctors", ""}, + {testdata.OpenTicketsGroup.ID, testdata.OpenTicketsGroup.UUID, "Open Tickets", "tickets > 0"}, + {testdata.StoppedGroup.ID, testdata.StoppedGroup.UUID, "Stopped", ""}, {testdata.TestersGroup.ID, testdata.TestersGroup.UUID, "Testers", ""}, } - assert.Equal(t, 2, len(groups)) + assert.Equal(t, 7, len(groups)) + for i, tc := range tcs { group := groups[i].(*models.Group) - assert.Equal(t, tc.UUID, group.UUID()) - assert.Equal(t, tc.ID, group.ID()) - assert.Equal(t, tc.Name, group.Name()) - assert.Equal(t, tc.Query, group.Query()) - } -} - -func TestDynamicGroups(t *testing.T) { - ctx, rt, db, _ := testsuite.Get() - - defer testsuite.Reset(testsuite.ResetAll) - - // insert an event on our campaign - newEvent := testdata.InsertCampaignFlowEvent(db, testdata.RemindersCampaign, testdata.Favorites, testdata.JoinedField, 1000, "W") - - // clear Cathy's value - db.MustExec( - `update contacts_contact set fields = fields - $2 - WHERE id = $1`, testdata.Cathy.ID, testdata.JoinedField.UUID) - - // and populate Bob's - db.MustExec( - fmt.Sprintf(`update contacts_contact set fields = fields || - '{"%s": { "text": "2029-09-15T12:00:00+00:00", "datetime": "2029-09-15T12:00:00+00:00" }}'::jsonb - WHERE id = $1`, testdata.JoinedField.UUID), testdata.Bob.ID) - - oa, err := models.GetOrgAssetsWithRefresh(ctx, rt, testdata.Org1.ID, models.RefreshCampaigns|models.RefreshGroups) - assert.NoError(t, err) - - esServer := testsuite.NewMockElasticServer() - defer esServer.Close() - - es, err := elastic.NewClient( - elastic.SetURL(esServer.URL()), - elastic.SetHealthcheck(false), - elastic.SetSniff(false), - ) - assert.NoError(t, err) - - contactHit := `{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }` - - cathyHit := fmt.Sprintf(contactHit, testdata.Cathy.ID) - bobHit := fmt.Sprintf(contactHit, testdata.Bob.ID) - - tcs := []struct { - Query string - ESResponse string - ContactIDs []models.ContactID - EventContactIDs []models.ContactID - }{ - { - "cathy", - cathyHit, - []models.ContactID{testdata.Cathy.ID}, - []models.ContactID{}, - }, - { - "bob", - bobHit, - []models.ContactID{testdata.Bob.ID}, - []models.ContactID{testdata.Bob.ID}, - }, - { - "unchanged", - bobHit, - []models.ContactID{testdata.Bob.ID}, - []models.ContactID{testdata.Bob.ID}, - }, - } - - for _, tc := range tcs { - err := models.UpdateGroupStatus(ctx, db, testdata.DoctorsGroup.ID, models.GroupStatusInitializing) - assert.NoError(t, err) - - esServer.NextResponse = tc.ESResponse - count, err := models.PopulateDynamicGroup(ctx, db, es, oa, testdata.DoctorsGroup.ID, tc.Query) - assert.NoError(t, err, "error populating dynamic group for: %s", tc.Query) - - assert.Equal(t, count, len(tc.ContactIDs)) - - // assert the current group membership - contactIDs, err := models.ContactIDsForGroupIDs(ctx, db, []models.GroupID{testdata.DoctorsGroup.ID}) - assert.NoError(t, err) - assert.Equal(t, tc.ContactIDs, contactIDs) - - assertdb.Query(t, db, `SELECT count(*) from contacts_contactgroup WHERE id = $1 AND status = 'R'`, testdata.DoctorsGroup.ID). - Returns(1, "wrong number of contacts in group for query: %s", tc.Query) - - assertdb.Query(t, db, `SELECT count(*) from campaigns_eventfire WHERE event_id = $1`, newEvent.ID). - Returns(len(tc.EventContactIDs), "wrong number of contacts with events for query: %s", tc.Query) - - assertdb.Query(t, db, `SELECT count(*) from campaigns_eventfire WHERE event_id = $1 AND contact_id = ANY($2)`, newEvent.ID, pq.Array(tc.EventContactIDs)). - Returns(len(tc.EventContactIDs), "wrong contacts with events for query: %s", tc.Query) + assert.Equal(t, tc.uuid, group.UUID()) + assert.Equal(t, tc.id, group.ID()) + assert.Equal(t, tc.name, group.Name()) + assert.Equal(t, tc.query, group.Query()) } } diff --git a/core/models/http_logs.go b/core/models/http_logs.go index 96a9874be..c5f98c086 100644 --- a/core/models/http_logs.go +++ b/core/models/http_logs.go @@ -105,16 +105,7 @@ RETURNING id // InsertHTTPLogs inserts the passed in logs returning any errors encountered func InsertHTTPLogs(ctx context.Context, tx Queryer, logs []*HTTPLog) error { - if len(logs) == 0 { - return nil - } - - ls := make([]interface{}, len(logs)) - for i := range logs { - ls[i] = &logs[i] - } - - return BulkQuery(ctx, "inserted http logs", tx, insertHTTPLogsSQL, ls) + return BulkQuery(ctx, "inserted http logs", tx, insertHTTPLogsSQL, logs) } // MarshalJSON marshals into JSON. 0 values will become null diff --git a/core/models/http_logs_test.go b/core/models/http_logs_test.go index 922f1ef91..f120eb9ea 100644 --- a/core/models/http_logs_test.go +++ b/core/models/http_logs_test.go @@ -46,6 +46,8 @@ func TestHTTPLogs(t *testing.T) { func TestHTTPLogger(t *testing.T) { ctx, _, db, _ := testsuite.Get() + defer func() { db.MustExec(`DELETE FROM request_logs_httplog`) }() + defer httpx.SetRequestor(httpx.DefaultRequestor) httpx.SetRequestor(httpx.NewMockRequestor(map[string][]httpx.MockResponse{ "https://temba.io": { diff --git a/core/models/imports.go b/core/models/imports.go index 51785416c..bd2ada923 100644 --- a/core/models/imports.go +++ b/core/models/imports.go @@ -53,49 +53,34 @@ type ContactImport struct { BatchStatuses string `db:"batch_statuses"` } -var loadContactImportSQL = ` -SELECT - i.id AS "id", - i.org_id AS "org_id", - i.status AS "status", - i.created_by_id AS "created_by_id", - i.finished_on AS "finished_on", - array_to_string(array_agg(DISTINCT b.status), '') AS "batch_statuses" -FROM - contacts_contactimport i -LEFT OUTER JOIN - contacts_contactimportbatch b ON b.contact_import_id = i.id -WHERE - i.id = $1 -GROUP BY - i.id` +var sqlLoadContactImport = ` + SELECT i.id, i.org_id, i.status, i.created_by_id, i.finished_on, array_to_string(array_agg(DISTINCT b.status), '') AS "batch_statuses" + FROM contacts_contactimport i +LEFT OUTER JOIN contacts_contactimportbatch b ON b.contact_import_id = i.id + WHERE i.id = $1 + GROUP BY i.id` // LoadContactImport loads a contact import by ID func LoadContactImport(ctx context.Context, db Queryer, id ContactImportID) (*ContactImport, error) { i := &ContactImport{} - err := db.GetContext(ctx, i, loadContactImportSQL, id) + err := db.GetContext(ctx, i, sqlLoadContactImport, id) if err != nil { return nil, errors.Wrapf(err, "error loading contact import id=%d", id) } return i, nil } -var markContactImportFinishedSQL = ` -UPDATE - contacts_contactimport -SET - status = $2, - finished_on = $3 -WHERE - id = $1 -` +var sqlMarkContactImportFinished = ` +UPDATE contacts_contactimport + SET status = $2, finished_on = $3 + WHERE id = $1` func (i *ContactImport) MarkFinished(ctx context.Context, db Queryer, status ContactImportStatus) error { now := dates.Now() i.Status = status i.FinishedOn = &now - _, err := db.ExecContext(ctx, markContactImportFinishedSQL, i.ID, i.Status, i.FinishedOn) + _, err := db.ExecContext(ctx, sqlMarkContactImportFinished, i.ID, i.Status, i.FinishedOn) return errors.Wrap(err, "error marking import as finished") } diff --git a/core/models/imports_test.go b/core/models/imports_test.go index 414152379..3243fc4ab 100644 --- a/core/models/imports_test.go +++ b/core/models/imports_test.go @@ -60,7 +60,7 @@ func TestContactImports(t *testing.T) { }{} jsonx.MustUnmarshal(testJSON, &tcs) - oa, err := models.GetOrgAssets(ctx, rt, 1) + oa, err := models.GetOrgAssetsWithRefresh(ctx, rt, testdata.Org1.ID, models.RefreshOrg|models.RefreshChannels|models.RefreshGroups) require.NoError(t, err) uuids.SetGenerator(uuids.NewSeededGenerator(12345)) diff --git a/core/models/incident.go b/core/models/incident.go index fcd6fe644..e9b5299ab 100644 --- a/core/models/incident.go +++ b/core/models/incident.go @@ -97,13 +97,15 @@ func IncidentWebhooksUnhealthy(ctx context.Context, db Queryer, rp *redis.Pool, return id, nil } -const insertIncidentSQL = ` -INSERT INTO notifications_incident(org_id, incident_type, scope, started_on, channel_id) VALUES($1, $2, $3, $4, $5) -ON CONFLICT DO NOTHING RETURNING id` +const sqlInsertIncident = ` +INSERT INTO notifications_incident(org_id, incident_type, scope, started_on, channel_id) + VALUES($1, $2, $3, $4, $5) +ON CONFLICT DO NOTHING + RETURNING id` func getOrCreateIncident(ctx context.Context, db Queryer, oa *OrgAssets, incident *Incident) (IncidentID, error) { var incidentID IncidentID - err := db.GetContext(ctx, &incidentID, insertIncidentSQL, incident.OrgID, incident.Type, incident.Scope, incident.StartedOn, incident.ChannelID) + err := db.GetContext(ctx, &incidentID, sqlInsertIncident, incident.OrgID, incident.Type, incident.Scope, incident.StartedOn, incident.ChannelID) if err != nil && err != sql.ErrNoRows { return NilIncidentID, errors.Wrap(err, "error inserting incident") } @@ -125,13 +127,13 @@ func getOrCreateIncident(ctx context.Context, db Queryer, oa *OrgAssets, inciden return incidentID, nil } -const selectOpenIncidentsSQL = ` +const sqlSelectOpenIncidents = ` SELECT id, org_id, incident_type, scope, started_on, ended_on, channel_id -FROM notifications_incident -WHERE ended_on IS NULL AND incident_type = ANY($1)` + FROM notifications_incident + WHERE ended_on IS NULL AND incident_type = ANY($1)` func GetOpenIncidents(ctx context.Context, db Queryer, types []IncidentType) ([]*Incident, error) { - rows, err := db.QueryxContext(ctx, selectOpenIncidentsSQL, pq.Array(types)) + rows, err := db.QueryxContext(ctx, sqlSelectOpenIncidents, pq.Array(types)) if err != nil { return nil, errors.Wrap(err, "error querying open incidents") } diff --git a/core/models/labels.go b/core/models/labels.go index 9a1e45054..1472cda29 100644 --- a/core/models/labels.go +++ b/core/models/labels.go @@ -75,16 +75,8 @@ ORDER BY // AddMsgLabels inserts the passed in msg labels to our db func AddMsgLabels(ctx context.Context, tx *sqlx.Tx, adds []*MsgLabelAdd) error { - is := make([]interface{}, len(adds)) - for i := range adds { - is[i] = adds[i] - } - - err := BulkQuery(ctx, "inserting msg labels", tx, insertMsgLabelsSQL, is) - if err != nil { - return errors.Wrapf(err, "error inserting new msg labels") - } - return nil + err := BulkQuery(ctx, "inserting msg labels", tx, insertMsgLabelsSQL, adds) + return errors.Wrapf(err, "error inserting new msg labels") } const insertMsgLabelsSQL = ` diff --git a/core/models/msgs.go b/core/models/msgs.go index bf7582848..476aae12a 100644 --- a/core/models/msgs.go +++ b/core/models/msgs.go @@ -79,7 +79,8 @@ type MsgFailedReason null.String const ( NilMsgFailedReason = MsgFailedReason("") - MsgFailedSuspended = MsgFailedReason("S") + MsgFailedSuspended = MsgFailedReason("S") // workspace suspended + MsgFailedContact = MsgFailedReason("C") // contact blocked, stopped or archived MsgFailedLooping = MsgFailedReason("L") MsgFailedErrorLimit = MsgFailedReason("E") MsgFailedTooOld = MsgFailedReason("O") @@ -321,31 +322,31 @@ return count `) // GetMsgRepetitions gets the number of repetitions of this msg text for the given contact in the current 5 minute window -func GetMsgRepetitions(rp *redis.Pool, contactID ContactID, msg *flows.MsgOut) (int, error) { +func GetMsgRepetitions(rp *redis.Pool, contact *flows.Contact, msg *flows.MsgOut) (int, error) { rc := rp.Get() defer rc.Close() keyTime := dates.Now().UTC().Round(time.Minute * 5) key := fmt.Sprintf("msg_repetitions:%s", keyTime.Format("2006-01-02T15:04")) - return redis.Int(msgRepetitionsScript.Do(rc, key, contactID, msg.Text())) + return redis.Int(msgRepetitionsScript.Do(rc, key, contact.ID(), msg.Text())) } // NewOutgoingFlowMsg creates an outgoing message for the passed in flow message func NewOutgoingFlowMsg(rt *runtime.Runtime, org *Org, channel *Channel, session *Session, flow *Flow, out *flows.MsgOut, createdOn time.Time) (*Msg, error) { - return newOutgoingMsg(rt, org, channel, session.ContactID(), out, createdOn, session, flow, NilBroadcastID) + return newOutgoingMsg(rt, org, channel, session.Contact(), out, createdOn, session, flow, NilBroadcastID) } // NewOutgoingBroadcastMsg creates an outgoing message which is part of a broadcast -func NewOutgoingBroadcastMsg(rt *runtime.Runtime, org *Org, channel *Channel, contactID ContactID, out *flows.MsgOut, createdOn time.Time, broadcastID BroadcastID) (*Msg, error) { - return newOutgoingMsg(rt, org, channel, contactID, out, createdOn, nil, nil, broadcastID) +func NewOutgoingBroadcastMsg(rt *runtime.Runtime, org *Org, channel *Channel, contact *flows.Contact, out *flows.MsgOut, createdOn time.Time, broadcastID BroadcastID) (*Msg, error) { + return newOutgoingMsg(rt, org, channel, contact, out, createdOn, nil, nil, broadcastID) } -func newOutgoingMsg(rt *runtime.Runtime, org *Org, channel *Channel, contactID ContactID, out *flows.MsgOut, createdOn time.Time, session *Session, flow *Flow, broadcastID BroadcastID) (*Msg, error) { +func newOutgoingMsg(rt *runtime.Runtime, org *Org, channel *Channel, contact *flows.Contact, out *flows.MsgOut, createdOn time.Time, session *Session, flow *Flow, broadcastID BroadcastID) (*Msg, error) { msg := &Msg{} m := &msg.m m.UUID = out.UUID() m.OrgID = org.ID() - m.ContactID = contactID + m.ContactID = ContactID(contact.ID()) m.BroadcastID = broadcastID m.TopupID = NilTopupID m.Text = out.Text() @@ -364,13 +365,17 @@ func newOutgoingMsg(rt *runtime.Runtime, org *Org, channel *Channel, contactID C // we fail messages for suspended orgs right away m.Status = MsgStatusFailed m.FailedReason = MsgFailedSuspended + } else if contact.Status() != flows.ContactStatusActive { + // and blocked, stopped or archived contacts + m.Status = MsgStatusFailed + m.FailedReason = MsgFailedContact } else if msg.URN() == urns.NilURN || channel == nil { // if msg is missing the URN or channel, we also fail it m.Status = MsgStatusFailed m.FailedReason = MsgFailedNoDestination } else { // also fail right away if this looks like a loop - repetitions, err := GetMsgRepetitions(rt.RP, contactID, out) + repetitions, err := GetMsgRepetitions(rt.RP, contact, out) if err != nil { return nil, errors.Wrap(err, "error looking up msg repetitions") } @@ -378,7 +383,7 @@ func newOutgoingMsg(rt *runtime.Runtime, org *Org, channel *Channel, contactID C m.Status = MsgStatusFailed m.FailedReason = MsgFailedLooping - logrus.WithFields(logrus.Fields{"contact_id": contactID, "text": out.Text(), "repetitions": repetitions}).Error("too many repetitions, failing message") + logrus.WithFields(logrus.Fields{"contact_id": contact.ID(), "text": out.Text(), "repetitions": repetitions}).Error("too many repetitions, failing message") } } @@ -743,22 +748,24 @@ type BroadcastTranslation struct { // Broadcast represents a broadcast that needs to be sent type Broadcast struct { b struct { - BroadcastID BroadcastID `json:"broadcast_id,omitempty" db:"id"` + BroadcastID BroadcastID `json:"broadcast_id,omitempty" db:"id"` Translations map[envs.Language]*BroadcastTranslation `json:"translations"` - Text hstore.Hstore ` db:"text"` + Text hstore.Hstore ` db:"text"` TemplateState TemplateState `json:"template_state"` - BaseLanguage envs.Language `json:"base_language" db:"base_language"` + BaseLanguage envs.Language `json:"base_language" db:"base_language"` URNs []urns.URN `json:"urns,omitempty"` ContactIDs []ContactID `json:"contact_ids,omitempty"` GroupIDs []GroupID `json:"group_ids,omitempty"` - OrgID OrgID `json:"org_id" db:"org_id"` - ParentID BroadcastID `json:"parent_id,omitempty" db:"parent_id"` - TicketID TicketID `json:"ticket_id,omitempty" db:"ticket_id"` + OrgID OrgID `json:"org_id" db:"org_id"` + CreatedByID UserID `json:"created_by_id,omitempty" db:"created_by_id"` + ParentID BroadcastID `json:"parent_id,omitempty" db:"parent_id"` + TicketID TicketID `json:"ticket_id,omitempty" db:"ticket_id"` } } func (b *Broadcast) ID() BroadcastID { return b.b.BroadcastID } func (b *Broadcast) OrgID() OrgID { return b.b.OrgID } +func (b *Broadcast) CreatedByID() UserID { return b.b.CreatedByID } func (b *Broadcast) ContactIDs() []ContactID { return b.b.ContactIDs } func (b *Broadcast) GroupIDs() []GroupID { return b.b.GroupIDs } func (b *Broadcast) URNs() []urns.URN { return b.b.URNs } @@ -773,7 +780,7 @@ func (b *Broadcast) UnmarshalJSON(data []byte) error { return json.Unmarshal(dat // NewBroadcast creates a new broadcast with the passed in parameters func NewBroadcast( orgID OrgID, id BroadcastID, translations map[envs.Language]*BroadcastTranslation, - state TemplateState, baseLanguage envs.Language, urns []urns.URN, contactIDs []ContactID, groupIDs []GroupID, ticketID TicketID) *Broadcast { + state TemplateState, baseLanguage envs.Language, urns []urns.URN, contactIDs []ContactID, groupIDs []GroupID, ticketID TicketID, createdByID UserID) *Broadcast { bcast := &Broadcast{} bcast.b.OrgID = orgID @@ -785,6 +792,7 @@ func NewBroadcast( bcast.b.ContactIDs = contactIDs bcast.b.GroupIDs = groupIDs bcast.b.TicketID = ticketID + bcast.b.CreatedByID = createdByID return bcast } @@ -801,8 +809,8 @@ func InsertChildBroadcast(ctx context.Context, db Queryer, parent *Broadcast) (* parent.b.ContactIDs, parent.b.GroupIDs, parent.b.TicketID, + parent.b.CreatedByID, ) - // populate our parent id child.b.ParentID = parent.ID() // populate text from our translations @@ -940,59 +948,42 @@ func NewBroadcastFromEvent(ctx context.Context, tx Queryer, oa *OrgAssets, event } } - return NewBroadcast(oa.OrgID(), NilBroadcastID, translations, TemplateStateEvaluated, event.BaseLanguage, event.URNs, contactIDs, groupIDs, NilTicketID), nil + return NewBroadcast(oa.OrgID(), NilBroadcastID, translations, TemplateStateEvaluated, event.BaseLanguage, event.URNs, contactIDs, groupIDs, NilTicketID, NilUserID), nil } func (b *Broadcast) CreateBatch(contactIDs []ContactID) *BroadcastBatch { - batch := &BroadcastBatch{} - batch.b.BroadcastID = b.b.BroadcastID - batch.b.BaseLanguage = b.b.BaseLanguage - batch.b.Translations = b.b.Translations - batch.b.TemplateState = b.b.TemplateState - batch.b.OrgID = b.b.OrgID - batch.b.TicketID = b.b.TicketID - batch.b.ContactIDs = contactIDs - return batch + return &BroadcastBatch{ + BroadcastID: b.b.BroadcastID, + BaseLanguage: b.b.BaseLanguage, + Translations: b.b.Translations, + TemplateState: b.b.TemplateState, + OrgID: b.b.OrgID, + CreatedByID: b.b.CreatedByID, + TicketID: b.b.TicketID, + ContactIDs: contactIDs, + } } // BroadcastBatch represents a batch of contacts that need messages sent for type BroadcastBatch struct { - b struct { - BroadcastID BroadcastID `json:"broadcast_id,omitempty"` - Translations map[envs.Language]*BroadcastTranslation `json:"translations"` - BaseLanguage envs.Language `json:"base_language"` - TemplateState TemplateState `json:"template_state"` - URNs map[ContactID]urns.URN `json:"urns,omitempty"` - ContactIDs []ContactID `json:"contact_ids,omitempty"` - IsLast bool `json:"is_last"` - OrgID OrgID `json:"org_id"` - TicketID TicketID `json:"ticket_id"` - } -} - -func (b *BroadcastBatch) BroadcastID() BroadcastID { return b.b.BroadcastID } -func (b *BroadcastBatch) ContactIDs() []ContactID { return b.b.ContactIDs } -func (b *BroadcastBatch) URNs() map[ContactID]urns.URN { return b.b.URNs } -func (b *BroadcastBatch) SetURNs(urns map[ContactID]urns.URN) { b.b.URNs = urns } -func (b *BroadcastBatch) OrgID() OrgID { return b.b.OrgID } -func (b *BroadcastBatch) TicketID() TicketID { return b.b.TicketID } -func (b *BroadcastBatch) Translations() map[envs.Language]*BroadcastTranslation { - return b.b.Translations + BroadcastID BroadcastID `json:"broadcast_id,omitempty"` + Translations map[envs.Language]*BroadcastTranslation `json:"translations"` + BaseLanguage envs.Language `json:"base_language"` + TemplateState TemplateState `json:"template_state"` + URNs map[ContactID]urns.URN `json:"urns,omitempty"` + ContactIDs []ContactID `json:"contact_ids,omitempty"` + IsLast bool `json:"is_last"` + OrgID OrgID `json:"org_id"` + CreatedByID UserID `json:"created_by_id"` + TicketID TicketID `json:"ticket_id"` } -func (b *BroadcastBatch) TemplateState() TemplateState { return b.b.TemplateState } -func (b *BroadcastBatch) BaseLanguage() envs.Language { return b.b.BaseLanguage } -func (b *BroadcastBatch) IsLast() bool { return b.b.IsLast } -func (b *BroadcastBatch) SetIsLast(last bool) { b.b.IsLast = last } - -func (b *BroadcastBatch) MarshalJSON() ([]byte, error) { return json.Marshal(b.b) } -func (b *BroadcastBatch) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &b.b) } -func CreateBroadcastMessages(ctx context.Context, rt *runtime.Runtime, oa *OrgAssets, bcast *BroadcastBatch) ([]*Msg, error) { +func (b *BroadcastBatch) CreateMessages(ctx context.Context, rt *runtime.Runtime, oa *OrgAssets) ([]*Msg, error) { repeatedContacts := make(map[ContactID]bool) - broadcastURNs := bcast.URNs() + broadcastURNs := b.URNs // build our list of contact ids - contactIDs := bcast.ContactIDs() + contactIDs := b.ContactIDs // build a map of the contacts that are present both in our URN list and our contact id list if broadcastURNs != nil { @@ -1085,7 +1076,7 @@ func CreateBroadcastMessages(ctx context.Context, rt *runtime.Runtime, oa *OrgAs } // have a valid contact language, try that - trans := bcast.Translations() + trans := b.Translations t := trans[lang] // not found? try org default language @@ -1095,20 +1086,20 @@ func CreateBroadcastMessages(ctx context.Context, rt *runtime.Runtime, oa *OrgAs // not found? use broadcast base language if t == nil { - t = trans[bcast.BaseLanguage()] + t = trans[b.BaseLanguage] } if t == nil { - logrus.WithField("base_language", bcast.BaseLanguage()).WithField("translations", trans).Error("unable to find translation for broadcast") + logrus.WithField("base_language", b.BaseLanguage).WithField("translations", trans).Error("unable to find translation for broadcast") return nil, nil } template := "" // if this is a legacy template, migrate it forward - if bcast.TemplateState() == TemplateStateLegacy { + if b.TemplateState == TemplateStateLegacy { template, _ = expressions.MigrateTemplate(t.Text, nil) - } else if bcast.TemplateState() == TemplateStateUnevaluated { + } else if b.TemplateState == TemplateStateUnevaluated { template = t.Text } @@ -1133,7 +1124,7 @@ func CreateBroadcastMessages(ctx context.Context, rt *runtime.Runtime, oa *OrgAs // create our outgoing message out := flows.NewMsgOut(urn, channel.ChannelReference(), text, t.Attachments, t.QuickReplies, nil, flows.NilMsgTopic) - msg, err := NewOutgoingBroadcastMsg(rt, oa.Org(), channel, c.ID(), out, time.Now(), bcast.BroadcastID()) + msg, err := NewOutgoingBroadcastMsg(rt, oa.Org(), channel, contact, out, time.Now(), b.BroadcastID) if err != nil { return nil, errors.Wrapf(err, "error creating outgoing message") } @@ -1187,16 +1178,46 @@ func CreateBroadcastMessages(ctx context.Context, rt *runtime.Runtime, oa *OrgAs } // if the broadcast was a ticket reply, update the ticket - if bcast.TicketID() != NilTicketID { - err = updateTicketLastActivity(ctx, rt.DB, []TicketID{bcast.TicketID()}, dates.Now()) - if err != nil { - return nil, errors.Wrapf(err, "error updating broadcast ticket") + if b.TicketID != NilTicketID { + if err := b.updateTicket(ctx, rt.DB, oa); err != nil { + return nil, err } } return msgs, nil } +func (b *BroadcastBatch) updateTicket(ctx context.Context, db Queryer, oa *OrgAssets) error { + firstReplySeconds, err := TicketRecordReplied(ctx, db, b.TicketID, dates.Now()) + if err != nil { + return err + } + + // record reply counts for org, user and team + replyCounts := map[string]int{scopeOrg(oa): 1} + + if b.CreatedByID != NilUserID { + user := oa.UserByID(b.CreatedByID) + if user != nil { + replyCounts[scopeUser(oa, user)] = 1 + if user.Team() != nil { + replyCounts[scopeTeam(user.Team())] = 1 + } + } + } + + if err := insertTicketDailyCounts(ctx, db, TicketDailyCountReply, oa.Org().Timezone(), replyCounts); err != nil { + return err + } + + if firstReplySeconds >= 0 { + if err := insertTicketDailyTiming(ctx, db, TicketDailyTimingFirstReply, oa.Org().Timezone(), scopeOrg(oa), firstReplySeconds); err != nil { + return err + } + } + return nil +} + const sqlUpdateMsgForResending = ` UPDATE msgs_msg m SET channel_id = r.channel_id::int, diff --git a/core/models/msgs_test.go b/core/models/msgs_test.go index de41a7d4a..68dbb4f6c 100644 --- a/core/models/msgs_test.go +++ b/core/models/msgs_test.go @@ -32,6 +32,9 @@ func TestNewOutgoingFlowMsg(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) + blake := testdata.InsertContact(db, testdata.Org1, "79b94a23-6d13-43f4-95fe-c733ee457857", "Blake", envs.NilLanguage, models.ContactStatusBlocked) + blakeURNID := testdata.InsertContactURN(db, testdata.Org1, blake, "tel:++250700000007", 1) + tcs := []struct { ChannelUUID assets.ChannelUUID Text string @@ -118,7 +121,6 @@ func TestNewOutgoingFlowMsg(t *testing.T) { URN: urns.NilURN, URNID: models.URNID(0), Flow: testdata.Favorites, - SuspendedOrg: false, ExpectedStatus: models.MsgStatusFailed, ExpectedFailedReason: models.MsgFailedNoDestination, ExpectedMetadata: map[string]interface{}{}, @@ -129,16 +131,28 @@ func TestNewOutgoingFlowMsg(t *testing.T) { ChannelUUID: "", Text: "missing Channel", Contact: testdata.Cathy, - URN: urns.NilURN, - URNID: models.URNID(0), + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", testdata.Cathy.URNID)), + URNID: testdata.Cathy.URNID, Flow: testdata.Favorites, - SuspendedOrg: false, ExpectedStatus: models.MsgStatusFailed, ExpectedFailedReason: models.MsgFailedNoDestination, ExpectedMetadata: map[string]interface{}{}, ExpectedMsgCount: 1, ExpectedPriority: false, }, + { + ChannelUUID: "74729f45-7f29-4868-9dc4-90e491e3c7d8", + Text: "blocked contact", + Contact: blake, + URN: urns.URN(fmt.Sprintf("tel:+250700000007?id=%d", blakeURNID)), + URNID: blakeURNID, + Flow: testdata.Favorites, + ExpectedStatus: models.MsgStatusFailed, + ExpectedFailedReason: models.MsgFailedContact, + ExpectedMetadata: map[string]interface{}{}, + ExpectedMsgCount: 1, + ExpectedPriority: false, + }, } now := time.Now() @@ -152,7 +166,7 @@ func TestNewOutgoingFlowMsg(t *testing.T) { channel := oa.ChannelByUUID(tc.ChannelUUID) flow, _ := oa.FlowByID(tc.Flow.ID) - session := insertTestSession(t, ctx, rt, testdata.Org1, testdata.Cathy, testdata.Favorites) + session := insertTestSession(t, ctx, rt, testdata.Org1, tc.Contact, testdata.Favorites) if tc.ResponseTo != models.NilMsgID { session.SetIncomingMsg(flows.MsgID(tc.ResponseTo), null.NullString) } @@ -188,7 +202,7 @@ func TestNewOutgoingFlowMsg(t *testing.T) { } // check nil failed reasons are saved as NULLs - assertdb.Query(t, db, `SELECT count(*) FROM msgs_msg WHERE failed_reason IS NOT NULL`).Returns(3) + assertdb.Query(t, db, `SELECT count(*) FROM msgs_msg WHERE failed_reason IS NOT NULL`).Returns(4) // ensure org is unsuspended db.MustExec(`UPDATE orgs_org SET is_suspended = FALSE`) @@ -254,6 +268,8 @@ func TestMarshalMsg(t *testing.T) { msg1, err := models.NewOutgoingFlowMsg(rt, oa.Org(), channel, session, flow, flowMsg1, time.Date(2021, 11, 9, 14, 3, 30, 0, time.UTC)) require.NoError(t, err) + cathy := session.Contact() + err = models.InsertMessages(ctx, db, []*models.Msg{msg1}) require.NoError(t, err) @@ -344,7 +360,7 @@ func TestMarshalMsg(t *testing.T) { // try a broadcast message which won't have session and flow fields set bcastID := testdata.InsertBroadcast(db, testdata.Org1, `eng`, map[envs.Language]string{`eng`: "Blast"}, models.NilScheduleID, []*testdata.Contact{testdata.Cathy}, nil) bcastMsg1 := flows.NewMsgOut(urn, assets.NewChannelReference(testdata.TwilioChannel.UUID, "Test Channel"), "Blast", nil, nil, nil, flows.NilMsgTopic) - msg3, err := models.NewOutgoingBroadcastMsg(rt, oa.Org(), channel, testdata.Cathy.ID, bcastMsg1, time.Date(2021, 11, 9, 14, 3, 30, 0, time.UTC), bcastID) + msg3, err := models.NewOutgoingBroadcastMsg(rt, oa.Org(), channel, cathy, bcastMsg1, time.Date(2021, 11, 9, 14, 3, 30, 0, time.UTC), bcastID) require.NoError(t, err) err = models.InsertMessages(ctx, db, []*models.Msg{msg2}) @@ -466,18 +482,21 @@ func TestResendMessages(t *testing.T) { } func TestGetMsgRepetitions(t *testing.T) { - _, _, _, rp := testsuite.Get() + _, rt, db, rp := testsuite.Get() defer testsuite.Reset(testsuite.ResetRedis) defer dates.SetNowSource(dates.DefaultNowSource) dates.SetNowSource(dates.NewFixedNowSource(time.Date(2021, 11, 18, 12, 13, 3, 234567, time.UTC))) + oa := testdata.Org1.Load(rt) + _, cathy := testdata.Cathy.Load(db, oa) + msg1 := flows.NewMsgOut(testdata.Cathy.URN, nil, "foo", nil, nil, nil, flows.NilMsgTopic) msg2 := flows.NewMsgOut(testdata.Cathy.URN, nil, "bar", nil, nil, nil, flows.NilMsgTopic) assertRepetitions := func(m *flows.MsgOut, expected int) { - count, err := models.GetMsgRepetitions(rp, testdata.Cathy.ID, m) + count, err := models.GetMsgRepetitions(rp, cathy, m) require.NoError(t, err) assert.Equal(t, expected, count) } @@ -576,7 +595,7 @@ func TestNonPersistentBroadcasts(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Bob, testdata.Mailgun, testdata.DefaultTopic, "", "", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Bob, testdata.Mailgun, testdata.DefaultTopic, "", "", time.Now(), nil) modelTicket := ticket.Load(db) translations := map[envs.Language]*models.BroadcastTranslation{envs.Language("eng"): {Text: "Hi there"}} @@ -592,6 +611,7 @@ func TestNonPersistentBroadcasts(t *testing.T) { []models.ContactID{testdata.Alexandria.ID, testdata.Bob.ID, testdata.Cathy.ID}, []models.GroupID{testdata.DoctorsGroup.ID}, ticket.ID, + models.NilUserID, ) assert.Equal(t, models.NilBroadcastID, bcast.ID()) @@ -606,18 +626,18 @@ func TestNonPersistentBroadcasts(t *testing.T) { batch := bcast.CreateBatch([]models.ContactID{testdata.Alexandria.ID, testdata.Bob.ID}) - assert.Equal(t, models.NilBroadcastID, batch.BroadcastID()) - assert.Equal(t, testdata.Org1.ID, batch.OrgID()) - assert.Equal(t, envs.Language("eng"), batch.BaseLanguage()) - assert.Equal(t, translations, batch.Translations()) - assert.Equal(t, models.TemplateStateUnevaluated, batch.TemplateState()) - assert.Equal(t, ticket.ID, batch.TicketID()) - assert.Equal(t, []models.ContactID{testdata.Alexandria.ID, testdata.Bob.ID}, batch.ContactIDs()) + assert.Equal(t, models.NilBroadcastID, batch.BroadcastID) + assert.Equal(t, testdata.Org1.ID, batch.OrgID) + assert.Equal(t, envs.Language("eng"), batch.BaseLanguage) + assert.Equal(t, translations, batch.Translations) + assert.Equal(t, models.TemplateStateUnevaluated, batch.TemplateState) + assert.Equal(t, ticket.ID, batch.TicketID) + assert.Equal(t, []models.ContactID{testdata.Alexandria.ID, testdata.Bob.ID}, batch.ContactIDs) oa, err := models.GetOrgAssets(ctx, rt, testdata.Org1.ID) require.NoError(t, err) - msgs, err := models.CreateBroadcastMessages(ctx, rt, oa, batch) + msgs, err := batch.CreateMessages(ctx, rt, oa) require.NoError(t, err) assert.Equal(t, 2, len(msgs)) diff --git a/core/models/notifications.go b/core/models/notifications.go index 90efc1ff2..ad0b7af51 100644 --- a/core/models/notifications.go +++ b/core/models/notifications.go @@ -140,12 +140,7 @@ INSERT INTO notifications_notification(org_id, notification_type, scope, user ON CONFLICT DO NOTHING` func insertNotifications(ctx context.Context, db Queryer, notifications []*Notification) error { - is := make([]interface{}, len(notifications)) - for i := range notifications { - is[i] = notifications[i] - } - - err := dbutil.BulkQuery(ctx, db, insertNotificationSQL, is) + err := dbutil.BulkQuery(ctx, db, insertNotificationSQL, notifications) return errors.Wrap(err, "error inserting notifications") } diff --git a/core/models/notifications_test.go b/core/models/notifications_test.go index 934e32f1c..f6e985786 100644 --- a/core/models/notifications_test.go +++ b/core/models/notifications_test.go @@ -165,7 +165,7 @@ func assertNotifications(t *testing.T, ctx context.Context, db *sqlx.DB, after t } func openTicket(t *testing.T, ctx context.Context, db *sqlx.DB, openedBy *testdata.User, assignee *testdata.User) (*models.Ticket, *models.TicketEvent) { - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.SupportTopic, "Where my pants", "", assignee) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.SupportTopic, "Where my pants", "", time.Now(), assignee) modelTicket := ticket.Load(db) openedEvent := models.NewTicketOpenedEvent(modelTicket, openedBy.SafeID(), assignee.SafeID()) diff --git a/core/models/orgs.go b/core/models/orgs.go index 41d4bc59f..bf623166f 100644 --- a/core/models/orgs.go +++ b/core/models/orgs.go @@ -181,7 +181,7 @@ func (o *Org) StoreAttachment(ctx context.Context, rt *runtime.Runtime, filename content.Close() if contentType == "" { - contentType = http.DetectContentType(contentBytes) + contentType = httpx.DetectContentType(contentBytes) contentType, _, _ = mime.ParseMediaType(contentType) } diff --git a/core/models/resthooks.go b/core/models/resthooks.go index 970c05a76..0035fa73a 100644 --- a/core/models/resthooks.go +++ b/core/models/resthooks.go @@ -86,17 +86,8 @@ ORDER BY // UnsubscribeResthooks unsubscribles all the resthooks passed in func UnsubscribeResthooks(ctx context.Context, tx *sqlx.Tx, unsubs []*ResthookUnsubscribe) error { - is := make([]interface{}, len(unsubs)) - for i := range unsubs { - is[i] = unsubs[i] - } - - err := BulkQuery(ctx, "unsubscribing resthooks", tx, unsubscribeResthooksSQL, is) - if err != nil { - return errors.Wrapf(err, "error unsubscribing from resthooks") - } - - return nil + err := BulkQuery(ctx, "unsubscribing resthooks", tx, sqlUnsubscribeResthooks, unsubs) + return errors.Wrapf(err, "error unsubscribing from resthooks") } type ResthookUnsubscribe struct { @@ -105,24 +96,12 @@ type ResthookUnsubscribe struct { URL string `db:"url"` } -const unsubscribeResthooksSQL = ` -UPDATE - api_resthooksubscriber -SET - is_active = FALSE, - modified_on = NOW() -WHERE - id = ANY( - SELECT - s.id - FROM - api_resthooksubscriber s - JOIN api_resthook r ON s.resthook_id = r.id, - (VALUES(:org_id, :slug, :url)) AS u(org_id, slug, url) - WHERE - s.is_active = TRUE AND - r.org_id = u.org_id::int AND - r.slug = u.slug AND - s.target_url = u.url - ) -` +const sqlUnsubscribeResthooks = ` +UPDATE api_resthooksubscriber + SET is_active = FALSE, modified_on = NOW() + WHERE id = ANY( + SELECT s.id + FROM api_resthooksubscriber s + JOIN api_resthook r ON s.resthook_id = r.id, (VALUES(:org_id, :slug, :url)) AS u(org_id, slug, url) + WHERE s.is_active = TRUE AND r.org_id = u.org_id::int AND r.slug = u.slug AND s.target_url = u.url +)` diff --git a/core/models/sessions.go b/core/models/sessions.go index 647b2c512..e32e39c20 100644 --- a/core/models/sessions.go +++ b/core/models/sessions.go @@ -108,7 +108,6 @@ func (s *Session) WaitStartedOn() *time.Time { return s.s.WaitStartedOn func (s *Session) WaitTimeoutOn() *time.Time { return s.s.WaitTimeoutOn } func (s *Session) WaitExpiresOn() *time.Time { return s.s.WaitExpiresOn } func (s *Session) WaitResumeOnExpire() bool { return s.s.WaitResumeOnExpire } -func (s *Session) ClearTimeoutOn() { s.s.WaitTimeoutOn = nil } func (s *Session) CurrentFlowID() FlowID { return s.s.CurrentFlowID } func (s *Session) ConnectionID() *ConnectionID { return s.s.ConnectionID } func (s *Session) IncomingMsgID() MsgID { return s.incomingMsgID } @@ -447,10 +446,7 @@ func (s *Session) Update(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, eventsToHandle = append(eventsToHandle, sprint.Events()...) } - // if contact's current flow has changed, add pseudo event to handle that - if s.SessionType().Interrupts() && contact.CurrentFlowID() != s.CurrentFlowID() { - eventsToHandle = append(eventsToHandle, NewContactFlowChangedEvent(s.CurrentFlowID())) - } + eventsToHandle = append(eventsToHandle, NewSprintEndedEvent(contact, true)) // apply all our events to generate hooks err = HandleEvents(ctx, rt, tx, oa, s.scene, eventsToHandle) @@ -467,6 +463,19 @@ func (s *Session) Update(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, return nil } +// ClearWaitTimeout clears the timeout on the wait on this session and is used if the engine tells us +// that the flow no longer has a timeout on that wait. It can be called without updating the session +// in the database which is used when handling msg_created events before session is updated anyway. +func (s *Session) ClearWaitTimeout(ctx context.Context, db *sqlx.DB) error { + s.s.WaitTimeoutOn = nil + + if db != nil { + _, err := db.ExecContext(ctx, `UPDATE flows_flowsession SET timeout_on = NULL WHERE id = $1`, s.ID()) + return errors.Wrap(err, "error clearing wait timeout") + } + return nil +} + // MarshalJSON is our custom marshaller so that our inner struct get output func (s *Session) MarshalJSON() ([]byte, error) { return json.Marshal(s.s) @@ -696,10 +705,7 @@ func InsertSessions(ctx context.Context, rt *runtime.Runtime, tx *sqlx.Tx, oa *O eventsToHandle = append(eventsToHandle, sprints[i].Events()...) } - // if contact's current flow has changed, add pseudo event to handle that - if s.SessionType().Interrupts() && contacts[i].CurrentFlowID() != s.CurrentFlowID() { - eventsToHandle = append(eventsToHandle, NewContactFlowChangedEvent(s.CurrentFlowID())) - } + eventsToHandle = append(eventsToHandle, NewSprintEndedEvent(contacts[i], false)) err = HandleEvents(ctx, rt, tx, oa, s.Scene(), eventsToHandle) if err != nil { @@ -850,7 +856,7 @@ func ExitSessions(ctx context.Context, db *sqlx.DB, sessionIDs []SessionID, stat } // split into batches and exit each batch in a transaction - for _, idBatch := range chunkSessionIDs(sessionIDs, 100) { + for _, idBatch := range chunkSlice(sessionIDs, 100) { tx, err := db.BeginTxx(ctx, nil) if err != nil { return errors.Wrapf(err, "error starting transaction to exit sessions") @@ -877,7 +883,7 @@ RETURNING contact_id` const sqlExitSessionRuns = ` UPDATE flows_flowrun SET exited_on = $2, status = $3, modified_on = NOW() - WHERE id = ANY (SELECT id FROM flows_flowrun WHERE session_id = ANY($1) AND status IN ('A', 'W'))` + WHERE session_id = ANY($1) AND status IN ('A', 'W')` const sqlExitSessionContacts = ` UPDATE contacts_contact diff --git a/core/models/sessions_test.go b/core/models/sessions_test.go index 9cdc4ef06..e06b18aff 100644 --- a/core/models/sessions_test.go +++ b/core/models/sessions_test.go @@ -435,6 +435,34 @@ func TestGetSessionWaitExpiresOn(t *testing.T) { assert.Nil(t, s2Actual) } +func TestClearWaitTimeout(t *testing.T) { + ctx, rt, db, _ := testsuite.Get() + + defer testsuite.Reset(testsuite.ResetData) + + oa := testdata.Org1.Load(rt) + + _, cathy := testdata.Cathy.Load(db, oa) + + expiresOn := time.Now().Add(time.Hour) + timeoutOn := time.Now().Add(time.Minute) + testdata.InsertWaitingSession(db, testdata.Org1, testdata.Cathy, models.FlowTypeMessaging, testdata.Favorites, models.NilConnectionID, time.Now(), expiresOn, true, &timeoutOn) + + session, err := models.FindWaitingSessionForContact(ctx, db, nil, oa, models.FlowTypeMessaging, cathy) + require.NoError(t, err) + + // can be called without db connection to clear without updating db + session.ClearWaitTimeout(ctx, nil) + assert.Nil(t, session.WaitTimeoutOn()) + assert.NotNil(t, session.WaitExpiresOn()) // unaffected + + // and called with one to clear in the database as well + session.ClearWaitTimeout(ctx, db) + assert.Nil(t, session.WaitTimeoutOn()) + + assertdb.Query(t, db, `SELECT timeout_on FROM flows_flowsession WHERE id = $1`, session.ID()).Returns(nil) +} + func insertSessionAndRun(db *sqlx.DB, contact *testdata.Contact, sessionType models.FlowType, status models.SessionStatus, flow *testdata.Flow, connID models.ConnectionID) (models.SessionID, models.FlowRunID) { // create session and add a run with same status sessionID := testdata.InsertFlowSession(db, testdata.Org1, contact, sessionType, status, flow, connID) diff --git a/core/models/starts.go b/core/models/starts.go index fd9c15091..db9013037 100644 --- a/core/models/starts.go +++ b/core/models/starts.go @@ -65,7 +65,7 @@ func MarkStartStarted(ctx context.Context, db Queryer, startID StartID, contactC ContactID ContactID `db:"contact_id"` } - args := make([]interface{}, len(createdContactIDs)) + args := make([]*startContact, len(createdContactIDs)) for i, id := range createdContactIDs { args[i] = &startContact{StartID: startID, ContactID: id} } @@ -112,16 +112,16 @@ type FlowStartBatch struct { } } -func (b *FlowStartBatch) StartID() StartID { return b.b.StartID } -func (b *FlowStartBatch) StartType() StartType { return b.b.StartType } -func (b *FlowStartBatch) OrgID() OrgID { return b.b.OrgID } -func (b *FlowStartBatch) CreatedByID() UserID { return b.b.CreatedByID } -func (b *FlowStartBatch) FlowID() FlowID { return b.b.FlowID } -func (b *FlowStartBatch) ContactIDs() []ContactID { return b.b.ContactIDs } -func (b *FlowStartBatch) RestartParticipants() bool { return b.b.RestartParticipants } -func (b *FlowStartBatch) IncludeActive() bool { return b.b.IncludeActive } -func (b *FlowStartBatch) IsLast() bool { return b.b.IsLast } -func (b *FlowStartBatch) TotalContacts() int { return b.b.TotalContacts } +func (b *FlowStartBatch) StartID() StartID { return b.b.StartID } +func (b *FlowStartBatch) StartType() StartType { return b.b.StartType } +func (b *FlowStartBatch) OrgID() OrgID { return b.b.OrgID } +func (b *FlowStartBatch) CreatedByID() UserID { return b.b.CreatedByID } +func (b *FlowStartBatch) FlowID() FlowID { return b.b.FlowID } +func (b *FlowStartBatch) ContactIDs() []ContactID { return b.b.ContactIDs } +func (b *FlowStartBatch) ExcludeStartedPreviously() bool { return !b.b.RestartParticipants } +func (b *FlowStartBatch) ExcludeInAFlow() bool { return !b.b.IncludeActive } +func (b *FlowStartBatch) IsLast() bool { return b.b.IsLast } +func (b *FlowStartBatch) TotalContacts() int { return b.b.TotalContacts } func (b *FlowStartBatch) ParentSummary() json.RawMessage { return json.RawMessage(b.b.ParentSummary) } func (b *FlowStartBatch) SessionHistory() json.RawMessage { return json.RawMessage(b.b.SessionHistory) } @@ -193,8 +193,17 @@ func (s *FlowStart) WithQuery(query string) *FlowStart { return s } -func (s *FlowStart) RestartParticipants() bool { return s.s.RestartParticipants } -func (s *FlowStart) IncludeActive() bool { return s.s.IncludeActive } +func (s *FlowStart) ExcludeStartedPreviously() bool { return !s.s.RestartParticipants } +func (s *FlowStart) WithExcludeStartedPreviously(exclude bool) *FlowStart { + s.s.RestartParticipants = !exclude + return s +} + +func (s *FlowStart) ExcludeInAFlow() bool { return !s.s.IncludeActive } +func (s *FlowStart) WithExcludeInAFlow(exclude bool) *FlowStart { + s.s.IncludeActive = !exclude + return s +} func (s *FlowStart) CreateContact() bool { return s.s.CreateContact } func (s *FlowStart) WithCreateContact(create bool) *FlowStart { @@ -234,16 +243,15 @@ func GetFlowStartAttributes(ctx context.Context, db Queryer, startID StartID) (* } // NewFlowStart creates a new flow start objects for the passed in parameters -func NewFlowStart(orgID OrgID, startType StartType, flowType FlowType, flowID FlowID, restartParticipants, includeActive bool) *FlowStart { +func NewFlowStart(orgID OrgID, startType StartType, flowType FlowType, flowID FlowID) *FlowStart { s := &FlowStart{} s.s.UUID = uuids.New() s.s.OrgID = orgID s.s.StartType = startType s.s.FlowType = flowType s.s.FlowID = flowID - s.s.RestartParticipants = restartParticipants - s.s.IncludeActive = includeActive - + s.s.RestartParticipants = true + s.s.IncludeActive = true return s } @@ -335,8 +343,8 @@ func (s *FlowStart) CreateBatch(contactIDs []ContactID, last bool, totalContacts b.b.FlowID = s.FlowID() b.b.FlowType = s.FlowType() b.b.ContactIDs = contactIDs - b.b.RestartParticipants = s.RestartParticipants() - b.b.IncludeActive = s.IncludeActive() + b.b.RestartParticipants = s.s.RestartParticipants + b.b.IncludeActive = s.s.IncludeActive b.b.ParentSummary = null.JSON(s.ParentSummary()) b.b.SessionHistory = null.JSON(s.SessionHistory()) b.b.Extra = null.JSON(s.Extra()) diff --git a/core/models/starts_test.go b/core/models/starts_test.go index cfa4780c4..b9e23fb8e 100644 --- a/core/models/starts_test.go +++ b/core/models/starts_test.go @@ -52,8 +52,8 @@ func TestStarts(t *testing.T) { assert.Equal(t, testdata.SingleMessage.ID, start.FlowID()) assert.Equal(t, models.FlowTypeMessaging, start.FlowType()) assert.Equal(t, "", start.Query()) - assert.True(t, start.RestartParticipants()) - assert.True(t, start.IncludeActive()) + assert.False(t, start.ExcludeStartedPreviously()) + assert.False(t, start.ExcludeInAFlow()) assert.Equal(t, []models.ContactID{testdata.Cathy.ID, testdata.Bob.ID}, start.ContactIDs()) assert.Equal(t, []models.GroupID{testdata.DoctorsGroup.ID}, start.GroupIDs()) assert.Equal(t, []models.GroupID{testdata.TestersGroup.ID}, start.ExcludeGroupIDs()) @@ -73,8 +73,8 @@ func TestStarts(t *testing.T) { assert.Equal(t, models.StartTypeManual, batch.StartType()) assert.Equal(t, testdata.SingleMessage.ID, batch.FlowID()) assert.Equal(t, []models.ContactID{testdata.Cathy.ID, testdata.Bob.ID}, batch.ContactIDs()) - assert.True(t, batch.RestartParticipants()) - assert.True(t, batch.IncludeActive()) + assert.False(t, batch.ExcludeStartedPreviously()) + assert.False(t, batch.ExcludeInAFlow()) assert.Equal(t, testdata.Admin.ID, batch.CreatedByID()) assert.False(t, batch.IsLast()) assert.Equal(t, 3, batch.TotalContacts()) @@ -100,7 +100,7 @@ func TestStartsBuilding(t *testing.T) { uuids.SetGenerator(uuids.NewSeededGenerator(12345)) defer uuids.SetGenerator(uuids.DefaultGenerator) - start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeManual, models.FlowTypeMessaging, testdata.Favorites.ID, true, true). + start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeManual, models.FlowTypeMessaging, testdata.Favorites.ID). WithGroupIDs([]models.GroupID{testdata.DoctorsGroup.ID}). WithExcludeGroupIDs([]models.GroupID{testdata.TestersGroup.ID}). WithContactIDs([]models.ContactID{testdata.Cathy.ID, testdata.Bob.ID}). diff --git a/core/models/teams.go b/core/models/teams.go new file mode 100644 index 000000000..a6aeec210 --- /dev/null +++ b/core/models/teams.go @@ -0,0 +1,43 @@ +package models + +import ( + "database/sql/driver" + + "github.com/nyaruka/null" +) + +const ( + // NilTeamID is the id 0 considered as nil user id + NilTeamID = TeamID(0) +) + +// TeamID is our type for team ids, which can be null +type TeamID null.Int + +type TeamUUID string + +type Team struct { + ID TeamID `json:"id"` + UUID TeamUUID `json:"uuid"` + Name string `json:"name"` +} + +// MarshalJSON marshals into JSON. 0 values will become null +func (i TeamID) MarshalJSON() ([]byte, error) { + return null.Int(i).MarshalJSON() +} + +// UnmarshalJSON unmarshals from JSON. null values become 0 +func (i *TeamID) UnmarshalJSON(b []byte) error { + return null.UnmarshalInt(b, (*null.Int)(i)) +} + +// Value returns the db value, null is returned for 0 +func (i TeamID) Value() (driver.Value, error) { + return null.Int(i).Value() +} + +// Scan scans from the db value. null values become 0 +func (i *TeamID) Scan(value interface{}) error { + return null.ScanInt(value, (*null.Int)(i)) +} diff --git a/core/models/ticket_events_test.go b/core/models/ticket_events_test.go index 34ca21ca4..eab35dd73 100644 --- a/core/models/ticket_events_test.go +++ b/core/models/ticket_events_test.go @@ -2,6 +2,7 @@ package models_test import ( "testing" + "time" "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/nyaruka/mailroom/core/models" @@ -17,7 +18,7 @@ func TestTicketEvents(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "17", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "17", time.Now(), nil) modelTicket := ticket.Load(db) e1 := models.NewTicketOpenedEvent(modelTicket, testdata.Admin.ID, testdata.Agent.ID) diff --git a/core/models/tickets.go b/core/models/tickets.go index b75a81a6f..852334160 100644 --- a/core/models/tickets.go +++ b/core/models/tickets.go @@ -51,10 +51,19 @@ func (i *TicketID) Scan(value interface{}) error { type TicketerID null.Int type TicketStatus string +type TicketDailyCountType string +type TicketDailyTimingType string const ( TicketStatusOpen = TicketStatus("O") TicketStatusClosed = TicketStatus("C") + + TicketDailyCountOpening = TicketDailyCountType("O") + TicketDailyCountAssignment = TicketDailyCountType("A") + TicketDailyCountReply = TicketDailyCountType("R") + + TicketDailyTimingFirstReply = TicketDailyTimingType("R") + TicketDailyTimingLastClose = TicketDailyTimingType("C") ) // Register a ticket service factory with the engine @@ -82,6 +91,7 @@ type Ticket struct { AssigneeID UserID `db:"assignee_id"` Config null.Map `db:"config"` OpenedOn time.Time `db:"opened_on"` + RepliedOn *time.Time `db:"replied_on"` ModifiedOn time.Time `db:"modified_on"` ClosedOn *time.Time `db:"closed_on"` LastActivityOn time.Time `db:"last_activity_on"` @@ -114,6 +124,7 @@ func (t *Ticket) Status() TicketStatus { return t.t.Status } func (t *Ticket) TopicID() TopicID { return t.t.TopicID } func (t *Ticket) Body() string { return t.t.Body } func (t *Ticket) AssigneeID() UserID { return t.t.AssigneeID } +func (t *Ticket) RepliedOn() *time.Time { return t.t.RepliedOn } func (t *Ticket) LastActivityOn() time.Time { return t.t.LastActivityOn } func (t *Ticket) Config(key string) string { return t.t.Config.GetString(key, "") @@ -185,6 +196,7 @@ SELECT t.assignee_id AS assignee_id, t.config AS config, t.opened_on AS opened_on, + t.replied_on, t.modified_on AS modified_on, t.closed_on AS closed_on, t.last_activity_on AS last_activity_on @@ -214,6 +226,7 @@ SELECT t.assignee_id AS assignee_id, t.config AS config, t.opened_on AS opened_on, + t.replied_on, t.modified_on AS modified_on, t.closed_on AS closed_on, t.last_activity_on AS last_activity_on @@ -262,6 +275,7 @@ SELECT t.assignee_id AS assignee_id, t.config AS config, t.opened_on AS opened_on, + t.replied_on, t.modified_on AS modified_on, t.closed_on AS closed_on, t.last_activity_on AS last_activity_on @@ -290,6 +304,7 @@ SELECT t.assignee_id AS assignee_id, t.config AS config, t.opened_on AS opened_on, + t.replied_on, t.modified_on AS modified_on, t.closed_on AS closed_on, t.last_activity_on AS last_activity_on @@ -334,17 +349,38 @@ RETURNING ` // InsertTickets inserts the passed in tickets returning any errors encountered -func InsertTickets(ctx context.Context, tx Queryer, tickets []*Ticket) error { +func InsertTickets(ctx context.Context, tx Queryer, oa *OrgAssets, tickets []*Ticket) error { if len(tickets) == 0 { return nil } + openingCounts := map[string]int{scopeOrg(oa): len(tickets)} // all new tickets are open + assignmentCounts := make(map[string]int) + ts := make([]interface{}, len(tickets)) - for i := range tickets { - ts[i] = &tickets[i].t + for i, t := range tickets { + ts[i] = &t.t + + if t.AssigneeID() != NilUserID { + assignee := oa.UserByID(t.AssigneeID()) + if assignee != nil { + assignmentCounts[scopeUser(oa, assignee)]++ + } + } } - return BulkQuery(ctx, "inserted tickets", tx, sqlInsertTicket, ts) + if err := BulkQuery(ctx, "inserted tickets", tx, sqlInsertTicket, ts); err != nil { + return err + } + + if err := insertTicketDailyCounts(ctx, tx, TicketDailyCountOpening, oa.Org().Timezone(), openingCounts); err != nil { + return err + } + if err := insertTicketDailyCounts(ctx, tx, TicketDailyCountAssignment, oa.Org().Timezone(), assignmentCounts); err != nil { + return err + } + + return nil } // UpdateTicketExternalID updates the external ID of the given ticket @@ -391,8 +427,19 @@ func TicketsAssign(ctx context.Context, db Queryer, oa *OrgAssets, userID UserID eventsByTicket := make(map[*Ticket]*TicketEvent, len(tickets)) now := dates.Now() + assignmentCounts := make(map[string]int) + for _, ticket := range tickets { if ticket.AssigneeID() != assigneeID { + + // if this is an initial assignment record count for user + if ticket.AssigneeID() == NilUserID && assigneeID != NilUserID { + assignee := oa.UserByID(assigneeID) + if assignee != nil { + assignmentCounts[scopeUser(oa, assignee)]++ + } + } + ids = append(ids, ticket.ID()) t := &ticket.t t.AssigneeID = assigneeID @@ -421,6 +468,11 @@ func TicketsAssign(ctx context.Context, db Queryer, oa *OrgAssets, userID UserID return nil, errors.Wrap(err, "error inserting notifications") } + err = insertTicketDailyCounts(ctx, db, TicketDailyCountAssignment, oa.Org().Timezone(), assignmentCounts) + if err != nil { + return nil, errors.Wrap(err, "error inserting assignment counts") + } + return eventsByTicket, nil } @@ -647,6 +699,41 @@ func recalcGroupsForTicketChanges(ctx context.Context, db Queryer, oa *OrgAssets return CalculateDynamicGroups(ctx, db, oa, flowContacts) } +const sqlUpdateTicketRepliedOn = ` + UPDATE tickets_ticket t1 + SET last_activity_on = $2, replied_on = LEAST(t1.replied_on, $2) + FROM tickets_ticket t2 + WHERE t1.id = t2.id AND t1.id = $1 +RETURNING CASE WHEN t2.replied_on IS NULL THEN EXTRACT(EPOCH FROM (t1.replied_on - t1.opened_on)) ELSE NULL END` + +// TicketRecordReplied records a ticket as being replied to, updating last_activity_on. If this is the first reply +// to this ticket then replied_on is updated and the function returns the number of seconds between that and when +// the ticket was opened. +func TicketRecordReplied(ctx context.Context, db Queryer, ticketID TicketID, when time.Time) (time.Duration, error) { + rows, err := db.QueryxContext(ctx, sqlUpdateTicketRepliedOn, ticketID, when) + if err != nil && err != sql.ErrNoRows { + return -1, err + } + + defer rows.Close() + + // if we didn't get anything back then we didn't change the ticket because it was already replied to + if err == sql.ErrNoRows || !rows.Next() { + return -1, nil + } + + var seconds *float64 + if err := rows.Scan(&seconds); err != nil { + return -1, err + } + + if seconds != nil { + return time.Duration(*seconds * float64(time.Second)), nil + } + + return time.Duration(-1), nil +} + // Ticketer is our type for a ticketer asset type Ticketer struct { t struct { @@ -830,3 +917,11 @@ func (i TicketerID) Value() (driver.Value, error) { func (i *TicketerID) Scan(value interface{}) error { return null.ScanInt(value, (*null.Int)(i)) } + +func insertTicketDailyCounts(ctx context.Context, tx Queryer, countType TicketDailyCountType, tz *time.Location, scopeCounts map[string]int) error { + return insertDailyCounts(ctx, tx, "tickets_ticketdailycount", countType, tz, scopeCounts) +} + +func insertTicketDailyTiming(ctx context.Context, tx Queryer, countType TicketDailyTimingType, tz *time.Location, scope string, duration time.Duration) error { + return insertDailyTiming(ctx, tx, "tickets_ticketdailytiming", countType, tz, scope, duration) +} diff --git a/core/models/tickets_test.go b/core/models/tickets_test.go index 10eaeb4ce..e4541f013 100644 --- a/core/models/tickets_test.go +++ b/core/models/tickets_test.go @@ -1,9 +1,11 @@ package models_test import ( + "fmt" "testing" "time" + "github.com/jmoiron/sqlx" "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/nyaruka/gocommon/httpx" @@ -62,6 +64,8 @@ func TestTickets(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) + oa := testdata.Org1.Load(rt) + ticket1 := models.NewTicket( "2ef57efc-d85f-4291-b330-e4afe68af5fe", testdata.Org1.ID, @@ -88,13 +92,13 @@ func TestTickets(t *testing.T) { ) ticket3 := models.NewTicket( "28ef8ddc-b221-42f3-aeae-ee406fc9d716", - testdata.Org2.ID, + testdata.Org1.ID, testdata.Alexandria.ID, testdata.Zendesk.ID, "EX6677", testdata.SupportTopic.ID, "Where are my pants?", - testdata.Org2Admin.ID, + testdata.Admin.ID, nil, ) @@ -108,12 +112,18 @@ func TestTickets(t *testing.T) { assert.Equal(t, testdata.Admin.ID, ticket1.AssigneeID()) assert.Equal(t, "", ticket1.Config("xyz")) - err := models.InsertTickets(ctx, db, []*models.Ticket{ticket1, ticket2, ticket3}) + err := models.InsertTickets(ctx, db, oa, []*models.Ticket{ticket1, ticket2, ticket3}) assert.NoError(t, err) // check all tickets were created assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticket WHERE status = 'O' AND closed_on IS NULL`).Returns(3) + // check counts were added + assertTicketDailyCount(t, db, models.TicketDailyCountOpening, fmt.Sprintf("o:%d", testdata.Org1.ID), 3) + assertTicketDailyCount(t, db, models.TicketDailyCountOpening, fmt.Sprintf("o:%d", testdata.Org2.ID), 0) + assertTicketDailyCount(t, db, models.TicketDailyCountAssignment, fmt.Sprintf("o:%d:u:%d", testdata.Org1.ID, testdata.Admin.ID), 2) + assertTicketDailyCount(t, db, models.TicketDailyCountAssignment, fmt.Sprintf("o:%d:u:%d", testdata.Org1.ID, testdata.Editor.ID), 0) + // can lookup a ticket by UUID tk1, err := models.LookupTicketByUUID(ctx, db, "2ef57efc-d85f-4291-b330-e4afe68af5fe") assert.NoError(t, err) @@ -140,7 +150,7 @@ func TestUpdateTicketConfig(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", time.Now(), nil) modelTicket := ticket.Load(db) // empty configs are null @@ -161,19 +171,19 @@ func TestUpdateTicketLastActivity(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) - now := time.Date(2021, 6, 22, 15, 59, 30, 123456789, time.UTC) + now := time.Date(2021, 6, 22, 15, 59, 30, 123456000, time.UTC) defer dates.SetNowSource(dates.DefaultNowSource) dates.SetNowSource(dates.NewFixedNowSource(now)) - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", time.Now(), nil) modelTicket := ticket.Load(db) models.UpdateTicketLastActivity(ctx, db, []*models.Ticket{modelTicket}) assert.Equal(t, now, modelTicket.LastActivityOn()) - assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticket WHERE id = $1 AND last_activity_on = $2`, ticket.ID, modelTicket.LastActivityOn()).Returns(1) + assertdb.Query(t, db, `SELECT last_activity_on FROM tickets_ticket WHERE id = $1`, ticket.ID).Returns(modelTicket.LastActivityOn()) } @@ -188,25 +198,34 @@ func TestTicketsAssign(t *testing.T) { ticket1 := testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", nil) modelTicket1 := ticket1.Load(db) - ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "234", nil) + ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "234", time.Now(), nil) modelTicket2 := ticket2.Load(db) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", nil) + // create ticket already assigned to a user + ticket3 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my glasses", "", time.Now(), testdata.Admin) + modelTicket3 := ticket3.Load(db) + + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", time.Now(), nil) - evts, err := models.TicketsAssign(ctx, db, oa, testdata.Admin.ID, []*models.Ticket{modelTicket1, modelTicket2}, testdata.Agent.ID, "please handle these") + evts, err := models.TicketsAssign(ctx, db, oa, testdata.Admin.ID, []*models.Ticket{modelTicket1, modelTicket2, modelTicket3}, testdata.Agent.ID, "please handle these") require.NoError(t, err) - assert.Equal(t, 2, len(evts)) + assert.Equal(t, 3, len(evts)) assert.Equal(t, models.TicketEventTypeAssigned, evts[modelTicket1].EventType()) assert.Equal(t, models.TicketEventTypeAssigned, evts[modelTicket2].EventType()) + assert.Equal(t, models.TicketEventTypeAssigned, evts[modelTicket3].EventType()) // check tickets are now assigned assertdb.Query(t, db, `SELECT assignee_id FROM tickets_ticket WHERE id = $1`, ticket1.ID).Columns(map[string]interface{}{"assignee_id": int64(testdata.Agent.ID)}) assertdb.Query(t, db, `SELECT assignee_id FROM tickets_ticket WHERE id = $1`, ticket2.ID).Columns(map[string]interface{}{"assignee_id": int64(testdata.Agent.ID)}) + assertdb.Query(t, db, `SELECT assignee_id FROM tickets_ticket WHERE id = $1`, ticket3.ID).Columns(map[string]interface{}{"assignee_id": int64(testdata.Agent.ID)}) - // and there are new assigned events - assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticketevent WHERE event_type = 'A' AND note = 'please handle these'`).Returns(2) - + // and there are new assigned events with notifications + assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticketevent WHERE event_type = 'A' AND note = 'please handle these'`).Returns(3) assertdb.Query(t, db, `SELECT count(*) FROM notifications_notification WHERE user_id = $1 AND notification_type = 'tickets:activity'`, testdata.Agent.ID).Returns(1) + + // and daily counts (we only count first assignments of a ticket) + assertTicketDailyCount(t, db, models.TicketDailyCountAssignment, fmt.Sprintf("o:%d:u:%d", testdata.Org1.ID, testdata.Agent.ID), 2) + assertTicketDailyCount(t, db, models.TicketDailyCountAssignment, fmt.Sprintf("o:%d:u:%d", testdata.Org1.ID, testdata.Admin.ID), 0) } func TestTicketsAddNote(t *testing.T) { @@ -220,10 +239,10 @@ func TestTicketsAddNote(t *testing.T) { ticket1 := testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", nil) modelTicket1 := ticket1.Load(db) - ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "234", testdata.Agent) + ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "234", time.Now(), testdata.Agent) modelTicket2 := ticket2.Load(db) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", nil) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", time.Now(), nil) evts, err := models.TicketsAddNote(ctx, db, oa, testdata.Admin.ID, []*models.Ticket{modelTicket1, modelTicket2}, "spam") require.NoError(t, err) @@ -248,13 +267,13 @@ func TestTicketsChangeTopic(t *testing.T) { ticket1 := testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.SalesTopic, "Where my shoes", "123", nil) modelTicket1 := ticket1.Load(db) - ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.SupportTopic, "Where my pants", "234", nil) + ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.SupportTopic, "Where my pants", "234", time.Now(), nil) modelTicket2 := ticket2.Load(db) - ticket3 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "345", nil) + ticket3 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "345", time.Now(), nil) modelTicket3 := ticket3.Load(db) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", nil) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", time.Now(), nil) evts, err := models.TicketsChangeTopic(ctx, db, oa, testdata.Admin.ID, []*models.Ticket{modelTicket1, modelTicket2, modelTicket3}, testdata.SupportTopic.ID) require.NoError(t, err) @@ -282,12 +301,10 @@ func TestCloseTickets(t *testing.T) { }, })) - testdata.InsertContactGroup(db, testdata.Org1, "94c816d7-cc87-42db-a577-ce072ceaab80", "Tickets", "tickets > 0") - oa, err := models.GetOrgAssetsWithRefresh(ctx, rt, testdata.Org1.ID, models.RefreshTicketers|models.RefreshGroups) require.NoError(t, err) - ticket1 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", nil) + ticket1 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", time.Now(), nil) modelTicket1 := ticket1.Load(db) ticket2 := testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "234", nil) @@ -299,7 +316,7 @@ func TestCloseTickets(t *testing.T) { require.NoError(t, err) assert.Equal(t, "Doctors", cathy.Groups().All()[0].Name()) - assert.Equal(t, "Tickets", cathy.Groups().All()[1].Name()) + assert.Equal(t, "Open Tickets", cathy.Groups().All()[1].Name()) logger := &models.HTTPLogger{} evts, err := models.CloseTickets(ctx, rt, oa, testdata.Admin.ID, []*models.Ticket{modelTicket1, modelTicket2}, true, false, logger) @@ -328,7 +345,7 @@ func TestCloseTickets(t *testing.T) { assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticketevent WHERE ticket_id = $1 AND event_type = 'C'`, ticket2.ID).Returns(0) // can close tickets without a user - ticket3 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", nil) + ticket3 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", time.Now(), nil) modelTicket3 := ticket3.Load(db) evts, err = models.CloseTickets(ctx, rt, oa, models.NilUserID, []*models.Ticket{modelTicket3}, false, false, logger) @@ -354,15 +371,13 @@ func TestReopenTickets(t *testing.T) { }, })) - testdata.InsertContactGroup(db, testdata.Org1, "94c816d7-cc87-42db-a577-ce072ceaab80", "Two Tickets", "tickets = 2") - oa, err := models.GetOrgAssetsWithRefresh(ctx, rt, testdata.Org1.ID, models.RefreshTicketers|models.RefreshGroups) require.NoError(t, err) ticket1 := testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", nil) modelTicket1 := ticket1.Load(db) - ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "234", nil) + ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Where my pants", "234", time.Now(), nil) modelTicket2 := ticket2.Load(db) logger := &models.HTTPLogger{} @@ -385,9 +400,52 @@ func TestReopenTickets(t *testing.T) { // but no events for ticket #2 which waas already open assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticketevent WHERE ticket_id = $1 AND event_type = 'R'`, ticket2.ID).Returns(0) - // check Cathy is now in the two tickets group + // check Cathy is now in the open tickets group _, cathy := testdata.Cathy.Load(db, oa) assert.Equal(t, 2, len(cathy.Groups().All())) assert.Equal(t, "Doctors", cathy.Groups().All()[0].Name()) - assert.Equal(t, "Two Tickets", cathy.Groups().All()[1].Name()) + assert.Equal(t, "Open Tickets", cathy.Groups().All()[1].Name()) + + // reopening doesn't change opening daily counts + assertTicketDailyCount(t, db, models.TicketDailyCountOpening, fmt.Sprintf("o:%d", testdata.Org1.ID), 0) +} + +func TestTicketRecordReply(t *testing.T) { + ctx, _, db, _ := testsuite.Get() + + defer testsuite.Reset(testsuite.ResetData) + + openedOn := time.Date(2022, 5, 18, 14, 21, 0, 0, time.UTC) + repliedOn := time.Date(2022, 5, 18, 15, 0, 0, 0, time.UTC) + + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Where my shoes", "123", openedOn, nil) + + timing, err := models.TicketRecordReplied(ctx, db, ticket.ID, repliedOn) + assert.NoError(t, err) + assert.Equal(t, 2340*time.Second, timing) + + modelTicket := ticket.Load(db) + assert.Equal(t, repliedOn, *modelTicket.RepliedOn()) + assert.Equal(t, repliedOn, modelTicket.LastActivityOn()) + + assertdb.Query(t, db, `SELECT replied_on FROM tickets_ticket WHERE id = $1`, ticket.ID).Returns(repliedOn) + assertdb.Query(t, db, `SELECT last_activity_on FROM tickets_ticket WHERE id = $1`, ticket.ID).Returns(repliedOn) + + repliedAgainOn := time.Date(2022, 5, 18, 15, 5, 0, 0, time.UTC) + + // if we call it again, it won't change replied_on again but it will update last_activity_on + timing, err = models.TicketRecordReplied(ctx, db, ticket.ID, repliedAgainOn) + assert.NoError(t, err) + assert.Equal(t, time.Duration(-1), timing) + + modelTicket = ticket.Load(db) + assert.Equal(t, repliedOn, *modelTicket.RepliedOn()) + assert.Equal(t, repliedAgainOn, modelTicket.LastActivityOn()) + + assertdb.Query(t, db, `SELECT replied_on FROM tickets_ticket WHERE id = $1`, ticket.ID).Returns(repliedOn) + assertdb.Query(t, db, `SELECT last_activity_on FROM tickets_ticket WHERE id = $1`, ticket.ID).Returns(repliedAgainOn) +} + +func assertTicketDailyCount(t *testing.T, db *sqlx.DB, countType models.TicketDailyCountType, scope string, expected int) { + assertdb.Query(t, db, `SELECT COALESCE(SUM(count), 0) FROM tickets_ticketdailycount WHERE count_type = $1 AND scope = $2`, countType, scope).Returns(expected) } diff --git a/core/models/users.go b/core/models/users.go index 551fc0442..8eb41f2f7 100644 --- a/core/models/users.go +++ b/core/models/users.go @@ -60,7 +60,8 @@ type User struct { Email string `json:"email"` FirstName string `json:"first_name"` LastName string `json:"last_name"` - Role UserRole `json:"role"` + Role UserRole `json:"role_code"` + Team *Team `json:"team"` } } @@ -85,32 +86,22 @@ func (u *User) Name() string { return strings.Join(names, " ") } +// Team returns the user's ticketing team if any +func (u *User) Team() *Team { + return u.u.Team +} + var _ assets.User = (*User)(nil) const selectOrgUsersSQL = ` -SELECT ROW_TO_JSON(r) FROM (SELECT - u.id AS "id", - u.email AS "email", - u.first_name as "first_name", - u.last_name as "last_name", - o.role AS "role" -FROM - auth_user u -INNER JOIN ( - SELECT user_id, 'A' AS "role" FROM orgs_org_administrators WHERE org_id = $1 - UNION - SELECT user_id, 'E' AS "role" FROM orgs_org_editors WHERE org_id = $1 - UNION - SELECT user_id, 'V' AS "role" FROM orgs_org_viewers WHERE org_id = $1 - UNION - SELECT user_id, 'T' AS "role" FROM orgs_org_agents WHERE org_id = $1 - UNION - SELECT user_id, 'S' AS "role" FROM orgs_org_surveyors WHERE org_id = $1 -) o ON o.user_id = u.id -WHERE - u.is_active = TRUE -ORDER BY - u.email ASC +SELECT ROW_TO_JSON(r) FROM ( + SELECT u.id, u.email, u.first_name, u.last_name, m.role_code, row_to_json(team_struct) AS team + FROM orgs_orgmembership m + INNER JOIN auth_user u ON u.id = m.user_id + LEFT JOIN orgs_usersettings s ON s.user_id = u.id +LEFT JOIN LATERAL (SELECT id, uuid, name FROM tickets_team WHERE tickets_team.id = s.team_id) AS team_struct ON True + WHERE m.org_id = $1 AND u.is_active = TRUE + ORDER BY u.email ASC ) r;` // loadUsers loads all the users for the passed in org diff --git a/core/models/users_test.go b/core/models/users_test.go index 28103ba05..dee438067 100644 --- a/core/models/users_test.go +++ b/core/models/users_test.go @@ -19,17 +19,21 @@ func TestLoadUsers(t *testing.T) { users, err := oa.Users() require.NoError(t, err) + partners := &models.Team{testdata.Partners.ID, testdata.Partners.UUID, "Partners"} + office := &models.Team{testdata.Office.ID, testdata.Office.UUID, "Office"} + expectedUsers := []struct { id models.UserID email string name string role models.UserRole + team *models.Team }{ - {testdata.Admin.ID, testdata.Admin.Email, "Andy Admin", models.UserRoleAdministrator}, - {testdata.Agent.ID, testdata.Agent.Email, "Ann D'Agent", models.UserRoleAgent}, - {testdata.Editor.ID, testdata.Editor.Email, "Ed McEditor", models.UserRoleEditor}, - {testdata.Surveyor.ID, testdata.Surveyor.Email, "Steve Surveys", models.UserRoleSurveyor}, - {testdata.Viewer.ID, testdata.Viewer.Email, "Veronica Views", models.UserRoleViewer}, + {id: testdata.Admin.ID, email: testdata.Admin.Email, name: "Andy Admin", role: models.UserRoleAdministrator, team: office}, + {id: testdata.Agent.ID, email: testdata.Agent.Email, name: "Ann D'Agent", role: models.UserRoleAgent, team: partners}, + {id: testdata.Editor.ID, email: testdata.Editor.Email, name: "Ed McEditor", role: models.UserRoleEditor, team: office}, + {id: testdata.Surveyor.ID, email: testdata.Surveyor.Email, name: "Steve Surveys", role: models.UserRoleSurveyor, team: nil}, + {id: testdata.Viewer.ID, email: testdata.Viewer.Email, name: "Veronica Views", role: models.UserRoleViewer, team: nil}, } require.Equal(t, len(expectedUsers), len(users)) @@ -43,6 +47,7 @@ func TestLoadUsers(t *testing.T) { assert.Equal(t, expected.id, modelUser.ID()) assert.Equal(t, expected.email, modelUser.Email()) assert.Equal(t, expected.role, modelUser.Role()) + assert.Equal(t, expected.team, modelUser.Team()) assert.Equal(t, modelUser, oa.UserByID(expected.id)) assert.Equal(t, modelUser, oa.UserByEmail(expected.email)) diff --git a/core/models/utils.go b/core/models/utils.go index 3a1a03e52..d2cb5b987 100644 --- a/core/models/utils.go +++ b/core/models/utils.go @@ -44,7 +44,7 @@ func Exec(ctx context.Context, label string, tx Queryer, sql string, args ...int } // BulkQuery runs the given query as a bulk operation -func BulkQuery(ctx context.Context, label string, tx Queryer, sql string, structs []interface{}) error { +func BulkQuery[T any](ctx context.Context, label string, tx Queryer, sql string, structs []T) error { // no values, nothing to do if len(structs) == 0 { return nil @@ -79,8 +79,8 @@ func BulkQueryBatches(ctx context.Context, label string, tx Queryer, sql string, return nil } -func chunkSlice(slice []interface{}, size int) [][]interface{} { - chunks := make([][]interface{}, 0, len(slice)/size+1) +func chunkSlice[T any](slice []T, size int) [][]T { + chunks := make([][]T, 0, len(slice)/size+1) for i := 0; i < len(slice); i += size { end := i + size @@ -91,31 +91,3 @@ func chunkSlice(slice []interface{}, size int) [][]interface{} { } return chunks } - -// chunks a slice of session IDs.. hurry up go generics -func chunkSessionIDs(ids []SessionID, size int) [][]SessionID { - chunks := make([][]SessionID, 0, len(ids)/size+1) - - for i := 0; i < len(ids); i += size { - end := i + size - if end > len(ids) { - end = len(ids) - } - chunks = append(chunks, ids[i:end]) - } - return chunks -} - -// chunks a slice of contact IDs.. hurry up go generics -func chunkContactIDs(ids []ContactID, size int) [][]ContactID { - chunks := make([][]ContactID, 0, len(ids)/size+1) - - for i := 0; i < len(ids); i += size { - end := i + size - if end > len(ids) { - end = len(ids) - } - chunks = append(chunks, ids[i:end]) - } - return chunks -} diff --git a/core/models/webhook_event.go b/core/models/webhook_event.go index e11ded7ad..5991e8a6e 100644 --- a/core/models/webhook_event.go +++ b/core/models/webhook_event.go @@ -33,11 +33,10 @@ func NewWebhookEvent(orgID OrgID, resthookID ResthookID, data string, createdOn return event } -const insertWebhookEventsSQL = ` -INSERT INTO api_webhookevent(data, resthook_id, org_id, created_on, action) - VALUES(:data, :resthook_id, :org_id, :created_on, 'POST') -RETURNING id -` +const sqlInsertWebhookEvents = ` +INSERT INTO api_webhookevent(data, resthook_id, org_id, created_on, action) + VALUES(:data, :resthook_id, :org_id, :created_on, 'POST') + RETURNING id` // InsertWebhookEvents inserts the passed in webhook events, assigning them ids func InsertWebhookEvents(ctx context.Context, db Queryer, events []*WebhookEvent) error { @@ -50,5 +49,5 @@ func InsertWebhookEvents(ctx context.Context, db Queryer, events []*WebhookEvent is[i] = &events[i].e } - return BulkQuery(ctx, "inserted webhook events", db, insertWebhookEventsSQL, is) + return BulkQuery(ctx, "inserted webhook events", db, sqlInsertWebhookEvents, is) } diff --git a/core/msgio/send.go b/core/msgio/send.go index c118048e9..737d596be 100644 --- a/core/msgio/send.go +++ b/core/msgio/send.go @@ -6,8 +6,7 @@ import ( "github.com/edganiukov/fcm" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/runtime" - - "github.com/apex/log" + "github.com/sirupsen/logrus" ) // SendMessages tries to send the given messages via Courier or Android syncing @@ -54,7 +53,7 @@ func SendMessages(ctx context.Context, rt *runtime.Runtime, tx models.Queryer, f // not being able to queue a message isn't the end of the world, log but don't return an error if err != nil { - log.WithField("messages", contactMsgs).WithField("contact", contactID).WithError(err).Error("error queuing messages") + logrus.WithField("messages", contactMsgs).WithField("contact", contactID).WithError(err).Error("error queuing messages") // in the case of errors we do want to change the messages back to pending however so they // get queued later. (for the common case messages are only inserted and queued, without a status update) @@ -76,7 +75,7 @@ func SendMessages(ctx context.Context, rt *runtime.Runtime, tx models.Queryer, f if len(pending) > 0 { err := models.MarkMessagesPending(ctx, tx, pending) if err != nil { - log.WithError(err).Error("error marking message as pending") + logrus.WithError(err).Error("error marking message as pending") } } } diff --git a/core/runner/runner.go b/core/runner/runner.go index 23e99fb57..a02cda02e 100644 --- a/core/runner/runner.go +++ b/core/runner/runner.go @@ -6,16 +6,16 @@ import ( "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/triggers" - "github.com/nyaruka/librato" "github.com/nyaruka/mailroom/core/goflow" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/locker" + "github.com/nyaruka/redisx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -33,11 +33,11 @@ var startTypeToOrigin = map[models.StartType]string{ // StartOptions define the various parameters that can be used when starting a flow type StartOptions struct { - // ExcludeWaiting excludes contacts with waiting sessions which would otherwise have to be interrupted - ExcludeWaiting bool + // ExcludeInAFlow excludes contacts with waiting sessions which would otherwise have to be interrupted + ExcludeInAFlow bool - // ExcludeReruns excludes contacts who have been in this flow previously (at least as long as we have runs for) - ExcludeReruns bool + // ExcludeStartedPreviously excludes contacts who have been in this flow previously (at least as long as we have runs for) + ExcludeStartedPreviously bool // Interrupt should be true if we want to interrupt the flows runs for any contact started in this flow Interrupt bool @@ -52,9 +52,9 @@ type StartOptions struct { // NewStartOptions creates and returns the default start options to be used for flow starts func NewStartOptions() *StartOptions { return &StartOptions{ - ExcludeWaiting: false, - ExcludeReruns: false, - Interrupt: true, + ExcludeInAFlow: false, + ExcludeStartedPreviously: false, + Interrupt: true, } } @@ -231,8 +231,8 @@ func StartFlowBatch( // options for our flow start options := NewStartOptions() - options.ExcludeReruns = !batch.RestartParticipants() - options.ExcludeWaiting = !batch.IncludeActive() + options.ExcludeStartedPreviously = batch.ExcludeStartedPreviously() + options.ExcludeInAFlow = batch.ExcludeInAFlow() options.Interrupt = flow.FlowType().Interrupts() options.TriggerBuilder = triggerBuilder options.CommitHook = updateStartID @@ -243,8 +243,8 @@ func StartFlowBatch( } // log both our total and average - librato.Gauge("mr.flow_batch_start_elapsed", float64(time.Since(start))/float64(time.Second)) - librato.Gauge("mr.flow_batch_start_count", float64(len(sessions))) + analytics.Gauge("mr.flow_batch_start_elapsed", float64(time.Since(start))/float64(time.Second)) + analytics.Gauge("mr.flow_batch_start_count", float64(len(sessions))) return sessions, nil } @@ -306,16 +306,16 @@ func FireCampaignEvents( options := NewStartOptions() switch dbEvent.StartMode() { case models.StartModeInterrupt: - options.ExcludeWaiting = false - options.ExcludeReruns = false + options.ExcludeInAFlow = false + options.ExcludeStartedPreviously = false options.Interrupt = true case models.StartModePassive: - options.ExcludeWaiting = false - options.ExcludeReruns = false + options.ExcludeInAFlow = false + options.ExcludeStartedPreviously = false options.Interrupt = false case models.StartModeSkip: - options.ExcludeWaiting = true - options.ExcludeReruns = false + options.ExcludeInAFlow = true + options.ExcludeStartedPreviously = false options.Interrupt = true default: return nil, errors.Errorf("unknown start mode: %s", dbEvent.StartMode()) @@ -393,8 +393,8 @@ func FireCampaignEvents( } // log both our total and average - librato.Gauge("mr.campaign_event_elapsed", float64(time.Since(start))/float64(time.Second)) - librato.Gauge("mr.campaign_event_count", float64(len(sessions))) + analytics.Gauge("mr.campaign_event_elapsed", float64(time.Since(start))/float64(time.Second)) + analytics.Gauge("mr.campaign_event_count", float64(len(sessions))) // build the list of contacts actually started startedContacts := make([]models.ContactID, len(sessions)) @@ -417,7 +417,7 @@ func StartFlow( exclude := make(map[models.ContactID]bool, 5) // filter out anybody who has has a flow run in this flow if appropriate - if options.ExcludeReruns { + if options.ExcludeStartedPreviously { // find all participants that have been in this flow started, err := models.FindFlowStartedOverlap(ctx, rt.DB, flow.ID(), contactIDs) if err != nil { @@ -429,7 +429,7 @@ func StartFlow( } // filter out our list of contacts to only include those that should be started - if options.ExcludeWaiting { + if options.ExcludeInAFlow { // find all participants active in any flow active, err := models.FilterByWaitingSession(ctx, rt.DB, contactIDs) if err != nil { @@ -461,7 +461,7 @@ func StartFlow( start := time.Now() // map of locks we've released - released := make(map[string]bool) + released := make(map[*redisx.Locker]bool) for len(remaining) > 0 && time.Since(start) < time.Minute*5 { locked := make([]models.ContactID, 0, len(remaining)) @@ -470,8 +470,9 @@ func StartFlow( // try up to a second to get a lock for a contact for _, contactID := range remaining { - lockID := models.ContactLock(oa.OrgID(), contactID) - lock, err := locker.GrabLock(rt.RP, lockID, time.Minute*5, time.Second) + locker := models.GetContactLocker(oa.OrgID(), contactID) + + lock, err := locker.Grab(rt.RP, time.Second) if err != nil { return nil, errors.Wrapf(err, "error attempting to grab lock") } @@ -484,8 +485,8 @@ func StartFlow( // defer unlocking if we exit due to error defer func() { - if !released[lockID] { - locker.ReleaseLock(rt.RP, lockID, lock) + if !released[locker] { + locker.Release(rt.RP, lock) } }() } @@ -517,9 +518,9 @@ func StartFlow( // release all our locks for i := range locked { - lockID := models.ContactLock(oa.OrgID(), locked[i]) - locker.ReleaseLock(rt.RP, lockID, locks[i]) - released[lockID] = true + locker := models.GetContactLocker(oa.OrgID(), locked[i]) + locker.Release(rt.RP, locks[i]) + released[locker] = true } // skipped are now our remaining @@ -558,7 +559,7 @@ func StartFlowForContacts( continue } log.WithField("elapsed", time.Since(start)).Info("flow engine start") - librato.Gauge("mr.flow_start_elapsed", float64(time.Since(start))) + analytics.Gauge("mr.flow_start_elapsed", float64(time.Since(start))) sessions = append(sessions, session) sprints = append(sprints, sprint) @@ -718,7 +719,7 @@ func TriggerIVRFlow(ctx context.Context, rt *runtime.Runtime, orgID models.OrgID tx, _ := rt.DB.BeginTxx(ctx, nil) // create our start - start := models.NewFlowStart(orgID, models.StartTypeTrigger, models.FlowTypeVoice, flowID, true, true). + start := models.NewFlowStart(orgID, models.StartTypeTrigger, models.FlowTypeVoice, flowID). WithContactIDs(contactIDs) // insert it diff --git a/core/runner/runner_test.go b/core/runner/runner_test.go index a12bd6cd1..598b50763 100644 --- a/core/runner/runner_test.go +++ b/core/runner/runner_test.go @@ -192,34 +192,36 @@ func TestBatchStart(t *testing.T) { contactIDs := []models.ContactID{testdata.Cathy.ID, testdata.Bob.ID} tcs := []struct { - Flow models.FlowID - Restart bool - IncludeActive bool - Extra json.RawMessage - Msg string - Count int - TotalCount int + Flow models.FlowID + ExcludeStartedPreviously bool + ExcludeInAFlow bool + Extra json.RawMessage + Msg string + Count int + TotalCount int }{ - {testdata.SingleMessage.ID, true, true, nil, "Hey, how are you?", 2, 2}, - {testdata.SingleMessage.ID, false, true, nil, "Hey, how are you?", 0, 2}, - {testdata.SingleMessage.ID, false, false, nil, "Hey, how are you?", 0, 2}, - {testdata.SingleMessage.ID, true, false, nil, "Hey, how are you?", 2, 4}, + {testdata.SingleMessage.ID, false, false, nil, "Hey, how are you?", 2, 2}, + {testdata.SingleMessage.ID, true, false, nil, "Hey, how are you?", 0, 2}, + {testdata.SingleMessage.ID, true, true, nil, "Hey, how are you?", 0, 2}, + {testdata.SingleMessage.ID, false, true, nil, "Hey, how are you?", 2, 4}, { - Flow: testdata.IncomingExtraFlow.ID, - Restart: true, - IncludeActive: false, - Extra: json.RawMessage([]byte(`{"name":"Fred", "age":33}`)), - Msg: "Great to meet you Fred. Your age is 33.", - Count: 2, - TotalCount: 2, + Flow: testdata.IncomingExtraFlow.ID, + ExcludeStartedPreviously: false, + ExcludeInAFlow: true, + Extra: json.RawMessage([]byte(`{"name":"Fred", "age":33}`)), + Msg: "Great to meet you Fred. Your age is 33.", + Count: 2, + TotalCount: 2, }, } last := time.Now() for i, tc := range tcs { - start := models.NewFlowStart(models.OrgID(1), models.StartTypeManual, models.FlowTypeMessaging, tc.Flow, tc.Restart, tc.IncludeActive). + start := models.NewFlowStart(models.OrgID(1), models.StartTypeManual, models.FlowTypeMessaging, tc.Flow). WithContactIDs(contactIDs). + WithExcludeInAFlow(tc.ExcludeInAFlow). + WithExcludeStartedPreviously(tc.ExcludeStartedPreviously). WithExtra(tc.Extra) batch := start.CreateBatch(contactIDs, true, len(contactIDs)) @@ -344,8 +346,8 @@ func TestStartFlowConcurrency(t *testing.T) { } options := &runner.StartOptions{ - ExcludeReruns: false, - ExcludeWaiting: false, + ExcludeStartedPreviously: false, + ExcludeInAFlow: false, TriggerBuilder: func(contact *flows.Contact) flows.Trigger { return triggers.NewBuilder(oa.Env(), flowRef, contact).Manual().Build() }, diff --git a/core/search/groups.go b/core/search/groups.go new file mode 100644 index 000000000..8e5d9b698 --- /dev/null +++ b/core/search/groups.go @@ -0,0 +1,102 @@ +package search + +import ( + "context" + "time" + + "github.com/jmoiron/sqlx" + "github.com/nyaruka/mailroom/core/models" + "github.com/olivere/elastic/v7" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// PopulateSmartGroup calculates which members should be part of a group and populates the contacts +// for that group by performing the minimum number of inserts / deletes. +func PopulateSmartGroup(ctx context.Context, db *sqlx.DB, es *elastic.Client, oa *models.OrgAssets, groupID models.GroupID, query string) (int, error) { + err := models.UpdateGroupStatus(ctx, db, groupID, models.GroupStatusEvaluating) + if err != nil { + return 0, errors.Wrapf(err, "error marking dynamic group as evaluating") + } + + start := time.Now() + + // we have a bit of a race with the indexer process.. we want to make sure that any contacts that changed + // before this group was updated but after the last index are included, so if a contact was modified + // more recently than 10 seconds ago, we wait that long before starting in populating our group + newest, err := models.GetNewestContactModifiedOn(ctx, db, oa) + if err != nil { + return 0, errors.Wrapf(err, "error getting most recent contact modified_on for org: %d", oa.OrgID()) + } + if newest != nil { + n := *newest + + // if it was more recent than 10 seconds ago, sleep until it has been 10 seconds + if n.Add(time.Second * 10).After(start) { + sleep := n.Add(time.Second * 10).Sub(start) + logrus.WithField("sleep", sleep).Info("sleeping before evaluating dynamic group") + time.Sleep(sleep) + } + } + + // get current set of contacts in our group + ids, err := models.ContactIDsForGroupIDs(ctx, db, []models.GroupID{groupID}) + if err != nil { + return 0, errors.Wrapf(err, "unable to look up contact ids for group: %d", groupID) + } + present := make(map[models.ContactID]bool, len(ids)) + for _, i := range ids { + present[i] = true + } + + // calculate new set of ids + new, err := GetContactIDsForQuery(ctx, es, oa, query, -1) + if err != nil { + return 0, errors.Wrapf(err, "error performing query: %s for group: %d", query, groupID) + } + + // find which contacts need to be added or removed + adds := make([]models.ContactID, 0, 100) + for _, id := range new { + if !present[id] { + adds = append(adds, id) + } + delete(present, id) + } + + // build our list of removals + removals := make([]models.ContactID, 0, len(present)) + for id := range present { + removals = append(removals, id) + } + + // first remove all the contacts + err = models.RemoveContactsFromGroupAndCampaigns(ctx, db, oa, groupID, removals) + if err != nil { + return 0, errors.Wrapf(err, "error removing contacts from group: %d", groupID) + } + + // then add them all + err = models.AddContactsToGroupAndCampaigns(ctx, db, oa, groupID, adds) + if err != nil { + return 0, errors.Wrapf(err, "error adding contacts to group: %d", groupID) + } + + // mark our group as no longer evaluating + err = models.UpdateGroupStatus(ctx, db, groupID, models.GroupStatusReady) + if err != nil { + return 0, errors.Wrapf(err, "error marking dynamic group as ready") + } + + // finally update modified_on for all affected contacts to ensure these changes are seen by rp-indexer + changed := make([]models.ContactID, 0, len(adds)) + changed = append(changed, adds...) + changed = append(changed, removals...) + + err = models.UpdateContactModifiedOn(ctx, db, changed) + if err != nil { + return 0, errors.Wrapf(err, "error updating contact modified_on after group population") + } + + return len(new), nil +} diff --git a/core/search/groups_test.go b/core/search/groups_test.go new file mode 100644 index 000000000..25c22931b --- /dev/null +++ b/core/search/groups_test.go @@ -0,0 +1,92 @@ +package search_test + +import ( + "fmt" + "testing" + + "github.com/lib/pq" + "github.com/nyaruka/gocommon/dbutil/assertdb" + "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/core/search" + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/testsuite/testdata" + "github.com/stretchr/testify/assert" +) + +func TestSmartGroups(t *testing.T) { + ctx, rt, db, _ := testsuite.Get() + + defer testsuite.Reset(testsuite.ResetAll) + + // insert an event on our campaign + newEvent := testdata.InsertCampaignFlowEvent(db, testdata.RemindersCampaign, testdata.Favorites, testdata.JoinedField, 1000, "W") + + // clear Cathy's value + db.MustExec( + `update contacts_contact set fields = fields - $2 + WHERE id = $1`, testdata.Cathy.ID, testdata.JoinedField.UUID) + + // and populate Bob's + db.MustExec( + fmt.Sprintf(`update contacts_contact set fields = fields || + '{"%s": { "text": "2029-09-15T12:00:00+00:00", "datetime": "2029-09-15T12:00:00+00:00" }}'::jsonb + WHERE id = $1`, testdata.JoinedField.UUID), testdata.Bob.ID) + + oa, err := models.GetOrgAssetsWithRefresh(ctx, rt, testdata.Org1.ID, models.RefreshCampaigns|models.RefreshGroups) + assert.NoError(t, err) + + mockES := testsuite.NewMockElasticServer() + defer mockES.Close() + + es := mockES.Client() + + mockES.AddResponse(testdata.Cathy.ID) + mockES.AddResponse(testdata.Bob.ID) + mockES.AddResponse(testdata.Bob.ID) + + tcs := []struct { + Query string + ContactIDs []models.ContactID + EventContactIDs []models.ContactID + }{ + { + "cathy", + []models.ContactID{testdata.Cathy.ID}, + []models.ContactID{}, + }, + { + "bob", + []models.ContactID{testdata.Bob.ID}, + []models.ContactID{testdata.Bob.ID}, + }, + { + "unchanged", + []models.ContactID{testdata.Bob.ID}, + []models.ContactID{testdata.Bob.ID}, + }, + } + + for _, tc := range tcs { + err := models.UpdateGroupStatus(ctx, db, testdata.DoctorsGroup.ID, models.GroupStatusInitializing) + assert.NoError(t, err) + + count, err := search.PopulateSmartGroup(ctx, db, es, oa, testdata.DoctorsGroup.ID, tc.Query) + assert.NoError(t, err, "error populating smart group for: %s", tc.Query) + + assert.Equal(t, count, len(tc.ContactIDs)) + + // assert the current group membership + contactIDs, err := models.ContactIDsForGroupIDs(ctx, db, []models.GroupID{testdata.DoctorsGroup.ID}) + assert.NoError(t, err) + assert.Equal(t, tc.ContactIDs, contactIDs) + + assertdb.Query(t, db, `SELECT count(*) from contacts_contactgroup WHERE id = $1 AND status = 'R'`, testdata.DoctorsGroup.ID). + Returns(1, "wrong number of contacts in group for query: %s", tc.Query) + + assertdb.Query(t, db, `SELECT count(*) from campaigns_eventfire WHERE event_id = $1`, newEvent.ID). + Returns(len(tc.EventContactIDs), "wrong number of contacts with events for query: %s", tc.Query) + + assertdb.Query(t, db, `SELECT count(*) from campaigns_eventfire WHERE event_id = $1 AND contact_id = ANY($2)`, newEvent.ID, pq.Array(tc.EventContactIDs)). + Returns(len(tc.EventContactIDs), "wrong contacts with events for query: %s", tc.Query) + } +} diff --git a/core/search/queries.go b/core/search/queries.go new file mode 100644 index 000000000..94fc81538 --- /dev/null +++ b/core/search/queries.go @@ -0,0 +1,81 @@ +package search + +import ( + "time" + + "github.com/nyaruka/gocommon/dates" + "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/contactql" + "github.com/nyaruka/goflow/envs" + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom/core/models" + "github.com/pkg/errors" +) + +// Exclusions are preset exclusion conditions +type Exclusions struct { + NonActive bool `json:"non_active"` // contacts who are blocked, stopped or archived + InAFlow bool `json:"in_a_flow"` // contacts who are currently in a flow (including this one) + StartedPreviously bool `json:"started_previously"` // contacts who have been in this flow in the last 90 days + NotSeenSinceDays int `json:"not_seen_since_days"` // contacts who have not been seen for more than this number of days +} + +// BuildStartQuery builds a start query for the given flow and start options +func BuildStartQuery(oa *models.OrgAssets, flow *models.Flow, groups []*models.Group, contactUUIDs []flows.ContactUUID, urnz []urns.URN, userQuery string, excs Exclusions) (string, error) { + var parsedQuery *contactql.ContactQuery + var err error + + if userQuery != "" { + parsedQuery, err = contactql.ParseQuery(oa.Env(), userQuery, oa.SessionAssets()) + if err != nil { + return "", errors.Wrap(err, "invalid user query") + } + } + + return contactql.Stringify(buildStartQuery(oa.Env(), flow, groups, contactUUIDs, urnz, parsedQuery, excs)), nil +} + +func buildStartQuery(env envs.Environment, flow *models.Flow, groups []*models.Group, contactUUIDs []flows.ContactUUID, urnz []urns.URN, userQuery *contactql.ContactQuery, excs Exclusions) contactql.QueryNode { + inclusions := make([]contactql.QueryNode, 0, 10) + + for _, group := range groups { + inclusions = append(inclusions, contactql.NewCondition("group", contactql.PropertyTypeAttribute, contactql.OpEqual, group.Name())) + } + for _, contactUUID := range contactUUIDs { + inclusions = append(inclusions, contactql.NewCondition("uuid", contactql.PropertyTypeAttribute, contactql.OpEqual, string(contactUUID))) + } + for _, urn := range urnz { + scheme, path, _, _ := urn.ToParts() + inclusions = append(inclusions, contactql.NewCondition(scheme, contactql.PropertyTypeScheme, contactql.OpEqual, path)) + } + if userQuery != nil { + inclusions = append(inclusions, userQuery.Root()) + } + + exclusions := make([]contactql.QueryNode, 0, 10) + if excs.NonActive { + exclusions = append(exclusions, contactql.NewCondition("status", contactql.PropertyTypeAttribute, contactql.OpEqual, "active")) + } + if excs.InAFlow { + exclusions = append(exclusions, contactql.NewCondition("flow", contactql.PropertyTypeAttribute, contactql.OpEqual, "")) + } + if excs.StartedPreviously { + exclusions = append(exclusions, contactql.NewCondition("history", contactql.PropertyTypeAttribute, contactql.OpNotEqual, flow.Name())) + } + if excs.NotSeenSinceDays > 0 { + seenSince := dates.Now().Add(-time.Hour * time.Duration(24*excs.NotSeenSinceDays)) + exclusions = append(exclusions, contactql.NewCondition("last_seen_on", contactql.PropertyTypeAttribute, contactql.OpGreaterThan, formatQueryDate(env, seenSince))) + } + + return contactql.NewBoolCombination(contactql.BoolOperatorAnd, + contactql.NewBoolCombination(contactql.BoolOperatorOr, inclusions...), + contactql.NewBoolCombination(contactql.BoolOperatorAnd, exclusions...), + ).Simplify() +} + +// formats a date for use in a query +func formatQueryDate(env envs.Environment, t time.Time) string { + d := dates.ExtractDate(t.In(env.Timezone())) + s, _ := d.Format(string(env.DateFormat()), env.DefaultLocale().ToBCP47()) + return s +} diff --git a/core/search/queries_test.go b/core/search/queries_test.go new file mode 100644 index 000000000..e846ecd56 --- /dev/null +++ b/core/search/queries_test.go @@ -0,0 +1,123 @@ +package search_test + +import ( + "testing" + "time" + + "github.com/nyaruka/gocommon/dates" + "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/core/search" + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/testsuite/testdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildStartQuery(t *testing.T) { + _, rt, _, _ := testsuite.Get() + + dates.SetNowSource(dates.NewFixedNowSource(time.Date(2022, 4, 20, 15, 30, 45, 0, time.UTC))) + defer dates.SetNowSource(dates.DefaultNowSource) + + oa := testdata.Org1.Load(rt) + flow, err := oa.FlowByID(testdata.Favorites.ID) + require.NoError(t, err) + + doctors := oa.GroupByID(testdata.DoctorsGroup.ID) + testers := oa.GroupByID(testdata.TestersGroup.ID) + + tcs := []struct { + groups []*models.Group + contactUUIDs []flows.ContactUUID + urns []urns.URN + userQuery string + exclusions search.Exclusions + expected string + err string + }{ + { + groups: []*models.Group{doctors, testers}, + contactUUIDs: []flows.ContactUUID{testdata.Cathy.UUID, testdata.George.UUID}, + urns: []urns.URN{"tel:+1234567890", "telegram:9876543210"}, + exclusions: search.Exclusions{}, + expected: `group = "Doctors" OR group = "Testers" OR uuid = "6393abc0-283d-4c9b-a1b3-641a035c34bf" OR uuid = "8d024bcd-f473-4719-a00a-bd0bb1190135" OR tel = "+1234567890" OR telegram = 9876543210`, + }, + { + groups: []*models.Group{doctors}, + contactUUIDs: []flows.ContactUUID{testdata.Cathy.UUID}, + urns: []urns.URN{"tel:+1234567890"}, + exclusions: search.Exclusions{ + NonActive: true, + InAFlow: true, + StartedPreviously: true, + NotSeenSinceDays: 90, + }, + expected: `(group = "Doctors" OR uuid = "6393abc0-283d-4c9b-a1b3-641a035c34bf" OR tel = "+1234567890") AND status = "active" AND flow = "" AND history != "Favorites" AND last_seen_on > "20-01-2022"`, + }, + { + contactUUIDs: []flows.ContactUUID{testdata.Cathy.UUID}, + exclusions: search.Exclusions{ + NonActive: true, + }, + expected: `uuid = "6393abc0-283d-4c9b-a1b3-641a035c34bf" AND status = "active"`, + }, + { + userQuery: `gender = "M"`, + exclusions: search.Exclusions{}, + expected: `gender = "M"`, + }, + { + userQuery: `gender = "M"`, + exclusions: search.Exclusions{ + NonActive: true, + InAFlow: true, + StartedPreviously: true, + NotSeenSinceDays: 30, + }, + expected: `gender = "M" AND status = "active" AND flow = "" AND history != "Favorites" AND last_seen_on > "21-03-2022"`, + }, + { + userQuery: `name ~ ben`, + exclusions: search.Exclusions{ + NonActive: false, + InAFlow: false, + StartedPreviously: false, + NotSeenSinceDays: 30, + }, + expected: `name ~ "ben" AND last_seen_on > "21-03-2022"`, + }, + { + userQuery: `name ~ ben OR name ~ eric`, + exclusions: search.Exclusions{ + NonActive: false, + InAFlow: false, + StartedPreviously: false, + NotSeenSinceDays: 30, + }, + expected: `(name ~ "ben" OR name ~ "eric") AND last_seen_on > "21-03-2022"`, + }, + { + userQuery: `name ~`, // syntactically invalid user query + exclusions: search.Exclusions{}, + err: "invalid user query: mismatched input '' expecting {TEXT, STRING}", + }, + { + userQuery: `goats > 14`, // no such field + exclusions: search.Exclusions{}, + err: "invalid user query: can't resolve 'goats' to attribute, scheme or field", + }, + } + + for _, tc := range tcs { + actual, err := search.BuildStartQuery(oa, flow, tc.groups, tc.contactUUIDs, tc.urns, tc.userQuery, tc.exclusions) + if tc.err != "" { + assert.Equal(t, "", actual) + assert.EqualError(t, err, tc.err) + } else { + assert.Equal(t, tc.expected, actual) + assert.NoError(t, err) + } + } +} diff --git a/core/models/search.go b/core/search/search.go similarity index 71% rename from core/models/search.go rename to core/search/search.go index 09c646313..dc74525bb 100644 --- a/core/models/search.go +++ b/core/search/search.go @@ -1,4 +1,4 @@ -package models +package search import ( "context" @@ -10,14 +10,28 @@ import ( "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/contactql" "github.com/nyaruka/goflow/contactql/es" - + "github.com/nyaruka/mailroom/core/models" "github.com/olivere/elastic/v7" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +// AssetMapper maps resolved assets in queries to how we identify them in ES which in the case +// of flows and groups is their ids. We can do this by just type cracking them to their models. +type AssetMapper struct{} + +func (m *AssetMapper) Flow(f assets.Flow) int64 { + return int64(f.(*models.Flow).ID()) +} + +func (m *AssetMapper) Group(g assets.Group) int64 { + return int64(g.(*models.Group).ID()) +} + +var assetMapper = &AssetMapper{} + // BuildElasticQuery turns the passed in contact ql query into an elastic query -func BuildElasticQuery(oa *OrgAssets, group assets.GroupUUID, status ContactStatus, excludeIDs []ContactID, query *contactql.ContactQuery) elastic.Query { +func BuildElasticQuery(oa *models.OrgAssets, group *models.Group, status models.ContactStatus, excludeIDs []models.ContactID, query *contactql.ContactQuery) elastic.Query { // filter by org and active contacts eq := elastic.NewBoolQuery().Must( elastic.NewTermQuery("org_id", oa.OrgID()), @@ -25,12 +39,12 @@ func BuildElasticQuery(oa *OrgAssets, group assets.GroupUUID, status ContactStat ) // our group if present - if group != "" { - eq = eq.Must(elastic.NewTermQuery("groups", group)) + if group != nil { + eq = eq.Must(elastic.NewTermQuery("group_ids", group.ID())) } // our status is present - if status != NilContactStatus { + if status != models.NilContactStatus { eq = eq.Must(elastic.NewTermQuery("status", status)) } @@ -45,7 +59,7 @@ func BuildElasticQuery(oa *OrgAssets, group assets.GroupUUID, status ContactStat // and by our query if present if query != nil { - q := es.ToElasticQuery(oa.Env(), query) + q := es.ToElasticQuery(oa.Env(), assetMapper, query) eq = eq.Must(q) } @@ -53,7 +67,7 @@ func BuildElasticQuery(oa *OrgAssets, group assets.GroupUUID, status ContactStat } // GetContactIDsForQueryPage returns a page of contact ids for the given query and sort -func GetContactIDsForQueryPage(ctx context.Context, client *elastic.Client, oa *OrgAssets, group assets.GroupUUID, excludeIDs []ContactID, query string, sort string, offset int, pageSize int) (*contactql.ContactQuery, []ContactID, int64, error) { +func GetContactIDsForQueryPage(ctx context.Context, client *elastic.Client, oa *models.OrgAssets, group *models.Group, excludeIDs []models.ContactID, query string, sort string, offset int, pageSize int) (*contactql.ContactQuery, []models.ContactID, int64, error) { env := oa.Env() start := time.Now() var parsed *contactql.ContactQuery @@ -70,7 +84,7 @@ func GetContactIDsForQueryPage(ctx context.Context, client *elastic.Client, oa * } } - eq := BuildElasticQuery(oa, group, NilContactStatus, excludeIDs, parsed) + eq := BuildElasticQuery(oa, group, models.NilContactStatus, excludeIDs, parsed) fieldSort, err := es.ToElasticFieldSort(sort, oa.SessionAssets()) if err != nil { @@ -91,27 +105,19 @@ func GetContactIDsForQueryPage(ctx context.Context, client *elastic.Client, oa * return nil, nil, 0, errors.Wrapf(err, "error performing query: %s", ee.Details.Reason) } - ids := make([]ContactID, 0, pageSize) + ids := make([]models.ContactID, 0, pageSize) ids, err = appendIDsFromHits(ids, results.Hits.Hits) if err != nil { return nil, nil, 0, err } - logrus.WithFields(logrus.Fields{ - "org_id": oa.OrgID(), - "parsed": parsed, - "group_uuid": group, - "query": query, - "elapsed": time.Since(start), - "page_count": len(ids), - "total_count": results.Hits.TotalHits, - }).Debug("paged contact query complete") + logrus.WithFields(logrus.Fields{"org_id": oa.OrgID(), "query": query, "elapsed": time.Since(start), "page_count": len(ids), "total_count": results.Hits.TotalHits.Value}).Debug("paged contact query complete") return parsed, ids, results.Hits.TotalHits.Value, nil } // GetContactIDsForQuery returns up to limit the contact ids that match the given query without sorting. Limit of -1 means return all. -func GetContactIDsForQuery(ctx context.Context, client *elastic.Client, oa *OrgAssets, query string, limit int) ([]ContactID, error) { +func GetContactIDsForQuery(ctx context.Context, client *elastic.Client, oa *models.OrgAssets, query string, limit int) ([]models.ContactID, error) { env := oa.Env() start := time.Now() @@ -126,8 +132,8 @@ func GetContactIDsForQuery(ctx context.Context, client *elastic.Client, oa *OrgA } routing := strconv.FormatInt(int64(oa.OrgID()), 10) - eq := BuildElasticQuery(oa, "", ContactStatusActive, nil, parsed) - ids := make([]ContactID, 0, 100) + eq := BuildElasticQuery(oa, nil, models.ContactStatusActive, nil, parsed) + ids := make([]models.ContactID, 0, 100) // if limit provided that can be done with regular search, do that if limit >= 0 && limit <= 10000 { @@ -165,14 +171,14 @@ func GetContactIDsForQuery(ctx context.Context, client *elastic.Client, oa *OrgA } // utility to convert search hits to contact IDs and append them to the given slice -func appendIDsFromHits(ids []ContactID, hits []*elastic.SearchHit) ([]ContactID, error) { +func appendIDsFromHits(ids []models.ContactID, hits []*elastic.SearchHit) ([]models.ContactID, error) { for _, hit := range hits { id, err := strconv.Atoi(hit.Id) if err != nil { return nil, errors.Wrapf(err, "unexpected non-integer contact id: %s", hit.Id) } - ids = append(ids, ContactID(id)) + ids = append(ids, models.ContactID(id)) } return ids, nil } diff --git a/core/models/search_test.go b/core/search/search_test.go similarity index 64% rename from core/models/search_test.go rename to core/search/search_test.go index 19f85dd80..c3cb5769e 100644 --- a/core/models/search_test.go +++ b/core/search/search_test.go @@ -1,15 +1,13 @@ -package models_test +package search_test import ( - "fmt" "testing" - "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/test" "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/core/search" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/testsuite/testdata" - "github.com/olivere/elastic/v7" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -18,32 +16,29 @@ import ( func TestGetContactIDsForQueryPage(t *testing.T) { ctx, rt, _, _ := testsuite.Get() - es := testsuite.NewMockElasticServer() - defer es.Close() + mockES := testsuite.NewMockElasticServer() + defer mockES.Close() - client, err := elastic.NewClient( - elastic.SetURL(es.URL()), - elastic.SetHealthcheck(false), - elastic.SetSniff(false), - ) - require.NoError(t, err) + mockES.AddResponse(testdata.George.ID) + mockES.AddResponse(testdata.George.ID) + + es := mockES.Client() oa, err := models.GetOrgAssets(ctx, rt, testdata.Org1.ID) require.NoError(t, err) tcs := []struct { - Group assets.GroupUUID + Group *testdata.Group ExcludeIDs []models.ContactID Query string Sort string ExpectedESRequest string - MockedESResponse string ExpectedContacts []models.ContactID ExpectedTotal int64 ExpectedError string }{ { - Group: testdata.AllContactsGroup.UUID, + Group: testdata.ActiveGroup, Query: "george", ExpectedESRequest: `{ "_source": false, @@ -63,7 +58,7 @@ func TestGetContactIDsForQueryPage(t *testing.T) { }, { "term": { - "groups": "d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008" + "group_ids": 1 } }, { @@ -86,38 +81,11 @@ func TestGetContactIDsForQueryPage(t *testing.T) { ], "track_total_hits": true }`, - MockedESResponse: fmt.Sprintf(`{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }`, testdata.George.ID), ExpectedContacts: []models.ContactID{testdata.George.ID}, ExpectedTotal: 1, }, { - Group: testdata.BlockedContactsGroup.UUID, + Group: testdata.BlockedGroup, ExcludeIDs: []models.ContactID{testdata.Bob.ID, testdata.Cathy.ID}, Query: "age > 32", Sort: "-age", @@ -139,7 +107,7 @@ func TestGetContactIDsForQueryPage(t *testing.T) { }, { "term": { - "groups": "9295ebab-5c2d-4eb1-86f9-7c15ed2f3219" + "group_ids": 2 } }, { @@ -198,46 +166,20 @@ func TestGetContactIDsForQueryPage(t *testing.T) { ], "track_total_hits": true }`, - MockedESResponse: fmt.Sprintf(`{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }`, testdata.George.ID), ExpectedContacts: []models.ContactID{testdata.George.ID}, ExpectedTotal: 1, }, { + Group: testdata.ActiveGroup, Query: "goats > 2", // no such contact field ExpectedError: "error parsing query: goats > 2: can't resolve 'goats' to attribute, scheme or field", }, } for i, tc := range tcs { - es.NextResponse = tc.MockedESResponse + group := oa.GroupByID(tc.Group.ID) - _, ids, total, err := models.GetContactIDsForQueryPage(ctx, client, oa, tc.Group, tc.ExcludeIDs, tc.Query, tc.Sort, 0, 50) + _, ids, total, err := search.GetContactIDsForQueryPage(ctx, es, oa, group, tc.ExcludeIDs, tc.Query, tc.Sort, 0, 50) if tc.ExpectedError != "" { assert.EqualError(t, err, tc.ExpectedError) @@ -246,7 +188,7 @@ func TestGetContactIDsForQueryPage(t *testing.T) { assert.Equal(t, tc.ExpectedContacts, ids, "%d: ids mismatch", i) assert.Equal(t, tc.ExpectedTotal, total, "%d: total mismatch", i) - test.AssertEqualJSON(t, []byte(tc.ExpectedESRequest), []byte(es.LastRequestBody), "%d: ES request mismatch", i) + test.AssertEqualJSON(t, []byte(tc.ExpectedESRequest), []byte(mockES.LastRequestBody), "%d: ES request mismatch", i) } } } @@ -254,14 +196,14 @@ func TestGetContactIDsForQueryPage(t *testing.T) { func TestGetContactIDsForQuery(t *testing.T) { ctx, rt, _, _ := testsuite.Get() - es := testsuite.NewMockElasticServer() - defer es.Close() + mockES := testsuite.NewMockElasticServer() + defer mockES.Close() - client, err := elastic.NewClient( - elastic.SetURL(es.URL()), - elastic.SetHealthcheck(false), - elastic.SetSniff(false), - ) + mockES.AddResponse(testdata.George.ID) + mockES.AddResponse() + mockES.AddResponse(testdata.George.ID) + + es, err := elastic.NewClient(elastic.SetURL(mockES.URL()), elastic.SetHealthcheck(false), elastic.SetSniff(false)) require.NoError(t, err) oa, err := models.GetOrgAssets(ctx, rt, 1) @@ -312,33 +254,6 @@ func TestGetContactIDsForQuery(t *testing.T) { }, "sort":["_doc"] }`, - mockedESResponse: fmt.Sprintf(`{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }`, testdata.George.ID), expectedContacts: []models.ContactID{testdata.George.ID}, }, { query: "nobody", @@ -376,22 +291,6 @@ func TestGetContactIDsForQuery(t *testing.T) { }, "sort":["_doc"] }`, - mockedESResponse: `{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 0, - "max_score": null, - "hits": [] - } - }`, expectedContacts: []models.ContactID{}, }, { @@ -431,24 +330,6 @@ func TestGetContactIDsForQuery(t *testing.T) { }, "size": 1 }`, - mockedESResponse: fmt.Sprintf(`{ - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }`, testdata.George.ID), expectedContacts: []models.ContactID{testdata.George.ID}, }, { @@ -459,9 +340,7 @@ func TestGetContactIDsForQuery(t *testing.T) { } for i, tc := range tcs { - es.NextResponse = tc.mockedESResponse - - ids, err := models.GetContactIDsForQuery(ctx, client, oa, tc.query, tc.limit) + ids, err := search.GetContactIDsForQuery(ctx, es, oa, tc.query, tc.limit) if tc.expectedError != "" { assert.EqualError(t, err, tc.expectedError) @@ -469,8 +348,8 @@ func TestGetContactIDsForQuery(t *testing.T) { assert.NoError(t, err, "%d: error encountered performing query", i) assert.Equal(t, tc.expectedContacts, ids, "%d: ids mismatch", i) - assert.Equal(t, tc.expectedRequestURL, es.LastRequestURL, "%d: request URL mismatch", i) - test.AssertEqualJSON(t, []byte(tc.expectedRequestBody), []byte(es.LastRequestBody), "%d: request body mismatch", i) + assert.Equal(t, tc.expectedRequestURL, mockES.LastRequestURL, "%d: request URL mismatch", i) + test.AssertEqualJSON(t, []byte(tc.expectedRequestBody), []byte(mockES.LastRequestBody), "%d: request body mismatch", i) } } } diff --git a/core/tasks/analytics/cron.go b/core/tasks/analytics/cron.go index 8774a81f1..a12647993 100644 --- a/core/tasks/analytics/cron.go +++ b/core/tasks/analytics/cron.go @@ -2,31 +2,17 @@ package analytics import ( "context" - "sync" "time" - "github.com/nyaruka/librato" + "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/sirupsen/logrus" ) func init() { - mailroom.AddInitFunction(StartAnalyticsCron) -} - -// StartAnalyticsCron starts our cron job of posting stats every minute -func StartAnalyticsCron(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, "stats", time.Second*60, true, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return reportAnalytics(ctx, rt) - }, - ) - return nil + mailroom.RegisterCron("analytics", time.Second*60, true, reportAnalytics) } var ( @@ -74,14 +60,14 @@ func reportAnalytics(ctx context.Context, rt *runtime.Runtime) error { redisWaitDuration = redisStats.WaitDuration redisWaitCount = redisStats.WaitCount - librato.Gauge("mr.db_busy", float64(dbStats.InUse)) - librato.Gauge("mr.db_idle", float64(dbStats.Idle)) - librato.Gauge("mr.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond)) - librato.Gauge("mr.db_wait_count", float64(dbWaitCountInPeriod)) - librato.Gauge("mr.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond)) - librato.Gauge("mr.redis_wait_count", float64(redisWaitCountInPeriod)) - librato.Gauge("mr.handler_queue", float64(handlerSize)) - librato.Gauge("mr.batch_queue", float64(batchSize)) + analytics.Gauge("mr.db_busy", float64(dbStats.InUse)) + analytics.Gauge("mr.db_idle", float64(dbStats.Idle)) + analytics.Gauge("mr.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond)) + analytics.Gauge("mr.db_wait_count", float64(dbWaitCountInPeriod)) + analytics.Gauge("mr.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond)) + analytics.Gauge("mr.redis_wait_count", float64(redisWaitCountInPeriod)) + analytics.Gauge("mr.handler_queue", float64(handlerSize)) + analytics.Gauge("mr.batch_queue", float64(batchSize)) logrus.WithFields(logrus.Fields{ "db_busy": dbStats.InUse, diff --git a/core/tasks/campaigns/cron.go b/core/tasks/campaigns/cron.go index 4eaf92bc0..c6bdd0008 100644 --- a/core/tasks/campaigns/cron.go +++ b/core/tasks/campaigns/cron.go @@ -3,17 +3,15 @@ package campaigns import ( "context" "fmt" - "sync" "time" "github.com/gomodule/redigo/redis" + "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/goflow/assets" - "github.com/nyaruka/librato" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/nyaruka/redisx" "github.com/pkg/errors" @@ -27,20 +25,7 @@ const ( var campaignsMarker = redisx.NewIntervalSet("campaign_event", time.Hour*24, 2) func init() { - mailroom.AddInitFunction(StartCampaignCron) -} - -// StartCampaignCron starts our cron job of firing expired campaign events -func StartCampaignCron(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, "campaign_event", time.Second*60, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return QueueEventFires(ctx, rt) - }, - ) - - return nil + mailroom.RegisterCron("campaign_event", time.Second*60, false, QueueEventFires) } // QueueEventFires looks for all due campaign event fires and queues them to be started @@ -122,8 +107,8 @@ func QueueEventFires(ctx context.Context, rt *runtime.Runtime) error { numTasks++ } - librato.Gauge("mr.campaign_event_cron_elapsed", float64(time.Since(start))/float64(time.Second)) - librato.Gauge("mr.campaign_event_cron_count", float64(numFires)) + analytics.Gauge("mr.campaign_event_cron_elapsed", float64(time.Since(start))/float64(time.Second)) + analytics.Gauge("mr.campaign_event_cron_count", float64(numFires)) log.WithFields(logrus.Fields{ "elapsed": time.Since(start), "fires": numFires, diff --git a/core/tasks/contacts/populate_dynamic_group.go b/core/tasks/contacts/populate_dynamic_group.go index 0ee825dd3..b12baf1fb 100644 --- a/core/tasks/contacts/populate_dynamic_group.go +++ b/core/tasks/contacts/populate_dynamic_group.go @@ -6,6 +6,7 @@ import ( "time" "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/core/search" "github.com/nyaruka/mailroom/core/tasks" "github.com/nyaruka/mailroom/runtime" "github.com/nyaruka/redisx" @@ -39,7 +40,7 @@ func (t *PopulateDynamicGroupTask) Perform(ctx context.Context, rt *runtime.Runt locker := redisx.NewLocker(fmt.Sprintf(populateLockKey, t.GroupID), time.Hour) lock, err := locker.Grab(rt.RP, time.Minute*5) if err != nil { - return errors.Wrapf(err, "error grabbing lock to repopulate dynamic group: %d", t.GroupID) + return errors.Wrapf(err, "error grabbing lock to repopulate smart group: %d", t.GroupID) } defer locker.Release(rt.RP, lock) @@ -50,18 +51,18 @@ func (t *PopulateDynamicGroupTask) Perform(ctx context.Context, rt *runtime.Runt "query": t.Query, }) - log.Info("starting population of dynamic group") + log.Info("starting population of smart group") oa, err := models.GetOrgAssets(ctx, rt, orgID) if err != nil { return errors.Wrapf(err, "unable to load org when populating group: %d", t.GroupID) } - count, err := models.PopulateDynamicGroup(ctx, rt.DB, rt.ES, oa, t.GroupID, t.Query) + count, err := search.PopulateSmartGroup(ctx, rt.DB, rt.ES, oa, t.GroupID, t.Query) if err != nil { - return errors.Wrapf(err, "error populating dynamic group: %d", t.GroupID) + return errors.Wrapf(err, "error populating smart group: %d", t.GroupID) } - logrus.WithField("elapsed", time.Since(start)).WithField("count", count).Info("completed populating dynamic group") + logrus.WithField("elapsed", time.Since(start)).WithField("count", count).Info("completed populating smart group") return nil } diff --git a/core/tasks/contacts/populate_dynamic_group_test.go b/core/tasks/contacts/populate_dynamic_group_test.go index 5398f0167..caefa03b0 100644 --- a/core/tasks/contacts/populate_dynamic_group_test.go +++ b/core/tasks/contacts/populate_dynamic_group_test.go @@ -1,7 +1,6 @@ package contacts_test import ( - "fmt" "testing" "github.com/nyaruka/gocommon/dates" @@ -10,7 +9,6 @@ import ( "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/testsuite/testdata" - "github.com/olivere/elastic/v7" "github.com/stretchr/testify/require" ) @@ -19,41 +17,12 @@ func TestPopulateTask(t *testing.T) { defer testsuite.Reset(testsuite.ResetAll) - mes := testsuite.NewMockElasticServer() - defer mes.Close() - es, err := elastic.NewClient( - elastic.SetURL(mes.URL()), - elastic.SetHealthcheck(false), - elastic.SetSniff(false), - ) - require.NoError(t, err) - rt.ES = es + mockES := testsuite.NewMockElasticServer() + defer mockES.Close() + + mockES.AddResponse(testdata.Cathy.ID) - mes.NextResponse = fmt.Sprintf(`{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [15124352] - } - ] - } - }`, testdata.Cathy.ID) + rt.ES = mockES.Client() group := testdata.InsertContactGroup(db, testdata.Org1, "e52fee05-2f95-4445-aef6-2fe7dac2fd56", "Women", "gender = F") start := dates.Now() @@ -62,7 +31,7 @@ func TestPopulateTask(t *testing.T) { GroupID: group.ID, Query: "gender = F", } - err = task.Perform(ctx, rt, testdata.Org1.ID) + err := task.Perform(ctx, rt, testdata.Org1.ID) require.NoError(t, err) assertdb.Query(t, db, `SELECT count(*) FROM contacts_contactgroup_contacts WHERE contactgroup_id = $1`, group.ID).Returns(1) diff --git a/core/tasks/expirations/cron.go b/core/tasks/expirations/cron.go index 442f19ffa..f7ed65365 100644 --- a/core/tasks/expirations/cron.go +++ b/core/tasks/expirations/cron.go @@ -3,7 +3,6 @@ package expirations import ( "context" "fmt" - "sync" "time" "github.com/nyaruka/mailroom" @@ -11,7 +10,6 @@ import ( "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/tasks/handler" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/nyaruka/redisx" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -24,28 +22,8 @@ const ( var expirationsMarker = redisx.NewIntervalSet("run_expirations", time.Hour*24, 2) func init() { - mailroom.AddInitFunction(StartExpirationCron) -} - -// StartExpirationCron starts our cron job of expiring runs every minute -func StartExpirationCron(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, "run_expirations", time.Minute, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return HandleWaitExpirations(ctx, rt) - }, - ) - - cron.Start(quit, rt, "expire_ivr_calls", time.Minute, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return ExpireVoiceSessions(ctx, rt) - }, - ) - - return nil + mailroom.RegisterCron("run_expirations", time.Minute, false, HandleWaitExpirations) + mailroom.RegisterCron("expire_ivr_calls", time.Minute, false, ExpireVoiceSessions) } // HandleWaitExpirations handles waiting messaging sessions whose waits have expired, resuming those that can be resumed, @@ -77,7 +55,7 @@ func HandleWaitExpirations(ctx context.Context, rt *runtime.Runtime) error { } // if it can't be resumed, add to batch to be expired - if !expiredWait.WaitResumes || expiredWait.ContactStatus != models.ContactStatusActive { + if !expiredWait.WaitResumes { expiredSessions = append(expiredSessions, expiredWait.SessionID) // batch is full? commit it @@ -135,20 +113,18 @@ func HandleWaitExpirations(ctx context.Context, rt *runtime.Runtime) error { } const sqlSelectExpiredWaits = ` - SELECT s.id as session_id, s.org_id, s.wait_expires_on, s.wait_resume_on_expire , c.id as contact_id, c.status as contact_status + SELECT s.id as session_id, s.org_id, s.wait_expires_on, s.wait_resume_on_expire , s.contact_id FROM flows_flowsession s -INNER JOIN contacts_contact c ON c.id = s.contact_id WHERE s.session_type = 'M' AND s.status = 'W' AND s.wait_expires_on <= NOW() ORDER BY s.wait_expires_on ASC LIMIT 25000` type ExpiredWait struct { - SessionID models.SessionID `db:"session_id"` - OrgID models.OrgID `db:"org_id"` - WaitExpiresOn time.Time `db:"wait_expires_on"` - WaitResumes bool `db:"wait_resume_on_expire"` - ContactID models.ContactID `db:"contact_id"` - ContactStatus models.ContactStatus `db:"contact_status"` + SessionID models.SessionID `db:"session_id"` + OrgID models.OrgID `db:"org_id"` + WaitExpiresOn time.Time `db:"wait_expires_on"` + WaitResumes bool `db:"wait_resume_on_expire"` + ContactID models.ContactID `db:"contact_id"` } // ExpireVoiceSessions looks for voice sessions that should be expired and ends them diff --git a/core/tasks/expirations/cron_test.go b/core/tasks/expirations/cron_test.go index 4282927a0..741cab636 100644 --- a/core/tasks/expirations/cron_test.go +++ b/core/tasks/expirations/cron_test.go @@ -1,11 +1,11 @@ package expirations_test import ( - "encoding/json" "testing" "time" "github.com/nyaruka/gocommon/dbutil/assertdb" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/goflow/envs" _ "github.com/nyaruka/mailroom/core/handlers" "github.com/nyaruka/mailroom/core/models" @@ -47,7 +47,7 @@ func TestExpirations(t *testing.T) { // create a parent/child session for the blocked contact s5ID := testdata.InsertWaitingSession(db, testdata.Org1, blake, models.FlowTypeMessaging, testdata.Favorites, models.NilConnectionID, time.Now(), time.Now(), true, nil) - r6ID := testdata.InsertFlowRun(db, testdata.Org1, s5ID, blake, testdata.Favorites, models.RunStatusWaiting) + r6ID := testdata.InsertFlowRun(db, testdata.Org1, s5ID, blake, testdata.Favorites, models.RunStatusActive) r7ID := testdata.InsertFlowRun(db, testdata.Org1, s5ID, blake, testdata.Favorites, models.RunStatusWaiting) time.Sleep(5 * time.Millisecond) @@ -73,23 +73,29 @@ func TestExpirations(t *testing.T) { assertdb.Query(t, db, `SELECT status FROM flows_flowsession WHERE id = $1;`, s4ID).Columns(map[string]interface{}{"status": "W"}) assertdb.Query(t, db, `SELECT status FROM flows_flowrun WHERE id = $1;`, r5ID).Columns(map[string]interface{}{"status": "W"}) - // blocked contact's session and runs should be expired because a blocked contact can't resume - assertdb.Query(t, db, `SELECT status FROM flows_flowsession WHERE id = $1;`, s5ID).Columns(map[string]interface{}{"status": "X"}) - assertdb.Query(t, db, `SELECT status FROM flows_flowrun WHERE id = $1;`, r6ID).Columns(map[string]interface{}{"status": "X"}) - assertdb.Query(t, db, `SELECT status FROM flows_flowrun WHERE id = $1;`, r7ID).Columns(map[string]interface{}{"status": "X"}) + // blocked contact's session and runs sshould be unchanged because it's been queued for resumption.. like any other contact + assertdb.Query(t, db, `SELECT status FROM flows_flowsession WHERE id = $1;`, s5ID).Columns(map[string]interface{}{"status": "W"}) + assertdb.Query(t, db, `SELECT status FROM flows_flowrun WHERE id = $1;`, r6ID).Columns(map[string]interface{}{"status": "A"}) + assertdb.Query(t, db, `SELECT status FROM flows_flowrun WHERE id = $1;`, r7ID).Columns(map[string]interface{}{"status": "W"}) - // should have created one task + // should have created two expiration tasks task, err := queue.PopNextTask(rc, queue.HandlerQueue) assert.NoError(t, err) assert.NotNil(t, task) - // decode the task + // check the first task eventTask := &handler.HandleEventTask{} - err = json.Unmarshal(task.Task, eventTask) + jsonx.MustUnmarshal(task.Task, eventTask) + assert.Equal(t, testdata.George.ID, eventTask.ContactID) + + task, err = queue.PopNextTask(rc, queue.HandlerQueue) assert.NoError(t, err) + assert.NotNil(t, task) - // assert its the right contact - assert.Equal(t, testdata.George.ID, eventTask.ContactID) + // check the second task + eventTask = &handler.HandleEventTask{} + jsonx.MustUnmarshal(task.Task, eventTask) + assert.Equal(t, blake.ID, eventTask.ContactID) // no other tasks task, err = queue.PopNextTask(rc, queue.HandlerQueue) diff --git a/core/tasks/handler/cron.go b/core/tasks/handler/cron.go index 944bd0c92..4816aa9f9 100644 --- a/core/tasks/handler/cron.go +++ b/core/tasks/handler/cron.go @@ -4,40 +4,21 @@ import ( "context" "encoding/json" "fmt" - "sync" "time" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/nyaruka/redisx" - "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -const ( - retryLock = "retry_msgs" -) - var retriedMsgs = redisx.NewIntervalSet("retried_msgs", time.Hour*24, 2) func init() { - mailroom.AddInitFunction(StartRetryCron) -} - -// StartRetryCron starts our cron job of retrying pending incoming messages -func StartRetryCron(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, retryLock, time.Minute*5, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return RetryPendingMsgs(ctx, rt) - }, - ) - return nil + mailroom.RegisterCron("retry_msgs", time.Minute*5, false, RetryPendingMsgs) } // RetryPendingMsgs looks for any pending msgs older than five minutes and queues them to be handled again diff --git a/core/tasks/handler/handler_test.go b/core/tasks/handler/handler_test.go index db13e5ebe..2e3497850 100644 --- a/core/tasks/handler/handler_test.go +++ b/core/tasks/handler/handler_test.go @@ -38,8 +38,8 @@ func TestMsgEvents(t *testing.T) { // give Cathy and Bob some tickets... openTickets := map[*testdata.Contact][]*testdata.Ticket{ testdata.Cathy: { - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Ok", "", nil), - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Ok", "", nil), + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Ok", "", time.Now(), nil), + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Ok", "", time.Now(), nil), }, } closedTickets := map[*testdata.Contact][]*testdata.Ticket{ @@ -71,6 +71,7 @@ func TestMsgEvents(t *testing.T) { expectedType models.MsgType expectedFlow *testdata.Flow }{ + // 0: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -79,6 +80,8 @@ func TestMsgEvents(t *testing.T) { expectedReply: "", expectedType: models.MsgTypeInbox, }, + + // 1: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -87,6 +90,8 @@ func TestMsgEvents(t *testing.T) { expectedReply: "", expectedType: models.MsgTypeInbox, }, + + // 2: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -96,6 +101,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Favorites, }, + + // 3: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -105,6 +112,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Favorites, }, + + // 4: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -114,6 +123,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Favorites, }, + + // 5: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -123,6 +134,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Favorites, }, + + // 6: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -132,6 +145,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Favorites, }, + + // 7: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -141,6 +156,7 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeInbox, }, + // 8: { org: testdata.Org2, channel: testdata.Org2Channel, @@ -150,6 +166,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Org2SingleMessage, }, + + // 9: { org: testdata.Org2, channel: testdata.Org2Channel, @@ -159,6 +177,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Org2Favorites, }, + + // 10: { org: testdata.Org2, channel: testdata.Org2Channel, @@ -168,6 +188,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Org2Favorites, }, + + // 11: { org: testdata.Org2, channel: testdata.Org2Channel, @@ -177,6 +199,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Org2Favorites, }, + + // 12: { org: testdata.Org2, channel: testdata.Org2Channel, @@ -186,6 +210,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Org2Favorites, }, + + // 13: { org: testdata.Org2, channel: testdata.Org2Channel, @@ -196,6 +222,7 @@ func TestMsgEvents(t *testing.T) { expectedFlow: testdata.Org2SingleMessage, }, + // 14: { org: testdata.Org1, channel: testdata.TwitterChannel, @@ -206,7 +233,21 @@ func TestMsgEvents(t *testing.T) { expectedFlow: testdata.IVRFlow, }, - // no URN on contact but handle event, session gets started but no message created + // 15: stopped contact should be unstopped + { + preHook: func() { + db.MustExec(`UPDATE contacts_contact SET status = 'S' WHERE id = $1`, testdata.George.ID) + }, + org: testdata.Org1, + channel: testdata.TwitterChannel, + contact: testdata.George, + text: "start", + expectedReply: "What is your favorite color?", + expectedType: models.MsgTypeFlow, + expectedFlow: testdata.Favorites, + }, + + // 16: no URN on contact but handle event, session gets started but no message created { org: testdata.Org1, channel: testdata.TwilioChannel, @@ -217,7 +258,7 @@ func TestMsgEvents(t *testing.T) { expectedFlow: testdata.Favorites, }, - // start Fred back in our favorite flow, then make it inactive, will be handled by catch-all + // 17: start Fred back in our favorite flow, then make it inactive, will be handled by catch-all { org: testdata.Org2, channel: testdata.Org2Channel, @@ -227,6 +268,8 @@ func TestMsgEvents(t *testing.T) { expectedType: models.MsgTypeFlow, expectedFlow: testdata.Org2Favorites, }, + + // 18: { preHook: func() { db.MustExec(`UPDATE flows_flow SET is_active = FALSE WHERE id = $1`, testdata.Org2Favorites.ID) @@ -240,7 +283,7 @@ func TestMsgEvents(t *testing.T) { expectedFlow: testdata.Org2SingleMessage, }, - // start Fred back in our favorites flow to test retries + // 19: start Fred back in our favorites flow to test retries { preHook: func() { db.MustExec(`UPDATE flows_flow SET is_active = TRUE WHERE id = $1`, testdata.Org2Favorites.ID) @@ -303,8 +346,8 @@ func TestMsgEvents(t *testing.T) { // if we are meant to have a reply, check it if tc.expectedReply != "" { - assertdb.Query(t, db, `SELECT text FROM msgs_msg WHERE contact_id = $1 AND created_on > $2 ORDER BY id DESC LIMIT 1`, tc.contact.ID, last). - Returns(tc.expectedReply, "%d: response mismatch", i) + assertdb.Query(t, db, `SELECT text, status FROM msgs_msg WHERE contact_id = $1 AND created_on > $2 ORDER BY id DESC LIMIT 1`, tc.contact.ID, last). + Columns(map[string]interface{}{"text": tc.expectedReply, "status": "Q"}, "%d: response mismatch", i) } // check any open tickets for this contact where updated @@ -331,7 +374,7 @@ func TestMsgEvents(t *testing.T) { // check messages queued to courier testsuite.AssertCourierQueues(t, map[string][]int{ - fmt.Sprintf("msgs:%s|10/1", testdata.TwitterChannel.UUID): {1, 1, 1, 1, 1}, + fmt.Sprintf("msgs:%s|10/1", testdata.TwitterChannel.UUID): {1, 1, 1, 1, 1, 1}, fmt.Sprintf("msgs:%s|10/1", testdata.Org2Channel.UUID): {1, 1, 1, 1, 1, 1, 1, 1, 1}, }) @@ -535,8 +578,9 @@ func TestTimedEvents(t *testing.T) { defer testsuite.Reset(testsuite.ResetAll) - // start to start our favorites flow + // create some keyword triggers testdata.InsertKeywordTrigger(db, testdata.Org1, testdata.Favorites, "start", models.MatchOnly, nil, nil) + testdata.InsertKeywordTrigger(db, testdata.Org1, testdata.PickANumber, "pick", models.MatchOnly, nil, nil) tcs := []struct { EventType string @@ -575,6 +619,15 @@ func TestTimedEvents(t *testing.T) { // 9: start our favorite flow again {handler.MsgEventType, testdata.Cathy, "start", "What is your favorite color?", testdata.TwitterChannel.ID, testdata.Org1.ID}, + + // 10: timeout on the color question + {handler.TimeoutEventType, testdata.Cathy, "", "Sorry you can't participate right now, I'll try again later.", testdata.TwitterChannel.ID, testdata.Org1.ID}, + + // 11: start the pick a number flow + {handler.MsgEventType, testdata.Cathy, "pick", "Pick a number between 1-10.", testdata.TwitterChannel.ID, testdata.Org1.ID}, + + // 12: try to resume with timeout even tho flow doesn't have one set + {handler.TimeoutEventType, testdata.Cathy, "", "", testdata.TwitterChannel.ID, testdata.Org1.ID}, } last := time.Now() @@ -585,25 +638,21 @@ func TestTimedEvents(t *testing.T) { time.Sleep(50 * time.Millisecond) var task *queue.Task - if tc.EventType == handler.MsgEventType { - event := &handler.MsgEvent{ - ContactID: tc.Contact.ID, - OrgID: tc.OrgID, - ChannelID: tc.ChannelID, - MsgID: flows.MsgID(1), - MsgUUID: flows.MsgUUID(uuids.New()), - URN: tc.Contact.URN, - URNID: tc.Contact.URNID, - Text: tc.Message, - } - - eventJSON, err := json.Marshal(event) - assert.NoError(t, err) + if tc.EventType == handler.MsgEventType { task = &queue.Task{ Type: tc.EventType, OrgID: int(tc.OrgID), - Task: eventJSON, + Task: jsonx.MustMarshal(&handler.MsgEvent{ + ContactID: tc.Contact.ID, + OrgID: tc.OrgID, + ChannelID: tc.ChannelID, + MsgID: flows.MsgID(1), + MsgUUID: flows.MsgUUID(uuids.New()), + URN: tc.Contact.URN, + URNID: tc.Contact.URNID, + Text: tc.Message, + }), } } else if tc.EventType == handler.ExpirationEventType { var expiration time.Time @@ -618,6 +667,14 @@ func TestTimedEvents(t *testing.T) { } task = handler.NewExpirationTask(tc.OrgID, tc.Contact.ID, sessionID, expiration) + + } else if tc.EventType == handler.TimeoutEventType { + timeoutOn := time.Now().Round(time.Millisecond) // so that there's no difference between this and what we read from the db + + // usually courier will set timeout_on after sending the last message + db.MustExec(`UPDATE flows_flowsession SET timeout_on = $2 WHERE id = $1`, sessionID, timeoutOn) + + task = handler.NewTimeoutTask(tc.OrgID, tc.Contact.ID, sessionID, timeoutOn) } err := handler.QueueHandleTask(rc, tc.Contact.ID, task) @@ -643,9 +700,10 @@ func TestTimedEvents(t *testing.T) { last = time.Now() } - // should only have a single waiting session/run per contact - assertdb.Query(t, db, `SELECT count(*) from flows_flowsession WHERE status = 'W' AND contact_id = $1`, testdata.Cathy.ID).Returns(1) - assertdb.Query(t, db, `SELECT count(*) from flows_flowrun WHERE status = 'W' AND contact_id = $1`, testdata.Cathy.ID).Returns(1) + // should only have a single waiting session/run with no timeout + assertdb.Query(t, db, `SELECT count(*) FROM flows_flowsession WHERE status = 'W' AND contact_id = $1`, testdata.Cathy.ID).Returns(1) + assertdb.Query(t, db, `SELECT timeout_on FROM flows_flowsession WHERE status = 'W' AND contact_id = $1`, testdata.Cathy.ID).Returns(nil) + assertdb.Query(t, db, `SELECT count(*) FROM flows_flowrun WHERE status = 'W' AND contact_id = $1`, testdata.Cathy.ID).Returns(1) // test the case of a run and session no longer being the most recent but somehow still active, expiration should still work r, err := db.QueryContext(ctx, `SELECT id, session_id from flows_flowrun WHERE contact_id = $1 and status = 'I' order by created_on asc limit 1`, testdata.Cathy.ID) diff --git a/core/tasks/handler/worker.go b/core/tasks/handler/worker.go index b03f5935d..4f3528a2c 100644 --- a/core/tasks/handler/worker.go +++ b/core/tasks/handler/worker.go @@ -8,21 +8,21 @@ import ( "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/flows/engine" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/flows/resumes" "github.com/nyaruka/goflow/flows/triggers" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/librato" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/core/runner" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/locker" "github.com/nyaruka/null" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -61,8 +61,9 @@ func handleContactEvent(ctx context.Context, rt *runtime.Runtime, task *queue.Ta } // acquire the lock for this contact - lockID := models.ContactLock(models.OrgID(task.OrgID), eventTask.ContactID) - lock, err := locker.GrabLock(rt.RP, lockID, time.Minute*5, time.Second*10) + locker := models.GetContactLocker(models.OrgID(task.OrgID), eventTask.ContactID) + + lock, err := locker.Grab(rt.RP, time.Second*10) if err != nil { return errors.Wrapf(err, "error acquiring lock for contact %d", eventTask.ContactID) } @@ -81,7 +82,7 @@ func handleContactEvent(ctx context.Context, rt *runtime.Runtime, task *queue.Ta }).Info("failed to get lock for contact, requeued and skipping") return nil } - defer locker.ReleaseLock(rt.RP, lockID, lock) + defer locker.Release(rt.RP, lock) // read all the events for this contact, one by one contactQ := fmt.Sprintf("c:%d:%d", task.OrgID, eventTask.ContactID) @@ -158,10 +159,10 @@ func handleContactEvent(ctx context.Context, rt *runtime.Runtime, task *queue.Ta } // log our processing time to librato - librato.Gauge(fmt.Sprintf("mr.%s_elapsed", contactEvent.Type), float64(time.Since(start))/float64(time.Second)) + analytics.Gauge(fmt.Sprintf("mr.%s_elapsed", contactEvent.Type), float64(time.Since(start))/float64(time.Second)) // and total latency for this task since it was queued - librato.Gauge(fmt.Sprintf("mr.%s_latency", contactEvent.Type), float64(time.Since(task.QueuedOn))/float64(time.Second)) + analytics.Gauge(fmt.Sprintf("mr.%s_latency", contactEvent.Type), float64(time.Since(task.QueuedOn))/float64(time.Second)) // if we get an error processing an event, requeue it for later and return our error if err != nil { @@ -210,11 +211,6 @@ func handleTimedEvent(ctx context.Context, rt *runtime.Runtime, eventType string return errors.Wrapf(err, "error loading contact") } - // contact has been deleted or is blocked/stopped/archived, ignore this event - if len(contacts) == 0 || contacts[0].Status() != models.ContactStatusActive { - return nil - } - modelContact := contacts[0] // build our flow contact @@ -278,7 +274,15 @@ func handleTimedEvent(ctx context.Context, rt *runtime.Runtime, eventType string _, err = runner.ResumeFlow(ctx, rt, oa, session, modelContact, resume, nil) if err != nil { - return errors.Wrapf(err, "error resuming flow for timeout") + // if we errored, and it's the wait rejecting the timeout event, it's because it no longer exists on the flow, so clear it + // on the session + var eerr *engine.Error + if errors.As(err, &eerr) && eerr.Code() == engine.ErrorResumeRejectedByWait && resume.Type() == resumes.TypeWaitTimeout { + log.WithField("session_id", session.ID()).Info("clearing session timeout which is no longer set in flow") + return errors.Wrap(session.ClearWaitTimeout(ctx, rt.DB), "error clearing session timeout") + } + + return errors.Wrap(err, "error resuming flow for timeout") } log.WithField("elapsed", time.Since(start)).Info("handled timed event") @@ -510,12 +514,6 @@ func handleMsgEvent(ctx context.Context, rt *runtime.Runtime, event *MsgEvent) e } } - // build our flow contact - contact, err := modelContact.FlowContact(oa) - if err != nil { - return errors.Wrapf(err, "error creating flow contact") - } - // if this channel is no longer active or this contact is blocked, ignore this message (mark it as handled) if channel == nil || modelContact.Status() == models.ContactStatusBlocked { err := models.UpdateMessage(ctx, rt.DB, event.MsgID, models.MsgStatusHandled, models.VisibilityArchived, models.MsgTypeInbox, models.NilFlowID, topupID) @@ -536,6 +534,12 @@ func handleMsgEvent(ctx context.Context, rt *runtime.Runtime, event *MsgEvent) e newContact = true } + // build our flow contact + contact, err := modelContact.FlowContact(oa) + if err != nil { + return errors.Wrapf(err, "error creating flow contact") + } + // if this is a new contact, we need to calculate dynamic groups and campaigns if newContact { err = models.CalculateDynamicGroups(ctx, rt.DB, oa, []*flows.Contact{contact}) diff --git a/core/tasks/incidents/end_incidents.go b/core/tasks/incidents/end_incidents.go index 9cc37442b..916d9bbe2 100644 --- a/core/tasks/incidents/end_incidents.go +++ b/core/tasks/incidents/end_incidents.go @@ -3,7 +3,6 @@ package incidents import ( "context" "fmt" - "sync" "time" "github.com/gomodule/redigo/redis" @@ -11,24 +10,12 @@ import ( "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) func init() { - mailroom.AddInitFunction(startEndCron) -} - -func startEndCron(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, "end_incidents", time.Minute*3, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return EndIncidents(ctx, rt) - }, - ) - return nil + mailroom.RegisterCron("end_incidents", time.Minute*3, false, EndIncidents) } // EndIncidents checks open incidents and end any that no longer apply diff --git a/core/tasks/ivr/cron.go b/core/tasks/ivr/cron.go index fd2f35a9d..3f628e966 100644 --- a/core/tasks/ivr/cron.go +++ b/core/tasks/ivr/cron.go @@ -2,37 +2,18 @@ package ivr import ( "context" - "sync" "time" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/ivr" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -const ( - retryIVRLock = "retry_ivr_calls" -) - func init() { - mailroom.AddInitFunction(StartIVRCron) -} - -// StartIVRCron starts our cron job of retrying errored calls -func StartIVRCron(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, retryIVRLock, time.Minute, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return RetryCalls(ctx, rt) - }, - ) - - return nil + mailroom.RegisterCron("retry_ivr_calls", time.Minute, false, RetryCalls) } // RetryCalls looks for calls that need to be retried and retries them diff --git a/core/tasks/ivr/cron_test.go b/core/tasks/ivr/cron_test.go index b585165d7..2dde012e0 100644 --- a/core/tasks/ivr/cron_test.go +++ b/core/tasks/ivr/cron_test.go @@ -29,7 +29,7 @@ func TestRetries(t *testing.T) { db.MustExec(`UPDATE channels_channel SET channel_type = 'ZZ', config = '{"max_concurrent_events": 1}' WHERE id = $1`, testdata.TwilioChannel.ID) // create a flow start for cathy - start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID, true, true). + start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID). WithContactIDs([]models.ContactID{testdata.Cathy.ID}) // call our master starter diff --git a/core/tasks/ivr/worker.go b/core/tasks/ivr/worker.go index 527dc4ef8..03fcc19d1 100644 --- a/core/tasks/ivr/worker.go +++ b/core/tasks/ivr/worker.go @@ -41,7 +41,7 @@ func HandleFlowStartBatch(bg context.Context, rt *runtime.Runtime, batch *models exclude := make(map[models.ContactID]bool, 5) // filter out anybody who has has a flow run in this flow if appropriate - if !batch.RestartParticipants() { + if batch.ExcludeStartedPreviously() { // find all participants that have been in this flow started, err := models.FindFlowStartedOverlap(ctx, rt.DB, batch.FlowID(), batch.ContactIDs()) if err != nil { @@ -53,7 +53,7 @@ func HandleFlowStartBatch(bg context.Context, rt *runtime.Runtime, batch *models } // filter out our list of contacts to only include those that should be started - if !batch.IncludeActive() { + if batch.ExcludeInAFlow() { // find all participants active in other sessions active, err := models.FilterByWaitingSession(ctx, rt.DB, batch.ContactIDs()) if err != nil { diff --git a/core/tasks/ivr/worker_test.go b/core/tasks/ivr/worker_test.go index 3c20744c5..2a9406624 100644 --- a/core/tasks/ivr/worker_test.go +++ b/core/tasks/ivr/worker_test.go @@ -36,7 +36,7 @@ func TestIVR(t *testing.T) { db.MustExec(`UPDATE channels_channel SET channel_type = 'ZZ', config = '{"max_concurrent_events": 1}' WHERE id = $1`, testdata.TwilioChannel.ID) // create a flow start for cathy - start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID, true, true). + start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID). WithContactIDs([]models.ContactID{testdata.Cathy.ID}) // call our master starter diff --git a/core/tasks/msgs/retries.go b/core/tasks/msgs/retries.go index 660963519..6c8de9617 100644 --- a/core/tasks/msgs/retries.go +++ b/core/tasks/msgs/retries.go @@ -2,36 +2,18 @@ package msgs import ( "context" - "sync" "time" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/msgio" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -const ( - retryMessagesLock = "retry_errored_messages" -) - func init() { - mailroom.AddInitFunction(startCrons) -} - -func startCrons(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, retryMessagesLock, time.Second*60, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return RetryErroredMessages(ctx, rt) - }, - ) - - return nil + mailroom.RegisterCron("retry_errored_messages", time.Second*60, false, RetryErroredMessages) } func RetryErroredMessages(ctx context.Context, rt *runtime.Runtime) error { diff --git a/core/tasks/msgs/send_broadcast.go b/core/tasks/msgs/send_broadcast.go index 529b3ccd2..09940d7e6 100644 --- a/core/tasks/msgs/send_broadcast.go +++ b/core/tasks/msgs/send_broadcast.go @@ -103,8 +103,8 @@ func CreateBroadcastBatches(ctx context.Context, rt *runtime.Runtime, bcast *mod // also set our URNs if isLast { - batch.SetIsLast(true) - batch.SetURNs(urnContacts) + batch.IsLast = true + batch.URNs = urnContacts } err = queue.AddTask(rc, q, queue.SendBroadcastBatch, int(bcast.OrgID()), batch, queue.DefaultPriority) @@ -151,21 +151,21 @@ func handleSendBroadcastBatch(ctx context.Context, rt *runtime.Runtime, task *qu func SendBroadcastBatch(ctx context.Context, rt *runtime.Runtime, bcast *models.BroadcastBatch) error { // always set our broadcast as sent if it is our last defer func() { - if bcast.IsLast() { - err := models.MarkBroadcastSent(ctx, rt.DB, bcast.BroadcastID()) + if bcast.IsLast { + err := models.MarkBroadcastSent(ctx, rt.DB, bcast.BroadcastID) if err != nil { logrus.WithError(err).Error("error marking broadcast as sent") } } }() - oa, err := models.GetOrgAssets(ctx, rt, bcast.OrgID()) + oa, err := models.GetOrgAssets(ctx, rt, bcast.OrgID) if err != nil { return errors.Wrapf(err, "error getting org assets") } // create this batch of messages - msgs, err := models.CreateBroadcastMessages(ctx, rt, oa, bcast) + msgs, err := bcast.CreateMessages(ctx, rt, oa) if err != nil { return errors.Wrapf(err, "error creating broadcast messages") } diff --git a/core/tasks/msgs/send_broadcast_test.go b/core/tasks/msgs/send_broadcast_test.go index 448e423f4..116f390ec 100644 --- a/core/tasks/msgs/send_broadcast_test.go +++ b/core/tasks/msgs/send_broadcast_test.go @@ -135,7 +135,7 @@ func TestBroadcastTask(t *testing.T) { // insert a broadcast so we can check it is being set to sent legacyID := testdata.InsertBroadcast(db, testdata.Org1, "base", map[envs.Language]string{"base": "hi @(PROPER(contact.name)) legacy"}, models.NilScheduleID, nil, nil) - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "", "", time.Now(), nil) modelTicket := ticket.Load(db) evaluated := map[envs.Language]*models.BroadcastTranslation{ @@ -177,6 +177,7 @@ func TestBroadcastTask(t *testing.T) { ContactIDs []models.ContactID URNs []urns.URN TicketID models.TicketID + CreatedByID models.UserID Queue string BatchCount int MsgCount int @@ -190,7 +191,8 @@ func TestBroadcastTask(t *testing.T) { doctorsOnly, cathyOnly, nil, - ticket.ID, + models.NilTicketID, + testdata.Admin.ID, queue.BatchQueue, 2, 121, @@ -205,6 +207,7 @@ func TestBroadcastTask(t *testing.T) { cathyOnly, nil, models.NilTicketID, + models.NilUserID, queue.HandlerQueue, 1, 1, @@ -218,7 +221,8 @@ func TestBroadcastTask(t *testing.T) { nil, cathyOnly, nil, - models.NilTicketID, + ticket.ID, + testdata.Agent.ID, queue.HandlerQueue, 1, 1, @@ -231,7 +235,7 @@ func TestBroadcastTask(t *testing.T) { for i, tc := range tcs { // handle our start task - bcast := models.NewBroadcast(oa.OrgID(), tc.BroadcastID, tc.Translations, tc.TemplateState, tc.BaseLanguage, tc.URNs, tc.ContactIDs, tc.GroupIDs, tc.TicketID) + bcast := models.NewBroadcast(oa.OrgID(), tc.BroadcastID, tc.Translations, tc.TemplateState, tc.BaseLanguage, tc.URNs, tc.ContactIDs, tc.GroupIDs, tc.TicketID, tc.CreatedByID) err = msgs.CreateBroadcastBatches(ctx, rt, bcast) assert.NoError(t, err) @@ -268,13 +272,20 @@ func TestBroadcastTask(t *testing.T) { Returns(1, "%d: broadcast not marked as sent", i) } - // if we had a ticket, make sure its last_activity_on was updated + // if we had a ticket, make sure its replied_on and last_activity_on were updated if tc.TicketID != models.NilTicketID { assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticket WHERE id = $1 AND last_activity_on > $2`, tc.TicketID, modelTicket.LastActivityOn()). Returns(1, "%d: ticket last_activity_on not updated", i) + assertdb.Query(t, db, `SELECT count(*) FROM tickets_ticket WHERE id = $1 AND replied_on IS NOT NULL`, tc.TicketID). + Returns(1, "%d: ticket replied_on not updated", i) } lastNow = time.Now() time.Sleep(10 * time.Millisecond) } + + assertdb.Query(t, db, `SELECT SUM(count) FROM tickets_ticketdailycount WHERE count_type = 'R' AND scope = CONCAT('o:', $1::text)`, testdata.Org1.ID).Returns(1) + assertdb.Query(t, db, `SELECT SUM(count) FROM tickets_ticketdailycount WHERE count_type = 'R' AND scope = CONCAT('o:', $1::text, ':u:', $2::text)`, testdata.Org1.ID, testdata.Agent.ID).Returns(1) + + assertdb.Query(t, db, `SELECT SUM(count) FROM tickets_ticketdailytiming WHERE count_type = 'R' AND scope = CONCAT('o:', $1::text)`, testdata.Org1.ID).Returns(1) } diff --git a/core/tasks/schedules/cron.go b/core/tasks/schedules/cron.go index 262c3999e..97e7ff77f 100644 --- a/core/tasks/schedules/cron.go +++ b/core/tasks/schedules/cron.go @@ -2,43 +2,26 @@ package schedules import ( "context" - "sync" "time" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -const ( - scheduleLock = "fire_schedules" -) - func init() { - mailroom.AddInitFunction(StartCheckSchedules) -} - -// StartCheckSchedules starts our cron job of firing schedules every minute -func StartCheckSchedules(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, scheduleLock, time.Minute*1, false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - // we sleep 1 second since we fire right on the minute and want to make sure to fire - // things that are schedules right at the minute as well (and DB time may be slightly drifted) - time.Sleep(time.Second * 1) - return checkSchedules(ctx, rt) - }, - ) - return nil + mailroom.RegisterCron("fire_schedules", time.Minute*1, false, checkSchedules) } // checkSchedules looks up any expired schedules and fires them, setting the next fire as needed func checkSchedules(ctx context.Context, rt *runtime.Runtime) error { + // we sleep 1 second since we fire right on the minute and want to make sure to fire + // things that are schedules right at the minute as well (and DB time may be slightly drifted) + time.Sleep(time.Second * 1) + log := logrus.WithField("comp", "schedules_cron") start := time.Now() diff --git a/core/tasks/starts/worker.go b/core/tasks/starts/worker.go index 45c3a509e..a652ee147 100644 --- a/core/tasks/starts/worker.go +++ b/core/tasks/starts/worker.go @@ -11,6 +11,7 @@ import ( "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/core/runner" + "github.com/nyaruka/mailroom/core/search" "github.com/nyaruka/mailroom/runtime" "github.com/lib/pq" @@ -120,7 +121,7 @@ func CreateFlowBatches(ctx context.Context, rt *runtime.Runtime, start *models.F if start.Type() == models.StartTypeFlowAction { limit = 1 } - matches, err := models.GetContactIDsForQuery(ctx, rt.ES, oa, start.Query(), limit) + matches, err := search.GetContactIDsForQuery(ctx, rt.ES, oa, start.Query(), limit) if err != nil { return errors.Wrapf(err, "error performing search for start: %d", start.ID()) } diff --git a/core/tasks/starts/worker_test.go b/core/tasks/starts/worker_test.go index 81d3e37cd..634490e3d 100644 --- a/core/tasks/starts/worker_test.go +++ b/core/tasks/starts/worker_test.go @@ -2,11 +2,10 @@ package starts import ( "encoding/json" - "fmt" "testing" + "time" "github.com/nyaruka/gocommon/dbutil/assertdb" - "github.com/nyaruka/gocommon/uuids" _ "github.com/nyaruka/mailroom/core/handlers" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/queue" @@ -14,7 +13,6 @@ import ( "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/testsuite/testdata" - "github.com/olivere/elastic/v7" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,190 +24,164 @@ func TestStarts(t *testing.T) { defer testsuite.Reset(testsuite.ResetAll) - mes := testsuite.NewMockElasticServer() - defer mes.Close() + mockES := testsuite.NewMockElasticServer() + defer mockES.Close() - es, err := elastic.NewClient( - elastic.SetURL(mes.URL()), - elastic.SetHealthcheck(false), - elastic.SetSniff(false), - ) - require.NoError(t, err) - rt.ES = es + rt.ES = mockES.Client() // convert our single message flow to an actual background flow that shouldn't interrupt db.MustExec(`UPDATE flows_flow SET flow_type = 'B' WHERE id = $1`, testdata.SingleMessage.ID) - // insert a flow run for one of our contacts - // TODO: can be replaced with a normal flow start of another flow once we support flows with waits - db.MustExec( - `INSERT INTO flows_flowrun(uuid, status, created_on, modified_on, responded, contact_id, flow_id, org_id) - VALUES($1, 'W', now(), now(), FALSE, $2, $3, 1);`, uuids.New(), testdata.George.ID, testdata.Favorites.ID) + sID := testdata.InsertWaitingSession(db, testdata.Org1, testdata.George, models.FlowTypeMessaging, testdata.Favorites, models.NilConnectionID, time.Now(), time.Now(), true, nil) + testdata.InsertFlowRun(db, testdata.Org1, sID, testdata.George, testdata.Favorites, models.RunStatusWaiting) tcs := []struct { - label string - flowID models.FlowID - groupIDs []models.GroupID - excludeGroupIDs []models.GroupID - contactIDs []models.ContactID - createContact bool - query string - queryResponse string - restartParticipants bool - includeActive bool - queue string - expectedContactCount int - expectedBatchCount int - expectedTotalCount int - expectedStatus models.StartStatus - expectedActiveRuns map[models.FlowID]int + label string + flowID models.FlowID + groupIDs []models.GroupID + excludeGroupIDs []models.GroupID + contactIDs []models.ContactID + createContact bool + query string + queryResult []models.ContactID + excludeInAFlow bool + excludeStartedPreviously bool + queue string + expectedContactCount int + expectedBatchCount int + expectedTotalCount int + expectedStatus models.StartStatus + expectedActiveRuns map[models.FlowID]int }{ { - label: "Empty flow start", - flowID: testdata.Favorites.ID, - queue: queue.BatchQueue, - expectedContactCount: 0, - expectedBatchCount: 0, - expectedTotalCount: 0, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 1, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Empty flow start", + flowID: testdata.Favorites.ID, + excludeInAFlow: true, + excludeStartedPreviously: true, + queue: queue.BatchQueue, + expectedContactCount: 0, + expectedBatchCount: 0, + expectedTotalCount: 0, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 1, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Single group", - flowID: testdata.Favorites.ID, - groupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, - queue: queue.BatchQueue, - expectedContactCount: 121, - expectedBatchCount: 2, - expectedTotalCount: 121, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 122, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Single group", + flowID: testdata.Favorites.ID, + groupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, + excludeInAFlow: true, + excludeStartedPreviously: true, + queue: queue.BatchQueue, + expectedContactCount: 121, + expectedBatchCount: 2, + expectedTotalCount: 121, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 122, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Group and Contact (but all already active)", - flowID: testdata.Favorites.ID, - groupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, - contactIDs: []models.ContactID{testdata.Cathy.ID}, - queue: queue.BatchQueue, - expectedContactCount: 121, - expectedBatchCount: 2, - expectedTotalCount: 0, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 122, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Group and Contact (but all already active)", + flowID: testdata.Favorites.ID, + groupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, + contactIDs: []models.ContactID{testdata.Cathy.ID}, + excludeInAFlow: true, + excludeStartedPreviously: true, + queue: queue.BatchQueue, + expectedContactCount: 121, + expectedBatchCount: 2, + expectedTotalCount: 0, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 122, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Contact restart", - flowID: testdata.Favorites.ID, - contactIDs: []models.ContactID{testdata.Cathy.ID}, - restartParticipants: true, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 1, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 122, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Contact restart", + flowID: testdata.Favorites.ID, + contactIDs: []models.ContactID{testdata.Cathy.ID}, + excludeInAFlow: false, + excludeStartedPreviously: false, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 1, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 122, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Previous group and one new contact", - flowID: testdata.Favorites.ID, - groupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, - contactIDs: []models.ContactID{testdata.Bob.ID}, - queue: queue.BatchQueue, - expectedContactCount: 122, - expectedBatchCount: 2, - expectedTotalCount: 1, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Previous group and one new contact", + flowID: testdata.Favorites.ID, + groupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, + contactIDs: []models.ContactID{testdata.Bob.ID}, + excludeStartedPreviously: true, + queue: queue.BatchQueue, + expectedContactCount: 122, + expectedBatchCount: 2, + expectedTotalCount: 1, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Single contact, no restart", - flowID: testdata.Favorites.ID, - contactIDs: []models.ContactID{testdata.Bob.ID}, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 0, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Single contact, no restart", + flowID: testdata.Favorites.ID, + contactIDs: []models.ContactID{testdata.Bob.ID}, + excludeStartedPreviously: true, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 0, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Single contact, include active, but no restart", - flowID: testdata.Favorites.ID, - contactIDs: []models.ContactID{testdata.Bob.ID}, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 0, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Single contact, include active, but no restart", + flowID: testdata.Favorites.ID, + contactIDs: []models.ContactID{testdata.Bob.ID}, + excludeInAFlow: false, + excludeStartedPreviously: true, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 0, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Single contact, include active and restart", - flowID: testdata.Favorites.ID, - contactIDs: []models.ContactID{testdata.Bob.ID}, - restartParticipants: true, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 1, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Single contact, include active and restart", + flowID: testdata.Favorites.ID, + contactIDs: []models.ContactID{testdata.Bob.ID}, + excludeInAFlow: false, + excludeStartedPreviously: false, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 1, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Query start", - flowID: testdata.Favorites.ID, - query: "bob", - queryResponse: fmt.Sprintf(`{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }`, testdata.Bob.ID), - restartParticipants: true, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 1, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Query start", + flowID: testdata.Favorites.ID, + query: "bob", + queryResult: []models.ContactID{testdata.Bob.ID}, + excludeInAFlow: false, + excludeStartedPreviously: false, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 1, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Query start with invalid query", - flowID: testdata.Favorites.ID, - query: "xyz = 45", - restartParticipants: true, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 0, - expectedBatchCount: 0, - expectedTotalCount: 0, - expectedStatus: models.StartStatusFailed, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Query start with invalid query", + flowID: testdata.Favorites.ID, + query: "xyz = 45", + excludeInAFlow: false, + excludeStartedPreviously: false, + queue: queue.HandlerQueue, + expectedContactCount: 0, + expectedBatchCount: 0, + expectedTotalCount: 0, + expectedStatus: models.StartStatusFailed, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { label: "New Contact", @@ -223,54 +195,60 @@ func TestStarts(t *testing.T) { expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 124, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, { - label: "Other messaging flow", - flowID: testdata.PickANumber.ID, - contactIDs: []models.ContactID{testdata.Bob.ID}, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 1, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 1, testdata.SingleMessage.ID: 0}, + label: "Other messaging flow", + flowID: testdata.PickANumber.ID, + contactIDs: []models.ContactID{testdata.Bob.ID}, + excludeInAFlow: false, + excludeStartedPreviously: true, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 1, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 1, testdata.SingleMessage.ID: 0}, }, { - label: "Background flow", - flowID: testdata.SingleMessage.ID, - contactIDs: []models.ContactID{testdata.Bob.ID}, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 1, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 1, testdata.SingleMessage.ID: 0}, + label: "Background flow", + flowID: testdata.SingleMessage.ID, + contactIDs: []models.ContactID{testdata.Bob.ID}, + excludeInAFlow: false, + excludeStartedPreviously: true, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 1, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 123, testdata.PickANumber.ID: 1, testdata.SingleMessage.ID: 0}, }, { - label: "Exclude group", - flowID: testdata.Favorites.ID, - contactIDs: []models.ContactID{testdata.Cathy.ID, testdata.Bob.ID}, - excludeGroupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, // should exclude Cathy - restartParticipants: true, - includeActive: true, - queue: queue.HandlerQueue, - expectedContactCount: 1, - expectedBatchCount: 1, - expectedTotalCount: 1, - expectedStatus: models.StartStatusComplete, - expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 124, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, + label: "Exclude group", + flowID: testdata.Favorites.ID, + contactIDs: []models.ContactID{testdata.Cathy.ID, testdata.Bob.ID}, + excludeGroupIDs: []models.GroupID{testdata.DoctorsGroup.ID}, // should exclude Cathy + excludeInAFlow: false, + excludeStartedPreviously: false, + queue: queue.HandlerQueue, + expectedContactCount: 1, + expectedBatchCount: 1, + expectedTotalCount: 1, + expectedStatus: models.StartStatusComplete, + expectedActiveRuns: map[models.FlowID]int{testdata.Favorites.ID: 124, testdata.PickANumber.ID: 0, testdata.SingleMessage.ID: 0}, }, } for _, tc := range tcs { - mes.NextResponse = tc.queryResponse + if tc.queryResult != nil { + mockES.AddResponse(tc.queryResult...) + } // handle our start task - start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeManual, models.FlowTypeMessaging, tc.flowID, tc.restartParticipants, tc.includeActive). + start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeManual, models.FlowTypeMessaging, tc.flowID). WithGroupIDs(tc.groupIDs). WithExcludeGroupIDs(tc.excludeGroupIDs). WithContactIDs(tc.contactIDs). WithQuery(tc.query). + WithExcludeInAFlow(tc.excludeInAFlow). + WithExcludeStartedPreviously(tc.excludeStartedPreviously). WithCreateContact(tc.createContact) err := models.InsertFlowStarts(ctx, db, []*models.FlowStart{start}) diff --git a/core/tasks/timeouts/cron.go b/core/tasks/timeouts/cron.go index 21c5e61aa..66796b4a5 100644 --- a/core/tasks/timeouts/cron.go +++ b/core/tasks/timeouts/cron.go @@ -3,40 +3,21 @@ package timeouts import ( "context" "fmt" - "sync" "time" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/core/models" "github.com/nyaruka/mailroom/core/tasks/handler" "github.com/nyaruka/mailroom/runtime" - "github.com/nyaruka/mailroom/utils/cron" "github.com/nyaruka/redisx" - "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -const ( - timeoutLock = "sessions_timeouts" -) - var marker = redisx.NewIntervalSet("session_timeouts", time.Hour*24, 2) func init() { - mailroom.AddInitFunction(StartTimeoutCron) -} - -// StartTimeoutCron starts our cron job of continuing timed out sessions every minute -func StartTimeoutCron(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { - cron.Start(quit, rt, timeoutLock, time.Second*time.Duration(rt.Config.TimeoutTime), false, - func() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - return timeoutSessions(ctx, rt) - }, - ) - return nil + mailroom.RegisterCron("sessions_timeouts", time.Second*60, false, timeoutSessions) } // timeoutRuns looks for any runs that have timed out and schedules for them to continue diff --git a/go.mod b/go.mod index 647c375f7..a21257c97 100644 --- a/go.mod +++ b/go.mod @@ -1,71 +1,72 @@ module github.com/nyaruka/mailroom +go 1.18 + require ( github.com/Masterminds/semver v1.5.0 - github.com/apex/log v1.1.4 - github.com/aws/aws-sdk-go v1.40.56 - github.com/buger/jsonparser v1.0.0 - github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect + github.com/aws/aws-sdk-go v1.44.44 + github.com/buger/jsonparser v1.1.1 github.com/edganiukov/fcm v0.4.0 - github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/golang-jwt/jwt v3.2.2+incompatible - github.com/golang/protobuf v1.4.0 - github.com/gomodule/redigo v2.0.0+incompatible - github.com/gorilla/schema v1.1.0 - github.com/jmoiron/sqlx v1.3.4 - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lib/pq v1.10.4 + github.com/golang/protobuf v1.5.2 + github.com/gomodule/redigo v1.8.8 + github.com/gorilla/schema v1.2.0 + github.com/jmoiron/sqlx v1.3.5 + github.com/lib/pq v1.10.6 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.17.1 - github.com/nyaruka/goflow v0.152.0 - github.com/nyaruka/librato v1.0.0 + github.com/nyaruka/gocommon v1.22.4 + github.com/nyaruka/goflow v0.163.0 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d github.com/nyaruka/null v1.2.0 github.com/nyaruka/redisx v0.2.1 - github.com/olivere/elastic/v7 v7.0.22 + github.com/olivere/elastic/v7 v7.0.32 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.9.1 - github.com/shopspring/decimal v1.2.0 - github.com/sirupsen/logrus v1.5.0 - github.com/stretchr/testify v1.7.0 + github.com/prometheus/common v0.35.0 + github.com/shopspring/decimal v1.3.1 + github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.8.0 gopkg.in/go-playground/validator.v9 v9.31.0 ) require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/antlr/antlr4 v0.0.0-20200701161529-3d9351f61e0f // indirect + github.com/Shopify/gomail v0.0.0-20220314142144-6897a5a5ba29 // indirect + github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220527190237-ee62e23da966 // indirect github.com/blevesearch/segment v0.9.0 // indirect + github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fatih/structs v1.0.0 // indirect - github.com/go-mail/mail v2.3.1+incompatible // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.0 // indirect + github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/gofrs/uuid v3.3.0+incompatible // indirect github.com/google/go-querystring v1.1.0 + github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect - github.com/nyaruka/phonenumbers v1.0.71 // indirect + github.com/nyaruka/librato v1.0.0 // indirect + github.com/nyaruka/phonenumbers v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/sergi/go-diff v1.1.0 // indirect - golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect - golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect - golang.org/x/text v0.3.6 // indirect - google.golang.org/protobuf v1.21.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect + golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d // indirect + golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect + golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.17 - replace github.com/nyaruka/gocommon => github.com/Ilhasoft/gocommon v1.17.1-weni replace github.com/gomodule/redigo => github.com/gomodule/redigo v1.8.8 diff --git a/go.sum b/go.sum index c307500e7..9fc2422f8 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,44 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Ilhasoft/gocommon v1.17.1-weni h1:cJfDP2Jg1OVmX8JWj2R3XYE3q2nal6CWN+7yMib+EA0= -github.com/Ilhasoft/gocommon v1.17.1-weni/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Shopify/gomail v0.0.0-20220314142144-6897a5a5ba29 h1:Bw1PtWrRByxjLQLawK/3lyeAbfIRzuDUnsBsh+0tuxk= +github.com/Shopify/gomail v0.0.0-20220314142144-6897a5a5ba29/go.mod h1:RS+Gaowa0M+gCuiFAiRMGBCMqxLrNA7TESTU/Wbblm8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -17,92 +50,156 @@ github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= -github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= -github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= -github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220527190237-ee62e23da966 h1:mEzJ8SH4M5wDL8C4a17yX2YeD/FIXV5w8FJekByaBi0= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220527190237-ee62e23da966/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/aws/aws-sdk-go v1.44.44 h1:XLEcUxILvVBYO/frO+TTCd8NIxklX/ZOzSJSBZ+b7B8= +github.com/aws/aws-sdk-go v1.44.44/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= -github.com/buger/jsonparser v1.0.0 h1:etJTGF5ESxjI0Ic2UaLQs2LQQpa8G9ykQScukbh4L8A= -github.com/buger/jsonparser v1.0.0/go.mod h1:tgcrVJ81GPSF0mz+0nu1Xaz0fazGPrmmJfJtxjbHhUQ= -github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= -github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= +github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/edganiukov/fcm v0.4.0 h1:PAZamwbiW2AegM5hGqYNv+djE1xxLyH7zMN6MwWpvoQ= github.com/edganiukov/fcm v0.4.0/go.mod h1:3gL1BLvC3w05anUsF2Wbd1Sz+ZdCu8qsNCa1LyRfwFo= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= +github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d h1:CIp8WnfXz70wJVQ0ytr3dswFYGoJbAxWgNvaLpiu3sY= github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-mail/mail v2.3.1+incompatible h1:UzNOn0k5lpfVtO31cK3hn6I4VEVGhe3lX8AJBAxXExM= -github.com/go-mail/mail v2.3.1+incompatible/go.mod h1:VPWjmmNyRsWXQZHVHT3g0YbIINUkSmuKOiLIDkWbL6M= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY= -github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= +github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= -github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -115,45 +212,43 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= -github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/goflow v0.152.0 h1:wPMhmfkmxFcFowyr+Z3MeSkTMwriqa++bBdvIPcqiKo= -github.com/nyaruka/goflow v0.152.0/go.mod h1:HhK+wn4aRji8qJgJR8l48hPiZxnwVDdWa0Ogy5ifnSQ= +github.com/nyaruka/gocommon v1.22.4 h1:NCAItnrQbXlipDeOszoYbjXEFa1J1M+alS8VSk/uero= +github.com/nyaruka/gocommon v1.22.4/go.mod h1:g6/d9drZXDUrtRSPe2Kf8lTUS+baHt/0G0dwHq3qeIU= +github.com/nyaruka/goflow v0.163.0 h1:dR8phssqATprCPVxjJj6U19yOpOgnCwNQvvqYzAuIcE= +github.com/nyaruka/goflow v0.163.0/go.mod h1:XUPFWNEgimo+hsOOonH8bN6I8V5LE2cxSYgSC0mdxas= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d/go.mod h1:FGdPJVDTNqbRAD+2RvnK9YoO2HcEW7ogSMPzc90b638= github.com/nyaruka/null v1.2.0 h1:uEbkyy4Z+zPB2Pr3ryQh/0N2965I9kEsXq/cGpyJ7PA= github.com/nyaruka/null v1.2.0/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= -github.com/nyaruka/phonenumbers v1.0.71 h1:itkCGhxkQkHrJ6OyZSApdjQVlPmrWs88MF283pPvbFU= -github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/phonenumbers v1.1.0 h1:OvNAOAl4A9a2kNpzziITbUVH4bBBeKHkHl0llPmkxaA= +github.com/nyaruka/phonenumbers v1.1.0/go.mod h1:cGaEsOrLjIL0iKGqJR5Rfywy86dSkbApEpXuM9KySNA= github.com/nyaruka/redisx v0.2.1 h1:BavpQRCsK5xV2uxPdJJ26yVmjSo+q6bdjWqeNNf0s5w= github.com/nyaruka/redisx v0.2.1/go.mod h1:cdbAm4y/+oFWu7qFzH2ERPeqRXJC2CtgRhwcBacM4Oc= -github.com/olivere/elastic/v7 v7.0.22 h1:esBA6JJwvYgfms0EVlH7Z+9J4oQ/WUADF2y/nCNDw7s= -github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= +github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -164,122 +259,348 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= -github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= -github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= -github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= -github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= -github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= -github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= -github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d h1:vtUKgx8dahOomfFzLREU8nSv25YHnTgLBn4rDnWZdU0= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk= gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/goreleaser.yml b/goreleaser.yml index 69f927f48..787b50d7c 100644 --- a/goreleaser.yml +++ b/goreleaser.yml @@ -6,6 +6,7 @@ build: - linux goarch: - amd64 + - arm64 archives: - files: diff --git a/mailroom.go b/mailroom.go index b17b7c4ae..07b691610 100644 --- a/mailroom.go +++ b/mailroom.go @@ -7,29 +7,37 @@ import ( "sync" "time" + "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/mailroom/core/queue" "github.com/nyaruka/mailroom/runtime" + "github.com/nyaruka/mailroom/utils/cron" "github.com/nyaruka/mailroom/web" "github.com/pkg/errors" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" - "github.com/nyaruka/librato" "github.com/olivere/elastic/v7" "github.com/sirupsen/logrus" ) // InitFunction is a function that will be called when mailroom starts -type InitFunction func(runtime *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error +type InitFunction func(*runtime.Runtime, *sync.WaitGroup, chan bool) error var initFunctions = make([]InitFunction, 0) -// AddInitFunction adds an init function that will be called on startup -func AddInitFunction(initFunc InitFunction) { +func addInitFunction(initFunc InitFunction) { initFunctions = append(initFunctions, initFunc) } +// RegisterCron registers a new cron function to run every interval +func RegisterCron(name string, interval time.Duration, allInstances bool, fn cron.Function) { + addInitFunction(func(rt *runtime.Runtime, wg *sync.WaitGroup, quit chan bool) error { + cron.Start(rt, wg, name, interval, allInstances, fn, time.Minute*5, quit) + return nil + }) +} + // TaskFunction is the function that will be called for a type of task type TaskFunction func(ctx context.Context, rt *runtime.Runtime, task *queue.Task) error @@ -155,7 +163,7 @@ func (mr *Mailroom) Start() error { // warn if we won't be doing FCM syncing if c.FCMKey == "" { - logrus.Error("fcm not configured, no syncing of android channels") + logrus.Warn("fcm not configured, no syncing of android channels") } for _, initFunc := range initFunctions { @@ -164,10 +172,11 @@ func (mr *Mailroom) Start() error { // if we have a librato token, configure it if c.LibratoToken != "" { - librato.Configure(c.LibratoUsername, c.LibratoToken, c.InstanceName, time.Second, mr.wg) - librato.Start() + analytics.RegisterBackend(analytics.NewLibrato(c.LibratoUsername, c.LibratoToken, c.InstanceName, time.Second, mr.wg)) } + analytics.Start() + // init our foremen and start it mr.batchForeman.Start() mr.handlerForeman.Start() @@ -186,7 +195,7 @@ func (mr *Mailroom) Stop() error { logrus.Info("mailroom stopping") mr.batchForeman.Stop() mr.handlerForeman.Stop() - librato.Stop() + analytics.Stop() close(mr.quit) mr.cancel() diff --git a/mailroom_test.dump b/mailroom_test.dump index b1caabf19..fba3599eb 100644 Binary files a/mailroom_test.dump and b/mailroom_test.dump differ diff --git a/services/tickets/mailgun/web_test.go b/services/tickets/mailgun/web_test.go index 98d544972..f831263f4 100644 --- a/services/tickets/mailgun/web_test.go +++ b/services/tickets/mailgun/web_test.go @@ -2,6 +2,7 @@ package mailgun import ( "testing" + "time" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/testsuite/testdata" @@ -14,7 +15,7 @@ func TestReceive(t *testing.T) { defer testsuite.Reset(testsuite.ResetData | testsuite.ResetStorage) // create a mailgun ticket for Cathy - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "", time.Now(), nil) web.RunWebTests(t, ctx, rt, "testdata/receive.json", map[string]string{"cathy_ticket_uuid": string(ticket.UUID)}) } diff --git a/services/tickets/rocketchat/web_test.go b/services/tickets/rocketchat/web_test.go index 595ea4ca6..b37567263 100644 --- a/services/tickets/rocketchat/web_test.go +++ b/services/tickets/rocketchat/web_test.go @@ -2,6 +2,7 @@ package rocketchat_test import ( "testing" + "time" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/testsuite/testdata" @@ -14,7 +15,7 @@ func TestEventCallback(t *testing.T) { defer testsuite.Reset(testsuite.ResetData | testsuite.ResetStorage) // create a rocketchat ticket for Cathy - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.RocketChat, testdata.DefaultTopic, "Have you seen my cookies?", "1234", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.RocketChat, testdata.DefaultTopic, "Have you seen my cookies?", "1234", time.Now(), nil) web.RunWebTests(t, ctx, rt, "testdata/event_callback.json", map[string]string{"cathy_ticket_uuid": string(ticket.UUID)}) } diff --git a/services/tickets/utils.go b/services/tickets/utils.go index 86ea8214d..805203c75 100644 --- a/services/tickets/utils.go +++ b/services/tickets/utils.go @@ -101,9 +101,9 @@ func SendReply(ctx context.Context, rt *runtime.Runtime, ticket *models.Ticket, translations := map[envs.Language]*models.BroadcastTranslation{envs.Language("base"): base} // we'll use a broadcast to send this message - bcast := models.NewBroadcast(oa.OrgID(), models.NilBroadcastID, translations, models.TemplateStateEvaluated, envs.Language("base"), nil, nil, nil, ticket.ID()) + bcast := models.NewBroadcast(oa.OrgID(), models.NilBroadcastID, translations, models.TemplateStateEvaluated, envs.Language("base"), nil, nil, nil, ticket.ID(), models.NilUserID) batch := bcast.CreateBatch([]models.ContactID{ticket.ContactID()}) - msgs, err := models.CreateBroadcastMessages(ctx, rt, oa, batch) + msgs, err := batch.CreateMessages(ctx, rt, oa) if err != nil { return nil, errors.Wrapf(err, "error creating message batch") } diff --git a/services/tickets/utils_test.go b/services/tickets/utils_test.go index 74453cca4..cc93cb41f 100644 --- a/services/tickets/utils_test.go +++ b/services/tickets/utils_test.go @@ -53,8 +53,8 @@ func TestFromTicketUUID(t *testing.T) { defer testsuite.Reset(testsuite.ResetAll) // create some tickets - ticket1 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "", nil) - ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my shoes?", "", nil) + ticket1 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "", time.Now(), nil) + ticket2 := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my shoes?", "", time.Now(), nil) // break mailgun configuration db.MustExec(`UPDATE tickets_ticketer SET config = '{"foo":"bar"}'::jsonb WHERE id = $1`, testdata.Mailgun.ID) @@ -124,7 +124,7 @@ func TestSendReply(t *testing.T) { image := &tickets.File{URL: "http://coolfiles.com/a.jpg", ContentType: "image/jpeg", Body: imageBody} // create a ticket - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "", time.Now(), nil) modelTicket := ticket.Load(db) msg, err := tickets.SendReply(ctx, rt, modelTicket, "I'll get back to you", []*tickets.File{image}) @@ -159,6 +159,8 @@ func TestCloseTicket(t *testing.T) { }, })) + oa := testdata.Org1.Load(rt) + // create an open ticket ticket1 := models.NewTicket( "2ef57efc-d85f-4291-b330-e4afe68af5fe", @@ -173,20 +175,17 @@ func TestCloseTicket(t *testing.T) { "contact-display": "Cathy", }, ) - err := models.InsertTickets(ctx, db, []*models.Ticket{ticket1}) + err := models.InsertTickets(ctx, db, oa, []*models.Ticket{ticket1}) require.NoError(t, err) // create a close ticket trigger testdata.InsertTicketClosedTrigger(db, testdata.Org1, testdata.Favorites) - oa, err := models.GetOrgAssets(ctx, rt, testdata.Org1.ID) - require.NoError(t, err) - logger := &models.HTTPLogger{} err = tickets.Close(ctx, rt, oa, ticket1, true, logger) require.NoError(t, err) testsuite.AssertContactTasks(t, 1, testdata.Cathy.ID, - []string{`{"type":"ticket_closed","org_id":1,"task":{"id":1,"org_id":1,"contact_id":10000,"ticket_id":1,"event_type":"C","created_on":"2021-06-08T16:40:32Z"},"queued_on":"2021-06-08T16:40:35Z"}`}) + []string{`{"type":"ticket_closed","org_id":1,"task":{"id":1,"org_id":1,"contact_id":10000,"ticket_id":1,"event_type":"C","created_on":"2021-06-08T16:40:34Z"},"queued_on":"2021-06-08T16:40:37Z"}`}) } diff --git a/services/tickets/zendesk/web_test.go b/services/tickets/zendesk/web_test.go index 1a189734f..c597b0da5 100644 --- a/services/tickets/zendesk/web_test.go +++ b/services/tickets/zendesk/web_test.go @@ -2,6 +2,7 @@ package zendesk import ( "testing" + "time" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/testsuite/testdata" @@ -14,7 +15,7 @@ func TestChannelback(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) // create a zendesk ticket for Cathy - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "1234", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "1234", time.Now(), nil) web.RunWebTests(t, ctx, rt, "testdata/channelback.json", map[string]string{"cathy_ticket_uuid": string(ticket.UUID)}) } @@ -25,7 +26,7 @@ func TestEventCallback(t *testing.T) { defer testsuite.Reset(testsuite.ResetAll) // tests include destroying ticketer // create a zendesk ticket for Cathy - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "1234", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "1234", time.Now(), nil) web.RunWebTests(t, ctx, rt, "testdata/event_callback.json", map[string]string{"cathy_ticket_uuid": string(ticket.UUID)}) } @@ -36,7 +37,7 @@ func TestTarget(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) // create a zendesk ticket for Cathy - ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "1234", nil) + ticket := testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "1234", time.Now(), nil) web.RunWebTests(t, ctx, rt, "testdata/target.json", map[string]string{"cathy_ticket_uuid": string(ticket.UUID)}) } diff --git a/testsuite/elastic.go b/testsuite/elastic.go index 0683fa2bc..d87e9357d 100644 --- a/testsuite/elastic.go +++ b/testsuite/elastic.go @@ -1,9 +1,14 @@ package testsuite import ( + "fmt" "io" "net/http" "net/http/httptest" + + "github.com/nyaruka/gocommon/jsonx" + "github.com/nyaruka/mailroom/core/models" + "github.com/olivere/elastic/v7" ) // MockElasticServer is a mock HTTP server/endpoint that can be used to test elastic queries @@ -11,7 +16,7 @@ type MockElasticServer struct { Server *httptest.Server LastRequestURL string LastRequestBody string - NextResponse string + Responses [][]byte } // NewMockElasticServer creates a new mock elastic server @@ -48,13 +53,24 @@ func NewMockElasticServer() *MockElasticServer { body, _ := io.ReadAll(r.Body) m.LastRequestBody = string(body) + if len(m.Responses) == 0 { + panic("mock elastic server has no more queued responses") + } + + var response []byte + response, m.Responses = m.Responses[0], m.Responses[1:] + w.WriteHeader(200) - w.Write([]byte(m.NextResponse)) - m.NextResponse = "" + w.Write(response) })) return m } +func (m *MockElasticServer) Client() *elastic.Client { + c, _ := elastic.NewClient(elastic.SetURL(m.URL()), elastic.SetHealthcheck(false), elastic.SetSniff(false)) + return c +} + // Close closes our HTTP server func (m *MockElasticServer) Close() { m.Server.Close() @@ -64,3 +80,36 @@ func (m *MockElasticServer) Close() { func (m *MockElasticServer) URL() string { return m.Server.URL } + +// AddResponse adds a mock response to the server's queue +func (m *MockElasticServer) AddResponse(ids ...models.ContactID) { + hits := make([]map[string]interface{}, len(ids)) + for i := range ids { + hits[i] = map[string]interface{}{ + "_index": "contacts", + "_type": "_doc", + "_id": fmt.Sprintf("%d", ids[i]), + "_score": nil, + "_routing": "1", + "sort": []int{15124352}, + } + } + + response := jsonx.MustMarshal(map[string]interface{}{ + "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", + "took": 2, + "timed_out": false, + "_shards": map[string]interface{}{ + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0, + }, + "hits": map[string]interface{}{ + "total": len(ids), + "max_score": nil, + "hits": hits, + }, + }) + m.Responses = append(m.Responses, response) +} diff --git a/testsuite/testdata/campaigns.go b/testsuite/testdata/campaigns.go index afa13668f..383380d17 100644 --- a/testsuite/testdata/campaigns.go +++ b/testsuite/testdata/campaigns.go @@ -22,8 +22,8 @@ func InsertCampaign(db *sqlx.DB, org *Org, name string, group *Group) *Campaign uuid := models.CampaignUUID(uuids.New()) var id models.CampaignID must(db.Get(&id, - `INSERT INTO campaigns_campaign(uuid, org_id, name, group_id, is_archived, is_active, created_on, modified_on, created_by_id, modified_by_id) - VALUES($1, $2, $3, $4, FALSE, TRUE, NOW(), NOW(), 1, 1) RETURNING id`, uuid, org.ID, name, group.ID, + `INSERT INTO campaigns_campaign(uuid, org_id, name, group_id, is_archived, is_system, is_active, created_on, modified_on, created_by_id, modified_by_id) + VALUES($1, $2, $3, $4, FALSE, FALSE, TRUE, NOW(), NOW(), 1, 1) RETURNING id`, uuid, org.ID, name, group.ID, )) return &Campaign{id, uuid} } diff --git a/testsuite/testdata/channels.go b/testsuite/testdata/channels.go index d28778809..df8b4585c 100644 --- a/testsuite/testdata/channels.go +++ b/testsuite/testdata/channels.go @@ -20,8 +20,8 @@ func InsertChannel(db *sqlx.DB, org *Org, channelType, name string, schemes []st uuid := assets.ChannelUUID(uuids.New()) var id models.ChannelID must(db.Get(&id, - `INSERT INTO channels_channel(uuid, org_id, channel_type, name, schemes, role, config, last_seen, is_active, created_on, modified_on, created_by_id, modified_by_id) - VALUES($1, $2, $3, $4, $5, $6, $7, NOW(), TRUE, NOW(), NOW(), 1, 1) RETURNING id`, uuid, org.ID, channelType, name, pq.Array(schemes), role, null.NewMap(config), + `INSERT INTO channels_channel(uuid, org_id, channel_type, name, schemes, role, config, last_seen, is_system, is_active, created_on, modified_on, created_by_id, modified_by_id) + VALUES($1, $2, $3, $4, $5, $6, $7, NOW(), FALSE, TRUE, NOW(), NOW(), 1, 1) RETURNING id`, uuid, org.ID, channelType, name, pq.Array(schemes), role, null.NewMap(config), )) return &Channel{id, uuid} } diff --git a/testsuite/testdata/constants.go b/testsuite/testdata/constants.go index 00e263ee8..ef16fd70c 100644 --- a/testsuite/testdata/constants.go +++ b/testsuite/testdata/constants.go @@ -27,21 +27,25 @@ var IncomingExtraFlow = &Flow{10006, "376d3de6-7f0e-408c-80d6-b1919738bc80"} var ParentTimeoutFlow = &Flow{10007, "81c0f323-7e06-4e0c-a960-19c20f17117c"} var CampaignFlow = &Flow{10009, "3a92a964-3a8d-420b-9206-2cd9d884ac30"} -var CreatedOnField = &Field{3, "53499958-0a0a-48a5-bb5f-8f9f4d8af77b"} -var LastSeenOnField = &Field{5, "4307df2e-b00b-42b6-922b-4a1dcfc268d8"} +var CreatedOnField = &Field{3, "fd18a69d-7514-4b76-9fad-072641995e17"} +var LanguageField = &Field{4, "4307df2e-b00b-42b6-922b-4a1dcfc268d8"} +var LastSeenOnField = &Field{5, "660ebe03-b717-4a80-aebf-9b7c718266e1"} var GenderField = &Field{6, "3a5891e4-756e-4dc9-8e12-b7a766168824"} var AgeField = &Field{7, "903f51da-2717-47c7-a0d3-f2f32877013d"} var JoinedField = &Field{8, "d83aae24-4bbf-49d0-ab85-6bfd201eac6d"} -var AllContactsGroup = &Group{1, "d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008"} -var BlockedContactsGroup = &Group{2, "9295ebab-5c2d-4eb1-86f9-7c15ed2f3219"} +var ActiveGroup = &Group{1, "b97f69f7-5edf-45c7-9fda-d37066eae91d"} +var BlockedGroup = &Group{2, "14f6ea01-456b-4417-b0b8-35e942f549f1"} +var StoppedGroup = &Group{3, "d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008"} +var ArchivedGroup = &Group{4, "9295ebab-5c2d-4eb1-86f9-7c15ed2f3219"} +var OpenTicketsGroup = &Group{5, "361838c4-2866-495a-8990-9f3c222a7604"} var DoctorsGroup = &Group{10000, "c153e265-f7c9-4539-9dbc-9b358714b638"} var TestersGroup = &Group{10001, "5e9d8fab-5e7e-4f51-b533-261af5dea70d"} var ReportingLabel = &Label{10000, "ebc4dedc-91c4-4ed4-9dd6-daa05ea82698"} var TestingLabel = &Label{10001, "a6338cdc-7938-4437-8b05-2d5d785e3a08"} -var DefaultTopic = &Topic{1, "ffc903f7-8cbb-443f-9627-87106842d1aa"} +var DefaultTopic = &Topic{1, "5cc1848a-357c-4de9-9720-45770ec18d11"} var SalesTopic = &Topic{2, "9ef2ff21-064a-41f1-8560-ccc990b4f937"} var SupportTopic = &Topic{3, "0a8f2e00-fef6-402c-bd79-d789446ec0e0"} @@ -51,6 +55,9 @@ var Zendesk = &Ticketer{3, "4ee6d4f3-f92b-439b-9718-8da90c05490b"} var RocketChat = &Ticketer{4, "6c50665f-b4ff-4e37-9625-bc464fe6a999"} var Twilioflex = &Ticketer{6, "12cc5dcf-44c2-4b25-9781-27275873e0df"} +var Partners = &Team{1, "4321c30b-b596-46fa-adb4-4a46d37923f6"} +var Office = &Team{2, "f14c1762-d38b-4072-ae63-2705332a3719"} + var Luis = &Classifier{1, "097e026c-ae79-4740-af67-656dbedf0263"} var Wit = &Classifier{2, "ff2a817c-040a-4eb2-8404-7d92e8b79dd0"} var Bothub = &Classifier{3, "859b436d-3005-4e43-9ad5-3de5f26ede4c"} diff --git a/testsuite/testdata/contacts.go b/testsuite/testdata/contacts.go index 6019b8f99..bbc742edc 100644 --- a/testsuite/testdata/contacts.go +++ b/testsuite/testdata/contacts.go @@ -58,10 +58,15 @@ func InsertContact(db *sqlx.DB, org *Org, uuid flows.ContactUUID, name string, l // InsertContactGroup inserts a contact group func InsertContactGroup(db *sqlx.DB, org *Org, uuid assets.GroupUUID, name, query string) *Group { + groupType := "M" + if query != "" { + groupType = "Q" + } + var id models.GroupID must(db.Get(&id, - `INSERT INTO contacts_contactgroup(uuid, org_id, group_type, name, query, status, is_active, created_by_id, created_on, modified_by_id, modified_on) - VALUES($1, $2, 'U', $3, $4, 'R', TRUE, 1, NOW(), 1, NOW()) RETURNING id`, uuid, org.ID, name, null.String(query), + `INSERT INTO contacts_contactgroup(uuid, org_id, group_type, name, query, status, is_system, is_active, created_by_id, created_on, modified_by_id, modified_on) + VALUES($1, $2, $3, $4, $5, 'R', FALSE, TRUE, 1, NOW(), 1, NOW()) RETURNING id`, uuid, org.ID, groupType, name, null.String(query), )) return &Group{id, uuid} } diff --git a/testsuite/testdata/flows.go b/testsuite/testdata/flows.go index e6ff80e13..6a3807706 100644 --- a/testsuite/testdata/flows.go +++ b/testsuite/testdata/flows.go @@ -71,8 +71,8 @@ func InsertFlowSession(db *sqlx.DB, org *Org, contact *Contact, sessionType mode var id models.SessionID must(db.Get(&id, - `INSERT INTO flows_flowsession(uuid, org_id, contact_id, status, responded, created_on, session_type, current_flow_id, connection_id, wait_started_on, wait_expires_on, wait_resume_on_expire) - VALUES($1, $2, $3, $4, TRUE, NOW(), $5, $6, $7, $8, $9, FALSE) RETURNING id`, uuids.New(), org.ID, contact.ID, status, sessionType, currentFlow.ID, connectionID, waitStartedOn, waitExpiresOn, + `INSERT INTO flows_flowsession(uuid, org_id, contact_id, status, output, responded, created_on, session_type, current_flow_id, connection_id, wait_started_on, wait_expires_on, wait_resume_on_expire) + VALUES($1, $2, $3, $4, '{}', TRUE, NOW(), $5, $6, $7, $8, $9, FALSE) RETURNING id`, uuids.New(), org.ID, contact.ID, status, sessionType, currentFlow.ID, connectionID, waitStartedOn, waitExpiresOn, )) return id } @@ -81,8 +81,8 @@ func InsertFlowSession(db *sqlx.DB, org *Org, contact *Contact, sessionType mode func InsertWaitingSession(db *sqlx.DB, org *Org, contact *Contact, sessionType models.FlowType, currentFlow *Flow, connectionID models.ConnectionID, waitStartedOn, waitExpiresOn time.Time, waitResumeOnExpire bool, waitTimeoutOn *time.Time) models.SessionID { var id models.SessionID must(db.Get(&id, - `INSERT INTO flows_flowsession(uuid, org_id, contact_id, status, responded, created_on, session_type, current_flow_id, connection_id, wait_started_on, wait_expires_on, wait_resume_on_expire, timeout_on) - VALUES($1, $2, $3, 'W', TRUE, NOW(), $4, $5, $6, $7, $8, $9, $10) RETURNING id`, uuids.New(), org.ID, contact.ID, sessionType, currentFlow.ID, connectionID, waitStartedOn, waitExpiresOn, waitResumeOnExpire, waitTimeoutOn, + `INSERT INTO flows_flowsession(uuid, org_id, contact_id, status, output, responded, created_on, session_type, current_flow_id, connection_id, wait_started_on, wait_expires_on, wait_resume_on_expire, timeout_on) + VALUES($1, $2, $3, 'W', '{"status":"waiting"}', TRUE, NOW(), $4, $5, $6, $7, $8, $9, $10) RETURNING id`, uuids.New(), org.ID, contact.ID, sessionType, currentFlow.ID, connectionID, waitStartedOn, waitExpiresOn, waitResumeOnExpire, waitTimeoutOn, )) return id } diff --git a/testsuite/testdata/tickets.go b/testsuite/testdata/tickets.go index bf0bf0225..e2173a3c7 100644 --- a/testsuite/testdata/tickets.go +++ b/testsuite/testdata/tickets.go @@ -23,6 +23,11 @@ type Ticket struct { UUID flows.TicketUUID } +type Team struct { + ID models.TeamID + UUID models.TeamUUID +} + func (k *Ticket) Load(db *sqlx.DB) *models.Ticket { tickets, err := models.LoadTickets(context.Background(), db, []models.TicketID{k.ID}) must(err, len(tickets) == 1) @@ -35,16 +40,16 @@ type Ticketer struct { } // InsertOpenTicket inserts an open ticket -func InsertOpenTicket(db *sqlx.DB, org *Org, contact *Contact, ticketer *Ticketer, topic *Topic, body, externalID string, assignee *User) *Ticket { - return insertTicket(db, org, contact, ticketer, models.TicketStatusOpen, topic, body, externalID, assignee) +func InsertOpenTicket(db *sqlx.DB, org *Org, contact *Contact, ticketer *Ticketer, topic *Topic, body, externalID string, openedOn time.Time, assignee *User) *Ticket { + return insertTicket(db, org, contact, ticketer, models.TicketStatusOpen, topic, body, externalID, openedOn, assignee) } // InsertClosedTicket inserts a closed ticket func InsertClosedTicket(db *sqlx.DB, org *Org, contact *Contact, ticketer *Ticketer, topic *Topic, body, externalID string, assignee *User) *Ticket { - return insertTicket(db, org, contact, ticketer, models.TicketStatusClosed, topic, body, externalID, assignee) + return insertTicket(db, org, contact, ticketer, models.TicketStatusClosed, topic, body, externalID, dates.Now(), assignee) } -func insertTicket(db *sqlx.DB, org *Org, contact *Contact, ticketer *Ticketer, status models.TicketStatus, topic *Topic, body, externalID string, assignee *User) *Ticket { +func insertTicket(db *sqlx.DB, org *Org, contact *Contact, ticketer *Ticketer, status models.TicketStatus, topic *Topic, body, externalID string, openedOn time.Time, assignee *User) *Ticket { uuid := flows.TicketUUID(uuids.New()) var closedOn *time.Time if status == models.TicketStatusClosed { @@ -55,7 +60,7 @@ func insertTicket(db *sqlx.DB, org *Org, contact *Contact, ticketer *Ticketer, s var id models.TicketID must(db.Get(&id, `INSERT INTO tickets_ticket(uuid, org_id, contact_id, ticketer_id, status, topic_id, body, external_id, opened_on, modified_on, closed_on, last_activity_on, assignee_id) - VALUES($1, $2, $3, $4, $5, $6, $7, $8, NOW(), NOW(), $9, NOW(), $10) RETURNING id`, uuid, org.ID, contact.ID, ticketer.ID, status, topic.ID, body, externalID, closedOn, assignee.SafeID(), + VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW(), $10, NOW(), $11) RETURNING id`, uuid, org.ID, contact.ID, ticketer.ID, status, topic.ID, body, externalID, openedOn, closedOn, assignee.SafeID(), )) return &Ticket{id, uuid} } diff --git a/testsuite/testsuite.go b/testsuite/testsuite.go index 18d7f9053..493717efc 100644 --- a/testsuite/testsuite.go +++ b/testsuite/testsuite.go @@ -178,12 +178,13 @@ func resetStorage() { must(os.RemoveAll(SessionStorageDir)) } -var resetDataSQL = ` +var sqlResetTestData = ` UPDATE contacts_contact SET current_flow_id = NULL; DELETE FROM notifications_notification; DELETE FROM notifications_incident; DELETE FROM request_logs_httplog; +DELETE FROM tickets_ticketdailycount; DELETE FROM tickets_ticketevent; DELETE FROM tickets_ticket; DELETE FROM triggers_trigger_contacts WHERE trigger_id >= 30000; @@ -226,7 +227,7 @@ ALTER SEQUENCE campaigns_campaignevent_id_seq RESTART WITH 30000;` // undo changes made to the contact data in the test database dump. func resetData() { db := getDB() - db.MustExec(resetDataSQL) + db.MustExec(sqlResetTestData) // because groups have changed models.FlushCache() diff --git a/utils/cron/cron.go b/utils/cron/cron.go index d673c1e62..efe365589 100644 --- a/utils/cron/cron.go +++ b/utils/cron/cron.go @@ -1,23 +1,26 @@ package cron import ( + "context" "fmt" + "sync" "time" - "github.com/apex/log" "github.com/nyaruka/mailroom/runtime" "github.com/nyaruka/redisx" "github.com/sirupsen/logrus" ) // Function is the function that will be called on our schedule -type Function func() error +type Function func(context.Context, *runtime.Runtime) error // Start calls the passed in function every interval, making sure it acquires a // lock so that only one process is running at once. Note that across processes // crons may be called more often than duration as there is no inter-process // coordination of cron fires. (this might be a worthy addition) -func Start(quit chan bool, rt *runtime.Runtime, name string, interval time.Duration, allInstances bool, cronFunc Function) { +func Start(rt *runtime.Runtime, wg *sync.WaitGroup, name string, interval time.Duration, allInstances bool, cronFunc Function, timeout time.Duration, quit chan bool) { + wg.Add(1) // add ourselves to the wait group + lockName := fmt.Sprintf("lock:%s_lock", name) // for historical reasons... // for jobs that run on all instances, the lock key is specific to this instance @@ -33,7 +36,10 @@ func Start(quit chan bool, rt *runtime.Runtime, name string, interval time.Durat log := logrus.WithField("cron", name).WithField("lockName", lockName) go func() { - defer log.Info("cron exiting") + defer func() { + log.Info("cron exiting") + wg.Done() + }() for { select { @@ -57,16 +63,23 @@ func Start(quit chan bool, rt *runtime.Runtime, name string, interval time.Durat } // ok, got the lock, run our cron function - err = fireCron(cronFunc, lockName, lock) + start := time.Now() + err = fireCron(rt, cronFunc, lockName, lock) if err != nil { log.WithError(err).Error("error while running cron") } + elapsed := time.Since(start) // release our lock err = locker.Release(rt.RP, lock) if err != nil { log.WithError(err).Error("error releasing lock") } + + // if cron too longer than a minute, log + if elapsed > time.Minute { + logrus.WithField("cron", name).WithField("elapsed", elapsed).Error("cron took too long") + } } // calculate our next fire time @@ -81,8 +94,11 @@ func Start(quit chan bool, rt *runtime.Runtime, name string, interval time.Durat // fireCron is just a wrapper around the cron function we will call for the purposes of // catching and logging panics -func fireCron(cronFunc Function, lockName string, lockValue string) error { - log := log.WithField("lockValue", lockValue).WithField("func", cronFunc) +func fireCron(rt *runtime.Runtime, cronFunc Function, lockName string, lockValue string) error { + log := logrus.WithField("lockValue", lockValue).WithField("func", cronFunc) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() defer func() { // catch any panics and recover @@ -92,7 +108,7 @@ func fireCron(cronFunc Function, lockName string, lockValue string) error { } }() - return cronFunc() + return cronFunc(ctx, rt) } // NextFire returns the next time we should fire based on the passed in time and interval diff --git a/utils/cron/cron_test.go b/utils/cron/cron_test.go index f0cb771f1..45460451a 100644 --- a/utils/cron/cron_test.go +++ b/utils/cron/cron_test.go @@ -1,9 +1,12 @@ package cron_test import ( + "context" + "sync" "testing" "time" + "github.com/nyaruka/mailroom/runtime" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/utils/cron" @@ -21,7 +24,7 @@ func TestCron(t *testing.T) { } createCronFunc := func(running *bool, fired *int, delays map[int]time.Duration, defaultDelay time.Duration) cron.Function { - return func() error { + return func(ctx context.Context, rt *runtime.Runtime) error { if *running { assert.Fail(t, "more than 1 thread is trying to run our cron job") } @@ -39,13 +42,14 @@ func TestCron(t *testing.T) { } fired := 0 + wg := &sync.WaitGroup{} quit := make(chan bool) running := false align() // start a job that takes ~100 ms and runs every 250ms - cron.Start(quit, rt, "test1", time.Millisecond*250, false, createCronFunc(&running, &fired, map[int]time.Duration{}, time.Millisecond*100)) + cron.Start(rt, wg, "test1", time.Millisecond*250, false, createCronFunc(&running, &fired, map[int]time.Duration{}, time.Millisecond*100), time.Minute, quit) // wait a bit, should only have fired three times (initial time + three repeats) time.Sleep(time.Millisecond * 875) // time for 3 delays between tasks plus half of another delay @@ -64,7 +68,7 @@ func TestCron(t *testing.T) { align() // simulate the job taking 400ms to run on the second fire, thus skipping the third fire - cron.Start(quit, rt, "test2", time.Millisecond*250, false, createCronFunc(&running, &fired, map[int]time.Duration{1: time.Millisecond * 400}, time.Millisecond*100)) + cron.Start(rt, wg, "test2", time.Millisecond*250, false, createCronFunc(&running, &fired, map[int]time.Duration{1: time.Millisecond * 400}, time.Millisecond*100), time.Minute, quit) time.Sleep(time.Millisecond * 875) assert.Equal(t, 3, fired) @@ -88,8 +92,8 @@ func TestCron(t *testing.T) { align() - cron.Start(quit, &rt1, "test3", time.Millisecond*250, false, createCronFunc(&running, &fired1, map[int]time.Duration{}, time.Millisecond*100)) - cron.Start(quit, &rt2, "test3", time.Millisecond*250, false, createCronFunc(&running, &fired2, map[int]time.Duration{}, time.Millisecond*100)) + cron.Start(&rt1, wg, "test3", time.Millisecond*250, false, createCronFunc(&running, &fired1, map[int]time.Duration{}, time.Millisecond*100), time.Minute, quit) + cron.Start(&rt2, wg, "test3", time.Millisecond*250, false, createCronFunc(&running, &fired2, map[int]time.Duration{}, time.Millisecond*100), time.Minute, quit) // same number of fires as if only a single instance was running it... time.Sleep(time.Millisecond * 875) @@ -106,8 +110,8 @@ func TestCron(t *testing.T) { align() // unless we start the cron with allInstances = true - cron.Start(quit, &rt1, "test4", time.Millisecond*250, true, createCronFunc(&running1, &fired1, map[int]time.Duration{}, time.Millisecond*100)) - cron.Start(quit, &rt2, "test4", time.Millisecond*250, true, createCronFunc(&running2, &fired2, map[int]time.Duration{}, time.Millisecond*100)) + cron.Start(&rt1, wg, "test4", time.Millisecond*250, true, createCronFunc(&running1, &fired1, map[int]time.Duration{}, time.Millisecond*100), time.Minute, quit) + cron.Start(&rt2, wg, "test4", time.Millisecond*250, true, createCronFunc(&running2, &fired2, map[int]time.Duration{}, time.Millisecond*100), time.Minute, quit) // now both instances fire 4 times time.Sleep(time.Millisecond * 875) diff --git a/utils/locker/locker.go b/utils/locker/locker.go deleted file mode 100644 index 36340f9d8..000000000 --- a/utils/locker/locker.go +++ /dev/null @@ -1,103 +0,0 @@ -package locker - -import ( - "fmt" - "math/rand" - "time" - - "github.com/gomodule/redigo/redis" - "github.com/pkg/errors" -) - -// GrabLock grabs the passed in lock from redis in an atomic operation. It returns the lock value -// if successful. It will retry until the retry period, returning empty string if not acquired -// in that time. -func GrabLock(rp *redis.Pool, key string, expiration time.Duration, retry time.Duration) (string, error) { - // generate our lock value - value := makeRandom(10) - - // convert our expiration to seconds - seconds := int(expiration / time.Second) - if seconds < 1 { - return "", errors.Errorf("can't grab lock with expiration less than a second") - } - - start := time.Now() - for { - rc := rp.Get() - success, err := rc.Do("SET", fmt.Sprintf("lock:%s", key), value, "EX", seconds, "NX") - rc.Close() - - if err != nil { - return "", errors.Wrapf(err, "error trying to get lock") - } - - if success == "OK" { - break - } - - if time.Since(start) > retry { - return "", nil - } - - time.Sleep(time.Second) - } - - return value, nil -} - -var releaseScript = redis.NewScript(2, ` - -- KEYS: [Key, Value] - if redis.call("get", KEYS[1]) == KEYS[2] then - return redis.call("del", KEYS[1]) - else - return 0 - end -`) - -// ReleaseLock releases the passed in lock, returning any error encountered while doing -// so. It is not considered an error to release a lock that is no longer present -func ReleaseLock(rp *redis.Pool, key string, value string) error { - rc := rp.Get() - defer rc.Close() - - // we use lua here because we only want to release the lock if we own it - _, err := releaseScript.Do(rc, fmt.Sprintf("lock:%s", key), value) - return err -} - -var expireScript = redis.NewScript(3, ` - -- KEYS: [Key, Value, Expiration] - if redis.call("get", KEYS[1]) == KEYS[2] then - return redis.call("expire", KEYS[1], KEYS[3]) - else - return 0 - end -`) - -// ExtendLock extends our lock expiration by the passed in number of seconds -func ExtendLock(rp *redis.Pool, key string, value string, expiration time.Duration) error { - rc := rp.Get() - defer rc.Close() - - // convert our expiration to seconds - seconds := int(expiration / time.Second) - if seconds < 1 { - return errors.Errorf("can't grab lock with expiration less than a second") - } - - // we use lua here because we only want to set the expiration time if we own it - _, err := expireScript.Do(rc, fmt.Sprintf("lock:%s", key), value, seconds) - return err -} - -const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - -// makeRandom creates a random key of the length passed in -func makeRandom(n int) string { - b := make([]byte, n) - for i := range b { - b[i] = letterBytes[rand.Intn(len(letterBytes))] - } - return string(b) -} diff --git a/utils/locker/locker_test.go b/utils/locker/locker_test.go deleted file mode 100644 index 18cd93464..000000000 --- a/utils/locker/locker_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package locker_test - -import ( - "testing" - "time" - - "github.com/nyaruka/mailroom/testsuite" - "github.com/nyaruka/mailroom/utils/locker" - - "github.com/stretchr/testify/assert" -) - -func TestLocker(t *testing.T) { - _, _, _, rp := testsuite.Get() - - defer testsuite.Reset(testsuite.ResetRedis) - - // acquire a lock, but have it expire in 5 seconds - v1, err := locker.GrabLock(rp, "test", time.Second*5, time.Second) - assert.NoError(t, err) - assert.NotZero(t, v1) - - // try to acquire the same lock, should fail - v2, err := locker.GrabLock(rp, "test", time.Second*5, time.Second) - assert.NoError(t, err) - assert.Zero(t, v2) - - // should succeed if we wait longer - v3, err := locker.GrabLock(rp, "test", time.Second*5, time.Second*5) - assert.NoError(t, err) - assert.NotZero(t, v3) - assert.NotEqual(t, v1, v3) - - // extend the lock - err = locker.ExtendLock(rp, "test", v3, time.Second*10) - assert.NoError(t, err) - - // trying to grab it should fail with a 5 second timeout - v4, err := locker.GrabLock(rp, "test", time.Second*5, time.Second*5) - assert.NoError(t, err) - assert.Zero(t, v4) - - // return the lock - err = locker.ReleaseLock(rp, "test", v3) - assert.NoError(t, err) - - // new grab should work - v5, err := locker.GrabLock(rp, "test", time.Second*5, time.Second*5) - assert.NoError(t, err) - assert.NotZero(t, v5) -} diff --git a/web/contact/search.go b/web/contact/search.go index 365733c80..aff32ae23 100644 --- a/web/contact/search.go +++ b/web/contact/search.go @@ -8,6 +8,7 @@ import ( "github.com/nyaruka/goflow/contactql" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/core/search" "github.com/nyaruka/mailroom/runtime" "github.com/nyaruka/mailroom/web" @@ -23,14 +24,15 @@ func init() { // // { // "org_id": 1, -// "group_uuid": "985a83fe-2e9f-478d-a3ec-fa602d5e7ddd", +// "group_id": 234, // "query": "age > 10", // "sort": "-age" // } // type searchRequest struct { OrgID models.OrgID `json:"org_id" validate:"required"` - GroupUUID assets.GroupUUID `json:"group_uuid" validate:"required"` + GroupID models.GroupID `json:"group_id"` + GroupUUID assets.GroupUUID `json:"group_uuid"` // deprecated ExcludeIDs []models.ContactID `json:"exclude_ids"` Query string `json:"query"` PageSize int `json:"page_size"` @@ -78,9 +80,15 @@ func handleSearch(ctx context.Context, rt *runtime.Runtime, r *http.Request) (in return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") } + var group *models.Group + if request.GroupID != 0 { + group = oa.GroupByID(request.GroupID) + } else if request.GroupUUID != "" { + group = oa.GroupByUUID(request.GroupUUID) + } + // perform our search - parsed, hits, total, err := models.GetContactIDsForQueryPage(ctx, rt.ES, oa, - request.GroupUUID, request.ExcludeIDs, request.Query, request.Sort, request.Offset, request.PageSize) + parsed, hits, total, err := search.GetContactIDsForQueryPage(ctx, rt.ES, oa, group, request.ExcludeIDs, request.Query, request.Sort, request.Offset, request.PageSize) if err != nil { isQueryError, qerr := contactql.IsQueryError(err) @@ -117,14 +125,15 @@ func handleSearch(ctx context.Context, rt *runtime.Runtime, r *http.Request) (in // { // "org_id": 1, // "query": "age > 10", -// "group_uuid": "123123-123-123-" +// "group_id": 234 // } // type parseRequest struct { OrgID models.OrgID `json:"org_id" validate:"required"` Query string `json:"query" validate:"required"` ParseOnly bool `json:"parse_only"` - GroupUUID assets.GroupUUID `json:"group_uuid"` + GroupID models.GroupID `json:"group_id"` + GroupUUID assets.GroupUUID `json:"group_uuid"` // deprecated } // Response for a parse query request @@ -158,6 +167,13 @@ func handleParseQuery(ctx context.Context, rt *runtime.Runtime, r *http.Request) return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") } + var group *models.Group + if request.GroupID != 0 { + group = oa.GroupByID(request.GroupID) + } else if request.GroupUUID != "" { + group = oa.GroupByUUID(request.GroupUUID) + } + env := oa.Env() var resolver contactql.Resolver if !request.ParseOnly { @@ -179,7 +195,7 @@ func handleParseQuery(ctx context.Context, rt *runtime.Runtime, r *http.Request) var elasticSource interface{} if !request.ParseOnly { - eq := models.BuildElasticQuery(oa, request.GroupUUID, models.NilContactStatus, nil, parsed) + eq := search.BuildElasticQuery(oa, group, models.NilContactStatus, nil, parsed) elasticSource, err = eq.Source() if err != nil { return nil, http.StatusInternalServerError, errors.Wrap(err, "error getting elastic source") diff --git a/web/contact/search_test.go b/web/contact/search_test.go index 2ff3871ea..1713b0dfe 100644 --- a/web/contact/search_test.go +++ b/web/contact/search_test.go @@ -18,26 +18,18 @@ import ( "github.com/nyaruka/mailroom/testsuite/testdata" "github.com/nyaruka/mailroom/web" - "github.com/olivere/elastic/v7" "github.com/stretchr/testify/assert" ) -func TestSearch(t *testing.T) { +func TestContactSearch(t *testing.T) { ctx, rt, _, _ := testsuite.Get() wg := &sync.WaitGroup{} - es := testsuite.NewMockElasticServer() - defer es.Close() + mockES := testsuite.NewMockElasticServer() + defer mockES.Close() - client, err := elastic.NewClient( - elastic.SetURL(es.URL()), - elastic.SetHealthcheck(false), - elastic.SetSniff(false), - ) - assert.NoError(t, err) - - rt.ES = client + rt.ES = mockES.Client() server := web.NewServer(ctx, rt, wg) server.Start() @@ -47,39 +39,11 @@ func TestSearch(t *testing.T) { defer server.Stop() - singleESResponse := fmt.Sprintf(`{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }`, testdata.Cathy.ID) - tcs := []struct { method string url string body string - esResponse string + mockResult []models.ContactID expectedStatus int expectedError string expectedHits []models.ContactID @@ -99,22 +63,22 @@ func TestSearch(t *testing.T) { { method: "POST", url: "/mr/contact/search", - body: fmt.Sprintf(`{"org_id": 1, "query": "birthday = tomorrow", "group_uuid": "%s"}`, testdata.AllContactsGroup.UUID), + body: fmt.Sprintf(`{"org_id": 1, "query": "birthday = tomorrow", "group_uuid": "%s"}`, testdata.ActiveGroup.UUID), expectedStatus: 400, expectedError: "can't resolve 'birthday' to attribute, scheme or field", }, { method: "POST", url: "/mr/contact/search", - body: fmt.Sprintf(`{"org_id": 1, "query": "age > tomorrow", "group_uuid": "%s"}`, testdata.AllContactsGroup.UUID), + body: fmt.Sprintf(`{"org_id": 1, "query": "age > tomorrow", "group_uuid": "%s"}`, testdata.ActiveGroup.UUID), expectedStatus: 400, expectedError: "can't convert 'tomorrow' to a number", }, { method: "POST", url: "/mr/contact/search", - body: fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s"}`, testdata.AllContactsGroup.UUID), - esResponse: singleESResponse, + body: fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s"}`, testdata.ActiveGroup.UUID), + mockResult: []models.ContactID{testdata.Cathy.ID}, expectedStatus: 200, expectedHits: []models.ContactID{testdata.Cathy.ID}, expectedQuery: `name ~ "Cathy"`, @@ -126,10 +90,10 @@ func TestSearch(t *testing.T) { { method: "POST", url: "/mr/contact/search", - body: fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s", "exclude_ids": [%d, %d]}`, testdata.AllContactsGroup.UUID, testdata.Bob.ID, testdata.George.ID), - esResponse: singleESResponse, + body: fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s", "exclude_ids": [%d, %d]}`, testdata.ActiveGroup.UUID, testdata.Bob.ID, testdata.George.ID), + mockResult: []models.ContactID{testdata.George.ID}, expectedStatus: 200, - expectedHits: []models.ContactID{testdata.Cathy.ID}, + expectedHits: []models.ContactID{testdata.George.ID}, expectedQuery: `name ~ "Cathy"`, expectedAttributes: []string{"name"}, expectedFields: []*assets.FieldReference{}, @@ -153,7 +117,7 @@ func TestSearch(t *testing.T) { }, { "term": { - "groups": "d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008" + "group_ids": 1 } }, { @@ -188,8 +152,8 @@ func TestSearch(t *testing.T) { { method: "POST", url: "/mr/contact/search", - body: fmt.Sprintf(`{"org_id": 1, "query": "AGE = 10 and gender = M", "group_uuid": "%s"}`, testdata.AllContactsGroup.UUID), - esResponse: singleESResponse, + body: fmt.Sprintf(`{"org_id": 1, "query": "AGE = 10 and gender = M", "group_uuid": "%s"}`, testdata.ActiveGroup.UUID), + mockResult: []models.ContactID{testdata.Cathy.ID}, expectedStatus: 200, expectedHits: []models.ContactID{testdata.Cathy.ID}, expectedQuery: `age = 10 AND gender = "M"`, @@ -204,8 +168,8 @@ func TestSearch(t *testing.T) { { method: "POST", url: "/mr/contact/search", - body: fmt.Sprintf(`{"org_id": 1, "query": "", "group_uuid": "%s"}`, testdata.AllContactsGroup.UUID), - esResponse: singleESResponse, + body: fmt.Sprintf(`{"org_id": 1, "query": "", "group_uuid": "%s"}`, testdata.ActiveGroup.UUID), + mockResult: []models.ContactID{testdata.Cathy.ID}, expectedStatus: 200, expectedHits: []models.ContactID{testdata.Cathy.ID}, expectedQuery: ``, @@ -217,9 +181,11 @@ func TestSearch(t *testing.T) { } for i, tc := range tcs { - var body io.Reader - es.NextResponse = tc.esResponse + if tc.mockResult != nil { + mockES.AddResponse(tc.mockResult...) + } + var body io.Reader if tc.body != "" { body = bytes.NewReader([]byte(tc.body)) } @@ -251,7 +217,7 @@ func TestSearch(t *testing.T) { } if tc.expectedESRequest != "" { - test.AssertEqualJSON(t, []byte(tc.expectedESRequest), []byte(es.LastRequestBody), "elastic request mismatch") + test.AssertEqualJSON(t, []byte(tc.expectedESRequest), []byte(mockES.LastRequestBody), "elastic request mismatch") } } else { r := &web.ErrorResponse{} diff --git a/web/contact/testdata/parse_query.json b/web/contact/testdata/parse_query.json index a4e0d0b64..9a835e6e6 100644 --- a/web/contact/testdata/parse_query.json +++ b/web/contact/testdata/parse_query.json @@ -140,13 +140,13 @@ } }, { - "label": "valid query with group", + "label": "valid query with group by ID", "method": "POST", "path": "/mr/contact/parse_query", "body": { "org_id": 1, "query": "age > 10", - "group_uuid": "903f51da-2717-47c7-a0d3-f2f32877013d" + "group_id": 10000 }, "status": 200, "response": { @@ -166,7 +166,80 @@ }, { "term": { - "groups": "903f51da-2717-47c7-a0d3-f2f32877013d" + "group_ids": 10000 + } + }, + { + "nested": { + "path": "fields", + "query": { + "bool": { + "must": [ + { + "term": { + "fields.field": "903f51da-2717-47c7-a0d3-f2f32877013d" + } + }, + { + "range": { + "fields.number": { + "from": 10, + "include_lower": false, + "include_upper": true, + "to": null + } + } + } + ] + } + } + } + } + ] + } + }, + "metadata": { + "attributes": [], + "schemes": [], + "fields": [ + { + "key": "age", + "name": "Age" + } + ], + "groups": [], + "allow_as_group": true + } + } + }, + { + "label": "valid query with group by UUID", + "method": "POST", + "path": "/mr/contact/parse_query", + "body": { + "org_id": 1, + "query": "age > 10", + "group_uuid": "c153e265-f7c9-4539-9dbc-9b358714b638" + }, + "status": 200, + "response": { + "query": "age > 10", + "elastic_query": { + "bool": { + "must": [ + { + "term": { + "org_id": 1 + } + }, + { + "term": { + "is_active": true + } + }, + { + "term": { + "group_ids": 10000 } }, { @@ -238,7 +311,7 @@ }, { "term": { - "groups": "5e9d8fab-5e7e-4f51-b533-261af5dea70d" + "group_ids": 10001 } } ] diff --git a/web/flow/start.go b/web/flow/start.go new file mode 100644 index 000000000..7e44a4f4c --- /dev/null +++ b/web/flow/start.go @@ -0,0 +1,124 @@ +package flow + +import ( + "context" + "net/http" + + "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/contactql" + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/utils" + "github.com/nyaruka/mailroom/core/models" + "github.com/nyaruka/mailroom/core/search" + "github.com/nyaruka/mailroom/runtime" + "github.com/nyaruka/mailroom/web" + "github.com/pkg/errors" +) + +func init() { + web.RegisterJSONRoute(http.MethodPost, "/mr/flow/preview_start", web.RequireAuthToken(handlePreviewStart)) +} + +// Generates a preview of which contacts will be started in the given flow. +// +// { +// "org_id": 1, +// "flow_id": 2, +// "include": { +// "group_uuids": ["5fa925e4-edd8-4e2a-ab24-b3dbb5932ddd", "2912b95f-5b89-4d39-a2a8-5292602f357f"], +// "contact_uuids": ["e5bb9e6f-7703-4ba1-afba-0b12791de38b"], +// "urns": ["tel:+1234567890"], +// "user_query": "" +// }, +// "exclude": { +// "non_active": false, +// "in_a_flow": false, +// "started_previously": true, +// "not_seen_recently": false +// }, +// "sample_size": 5 +// } +// +// { +// "query": "(group = "No Age" OR group = "No Name" OR uuid = "e5bb9e6f-7703-4ba1-afba-0b12791de38b" OR tel = "+1234567890") AND history != \"Registration\"", +// "total": 567, +// "sample": [12, 34, 56, 67, 78], +// "metadata": { +// "fields": [ +// {"key": "age", "name": "Age"} +// ], +// "allow_as_group": true +// } +// } +// +type previewStartRequest struct { + OrgID models.OrgID `json:"org_id" validate:"required"` + FlowID models.FlowID `json:"flow_id" validate:"required"` + Include struct { + GroupUUIDs []assets.GroupUUID `json:"group_uuids"` + ContactUUIDs []flows.ContactUUID `json:"contact_uuids"` + URNs []urns.URN `json:"urns"` + Query string `json:"query"` + } `json:"include" validate:"required"` + Exclude search.Exclusions `json:"exclude"` + SampleSize int `json:"sample_size" validate:"required"` +} + +type previewStartResponse struct { + Query string `json:"query"` + Total int `json:"total"` + SampleIDs []models.ContactID `json:"sample_ids"` + Metadata *contactql.Inspection `json:"metadata,omitempty"` +} + +func handlePreviewStart(ctx context.Context, rt *runtime.Runtime, r *http.Request) (interface{}, int, error) { + request := &previewStartRequest{} + if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil { + return errors.Wrapf(err, "request failed validation"), http.StatusBadRequest, nil + } + + oa, err := models.GetOrgAssets(ctx, rt, request.OrgID) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") + } + + flow, err := oa.FlowByID(request.FlowID) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load flow") + } + + groups := make([]*models.Group, 0, len(request.Include.GroupUUIDs)) + for _, groupUUID := range request.Include.GroupUUIDs { + g := oa.GroupByUUID(groupUUID) + if g != nil { + groups = append(groups, g) + } + } + + query, err := search.BuildStartQuery(oa, flow, groups, request.Include.ContactUUIDs, request.Include.URNs, request.Include.Query, request.Exclude) + if err != nil { + isQueryError, qerr := contactql.IsQueryError(err) + if isQueryError { + return qerr, http.StatusBadRequest, nil + } + return nil, http.StatusInternalServerError, err + } + if query == "" { + return &previewStartResponse{SampleIDs: []models.ContactID{}}, http.StatusOK, nil + } + + parsedQuery, sampleIDs, total, err := search.GetContactIDsForQueryPage(ctx, rt.ES, oa, nil, nil, query, "", 0, request.SampleSize) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "error querying preview") + } + + inspection := contactql.Inspect(parsedQuery) + + return &previewStartResponse{ + Query: parsedQuery.String(), + Total: int(total), + SampleIDs: sampleIDs, + Metadata: inspection, + }, http.StatusOK, nil +} diff --git a/web/flow/start_test.go b/web/flow/start_test.go new file mode 100644 index 000000000..6e2fac073 --- /dev/null +++ b/web/flow/start_test.go @@ -0,0 +1,25 @@ +package flow_test + +import ( + "testing" + + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/testsuite/testdata" + "github.com/nyaruka/mailroom/web" +) + +func TestPreviewStart(t *testing.T) { + ctx, rt, _, _ := testsuite.Get() + + mockES := testsuite.NewMockElasticServer() + defer mockES.Close() + + rt.ES = mockES.Client() + + mockES.AddResponse(testdata.Cathy.ID) + mockES.AddResponse(testdata.Bob.ID) + mockES.AddResponse(testdata.George.ID) + mockES.AddResponse(testdata.Alexandria.ID) + + web.RunWebTests(t, ctx, rt, "testdata/preview_start.json", nil) +} diff --git a/web/flow/testdata/preview_start.json b/web/flow/testdata/preview_start.json new file mode 100644 index 000000000..8f037d917 --- /dev/null +++ b/web/flow/testdata/preview_start.json @@ -0,0 +1,282 @@ +[ + { + "label": "illegal method", + "method": "GET", + "path": "/mr/flow/preview_start", + "status": 405, + "response": { + "error": "illegal method: GET" + } + }, + { + "label": "missing org or flow id", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": {}, + "status": 400, + "response": { + "error": "request failed validation: field 'org_id' is required, field 'flow_id' is required, field 'sample_size' is required" + } + }, + { + "label": "no inclusions or exclusions", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": { + "org_id": 1, + "flow_id": 10001, + "include": {}, + "sample_size": 3 + }, + "status": 200, + "response": { + "query": "", + "total": 0, + "sample_ids": [] + } + }, + { + "label": "manual inclusions, no exclusions", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": { + "org_id": 1, + "flow_id": 10001, + "include": { + "group_uuids": [ + "c153e265-f7c9-4539-9dbc-9b358714b638", + "5e9d8fab-5e7e-4f51-b533-261af5dea70d" + ], + "contact_uuids": [ + "5a8345c1-514a-4d1b-aee5-6f39b2f53cfa", + "bd2aab59-5e28-4db4-b6e8-bbdb75fd7a0a" + ], + "urns": [ + "tel:+1234567890", + "facebook:9876543210" + ], + "query": "" + }, + "sample_size": 3 + }, + "status": 200, + "response": { + "query": "group = \"Doctors\" OR group = \"Testers\" OR uuid = \"5a8345c1-514a-4d1b-aee5-6f39b2f53cfa\" OR uuid = \"bd2aab59-5e28-4db4-b6e8-bbdb75fd7a0a\" OR tel = \"+1234567890\" OR facebook = 9876543210", + "total": 1, + "sample_ids": [ + 10000 + ], + "metadata": { + "attributes": [ + "group", + "uuid" + ], + "fields": [], + "groups": [ + { + "name": "Doctors", + "uuid": "c153e265-f7c9-4539-9dbc-9b358714b638" + }, + { + "name": "Testers", + "uuid": "5e9d8fab-5e7e-4f51-b533-261af5dea70d" + } + ], + "schemes": [ + "facebook", + "tel" + ], + "allow_as_group": false + } + } + }, + { + "label": "query inclusion, no exclusions", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": { + "org_id": 1, + "flow_id": 10001, + "include": { + "group_ids": [], + "contact_ids": [], + "urns": [], + "query": "gender = M" + }, + "sample_size": 3 + }, + "status": 200, + "response": { + "query": "gender = \"M\"", + "total": 1, + "sample_ids": [ + 10001 + ], + "metadata": { + "attributes": [], + "fields": [ + { + "key": "gender", + "name": "Gender" + } + ], + "groups": [], + "schemes": [], + "allow_as_group": true + } + } + }, + { + "label": "manual inclusions, all exclusions", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": { + "org_id": 1, + "flow_id": 10001, + "include": { + "group_uuids": [ + "c153e265-f7c9-4539-9dbc-9b358714b638", + "5e9d8fab-5e7e-4f51-b533-261af5dea70d" + ], + "contact_uuids": [ + "5a8345c1-514a-4d1b-aee5-6f39b2f53cfa", + "bd2aab59-5e28-4db4-b6e8-bbdb75fd7a0a" + ], + "urns": [ + "tel:+1234567890", + "facebook:9876543210" + ], + "query": "" + }, + "exclude": { + "non_active": true, + "in_a_flow": true, + "started_previously": true, + "not_seen_since_days": 90 + }, + "sample_size": 3 + }, + "status": 200, + "response": { + "query": "(group = \"Doctors\" OR group = \"Testers\" OR uuid = \"5a8345c1-514a-4d1b-aee5-6f39b2f53cfa\" OR uuid = \"bd2aab59-5e28-4db4-b6e8-bbdb75fd7a0a\" OR tel = \"+1234567890\" OR facebook = 9876543210) AND status = \"active\" AND flow = \"\" AND history != \"Pick a Number\" AND last_seen_on > \"07-04-2018\"", + "total": 1, + "sample_ids": [ + 10002 + ], + "metadata": { + "attributes": [ + "flow", + "group", + "history", + "last_seen_on", + "status", + "uuid" + ], + "fields": [], + "groups": [ + { + "name": "Doctors", + "uuid": "c153e265-f7c9-4539-9dbc-9b358714b638" + }, + { + "name": "Testers", + "uuid": "5e9d8fab-5e7e-4f51-b533-261af5dea70d" + } + ], + "schemes": [ + "facebook", + "tel" + ], + "allow_as_group": false + } + } + }, + { + "label": "query inclusion, all exclusions", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": { + "org_id": 1, + "flow_id": 10001, + "include": { + "query": "gender = M" + }, + "exclude": { + "non_active": true, + "in_a_flow": true, + "started_previously": true, + "not_seen_since_days": 90 + }, + "sample_size": 3 + }, + "status": 200, + "response": { + "query": "gender = \"M\" AND status = \"active\" AND flow = \"\" AND history != \"Pick a Number\" AND last_seen_on > \"07-04-2018\"", + "total": 1, + "sample_ids": [ + 10003 + ], + "metadata": { + "attributes": [ + "flow", + "history", + "last_seen_on", + "status" + ], + "fields": [ + { + "key": "gender", + "name": "Gender" + } + ], + "groups": [], + "schemes": [], + "allow_as_group": false + } + } + }, + { + "label": "invalid query inclusion (bad syntax)", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": { + "org_id": 1, + "flow_id": 10001, + "include": { + "query": "gender =" + }, + "exclude": {}, + "sample_size": 3 + }, + "status": 400, + "response": { + "code": "unexpected_token", + "error": "mismatched input '' expecting {TEXT, STRING}", + "extra": { + "token": "" + } + } + }, + { + "label": "invalid query inclusion (missing field)", + "method": "POST", + "path": "/mr/flow/preview_start", + "body": { + "org_id": 1, + "flow_id": 10001, + "include": { + "query": "goats > 10" + }, + "exclude": {}, + "sample_size": 3 + }, + "status": 400, + "response": { + "code": "unknown_property", + "error": "can't resolve 'goats' to attribute, scheme or field", + "extra": { + "property": "goats" + } + } + } +] \ No newline at end of file diff --git a/web/ivr/ivr.go b/web/ivr/ivr.go index 7120f8a6a..360f1bdad 100644 --- a/web/ivr/ivr.go +++ b/web/ivr/ivr.go @@ -362,7 +362,7 @@ func handleStatus(ctx context.Context, rt *runtime.Runtime, r *http.Request, w h return channel, nil, provider.WriteErrorResponse(w, errors.Wrapf(err, "error while preprocessing status")) } if len(body) > 0 { - contentType := http.DetectContentType(body) + contentType := httpx.DetectContentType(body) w.Header().Set("Content-Type", contentType) _, err := w.Write(body) return channel, nil, err diff --git a/web/ivr/ivr_test.go b/web/ivr/ivr_test.go index 156adf62b..8827f8f5a 100644 --- a/web/ivr/ivr_test.go +++ b/web/ivr/ivr_test.go @@ -94,7 +94,7 @@ func TestTwilioIVR(t *testing.T) { }, "results": {} }`) - start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID, true, true). + start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID). WithContactIDs([]models.ContactID{testdata.Cathy.ID, testdata.Bob.ID, testdata.George.ID}). WithParentSummary(parentSummary) @@ -402,7 +402,7 @@ func TestVonageIVR(t *testing.T) { // create a flow start for cathy and george extra := json.RawMessage(`{"ref_id":"123"}`) - start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID, true, true). + start := models.NewFlowStart(testdata.Org1.ID, models.StartTypeTrigger, models.FlowTypeVoice, testdata.IVRFlow.ID). WithContactIDs([]models.ContactID{testdata.Cathy.ID, testdata.George.ID}). WithExtra(extra) models.InsertFlowStarts(ctx, db, []*models.FlowStart{start}) diff --git a/web/org/metrics.go b/web/org/metrics.go index bcfe6be68..7f6617959 100644 --- a/web/org/metrics.go +++ b/web/org/metrics.go @@ -21,31 +21,18 @@ func init() { } const groupCountsSQL = ` -SELECT - g.id AS id, - g.name AS name, - g.uuid AS uuid, - g.group_type AS group_type, - COALESCE(SUM(c.count), 0) AS count -FROM - contacts_contactgroup g -LEFT OUTER JOIN - contacts_contactgroupcount c -ON - c.group_id = g.id -WHERE - g.org_id = $1 AND - g.is_active = TRUE -GROUP BY - g.id; -` + SELECT g.id, g.name, g.uuid, g.is_system, COALESCE(SUM(c.count), 0) AS count + FROM contacts_contactgroup g +LEFT OUTER JOIN contacts_contactgroupcount c ON c.group_id = g.id + WHERE g.org_id = $1 AND g.is_active = TRUE + GROUP BY g.id;` type groupCountRow struct { - ID models.GroupID `db:"id"` - Name string `db:"name"` - UUID assets.GroupUUID `db:"uuid"` - Type string `db:"group_type"` - Count int64 `db:"count"` + ID models.GroupID `db:"id"` + Name string `db:"name"` + UUID assets.GroupUUID `db:"uuid"` + IsSystem bool `db:"is_system"` + Count int64 `db:"count"` } func calculateGroupCounts(ctx context.Context, rt *runtime.Runtime, org *models.OrgReference) (*dto.MetricFamily, error) { @@ -70,26 +57,26 @@ func calculateGroupCounts(ctx context.Context, rt *runtime.Runtime, org *models. } groupType := "user" - if row.Type != "U" { + if row.IsSystem { groupType = "system" } family.Metric = append(family.Metric, &dto.Metric{ Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("group_name"), Value: proto.String(row.Name), }, - &dto.LabelPair{ + { Name: proto.String("group_uuid"), Value: proto.String(string(row.UUID)), }, - &dto.LabelPair{ + { Name: proto.String("group_type"), Value: proto.String(groupType), }, - &dto.LabelPair{ + { Name: proto.String("org"), Value: proto.String(org.Name), }, @@ -105,26 +92,11 @@ func calculateGroupCounts(ctx context.Context, rt *runtime.Runtime, org *models. } const channelCountsSQL = ` -SELECT - ch.id AS id, - ch.uuid AS uuid, - ch.name AS name, - ch.role AS role, - ch.channel_type AS channel_type, - c.count_type AS count_type, - COALESCE(SUM(c.count), 0) as count -FROM - channels_channel ch -LEFT OUTER JOIN - channels_channelcount c -ON - c.channel_id = ch.id -WHERE - ch.org_id = $1 AND - ch.is_active = TRUE -GROUP BY - (ch.id, c.count_type); -` + SELECT ch.id, ch.uuid, ch.name, ch.role, ch.channel_type, c.count_type, COALESCE(SUM(c.count), 0) as count + FROM channels_channel ch +LEFT OUTER JOIN channels_channelcount c ON c.channel_id = ch.id + WHERE ch.org_id = $1 AND ch.is_active = TRUE + GROUP BY ch.id, c.count_type;` type channelCountRow struct { ID models.ChannelID `db:"id"` @@ -225,27 +197,27 @@ func calculateChannelCounts(ctx context.Context, rt *runtime.Runtime, org *model family.Metric = append(family.Metric, &dto.Metric{ Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("channel_name"), Value: proto.String(channel.Name), }, - &dto.LabelPair{ + { Name: proto.String("channel_uuid"), Value: proto.String(string(channel.UUID)), }, - &dto.LabelPair{ + { Name: proto.String("channel_type"), Value: proto.String(channel.ChannelType), }, - &dto.LabelPair{ + { Name: proto.String("msg_direction"), Value: proto.String(direction), }, - &dto.LabelPair{ + { Name: proto.String("msg_type"), Value: proto.String(countType), }, - &dto.LabelPair{ + { Name: proto.String("org"), Value: proto.String(org.Name), }, diff --git a/web/ticket/base_test.go b/web/ticket/base_test.go index 714ef258e..c25ae085d 100644 --- a/web/ticket/base_test.go +++ b/web/ticket/base_test.go @@ -2,6 +2,7 @@ package ticket import ( "testing" + "time" _ "github.com/nyaruka/mailroom/services/tickets/mailgun" _ "github.com/nyaruka/mailroom/services/tickets/zendesk" @@ -15,8 +16,8 @@ func TestTicketAssign(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "17", testdata.Admin) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "21", testdata.Agent) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "17", time.Now(), testdata.Admin) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "21", time.Now(), testdata.Agent) testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "34", nil) testdata.InsertClosedTicket(db, testdata.Org1, testdata.Bob, testdata.Internal, testdata.DefaultTopic, "", "", nil) @@ -28,8 +29,8 @@ func TestTicketAddNote(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "17", testdata.Admin) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "21", testdata.Agent) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "17", time.Now(), testdata.Admin) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "21", time.Now(), testdata.Agent) testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "34", nil) web.RunWebTests(t, ctx, rt, "testdata/add_note.json", nil) @@ -40,8 +41,8 @@ func TestTicketChangeTopic(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "17", testdata.Admin) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.SupportTopic, "Have you seen my cookies?", "21", testdata.Agent) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.DefaultTopic, "Have you seen my cookies?", "17", time.Now(), testdata.Admin) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.SupportTopic, "Have you seen my cookies?", "21", time.Now(), testdata.Agent) testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Internal, testdata.SalesTopic, "Have you seen my cookies?", "34", nil) web.RunWebTests(t, ctx, rt, "testdata/change_topic.json", nil) @@ -53,10 +54,10 @@ func TestTicketClose(t *testing.T) { defer testsuite.Reset(testsuite.ResetData) // create 2 open tickets and 1 closed one for Cathy across two different ticketers - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "17", testdata.Admin) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "21", nil) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "17", time.Now(), testdata.Admin) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "21", time.Now(), nil) testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "34", testdata.Editor) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "21", nil) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "21", time.Now(), nil) web.RunWebTests(t, ctx, rt, "testdata/close.json", nil) } @@ -69,7 +70,7 @@ func TestTicketReopen(t *testing.T) { // create 2 closed tickets and 1 open one for Cathy testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Mailgun, testdata.DefaultTopic, "Have you seen my cookies?", "17", testdata.Admin) testdata.InsertClosedTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "21", nil) - testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "34", testdata.Editor) + testdata.InsertOpenTicket(db, testdata.Org1, testdata.Cathy, testdata.Zendesk, testdata.DefaultTopic, "Have you seen my cookies?", "34", time.Now(), testdata.Editor) web.RunWebTests(t, ctx, rt, "testdata/reopen.json", nil) }