merge/v5.6.0

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
       uses: codecov/codecov-action@v1
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        fail_ci_if_error: true
+        # fail_ci_if_error: true  # ideally this should be enabled but codecov uploads fail a lot
 
   release:
     name: Release
@@ -57,9 +57,11 @@ jobs:
       uses: actions/checkout@v1
 
     - name: Fetch GoFlow docs
+      # for now just grab en_US/ docs and bundle as docs/
       run: |
-        export GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2)
-        curl https://codeload.github.com/nyaruka/goflow/tar.gz/$GOFLOW_VERSION | tar --wildcards --strip=1 -zx "*/docs/*"
+        GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2)
+        curl https://codeload.github.com/nyaruka/goflow/tar.gz/$GOFLOW_VERSION | tar --wildcards --strip=2 -zx "*/docs/en_US/*"
+        mv en_US docs  
 
     - name: Install Go
       uses: actions/setup-go@v1

CHANGELOG.md

@@ -1,3 +1,70 @@
+v5.6.0
+----------
+ * 5.6.0 Release Candidate
+
+v5.5.38 
+----------
+ * Varible naming consistency
+
+v5.5.37
+----------
+ * Fix reading of modifiers so always ignore modifier that becomes noop
+
+v5.5.36
+----------
+ * Sead country from templates
+ * Ignore missing assets when reading modifiers
+ * Fail flow starts which can't be started
+
+v5.5.35
+----------
+ * Update to latest goflow and add tests for field modifiers
+
+v5.5.34
+----------
+ * Fix detaching URNs
+
+v5.5.33
+----------
+ * Update to latest goflow v0.93.0
+
+v5.5.32
+----------
+ * When blocking contacts archive any triggers which only apply to them
+
+v5.5.31
+----------
+ * Messages without topups should be queued
+ * Continue handling as normal for suspended orgs
+
+v5.5.30
+----------
+ * Org being suspended should stop message handling
+ * Make decrementing org credit optional
+
+v5.5.29
+----------
+ * Return query inspection results as new metadata field in responses
+ * Update to latest goflow v0.92.0
+
+v5.5.28
+----------
+ * Don't do any decoration of email ticket subjects
+
+v5.5.27
+----------
+ * Allow searching by UUID, as well != matches on ID and UUID
+ * Update to latest goflow v0.91.1 to fix clearing fields
+ * Maybe fix intermittently failing test
+
+v5.5.26
+----------
+ * Update to goflow v0.89.0
+
+v5.5.25
+----------
+ * Add endpoint to change a flow language
+
 v5.5.24
 ----------
  * Tickets fixes and improvements

go.mod

@@ -17,15 +17,14 @@ require (
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lib/pq v1.4.0
-	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/nyaruka/ezconf v0.2.1
 	github.com/nyaruka/gocommon v1.2.0
-	github.com/nyaruka/goflow v0.87.0
+	github.com/nyaruka/goflow v0.94.2
 	github.com/nyaruka/librato v1.0.0
 	github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d
 	github.com/nyaruka/null v1.2.0
-	github.com/olivere/elastic v6.2.30+incompatible
+	github.com/olivere/elastic v6.2.33+incompatible
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_model v0.2.0

go.sum

@@ -105,8 +105,8 @@ github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.4.0 h1:TmtCFbH+Aw0AixwyttznSMQDgbR5Yed/Gg6S8Funrhc=
 github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
+github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -128,8 +128,8 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
 github.com/nyaruka/gocommon v1.2.0 h1:gCmVCXYZFwKDMqQj8R1jNlK+7a06khKFq3zX8fBBbzw=
 github.com/nyaruka/gocommon v1.2.0/go.mod h1:9Y21Fd6iZXDLHWTRiZAc6b4LQSCi6HEEQK4SB45Yav4=
-github.com/nyaruka/goflow v0.87.0 h1:JwClty7iwg4iitzitDQulqjkbRMANa0Z//yd2wfqfBQ=
-github.com/nyaruka/goflow v0.87.0/go.mod h1:HBoTXbhrjhZENbCUlDvh8ZCR16ZvlgY4aTE/ABMS8uE=
+github.com/nyaruka/goflow v0.94.2 h1:NduUutwkEQIMNBV1XLTIVPVAbRFyCFtjD45lri04xRo=
+github.com/nyaruka/goflow v0.94.2/go.mod h1:PDah2hr5WzODnUFK4VWWQkg7SqnYclf7P9Ik5u/VOG0=
 github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
 github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc=
@@ -139,8 +139,8 @@ github.com/nyaruka/null v1.2.0/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBN
 github.com/nyaruka/phonenumbers v1.0.34/go.mod h1:GQ0cTHlrxPrhoLwyQ1blyN1hO794ygt6FTHWrFB5SSc=
 github.com/nyaruka/phonenumbers v1.0.55 h1:bj0nTO88Y68KeUQ/n3Lo2KgK7lM1hF7L9NFuwcCl3yg=
 github.com/nyaruka/phonenumbers v1.0.55/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U=
-github.com/olivere/elastic v6.2.30+incompatible h1:9JdhoNFfUF809qM1S5WLz3CZaxazd/mDty9XXwDRz4Q=
-github.com/olivere/elastic v6.2.30+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
+github.com/olivere/elastic v6.2.33+incompatible h1:SRPB2w2OhJ7iULftDEHsNPRoL2GLREqPMRalVmbZaEw=
+github.com/olivere/elastic v6.2.33+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=

goflow/modifiers.go

@@ -0,0 +1,38 @@
+package goflow
+
+import (
+	"encoding/json"
+
+	"github.com/nyaruka/goflow/assets"
+	"github.com/nyaruka/goflow/flows"
+	"github.com/nyaruka/goflow/flows/actions/modifiers"
+
+	"github.com/pkg/errors"
+)
+
+// MissingAssets is the type for defining missing assets behavior
+type MissingAssets int
+
+// missing assets constants
+const (
+	IgnoreMissing  MissingAssets = 0
+	ErrorOnMissing MissingAssets = 1
+)
+
+// ReadModifiers reads modifiers from the given JSON
+func ReadModifiers(sa flows.SessionAssets, data []json.RawMessage, missing MissingAssets) ([]flows.Modifier, error) {
+	mods := make([]flows.Modifier, 0, len(data))
+	for _, m := range data {
+		mod, err := modifiers.ReadModifier(sa, m, assets.IgnoreMissing)
+
+		// if this modifier turned into a no-op, ignore
+		if err == modifiers.ErrNoModifier && missing == IgnoreMissing {
+			continue
+		}
+		if err != nil {
+			return nil, errors.Wrapf(err, "error reading modifier: %s", string(m))
+		}
+		mods = append(mods, mod)
+	}
+	return mods, nil
+}

goflow/modifiers_test.go

@@ -0,0 +1,63 @@
+package goflow_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/nyaruka/mailroom/goflow"
+	"github.com/nyaruka/mailroom/models"
+	"github.com/nyaruka/mailroom/testsuite"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadModifiers(t *testing.T) {
+	ctx := testsuite.CTX()
+	db := testsuite.DB()
+
+	oa, err := models.GetOrgAssets(ctx, db, models.Org1)
+	assert.NoError(t, err)
+
+	// can read empty list
+	mods, err := goflow.ReadModifiers(oa.SessionAssets(), []json.RawMessage{}, goflow.IgnoreMissing)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(mods))
+
+	// can read non-empty list
+	mods, err = goflow.ReadModifiers(oa.SessionAssets(), []json.RawMessage{
+		[]byte(`{"type": "name", "name": "Bob"}`),
+		[]byte(`{"type": "field", "field": {"key": "gender", "name": "Gender"}, "value": "M"}`),
+		[]byte(`{"type": "language", "language": "spa"}`),
+	}, goflow.IgnoreMissing)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(mods))
+	assert.Equal(t, "name", mods[0].Type())
+	assert.Equal(t, "field", mods[1].Type())
+	assert.Equal(t, "language", mods[2].Type())
+
+	// modifier with missing asset can be ignored
+	mods, err = goflow.ReadModifiers(oa.SessionAssets(), []json.RawMessage{
+		[]byte(`{"type": "name", "name": "Bob"}`),
+		[]byte(`{"type": "field", "field": {"key": "blood_type", "name": "Blood Type"}, "value": "O"}`),
+		[]byte(`{"type": "language", "language": "spa"}`),
+	}, goflow.IgnoreMissing)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(mods))
+	assert.Equal(t, "name", mods[0].Type())
+	assert.Equal(t, "language", mods[1].Type())
+
+	// modifier with missing asset or an error if allowMissing is false
+	mods, err = goflow.ReadModifiers(oa.SessionAssets(), []json.RawMessage{
+		[]byte(`{"type": "name", "name": "Bob"}`),
+		[]byte(`{"type": "field", "field": {"key": "blood_type", "name": "Blood Type"}, "value": "O"}`),
+		[]byte(`{"type": "language", "language": "spa"}`),
+	}, goflow.ErrorOnMissing)
+	assert.EqualError(t, err, `error reading modifier: {"type": "field", "field": {"key": "blood_type", "name": "Blood Type"}, "value": "O"}: no modifier to return because of missing assets`)
+
+	// error if any modifier structurally invalid
+	mods, err = goflow.ReadModifiers(oa.SessionAssets(), []json.RawMessage{
+		[]byte(`{"type": "field", "value": "O"}`),
+		[]byte(`{"type": "language", "language": "spa"}`),
+	}, goflow.ErrorOnMissing)
+	assert.EqualError(t, err, `error reading modifier: {"type": "field", "value": "O"}: field 'field' is required`)
+}

hooks/airtime_transferred.go

@@ -25,7 +25,7 @@ type InsertAirtimeTransfersHook struct{}
 var insertAirtimeTransfersHook = &InsertAirtimeTransfersHook{}
 
 // Apply inserts all the airtime transfers that were created
-func (h *InsertAirtimeTransfersHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *models.OrgAssets, scenes map[*models.Scene][]interface{}) error {
+func (h *InsertAirtimeTransfersHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error {
 	// gather all our transfers
 	transfers := make([]*models.AirtimeTransfer, 0, len(scenes))
 
@@ -62,7 +62,7 @@ func (h *InsertAirtimeTransfersHook) Apply(ctx context.Context, tx *sqlx.Tx, rp
 }
 
 // handleAirtimeTransferred is called for each airtime transferred event
-func handleAirtimeTransferred(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *models.OrgAssets, scene *models.Scene, e flows.Event) error {
+func handleAirtimeTransferred(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scene *models.Scene, e flows.Event) error {
 	event := e.(*events.AirtimeTransferredEvent)
 
 	status := models.AirtimeTransferStatusSuccess
@@ -71,7 +71,7 @@ func handleAirtimeTransferred(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool,
 	}
 
 	transfer := models.NewAirtimeTransfer(
-		org.OrgID(),
+		oa.OrgID(),
 		status,
 		scene.ContactID(),
 		event.Sender,
@@ -95,7 +95,7 @@ func handleAirtimeTransferred(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool,
 	// add a log for each HTTP call
 	for _, httpLog := range event.HTTPLogs {
 		transfer.AddLog(models.NewAirtimeTransferredLog(
-			org.OrgID(),
+			oa.OrgID(),
 			httpLog.URL,
 			httpLog.Request,
 			httpLog.Response,

hooks/broadcast_created.go

@@ -23,7 +23,7 @@ type StartBroadcastsHook struct{}
 var startBroadcastsHook = &StartBroadcastsHook{}
 
 // Apply queues up our broadcasts for sending
-func (h *StartBroadcastsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *models.OrgAssets, scenes map[*models.Scene][]interface{}) error {
+func (h *StartBroadcastsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error {
 	rc := rp.Get()
 	defer rc.Close()
 
@@ -32,7 +32,7 @@ func (h *StartBroadcastsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.
 		for _, e := range es {
 			event := e.(*events.BroadcastCreatedEvent)
 
-			bcast, err := models.NewBroadcastFromEvent(ctx, tx, org, event)
+			bcast, err := models.NewBroadcastFromEvent(ctx, tx, oa, event)
 			if err != nil {
 				return errors.Wrapf(err, "error creating broadcast")
 			}
@@ -46,7 +46,7 @@ func (h *StartBroadcastsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.
 				priority = queue.HighPriority
 			}
 
-			err = queue.AddTask(rc, taskQ, queue.SendBroadcast, int(org.OrgID()), bcast, priority)
+			err = queue.AddTask(rc, taskQ, queue.SendBroadcast, int(oa.OrgID()), bcast, priority)
 			if err != nil {
 				return errors.Wrapf(err, "error queuing broadcast")
 			}
@@ -57,7 +57,7 @@ func (h *StartBroadcastsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.
 }
 
 // handleBroadcastCreated is called for each broadcast created event across our scene
-func handleBroadcastCreated(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *models.OrgAssets, scene *models.Scene, e flows.Event) error {
+func handleBroadcastCreated(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scene *models.Scene, e flows.Event) error {
 	event := e.(*events.BroadcastCreatedEvent)
 	logrus.WithFields(logrus.Fields{
 		"contact_uuid": scene.ContactUUID(),

hooks/campaigns.go

@@ -18,7 +18,7 @@ type UpdateCampaignEventsHook struct{}
 var updateCampaignEventsHook = &UpdateCampaignEventsHook{}
 
 // Apply will update all the campaigns for the passed in scene, minimizing the number of queries to do so
-func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *models.OrgAssets, scenes map[*models.Scene][]interface{}) error {
+func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error {
 	// these are all the events we need to delete unfired fires for
 	deletes := make([]*models.FireDelete, 0, 5)
 
@@ -42,7 +42,7 @@ func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *r
 				delete(groupAdds, event.GroupID)
 
 			case *events.ContactFieldChangedEvent:
-				field := org.FieldByKey(event.Field.Key)
+				field := oa.FieldByKey(event.Field.Key)
 				if field == nil {
 					logrus.WithFields(logrus.Fields{
 						"field_key":  event.Field.Key,
@@ -63,7 +63,7 @@ func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *r
 
 		// for every group that was removed, we need to remove all event fires for them
 		for g := range groupRemoves {
-			for _, c := range org.CampaignByGroupID(g) {
+			for _, c := range oa.CampaignByGroupID(g) {
 				for _, e := range c.Events() {
 					// only delete events that we qualify for or that were changed
 					if e.QualifiesByField(s.Contact()) || fieldChanges[e.RelativeToID()] {
@@ -75,7 +75,7 @@ func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *r
 
 		// for every field that was changed, we need to also remove event fires and recalculate
 		for f := range fieldChanges {
-			fieldEvents := org.CampaignEventsByFieldID(f)
+			fieldEvents := oa.CampaignEventsByFieldID(f)
 			for _, e := range fieldEvents {
 				// only recalculate the events if this contact qualifies for this event or this group was removed
 				if e.QualifiesByGroup(s.Contact()) || groupRemoves[e.Campaign().GroupID()] {
@@ -95,15 +95,15 @@ func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *r
 
 		// add in all the events we qualify for in campaigns we are now part of
 		for g := range groupAdds {
-			for _, c := range org.CampaignByGroupID(g) {
+			for _, c := range oa.CampaignByGroupID(g) {
 				for _, e := range c.Events() {
 					addEvents[e] = true
 				}
 			}
 		}
 
 		// ok, for all the unique events we now calculate our fire date
-		tz := org.Env().Timezone()
+		tz := oa.Env().Timezone()
 		now := time.Now()
 		for ce := range addEvents {
 			scheduled, err := ce.ScheduleForContact(tz, now, s.Contact())