Merge tag 'v6.5.6' into update/v6.5.6

 * Update to latest goflow and add parse_only as param to parse_query to allow us to extract field dependencies even when they don't yet exist in the database

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: '1.14.x'
+  go-version: '1.16.x'
   postgis-version: '2.5'
 jobs:
   test:
@@ -51,7 +51,7 @@ jobs:
       if: success()
       uses: codecov/codecov-action@v1
       with:
-        fail_ci_if_error: true
+        fail_ci_if_error: false
 
   release:
     name: Release

CHANGELOG.md

@@ -1,3 +1,168 @@
+v6.5.6
+----------
+ * Update to latest goflow and add parse_only as param to parse_query to allow us to extract field dependencies even when they don't yet exist in the database
+
+v6.5.5
+----------
+ * Fix tests broken by recent db changes to msgs and broadcasts
+ * Populate ticket_count when creating new contacts
+
+v6.5.4
+----------
+ * Actually save IVR messages with sent_on set
+
+v6.5.3
+----------
+ * Update contact modified_on after populate dynamic group task
+ * Update to latest goflow
+
+v6.5.2
+----------
+ * Set sent_on for outgoing IVR messages
+
+v6.5.1
+----------
+ * Support flow config ivr_retry values of -1 meaning no retry
+ * Log error if marking event fire as fired fails
+
+v6.5.0
+----------
+ * Update to latest goflow and gocommon
+
+v6.4.3
+----------
+ * Fix triggering new IVR flow from a simulation resume so that it includes connection to test channel
+
+v6.4.2
+----------
+ * Latest goflow with latest localization
+
+v6.4.1
+----------
+ * Update to latest goflow to get fixes for nulls in webhook responses
+ * Add new error type for failed SQL queries
+
+v6.4.0
+----------
+ * move s3 session config error to a warning for the time being since not strictly required yet
+
+v6.3.31
+----------
+ * Support ticket open events with assignees
+ * Add endpoints for ticket assignment and adding notes
+
+v6.3.30
+----------
+ * Update to latest goflow
+
+v6.3.29
+----------
+ * Include args in BulkQuery error output
+
+v6.3.28
+----------
+ * Return more SQL when BulkQuery errors
+ * Update to latest goflow/gocommon
+
+v6.3.27
+----------
+ * Fix handling of inbox messages to also update open tickets
+
+v6.3.26
+----------
+ * Stop writing broadcast.is_active which is now nullable
+
+v6.3.25
+----------
+ * Update to latest goflow
+
+v6.3.24
+----------
+ * Update to latest goflow
+ * Load org users as assets and use for ticket assignees and manual trigger users
+ * Add ticket to broadcasts and update last_activity_on after creating messages for a broadcast with a ticket
+
+v6.3.23
+----------
+ * Add support for exclusion groups on scheduled triggers
+
+v6.3.22
+----------
+ * Update ticket last_activity_on when opening/closing and for incoming messages
+ * Set contact_id when creating new tickets events
+
+v6.3.21
+----------
+ * Update to latest goflow and which no longer takes default_language
+
+v6.3.20
+----------
+ * Have our session filename lead with timestamp so other objects can exist in contact dirs
+
+v6.3.19
+----------
+ * Parse URL to get path out for sessions
+
+v6.3.18
+----------
+ * Use s3 session prefix when building s3 paths, default to /
+ * Throw error upwards if we have no DB backdown
+ * Read session files from storage when org configured to do so
+
+v6.3.17
+----------
+ * Ignore contact tickets on ticketers which have been deleted
+
+v6.3.16
+----------
+ * Add ticket closed triggers and use to handle close ticket events
+ * Add ticket events and insert when opening/closing/reopening tickets
+
+v6.3.15
+----------
+ * Fix test which modifies org
+ * Update to latest goflow as last release was broken
+
+v6.3.14
+----------
+ * Update to latest goflow
+ * Write sessions to s3 on resumes (optionally)
+ * Add support for exclusion groups on triggers and generalize trigger matching
+
+v6.3.13
+----------
+ * Introduce runtime.Runtime
+ * Simplify testdata functions
+ * Various fixes from linter
+ * Simplify use of test contacts in handler tests
+ * Move test constants out of models package
+ * Remove reduntant resend_msgs task
+
+v6.3.12
+----------
+ * Update to latest goflow (legacy_extra is no longer an issue)
+ * Make Msg.next_attempt nullable
+ * Add web endpoint for msg resends so they can be a synchronous operation
+
+v6.3.11
+----------
+ * Expose open tickets as @contact.tickets
+
+v6.3.9
+----------
+ * Fix queueing of resent messages to courier and improve testing of queueing
+ * Update to latest goflow
+ * Add WA template translation namespace
+
+v6.3.8
+----------
+ * Add task to resend messages
+
+v6.3.7
+----------
+ * Update to latest goflow
+ * Update test database and rename Nexmo to Vonage
+
 v6.3.6
 ----------
  * Update to latest goflow

README.md

@@ -1,9 +1,9 @@
-# mailroom 
+# mailroom
 
-[![Build Status](https://github.com/nyaruka/mailroom/workflows/CI/badge.svg)](https://github.com/nyaruka/mailroom/actions?query=workflow%3ACI) 
-[![codecov](https://codecov.io/gh/nyaruka/mailroom/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/mailroom) 
+[![Build Status](https://github.com/nyaruka/mailroom/workflows/CI/badge.svg)](https://github.com/nyaruka/mailroom/actions?query=workflow%3ACI)
+[![codecov](https://codecov.io/gh/nyaruka/mailroom/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/mailroom)
 
-# About 
+# About
 
 Mailroom is the [RapidPro](https://github.com/rapidpro/rapidpro) component responsible for the execution of
 flows. It interacts directly with the RapidPro database and sends and receives messages with [Courier](https://github.com/nyaruka/courier) for handling via Redis.
@@ -18,9 +18,10 @@ behind a reverse proxy such as nginx or Elastic Load Balancer that provides HTTP
 # Configuration
 
 Mailroom uses a tiered configuration system, each option takes precendence over the ones above it:
- 1. The configuration file
- 2. Environment variables starting with `MAILROOM_` 
- 3. Command line parameters
+
+1.  The configuration file
+2.  Environment variables starting with `MAILROOM_`
+3.  Command line parameters
 
 We recommend running Mailroom with no changes to the configuration and no parameters, using only
 environment variables to configure it. You can use `% mailroom --help` to see a list of the
@@ -30,29 +31,34 @@ environment variables and parameters and for more details on each option.
 
 For use with RapidPro, you will want to configure these settings:
 
- * `MAILROOM_ADDRESS`: the address to bind our web server to (default "localhost")
- * `MAILROOM_DOMAIN`: the domain that mailroom is listening on
- * `MAILROOM_AUTH_TOKEN`: the token clients will need to authenticate web requests (should match setting in RapidPro)
- * `MAILROOM_ATTACHMENT_DOMAIN`: the domain that will be used for relative attachments in flows
- * `MAILROOM_DB`: URL describing how to connect to the RapidPro database (default "postgres://temba:temba@localhost/temba?sslmode=disable")
- * `MAILROOM_REDIS`: URL describing how to connect to Redis (default "redis://localhost:6379/15")
- * `MAILROOM_ELASTIC`: URL describing how to connect to ElasticSearch (default "http://localhost:9200")
- * `MAILROOM_SMTP_SERVER`: the smtp configuration for sending emails ex: smtp://user%40password@server:port/?from=foo%40gmail.com
- 
+- `MAILROOM_ADDRESS`: the address to bind our web server to (default "localhost")
+- `MAILROOM_DOMAIN`: the domain that mailroom is listening on
+- `MAILROOM_AUTH_TOKEN`: the token clients will need to authenticate web requests (should match setting in RapidPro)
+- `MAILROOM_ATTACHMENT_DOMAIN`: the domain that will be used for relative attachments in flows
+- `MAILROOM_DB`: URL describing how to connect to the RapidPro database (default "postgres://temba:temba@localhost/temba?sslmode=disable")
+- `MAILROOM_REDIS`: URL describing how to connect to Redis (default "redis://localhost:6379/15")
+- `MAILROOM_ELASTIC`: URL describing how to connect to ElasticSearch (default "http://localhost:9200")
+- `MAILROOM_SMTP_SERVER`: the smtp configuration for sending emails ex: smtp://user%40password@server:port/?from=foo%40gmail.com
+
 For writing of message attachments, Mailroom needs access to an S3 bucket, you can configure access to your bucket via:
 
- * `MAILROOM_S3_REGION`: The region for your S3 bucket (ex: `eu-west-1`)
- * `MAILROOM_S3_MEDIA_BUCKET`: The name of your S3 bucket (ex: `dl-mailroom`)
- * `MAILROOM_S3_MEDIA_PREFIX`: The prefix to use for filenames of attachments added to your bucket (ex: `attachments`)
- * `MAILROOM_AWS_ACCESS_KEY_ID`: The AWS access key id used to authenticate to AWS
- * `MAILROOM_AWS_SECRET_ACCESS_KEY` The AWS secret access key used to authenticate to AWS
+- `MAILROOM_S3_REGION`: The region for your S3 bucket (ex: `eu-west-1`)
+- `MAILROOM_S3_MEDIA_BUCKET`: The name of your S3 bucket (ex: `dl-mailroom`)
+- `MAILROOM_S3_MEDIA_PREFIX`: The prefix to use for filenames of attachments added to your bucket (ex: `attachments`)
+- `MAILROOM_AWS_ACCESS_KEY_ID`: The AWS access key id used to authenticate to AWS
+- `MAILROOM_AWS_SECRET_ACCESS_KEY` The AWS secret access key used to authenticate to AWS
+
+While still in beta, Mailroom will move to writing session data to S3 in 6.6, you can configure those buckets using:
+
+- `MAILROOM_S3_SESSION_BUCKET`: The name of your S3 bucket (ex: `rp-sessions`)
+- `MAILROOM_S3_SESSION_PREFIX`: The prefix to use for filenames of sessions added to your bucket (ex: ``)
 
 Recommended settings for error and performance monitoring:
 
- * `MAILROOM_LIBRATO_USERNAME`: The username to use for logging of events to Librato
- * `MAILROOM_LIBRATO_TOKEN`: The token to use for logging of events to Librato
- * `MAILROOM_SENTRY_DSN`: The DSN to use when logging errors to Sentry
- * `MAILROOM_LOG_LEVEL`: the logging level mailroom should use (default "error", use "debug" for more)
+- `MAILROOM_LIBRATO_USERNAME`: The username to use for logging of events to Librato
+- `MAILROOM_LIBRATO_TOKEN`: The token to use for logging of events to Librato
+- `MAILROOM_SENTRY_DSN`: The DSN to use when logging errors to Sentry
+- `MAILROOM_LOG_LEVEL`: the logging level mailroom should use (default "error", use "debug" for more)
 
 # Development
 

cmd/mailroom/main.go

@@ -20,7 +20,6 @@ import (
 	_ "github.com/nyaruka/mailroom/core/tasks/campaigns"
 	_ "github.com/nyaruka/mailroom/core/tasks/contacts"
 	_ "github.com/nyaruka/mailroom/core/tasks/expirations"
-	_ "github.com/nyaruka/mailroom/core/tasks/groups"
 	_ "github.com/nyaruka/mailroom/core/tasks/interrupts"
 	_ "github.com/nyaruka/mailroom/core/tasks/ivr"
 	_ "github.com/nyaruka/mailroom/core/tasks/schedules"
@@ -36,6 +35,7 @@ import (
 	_ "github.com/nyaruka/mailroom/web/expression"
 	_ "github.com/nyaruka/mailroom/web/flow"
 	_ "github.com/nyaruka/mailroom/web/ivr"
+	_ "github.com/nyaruka/mailroom/web/msg"
 	_ "github.com/nyaruka/mailroom/web/org"
 	_ "github.com/nyaruka/mailroom/web/po"
 	_ "github.com/nyaruka/mailroom/web/simulation"

config/config.go

@@ -47,12 +47,18 @@ type Config struct {
 	Domain           string `help:"the domain that mailroom is listening on"`
 	AttachmentDomain string `help:"the domain that will be used for relative attachment"`
 
-	S3Endpoint         string `help:"the S3 endpoint we will write attachments to"`
-	S3Region           string `help:"the S3 region we will write attachments to"`
-	S3MediaBucket      string `help:"the S3 bucket we will write attachments to"`
-	S3MediaPrefix      string `help:"the prefix that will be added to attachment filenames"`
-	S3DisableSSL       bool   `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"`
-	S3ForcePathStyle   bool   `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"`
+	S3Endpoint string `help:"the S3 endpoint we will write attachments to"`
+	S3Region   string `help:"the S3 region we will write attachments to"`
+
+	S3MediaBucket string `help:"the S3 bucket we will write attachments to"`
+	S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"`
+
+	S3SessionBucket string `help:"the S3 bucket we will write attachments to"`
+	S3SessionPrefix string `help:"the prefix that will be added to attachment filenames"`
+
+	S3DisableSSL     bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"`
+	S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"`
+
 	AWSAccessKeyID     string `help:"the access key id to use when authenticating S3"`
 	AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"`
 
@@ -92,6 +98,8 @@ func NewMailroomConfig() *Config {
 		S3Region:           "us-east-1",
 		S3MediaBucket:      "mailroom-media",
 		S3MediaPrefix:      "/media/",
+		S3SessionBucket:    "mailroom-sessions",
+		S3SessionPrefix:    "/",
 		S3DisableSSL:       false,
 		S3ForcePathStyle:   false,
 		AWSAccessKeyID:     "",

config/config_test.go

@@ -5,11 +5,15 @@ import (
 	"testing"
 
 	"github.com/nyaruka/mailroom/config"
+	"github.com/nyaruka/mailroom/testsuite"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestParseDisallowedNetworks(t *testing.T) {
+	// this is only here because this is the first test run.. should find a better way to ensure DB is in correct state for first test that needs it
+	testsuite.Reset()
+
 	cfg := config.NewMailroomConfig()
 
 	privateNetwork1 := &net.IPNet{IP: net.IPv4(10, 0, 0, 0).To4(), Mask: net.CIDRMask(8, 32)}