Update rp/6.1.5 rocketchat ticketer unofficial

.github/workflows/ci.yml

@@ -44,8 +44,7 @@ jobs:
       if: success()
       uses: codecov/codecov-action@v1
       with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        # fail_ci_if_error: true  # ideally this should be enabled but codecov uploads fail a lot
+        fail_ci_if_error: true
 
   release:
     name: Release
@@ -72,7 +71,7 @@ jobs:
       uses: goreleaser/goreleaser-action@v1
       if: ${{ !contains(github.ref, '-') }}
       with:
-        version: latest
+        version: v0.147.2
         args: release --rm-dist
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -81,7 +80,7 @@ jobs:
       uses: goreleaser/goreleaser-action@v1
       if: contains(github.ref, '-')
       with:
-        version: latest
+        version: v0.147.2
         args: release --rm-dist --skip-validate
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -1,3 +1,121 @@
+v6.1.5
+----------
+ * Update to goflow v0.107.1
+
+v6.1.4
+----------
+ * Variable timeout for fire campaign task
+
+v6.1.3
+----------
+ * Fix misreporting created contacts as updated during imports
+
+v6.1.2
+----------
+ * Ensure field and group assets used for imports are fresh
+ * Add support for internal type ticketers
+
+v6.1.1
+----------
+ * Update to latest goflow v0.106.3
+
+v6.1.0
+----------
+ * Configure engine to disallow HTTP requests to private networks
+
+v6.0.3
+----------
+ * correct name for completion.json in release
+
+v6.0.2
+----------
+ * pin goreleaser, explicit inclusion of /docs/*, completions and functions
+
+v6.0.1
+----------
+ * add log when queuing to courier
+
+v6.0.0
+----------
+ * Update test database
+
+v5.7.44
+----------
+ * Add ticket service for Rocket.Chat
+
+v5.7.43
+----------
+ * Update to latest goflow v0.106.1
+
+v5.7.42
+----------
+ * Prevent importing invalid URNs during import
+
+v5.7.41
+----------
+ * Update to latest goflow v0.106.0
+ * Don't write to flows_flowrun.timeout_on so it can be dropped
+
+v5.7.40
+----------
+ * Update to goflow v0.105.5
+
+v5.7.39
+----------
+ * Fix input_labels_added event handling when session input ID isn't set
+
+v5.7.38
+----------
+ * Always request Mp3 files from Twilio IVR recordings
+
+v5.7.37
+----------
+ * Add support for outgoing mailgun ticket attachments
+
+v5.7.36
+----------
+ * Incoming attachments for mailgun ticketers
+
+v5.7.33
+----------
+ * Update to goflow v0.105.4
+ * Fix cloning of flows during simulation so that ignore keyword triggers is cloned too
+
+v5.7.32
+----------
+ * Update to goflow v0.105.3 to get support for arabic numerals in has_number tests
+
+v5.7.31
+----------
+ * Update to latest goflow to get for normalizing numbers from The Gambia
+ * Enable retrying on the elastic client
+
+v5.7.30
+----------
+ * Reorganization of core packages
+
+v5.7.29
+----------
+ * Fix race condition when bulk getting/creating contacts
+
+v5.7.28
+----------
+ * Add contact/resolve endpoint to assist with channel events still handled in RP
+
+v5.7.27
+----------
+ * If a flow start task creates new contacts, save those back to the start
+
+v5.7.26
+----------
+ * Add mockable DB to enable testing database errors
+ * CreateContact also should do lookup before trying to create new contact with URNs
+ * Imports 2.0
+
+v5.7.25
+----------
+ * Pass org from the base task to task structs to remove need for duplicating it in the task body
+
 v5.7.24
 ----------
  * Add SessionStatus to messages queued to courier

README.md

@@ -53,19 +53,13 @@ Recommended settings for error and performance monitoring:
 
 # Development
 
-Install Mailroom source in your workspace with:
-
-```
-go get github.com/nyaruka/mailroom
-```
-
-Build Mailroom with:
+Once you've checked out the code, you can build Mailroom with:
 
 ```
 go build github.com/nyaruka/mailroom/cmd/mailroom
 ```
 
-This will create a new executable in your current directory `mailroom`
+This will create a new executable in $GOPATH/bin called `mailroom`.
 
 To run the tests you need to create the test database:
 

cmd/mailroom/main.go

@@ -12,21 +12,24 @@ import (
 	"github.com/nyaruka/mailroom"
 	"github.com/nyaruka/mailroom/config"
 
-	_ "github.com/nyaruka/mailroom/hooks"
-	_ "github.com/nyaruka/mailroom/ivr/nexmo"
-	_ "github.com/nyaruka/mailroom/ivr/twiml"
+	_ "github.com/nyaruka/mailroom/core/handlers"
+	_ "github.com/nyaruka/mailroom/core/hooks"
+	_ "github.com/nyaruka/mailroom/core/ivr/nexmo"
+	_ "github.com/nyaruka/mailroom/core/ivr/twiml"
+	_ "github.com/nyaruka/mailroom/core/tasks/broadcasts"
+	_ "github.com/nyaruka/mailroom/core/tasks/campaigns"
+	_ "github.com/nyaruka/mailroom/core/tasks/contacts"
+	_ "github.com/nyaruka/mailroom/core/tasks/expirations"
+	_ "github.com/nyaruka/mailroom/core/tasks/groups"
+	_ "github.com/nyaruka/mailroom/core/tasks/interrupts"
+	_ "github.com/nyaruka/mailroom/core/tasks/ivr"
+	_ "github.com/nyaruka/mailroom/core/tasks/schedules"
+	_ "github.com/nyaruka/mailroom/core/tasks/starts"
+	_ "github.com/nyaruka/mailroom/core/tasks/stats"
+	_ "github.com/nyaruka/mailroom/core/tasks/timeouts"
 	_ "github.com/nyaruka/mailroom/services/tickets/mailgun"
+	_ "github.com/nyaruka/mailroom/services/tickets/rocketchat"
 	_ "github.com/nyaruka/mailroom/services/tickets/zendesk"
-	_ "github.com/nyaruka/mailroom/tasks/broadcasts"
-	_ "github.com/nyaruka/mailroom/tasks/campaigns"
-	_ "github.com/nyaruka/mailroom/tasks/expirations"
-	_ "github.com/nyaruka/mailroom/tasks/groups"
-	_ "github.com/nyaruka/mailroom/tasks/interrupts"
-	_ "github.com/nyaruka/mailroom/tasks/ivr"
-	_ "github.com/nyaruka/mailroom/tasks/schedules"
-	_ "github.com/nyaruka/mailroom/tasks/starts"
-	_ "github.com/nyaruka/mailroom/tasks/stats"
-	_ "github.com/nyaruka/mailroom/tasks/timeouts"
 	_ "github.com/nyaruka/mailroom/web/contact"
 	_ "github.com/nyaruka/mailroom/web/docs"
 	_ "github.com/nyaruka/mailroom/web/expression"
@@ -53,6 +56,11 @@ func main() {
 	)
 	loader.MustLoad()
 
+	// ensure config is valid
+	if err := config.Validate(); err != nil {
+		logrus.Fatalf("invalid config: %s", err)
+	}
+
 	// if we have a custom version, use it
 	if version != "Dev" {
 		config.Version = version

config/config.go

@@ -37,7 +37,7 @@ type Config struct {
 	WebhooksInitialBackoff int     `help:"the initial backoff in milliseconds when retrying a failed webhook call"`
 	WebhooksBackoffJitter  float64 `help:"the amount of jitter to apply to backoff times"`
 	SMTPServer             string  `help:"the smtp configuration for sending emails ex: smtp://user%40password@server:port/?from=foo%40gmail.com"`
-	DisallowedIPs          string  `help:"comma separated list of IP addresses which engine can't make HTTP calls to"`
+	DisallowedNetworks     string  `help:"comma separated list of IP addresses and networks which engine can't make HTTP calls to"`
 	MaxStepsPerSprint      int     `help:"the maximum number of steps allowed per engine sprint"`
 	MaxValueLength         int     `help:"the maximum size in characters for contact field values and run result values"`
 
@@ -84,7 +84,7 @@ func NewMailroomConfig() *Config {
 		WebhooksInitialBackoff: 5000,
 		WebhooksBackoffJitter:  0.5,
 		SMTPServer:             "",
-		DisallowedIPs:          `127.0.0.1,::1`,
+		DisallowedNetworks:     `127.0.0.1,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`,
 		MaxStepsPerSprint:      100,
 		MaxValueLength:         640,
 
@@ -106,19 +106,40 @@ func NewMailroomConfig() *Config {
 	}
 }
 
-func (c *Config) ParseDisallowedIPs() ([]net.IP, error) {
-	addrs, err := csv.NewReader(strings.NewReader(c.DisallowedIPs)).Read()
+// Validate validates the config
+func (c *Config) Validate() error {
+	_, _, err := c.ParseDisallowedNetworks()
+	if err != nil {
+		return errors.Wrap(err, "unable to parse DisallowedNetworks")
+	}
+	return nil
+}
+
+// ParseDisallowedNetworks parses the list of IPs and IP networks (written in CIDR notation)
+func (c *Config) ParseDisallowedNetworks() ([]net.IP, []*net.IPNet, error) {
+	addrs, err := csv.NewReader(strings.NewReader(c.DisallowedNetworks)).Read()
 	if err != nil && err != io.EOF {
-		return nil, err
+		return nil, nil, err
 	}
+
 	ips := make([]net.IP, 0, len(addrs))
+	ipNets := make([]*net.IPNet, 0, len(addrs))
+
 	for _, addr := range addrs {
-		ip := net.ParseIP(addr)
-		if ip == nil {
-			return nil, errors.Errorf("couldn't parse '%s' as an IP address", addr)
+		if strings.Contains(addr, "/") {
+			_, ipNet, err := net.ParseCIDR(addr)
+			if err != nil {
+				return nil, nil, errors.Errorf("couldn't parse '%s' as an IP network", addr)
+			}
+			ipNets = append(ipNets, ipNet)
+		} else {
+			ip := net.ParseIP(addr)
+			if ip == nil {
+				return nil, nil, errors.Errorf("couldn't parse '%s' as an IP address", addr)
+			}
+			ips = append(ips, ip)
 		}
-		ips = append(ips, ip)
 	}
 
-	return ips, nil
+	return ips, ipNets, nil
 }

config/config_test.go

@@ -9,27 +9,38 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestParseDisallowedIPs(t *testing.T) {
+func TestParseDisallowedNetworks(t *testing.T) {
 	cfg := config.NewMailroomConfig()
 
+	privateNetwork1 := &net.IPNet{IP: net.IPv4(10, 0, 0, 0).To4(), Mask: net.CIDRMask(8, 32)}
+	privateNetwork2 := &net.IPNet{IP: net.IPv4(172, 16, 0, 0).To4(), Mask: net.CIDRMask(12, 32)}
+	privateNetwork3 := &net.IPNet{IP: net.IPv4(192, 168, 0, 0).To4(), Mask: net.CIDRMask(16, 32)}
+
 	// test with config defaults
-	ips, err := cfg.ParseDisallowedIPs()
+	ips, ipNets, err := cfg.ParseDisallowedNetworks()
 	assert.NoError(t, err)
 	assert.Equal(t, []net.IP{net.IPv4(127, 0, 0, 1), net.ParseIP(`::1`)}, ips)
+	assert.Equal(t, []*net.IPNet{privateNetwork1, privateNetwork2, privateNetwork3}, ipNets)
 
 	// test with empty
-	cfg.DisallowedIPs = ``
-	ips, err = cfg.ParseDisallowedIPs()
+	cfg.DisallowedNetworks = ``
+	ips, ipNets, err = cfg.ParseDisallowedNetworks()
 	assert.NoError(t, err)
 	assert.Equal(t, []net.IP{}, ips)
+	assert.Equal(t, []*net.IPNet{}, ipNets)
 
 	// test with invalid CSV
-	cfg.DisallowedIPs = `"127.0.0.1`
-	_, err = cfg.ParseDisallowedIPs()
+	cfg.DisallowedNetworks = `"127.0.0.1`
+	_, _, err = cfg.ParseDisallowedNetworks()
 	assert.EqualError(t, err, `record on line 1; parse error on line 2, column 0: extraneous or missing " in quoted-field`)
 
 	// test with invalid IP
-	cfg.DisallowedIPs = `127.0.1`
-	_, err = cfg.ParseDisallowedIPs()
+	cfg.DisallowedNetworks = `127.0.1`
+	_, _, err = cfg.ParseDisallowedNetworks()
 	assert.EqualError(t, err, `couldn't parse '127.0.1' as an IP address`)
+
+	// test with invalid network
+	cfg.DisallowedNetworks = `127.0.0.1/x`
+	_, _, err = cfg.ParseDisallowedNetworks()
+	assert.EqualError(t, err, `couldn't parse '127.0.0.1/x' as an IP network`)
 }

courier/courier.go → core/courier/courier.go

@@ -7,8 +7,9 @@ import (
 
 	"github.com/gomodule/redigo/redis"
 	"github.com/nyaruka/gocommon/urns"
-	"github.com/nyaruka/mailroom/models"
+	"github.com/nyaruka/mailroom/core/models"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -43,10 +44,16 @@ func QueueMessages(rc redis.Conn, msgs []*models.Msg) error {
 			if err != nil {
 				return err
 			}
+			start := time.Now()
 			_, err = queueMsg.Do(rc, epochMS, "msgs", currentChannel.UUID(), currentChannel.TPS(), priority, batchJSON)
 			if err != nil {
 				return err
 			}
+			logrus.WithFields(logrus.Fields{
+				"msgs":         len(batch),
+				"channel_uuid": currentChannel.UUID(),
+				"elapsed":      time.Since(start),
+			}).Info("msgs queued to courier")
 		}
 		return nil
 	}

courier/courier_test.go → core/courier/courier_test.go

@@ -9,8 +9,8 @@ import (
 	"github.com/nyaruka/gocommon/urns"
 	"github.com/nyaruka/goflow/assets"
 	"github.com/nyaruka/goflow/flows"
-	"github.com/nyaruka/mailroom/courier"
-	"github.com/nyaruka/mailroom/models"
+	"github.com/nyaruka/mailroom/core/courier"
+	"github.com/nyaruka/mailroom/core/models"
 	"github.com/nyaruka/mailroom/testsuite"
 
 	"github.com/stretchr/testify/assert"

goflow/engine_test.go → core/goflow/engine_test.go

@@ -7,8 +7,8 @@ import (
 	"github.com/nyaruka/gocommon/httpx"
 	"github.com/nyaruka/gocommon/urns"
 	"github.com/nyaruka/goflow/flows"
-	"github.com/nyaruka/mailroom/goflow"
-	"github.com/nyaruka/mailroom/models"
+	"github.com/nyaruka/mailroom/core/goflow"
+	"github.com/nyaruka/mailroom/core/models"
 	"github.com/nyaruka/mailroom/testsuite"
 
 	"github.com/shopspring/decimal"

goflow/flows_test.go → core/goflow/flows_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/nyaruka/goflow/envs"
 	"github.com/nyaruka/goflow/flows"
 	"github.com/nyaruka/goflow/test"
-	"github.com/nyaruka/mailroom/goflow"
+	"github.com/nyaruka/mailroom/core/goflow"
 
 	"github.com/Masterminds/semver"
 	"github.com/stretchr/testify/assert"