diff --git a/go.mod b/go.mod index 9abe62516..09bd5858f 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,12 @@ module github.com/nyaruka/mailroom require ( github.com/Masterminds/semver v1.5.0 github.com/apex/log v1.1.4 - github.com/aws/aws-sdk-go v1.34.17 github.com/buger/jsonparser v0.0.0-20200322175846-f7e751efca13 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/edganiukov/fcm v0.4.0 github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d // indirect - github.com/go-chi/chi v3.3.3+incompatible + github.com/go-chi/chi v4.1.2+incompatible github.com/golang/protobuf v1.4.0 github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/schema v1.1.0 @@ -18,8 +17,8 @@ require ( github.com/lib/pq v1.4.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.4.0 - github.com/nyaruka/goflow v0.103.0 + github.com/nyaruka/gocommon v1.5.0 + github.com/nyaruka/goflow v0.103.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d github.com/nyaruka/null v1.2.0 diff --git a/go.sum b/go.sum index 0060cab10..e378b543b 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.17 h1:Y8cyVjc7RWSJwt9uymwnsKZI4qnmamMkfYJJ806wHtA= -github.com/aws/aws-sdk-go v1.30.17/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.17 h1:9OzUgRrLmYm2mbfFx4v+2nBEg+Cvape1cvn9C3RNWTE= github.com/aws/aws-sdk-go v1.34.17/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= @@ -41,8 +39,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d h1:CIp8WnfXz70wJVQ0ytr3dswFYGoJbAxWgNvaLpiu3sY= github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/go-chi/chi v3.3.3+incompatible h1:KHkmBEMNkwKuK4FdQL7N2wOeB9jnIx7jR5wsuSBEFI8= -github.com/go-chi/chi v3.3.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= +github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -57,8 +55,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -132,14 +128,10 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.3.0 h1:IqaPT4KQ2oVq/2Ivp/c+RVCs8v71+RzPU2VhMoRrgpU= -github.com/nyaruka/gocommon v1.3.0/go.mod h1:w7lKxIkm/qLAoO9Y3aI1LV7EiYogn6+1C8MTEjxTC9M= -github.com/nyaruka/gocommon v1.4.0 h1:AgQqwyt9AV7bdpFEnjxjd2MARX2Qfnviw5nsLGyfbJE= -github.com/nyaruka/gocommon v1.4.0/go.mod h1:vgLYauW9cL19P/vmPZk49YIzuzW8ZdvKWoHVoEzju0w= -github.com/nyaruka/goflow v0.102.1 h1:7QX2jTwV7uIbaGnkkpmB+ao+E7Cmyar9g7sRQH4Bu3M= -github.com/nyaruka/goflow v0.102.1/go.mod h1:wuvXZTs6a6S1rjSRLaQGVxDfKomDJ/1XQoLXCqFekK4= -github.com/nyaruka/goflow v0.103.0 h1:OBLz2oG8osNDwU89rTg3QwmcPbsfvvkwuLB+aGokPUQ= -github.com/nyaruka/goflow v0.103.0/go.mod h1:MUIEX5UdP3sPd7EPAX3d9AcQI+a+7OfgM7kFkTYRuN4= +github.com/nyaruka/gocommon v1.5.0 h1:MYTSqfxun9QoZSTkX9vLF9s4vFVW37s+k/pELkknD8g= +github.com/nyaruka/gocommon v1.5.0/go.mod h1:tlsX12mmJMr/2tUKSdSm/6IFdMX2wp/s9GAeEAzZSU4= +github.com/nyaruka/goflow v0.103.1 h1:4JZMDmUTUQZ4JMyZQr9O0eijC0I49h+HahPSY6hgSr8= +github.com/nyaruka/goflow v0.103.1/go.mod h1:wwTXOxQoQAWpLxYm9hcmcvvlN30w7RR3CiNajHTAsr4= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc= @@ -231,6 +223,8 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/models/msgs.go b/models/msgs.go index d64b989e2..3f03b1457 100644 --- a/models/msgs.go +++ b/models/msgs.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/nyaruka/gocommon/gsm7" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" @@ -19,7 +20,6 @@ import ( "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/config" - "github.com/nyaruka/mailroom/utils/gsm7" "github.com/nyaruka/null" "github.com/gomodule/redigo/redis" diff --git a/utils/gsm7/gsm7.go b/utils/gsm7/gsm7.go deleted file mode 100644 index cd136fdcb..000000000 --- a/utils/gsm7/gsm7.go +++ /dev/null @@ -1,295 +0,0 @@ -package gsm7 - -// base gsm7 characters in our normal table -var baseGSM7 = map[rune]byte{ - '@': 0x00, - '£': 0x01, - '$': 0x02, - '¥': 0x03, - 'è': 0x04, - 'é': 0x05, - 'ù': 0x06, - 'ì': 0x07, - 'ò': 0x08, - 'Ç': 0x09, - '\n': 0x0A, - 'Ø': 0x0B, - 'ø': 0x0C, - '\r': 0x0D, - 'Å': 0x0E, - 'å': 0x0F, - 'Δ': 0x10, - '_': 0x11, - 'Φ': 0x12, - 'Γ': 0x13, - 'Λ': 0x14, - 'Ω': 0x15, - 'Π': 0x16, - 'Ψ': 0x17, - 'Σ': 0x18, - 'Θ': 0x19, - 'Ξ': 0x1A, - // 'ESC': 0x1B, // Escape control - 'Æ': 0x1C, - 'æ': 0x1D, - 'ß': 0x1E, - 'É': 0x1F, - ' ': 0x20, - '!': 0x21, - '"': 0x22, - '#': 0x23, - '¤': 0x24, - '%': 0x25, - '&': 0x26, - '\'': 0x27, - '(': 0x28, - ')': 0x29, - '*': 0x2A, - '+': 0x2B, - ',': 0x2C, - '-': 0x2D, - '.': 0x2E, - '/': 0x2F, - '0': 0x30, - '1': 0x31, - '2': 0x32, - '3': 0x33, - '4': 0x34, - '5': 0x35, - '6': 0x36, - '7': 0x37, - '8': 0x38, - '9': 0x39, - ':': 0x3A, - ';': 0x3B, - '<': 0x3C, - '=': 0x3D, - '>': 0x3E, - '?': 0x3F, - '¡': 0x40, - 'A': 0x41, - 'B': 0x42, - 'C': 0x43, - 'D': 0x44, - 'E': 0x45, - 'F': 0x46, - 'G': 0x47, - 'H': 0x48, - 'I': 0x49, - 'J': 0x4A, - 'K': 0x4B, - 'L': 0x4C, - 'M': 0x4D, - 'N': 0x4E, - 'O': 0x4F, - 'P': 0x50, - 'Q': 0x51, - 'R': 0x52, - 'S': 0x53, - 'T': 0x54, - 'U': 0x55, - 'V': 0x56, - 'W': 0x57, - 'X': 0x58, - 'Y': 0x59, - 'Z': 0x5A, - 'Ä': 0x5B, - 'Ö': 0x5C, - 'Ñ': 0x5D, - 'Ü': 0x5E, - '§': 0x5F, - '¿': 0x60, - 'a': 0x61, - 'b': 0x62, - 'c': 0x63, - 'd': 0x64, - 'e': 0x65, - 'f': 0x66, - 'g': 0x67, - 'h': 0x68, - 'i': 0x69, - 'j': 0x6A, - 'k': 0x6B, - 'l': 0x6C, - 'm': 0x6D, - 'n': 0x6E, - 'o': 0x6F, - 'p': 0x70, - 'q': 0x71, - 'r': 0x72, - 's': 0x73, - 't': 0x74, - 'u': 0x75, - 'v': 0x76, - 'w': 0x77, - 'x': 0x78, - 'y': 0x79, - 'z': 0x7A, - 'ä': 0x7B, - 'ö': 0x7C, - 'ñ': 0x7D, - 'ü': 0x7E, - 'à': 0x7F, -} - -// extended gsm7 characters, these my be preceded by our escape -var extendedGSM7 = map[rune]byte{ - ' ': 0x0A, - '^': 0x14, - '{': 0x28, - '}': 0x29, - '\\': 0x2F, - '[': 0x3C, - '~': 0x3D, - ']': 0x3E, - '|': 0x40, - '€': 0x65, -} - -// Characters we replace in GSM7 with versions that can actually be encoded -var gsm7Replacements = map[rune]rune{ - 'á': 'a', - 'ê': 'e', - 'ã': 'a', - 'â': 'a', - 'ç': 'c', - 'í': 'i', - 'î': 'i', - 'ú': 'u', - 'û': 'u', - 'õ': 'o', - 'ô': 'o', - 'ó': 'o', - - 'Á': 'A', - 'Â': 'A', - 'Ã': 'A', - 'À': 'A', - 'Ç': 'C', - 'È': 'E', - 'Ê': 'E', - 'Í': 'I', - 'Î': 'I', - 'Ì': 'I', - 'Ó': 'O', - 'Ô': 'O', - 'Ò': 'O', - 'Õ': 'O', - 'Ú': 'U', - 'Ù': 'U', - 'Û': 'U', - - // shit Word likes replacing automatically - '’': '\'', - '‘': '\'', - '“': '"', - '”': '"', - '–': '-', - '\xa0': ' ', -} - -// esc is our escape byte for the extended charset -const esc byte = 0x1B - -// unknown is the rune we replace invalid characters with -const unknown byte = '?' - -// max GSM7 value -const max byte = 0x7F - -// our reverse mapping from GSM7 byte to rune -var gsm7ToBase = make(map[byte]rune) -var gsm7ToExtended = make(map[byte]rune) - -// we create our reverse mappings in our init -func init() { - for r, b := range baseGSM7 { - gsm7ToBase[b] = r - } - - for r, b := range extendedGSM7 { - gsm7ToExtended[b] = r - } -} - -// IsValid returns whether the passed in string is made up of entirely GSM7 characters -func IsValid(text string) bool { - for _, r := range text { - _, present := baseGSM7[r] - if !present { - _, present = extendedGSM7[r] - if !present { - return false - } - } - } - return true -} - -// Segments calculates the number of SMS segments it will take to send the passed in text. -// This automatically figures out if the text is GSM7 or UCS2 and then calculates how many segments it -// will break up into. -// -// UCS2 messages can be 70 characters per segnment max, if more, each segment is 67 -// GSM7 messages can be 160 characters per segmend max, if more, each segment is 153 -// -// TODO: likely some optimizations could be made here by comparing ranges instead of map lookups -func Segments(text string) int { - // are we ucs2? - isGSM7 := IsValid(text) - - // first figure out if we are multipart - isMultipart := false - size := 0 - - for _, c := range text { - _, isExtended := extendedGSM7[c] - if isExtended && isGSM7 { - size += 2 - } else { - size += 1 - } - - if !isGSM7 && size > 70 { - isMultipart = true - break - } - - if isGSM7 && size > 160 { - isMultipart = true - break - } - } - - // we aren't multipart, so just return a single segment - if !isMultipart { - return 1 - } - - // our current segment count - segments := 1 - size = 0 - - // calculate our total number of segments, we can't do simple division because multibyte extended chars - // may land on a boundary between messages (`{` as character 153 will not fit and force another segment) - for _, c := range text { - _, isExtended := extendedGSM7[c] - if isExtended && isGSM7 { - size += 2 - } else { - size += 1 - } - - if isGSM7 && size > 153 { - size -= 153 - segments += 1 - } - - if !isGSM7 && size > 67 { - size -= 67 - segments += 1 - } - } - - return segments -} diff --git a/utils/gsm7/gsm7_test.go b/utils/gsm7/gsm7_test.go deleted file mode 100644 index 7aa50ee48..000000000 --- a/utils/gsm7/gsm7_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package gsm7_test - -import ( - "testing" - - "github.com/nyaruka/mailroom/utils/gsm7" - - "github.com/stretchr/testify/assert" -) - -func TestSegments(t *testing.T) { - // utility pads - tenChars := "0123456789" - unicodeTenChars := "☺123456789" - extendedTenChars := "[123456789" - fiftyChars := tenChars + tenChars + tenChars + tenChars + tenChars - hundredChars := fiftyChars + fiftyChars - unicode := "☺" - - tcs := []struct { - Text string - Segments int - }{ - {"", 1}, - {"hello", 1}, - {"“word”", 1}, - {hundredChars + fiftyChars + tenChars, 1}, - {hundredChars + fiftyChars + tenChars + "Z", 2}, - {hundredChars + fiftyChars + extendedTenChars, 2}, - {hundredChars + hundredChars + hundredChars + "123456", 2}, - {hundredChars + hundredChars + hundredChars + "1234567", 3}, - {fiftyChars + "zZ" + unicode, 1}, - {fiftyChars + tenChars + unicodeTenChars, 1}, - {fiftyChars + tenChars + unicodeTenChars + "z", 2}, - } - - for _, tc := range tcs { - assert.Equal(t, tc.Segments, gsm7.Segments(tc.Text), "unexpected num of segments for: %s", tc.Text) - } -} diff --git a/web/server.go b/web/server.go index aeb56d587..21db50dfd 100644 --- a/web/server.go +++ b/web/server.go @@ -1,6 +1,7 @@ package web import ( + "compress/flate" "context" "encoding/json" "fmt" @@ -75,7 +76,7 @@ func NewServer(ctx context.Context, config *config.Config, db *sqlx.DB, rp *redi router := chi.NewRouter() // set up our middlewares - router.Use(middleware.DefaultCompress) + router.Use(middleware.Compress(flate.DefaultCompression)) router.Use(middleware.RequestID) router.Use(middleware.RealIP) router.Use(panicRecovery)