Add support for incoming attachments on ticketing services (WIP)

models/tickets.go

@@ -98,7 +98,7 @@ func (t *Ticket) ForwardIncoming(ctx context.Context, db *sqlx.DB, org *OrgAsset
 	}
 
 	logger := &HTTPLogger{}
-	err = service.Forward(t, msgUUID, text, logger.Ticketer(ticketer))
+	err = service.Forward(t, msgUUID, text, attachments, logger.Ticketer(ticketer))
 
 	return logger.Insert(ctx, db)
 }
@@ -477,7 +477,7 @@ func (t *Ticketer) UpdateConfig(ctx context.Context, db *sqlx.DB, add map[string
 type TicketService interface {
 	flows.TicketService
 
-	Forward(*Ticket, flows.MsgUUID, string, flows.HTTPLogCallback) error
+	Forward(*Ticket, flows.MsgUUID, string, []utils.Attachment, flows.HTTPLogCallback) error
 	Close([]*Ticket, flows.HTTPLogCallback) error
 	Reopen([]*Ticket, flows.HTTPLogCallback) error
 }

services/tickets/mailgun/client.go

@@ -43,15 +43,25 @@ type messageResponse struct {
 	ID string `json:"id"`
 }
 
+type File struct {
+	Filename string
+	Data     []byte
+}
+
 // SendMessage sends a new email message and returns the ID
 // see https://documentation.mailgun.com/en/latest/api-sending.html
-func (c *Client) SendMessage(from, to, subject, text string, headers map[string]string) (string, *httpx.Trace, error) {
+func (c *Client) SendMessage(from, to, subject, text string, attachments []File, headers map[string]string) (string, *httpx.Trace, error) {
 	writeBody := func(w *multipart.Writer) {
 		w.WriteField("from", from)
 		w.WriteField("to", to)
 		w.WriteField("subject", subject)
 		w.WriteField("text", text)
 
+		for _, attachment := range attachments {
+			fw, _ := w.CreateFormFile("attachment", attachment.Filename)
+			fw.Write(attachment.Data)
+		}
+
 		// for the sake of tests, we want to output headers in consistent order
 		headerKeys := make([]string, 0, len(headers))
 		for k := range headers {

services/tickets/mailgun/client_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/nyaruka/gocommon/httpx"
+	"github.com/nyaruka/goflow/test"
 	"github.com/nyaruka/goflow/utils/uuids"
 	"github.com/nyaruka/mailroom/services/tickets/mailgun"
 
@@ -30,18 +31,26 @@ func TestSendMessage(t *testing.T) {
 
 	client := mailgun.NewClient(http.DefaultClient, nil, "tickets.rapidpro.io", "123456789")
 
-	_, _, err := client.SendMessage("Bob <ticket+12446@tickets.rapidpro.io>", "support@acme.com", "Need help", "Where are my cookies?", nil)
+	_, _, err := client.SendMessage("Bob <ticket+12446@tickets.rapidpro.io>", "support@acme.com", "Need help", "Where are my cookies?", nil, nil)
 	assert.EqualError(t, err, "unable to connect to server")
 
-	_, _, err = client.SendMessage("Bob <ticket+12446@tickets.rapidpro.io>", "support@acme.com", "Need help", "Where are my cookies?", nil)
+	_, _, err = client.SendMessage("Bob <ticket+12446@tickets.rapidpro.io>", "support@acme.com", "Need help", "Where are my cookies?", nil, nil)
 	assert.EqualError(t, err, "Something went wrong")
 
-	_, _, err = client.SendMessage("Bob <ticket+12446@tickets.rapidpro.io>", "support@acme.com", "Need help", "Where are my cookies?", nil)
+	_, _, err = client.SendMessage("Bob <ticket+12446@tickets.rapidpro.io>", "support@acme.com", "Need help", "Where are my cookies?", nil, nil)
 	assert.EqualError(t, err, "invalid character 'x' looking for beginning of value")
 
-	msgID, trace, err := client.SendMessage("Bob <ticket+12446@tickets.rapidpro.io>", "support@acme.com", "Need help", "Where are my cookies?", map[string]string{"In-Reply-To": "12415"})
+	msgID, trace, err := client.SendMessage(
+		"Bob <ticket+12446@tickets.rapidpro.io>",
+		"support@acme.com",
+		"Need help",
+		"Where are my cookies?",
+		[]mailgun.File{{"test.jpg", []byte(`IMANIMAGE`)}, {"test.mp4", []byte(`IMAVIDEO`)}},
+		map[string]string{"In-Reply-To": "12415"},
+	)
 	assert.NoError(t, err)
 	assert.Equal(t, "<20200426161758.1.590432020254B2BF@tickets.rapidpro.io>", msgID)
-	assert.Equal(t, "POST /v3/tickets.rapidpro.io/messages HTTP/1.1\r\nHost: api.mailgun.net\r\nUser-Agent: Go-http-client/1.1\r\nContent-Length: 586\r\nAuthorization: Basic YXBpOjEyMzQ1Njc4OQ==\r\nContent-Type: multipart/form-data; boundary=9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nAccept-Encoding: gzip\r\n\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"from\"\r\n\r\nBob <ticket+12446@tickets.rapidpro.io>\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"to\"\r\n\r\nsupport@acme.com\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"subject\"\r\n\r\nNeed help\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"text\"\r\n\r\nWhere are my cookies?\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"h:In-Reply-To\"\r\n\r\n12415\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d--\r\n", string(trace.RequestTrace))
 	assert.Equal(t, "HTTP/1.0 200 OK\r\nContent-Length: 111\r\n\r\n", string(trace.ResponseTrace))
+
+	test.AssertSnapshot(t, "mailgun_request", string(trace.RequestTrace))
 }

services/tickets/mailgun/service.go

@@ -115,7 +115,7 @@ func (s *service) Open(session flows.Session, subject, body string, logHTTP flow
 	context := s.templateContext(subject, body, "", string(session.Contact().UUID()), contactDisplay)
 	fullBody := evaluateTemplate(openBodyTemplate, context)
 
-	msgID, trace, err := s.client.SendMessage(from, s.toAddress, subject, fullBody, nil)
+	msgID, trace, err := s.client.SendMessage(from, s.toAddress, subject, fullBody, nil, nil)
 	if trace != nil {
 		logHTTP(flows.NewHTTPLog(trace, flows.HTTPStatusFromCode, s.redactor))
 	}
@@ -126,11 +126,11 @@ func (s *service) Open(session flows.Session, subject, body string, logHTTP flow
 	return flows.NewTicket(ticketUUID, s.ticketer.Reference(), subject, body, msgID), nil
 }
 
-func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, logHTTP flows.HTTPLogCallback) error {
+func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, attachments []utils.Attachment, logHTTP flows.HTTPLogCallback) error {
 	context := s.templateContext(ticket.Subject(), ticket.Body(), text, ticket.Config(ticketConfigContactUUID), ticket.Config(ticketConfigContactDisplay))
 	body := evaluateTemplate(forwardBodyTemplate, context)
 
-	_, err := s.sendInTicket(ticket, body, logHTTP)
+	_, err := s.sendInTicket(ticket, body, attachments, logHTTP)
 	return err
 }
 
@@ -139,7 +139,7 @@ func (s *service) Close(tickets []*models.Ticket, logHTTP flows.HTTPLogCallback)
 		context := s.templateContext(ticket.Subject(), ticket.Body(), "", ticket.Config(ticketConfigContactUUID), ticket.Config(ticketConfigContactDisplay))
 		body := evaluateTemplate(closedBodyTemplate, context)
 
-		_, err := s.sendInTicket(ticket, body, logHTTP)
+		_, err := s.sendInTicket(ticket, body, nil, logHTTP)
 		if err != nil {
 			return err
 		}
@@ -152,7 +152,7 @@ func (s *service) Reopen(tickets []*models.Ticket, logHTTP flows.HTTPLogCallback
 		context := s.templateContext(ticket.Subject(), ticket.Body(), "", ticket.Config(ticketConfigContactUUID), ticket.Config(ticketConfigContactDisplay))
 		body := evaluateTemplate(reopenedBodyTemplate, context)
 
-		_, err := s.sendInTicket(ticket, body, logHTTP)
+		_, err := s.sendInTicket(ticket, body, nil, logHTTP)
 		if err != nil {
 			return err
 		}
@@ -161,7 +161,7 @@ func (s *service) Reopen(tickets []*models.Ticket, logHTTP flows.HTTPLogCallback
 }
 
 // sends an email as part of the thread for the given ticket
-func (s *service) sendInTicket(ticket *models.Ticket, text string, logHTTP flows.HTTPLogCallback) (string, error) {
+func (s *service) sendInTicket(ticket *models.Ticket, text string, attachments []utils.Attachment, logHTTP flows.HTTPLogCallback) (string, error) {
 	contactDisplay := ticket.Config(ticketConfigContactDisplay)
 	lastMessageID := ticket.Config(ticketConfigLastMessageID)
 	if lastMessageID == "" {
@@ -173,11 +173,14 @@ func (s *service) sendInTicket(ticket *models.Ticket, text string, logHTTP flows
 	}
 	from := s.ticketAddress(contactDisplay, ticket.UUID())
 
-	return s.send(from, s.toAddress, ticket.Subject(), text, headers, logHTTP)
+	return s.send(from, s.toAddress, ticket.Subject(), text, attachments, headers, logHTTP)
 }
 
-func (s *service) send(from, to, subject, text string, headers map[string]string, logHTTP flows.HTTPLogCallback) (string, error) {
-	msgID, trace, err := s.client.SendMessage(from, to, subject, text, headers)
+func (s *service) send(from, to, subject, text string, attachments []utils.Attachment, headers map[string]string, logHTTP flows.HTTPLogCallback) (string, error) {
+	// TODO fetch attachments to send
+	var files []File
+
+	msgID, trace, err := s.client.SendMessage(from, to, subject, text, files, headers)
 	if trace != nil {
 		logHTTP(flows.NewHTTPLog(trace, flows.HTTPStatusFromCode, s.redactor))
 	}

services/tickets/mailgun/service_test.go

@@ -94,7 +94,7 @@ func TestOpenAndForward(t *testing.T) {
 	})
 
 	logger = &flows.HTTPLogger{}
-	err = svc.Forward(dbTicket, flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"), "It's urgent", logger.Log)
+	err = svc.Forward(dbTicket, flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"), "It's urgent", nil, logger.Log)
 
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(logger.Logs))

services/tickets/mailgun/testdata/TestSendMessage_mailgun_request.snap

@@ -0,0 +1,39 @@
+POST /v3/tickets.rapidpro.io/messages HTTP/1.1
+Host: api.mailgun.net
+User-Agent: Go-http-client/1.1
+Content-Length: 915
+Authorization: Basic YXBpOjEyMzQ1Njc4OQ==
+Content-Type: multipart/form-data; boundary=9688d21d-95aa-4bed-afc7-f31b35731a3d
+Accept-Encoding: gzip
+
+--9688d21d-95aa-4bed-afc7-f31b35731a3d
+Content-Disposition: form-data; name="from"
+
+Bob <ticket+12446@tickets.rapidpro.io>
+--9688d21d-95aa-4bed-afc7-f31b35731a3d
+Content-Disposition: form-data; name="to"
+
+support@acme.com
+--9688d21d-95aa-4bed-afc7-f31b35731a3d
+Content-Disposition: form-data; name="subject"
+
+Need help
+--9688d21d-95aa-4bed-afc7-f31b35731a3d
+Content-Disposition: form-data; name="text"
+
+Where are my cookies?
+--9688d21d-95aa-4bed-afc7-f31b35731a3d
+Content-Disposition: form-data; name="attachment"; filename="test.jpg"
+Content-Type: application/octet-stream
+
+IMANIMAGE
+--9688d21d-95aa-4bed-afc7-f31b35731a3d
+Content-Disposition: form-data; name="attachment"; filename="test.mp4"
+Content-Type: application/octet-stream
+
+IMAVIDEO
+--9688d21d-95aa-4bed-afc7-f31b35731a3d
+Content-Disposition: form-data; name="h:In-Reply-To"
+
+12415
+--9688d21d-95aa-4bed-afc7-f31b35731a3d--

services/tickets/mailgun/web.go

@@ -87,7 +87,7 @@ func handleReceive(ctx context.Context, s *web.Server, r *http.Request, l *model
 	if request.Sender != configuredAddress {
 		body := fmt.Sprintf("The address %s is not allowed to reply to this ticket\n", request.Sender)
 
-		mailgun.send(mailgun.noReplyAddress(), request.From, "Ticket reply rejected", body, nil, l.Ticketer(ticketer))
+		mailgun.send(mailgun.noReplyAddress(), request.From, "Ticket reply rejected", body, nil, nil, l.Ticketer(ticketer))
 
 		return &receiveResponse{Action: "rejected", TicketUUID: ticket.UUID()}, http.StatusOK, nil
 	}

services/tickets/zendesk/service.go

@@ -3,6 +3,7 @@ package zendesk
 import (
 	"fmt"
 	"net/http"
+	"net/url"
 
 	"github.com/nyaruka/gocommon/dates"
 	"github.com/nyaruka/gocommon/httpx"
@@ -94,10 +95,15 @@ func (s *service) Open(session flows.Session, subject, body string, logHTTP flow
 	return flows.NewTicket(ticketUUID, s.ticketer.Reference(), subject, body, ""), nil
 }
 
-func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, logHTTP flows.HTTPLogCallback) error {
+func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, attachments []utils.Attachment, logHTTP flows.HTTPLogCallback) error {
 	contactUUID := ticket.Config("contact-uuid")
 	contactDisplay := ticket.Config("contact-display")
 
+	fileURLs, err := s.convertAttachments(attachments)
+	if err != nil {
+		return errors.Wrap(err, "error converting attachments")
+	}
+
 	msg := &ExternalResource{
 		ExternalID: string(msgUUID),
 		Message:    text,
@@ -107,6 +113,7 @@ func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text str
 			ExternalID: contactUUID,
 			Name:       contactDisplay,
 		},
+		FileURLs:         fileURLs,
 		AllowChannelback: true,
 	}
 
@@ -233,3 +240,18 @@ func (s *service) push(msg *ExternalResource, logHTTP flows.HTTPLogCallback) err
 	}
 	return nil
 }
+
+// convert attachments to URLs which Zendesk can POST to
+func (s *service) convertAttachments(attachments []utils.Attachment) ([]string, error) {
+	fileURLs := make([]string, len(attachments))
+	for i, a := range attachments {
+		u, err := url.Parse(a.URL())
+		if err != nil {
+			return nil, err
+		}
+
+		// TODO generate URL of current instance??
+		fileURLs[i] = "https://temba.ngrok.io/mr/tickets/types/zendesk/file?path=" + url.QueryEscape(u.Path)
+	}
+	return fileURLs, nil
+}

services/tickets/zendesk/service_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/nyaruka/goflow/envs"
 	"github.com/nyaruka/goflow/flows"
 	"github.com/nyaruka/goflow/test"
+	"github.com/nyaruka/goflow/utils"
 	"github.com/nyaruka/goflow/utils/uuids"
 	"github.com/nyaruka/mailroom/models"
 	"github.com/nyaruka/mailroom/services/tickets/zendesk"
@@ -104,7 +105,13 @@ func TestOpenAndForward(t *testing.T) {
 	})
 
 	logger = &flows.HTTPLogger{}
-	err = svc.Forward(dbTicket, flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"), "It's urgent", logger.Log)
+	err = svc.Forward(
+		dbTicket,
+		flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"),
+		"It's urgent",
+		[]utils.Attachment{utils.Attachment("image/jpg:http://myfiles.com/attachments/0123/attachment1.jpg")},
+		logger.Log,
+	)
 
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(logger.Logs))

services/tickets/zendesk/testdata/TestOpenAndForward_forward_message.snap

@@ -1,9 +1,9 @@
 POST /api/v2/any_channel/push.json HTTP/1.1
 Host: nyaruka.zendesk.com
 User-Agent: Go-http-client/1.1
-Content-Length: 367
+Content-Length: 481
 Authorization: Bearer ****************
 Content-Type: application/json
 Accept-Encoding: gzip
 
-{"instance_push_id":"1234-abcd","request_id":"sesame:1570461699000000000","external_resources":[{"external_id":"ca5607f0-cba8-4c94-9cd5-c4fbc24aa767","message":"It's urgent","thread_id":"59d74b86-3e2f-4a93-aece-b05d2fdcde0c","created_at":"2019-10-07T15:21:38Z","author":{"external_id":"6393abc0-283d-4c9b-a1b3-641a035c34bf","name":"Cathy"},"allow_channelback":true}]}
+{"instance_push_id":"1234-abcd","request_id":"sesame:1570461699000000000","external_resources":[{"external_id":"ca5607f0-cba8-4c94-9cd5-c4fbc24aa767","message":"It's urgent","thread_id":"59d74b86-3e2f-4a93-aece-b05d2fdcde0c","created_at":"2019-10-07T15:21:38Z","author":{"external_id":"6393abc0-283d-4c9b-a1b3-641a035c34bf","name":"Cathy"},"allow_channelback":true,"file_urls":["https://temba.ngrok.io/mr/tickets/types/zendesk/file?path=%2Fattachments%2F0123%2Fattachment1.jpg"]}]}

services/tickets/zendesk/web.go

@@ -26,7 +26,8 @@ func init() {
 
 	web.RegisterJSONRoute(http.MethodPost, base+"/channelback", handleChannelback)
 	web.RegisterJSONRoute(http.MethodPost, base+"/event_callback", web.WithHTTPLogs(handleEventCallback))
-	web.RegisterJSONRoute(http.MethodPost, base+"/target/{ticketer:[a-f0-9\\-]+}", web.WithHTTPLogs(handleTicketerTarget))
+	web.RegisterJSONRoute(http.MethodPost, base+`/target/{ticketer:[a-f0-9\-]+}`, web.WithHTTPLogs(handleTicketerTarget))
+	web.RegisterRoute(http.MethodPost, base+`/file`, handleFileCallback)
 }
 
 type integrationMetadata struct {
@@ -278,3 +279,19 @@ func handleTicketerTarget(ctx context.Context, s *web.Server, r *http.Request, l
 
 	return map[string]string{"status": "handled"}, http.StatusOK, nil
 }
+
+func handleFileCallback(ctx context.Context, s *web.Server, r *http.Request, w http.ResponseWriter) error {
+	path := r.URL.Query().Get("path")
+
+	fmt.Printf("callback for file URL '%s'\n", path)
+
+	data, err := s.Storage.Get(path)
+	if err != nil {
+		http.NotFound(w, r)
+		return nil
+	}
+
+	w.WriteHeader(200)
+	w.Write(data)
+	return nil
+}

utils/storage/base.go

@@ -9,6 +9,7 @@ type Storage interface {
 	Name() string
 	Test() error
 	Put(path string, contentType string, contents []byte) (string, error)
+	Get(path string) ([]byte, error)
 }
 
 // New creates a new storage service

utils/storage/fs.go

@@ -45,3 +45,9 @@ func (s *fsStorage) Put(path string, contentType string, contents []byte) (strin
 
 	return fullPath, nil
 }
+
+func (s *fsStorage) Get(path string) ([]byte, error) {
+	fullPath := filepath.Join(s.directory, path)
+
+	return ioutil.ReadFile(fullPath)
+}

utils/storage/fs_test.go

@@ -1,7 +1,6 @@
 package storage_test
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -26,7 +25,7 @@ func TestFS(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "_testing/foo/bar.txt", url)
 
-	data, err := ioutil.ReadFile(url)
+	data, err := s.Get("/foo/bar.txt")
 	assert.NoError(t, err)
 	assert.Equal(t, []byte(`hello world`), data)
 

utils/storage/s3.go

@@ -69,5 +69,13 @@ func (s *s3Storage) Put(path string, contentType string, contents []byte) (strin
 		return "", err
 	}
 
-	return fmt.Sprintf(s3BucketURL, s.bucket, path), nil
+	return s.url(path), nil
+}
+
+func (s *s3Storage) Get(path string) ([]byte, error) {
+	return nil, nil // TODO
+}
+
+func (s *s3Storage) url(path string) string {
+	return fmt.Sprintf(s3BucketURL, s.bucket, path)
 }