Merge pull request #48 from Ilhasoft/update/v6.5.31

Update/v6.5.31

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: '1.16.x'
+  go-version: '1.17.x'
   postgis-version: '2.5'
 jobs:
   test:

CHANGELOG.md

@@ -1,3 +1,107 @@
+v6.5.31
+----------
+ * Recalculate dynamic groups after closing and reopening tickets
+ * Stop writing webhook results
+
+v6.5.30
+----------
+ * Fix handling webhook called events on resumes
+
+v6.5.29
+----------
+ * Add new fields to HTTPLog and save for webhook called events
+ * Stop writing ticket subjects
+
+v6.5.28
+----------
+ * Add warning log entry when task takes longer than 1 minute
+
+v6.5.27
+----------
+ * Update to latest goflow (fixes word_slice)
+
+v6.5.26
+----------
+ * Update to latest goflow
+
+v6.5.25
+----------
+ * Add notifications for contact imports and set contact import status
+
+v6.5.24
+----------
+ * Fix IVR for orgs using S3 sessions
+ * Ticket notifications (opened and activity)
+
+v6.5.23
+----------
+ * Add force param to close tickets endpoint to let us ignore errors on external ticket service when removing a ticketer
+
+v6.5.22
+----------
+ * Support Spanish status names passed back from Zendesk targets
+
+v6.5.21
+----------
+ * Read sessions from db or s3 depending on whether output_url is set
+ * Don't write output in db when writing to s3
+
+v6.5.20
+----------
+ * Add endpoint to change ticket topic
+ * Update to latest goflow/gocommon/phonenumbers
+
+v6.5.19
+----------
+ * Update to latest goflow to get ticket topic changes there
+ * Add ticket topics
+
+v6.5.18
+----------
+ * Switch to synchronous answering machine detection for Twilio channels
+
+v6.5.15
+----------
+ * Make IVR machine detection an option in channel config
+
+v6.5.14
+----------
+ * Stop reading/writing channelconnection.retry_count so that it can be dropped
+
+v6.5.13
+----------
+ * Don't let IVR status callbacks overwrite error status (otherwise calls aren't retried)
+
+v6.5.12
+----------
+ * Revert previous to query to fetch calls to retry so we only look as statuses Q and E
+ * Add ChannelConnection.errorReason and start populating
+ * Reinstate channel connection error_count and write it
+ * Fix retrying of calls where answering machine was detected
+
+v6.5.11
+----------
+ * Implement asynchronous AMD for Twilio IVR
+ * Enable answering machine detection for Vonage IVR
+
+v6.5.10
+----------
+ * Fix requests to twilio to start calls with machine detection
+
+v6.5.9
+----------
+ * Add support for answering machine detection on Twilio calls
+ * Update to latest goflow
+
+v6.5.8
+----------
+ * Use new config keys for LUIS classifiers
+
+v6.5.7
+----------
+ * Switch from abandoned dgrijalva/jwt-go to golang-jwt/jwt
+ * Update to latest goflow (adds support for queries on tickets, fixes LUIS classifiers)
+
 v6.5.6
 ----------
  * Update to latest goflow and add parse_only as param to parse_query to allow us to extract field dependencies even when they don't yet exist in the database

config/config_test.go

@@ -39,7 +39,7 @@ func TestParseDisallowedNetworks(t *testing.T) {
 	// test with invalid CSV
 	cfg.DisallowedNetworks = `"127.0.0.1`
 	_, _, err = cfg.ParseDisallowedNetworks()
-	assert.EqualError(t, err, `record on line 1; parse error on line 2, column 0: extraneous or missing " in quoted-field`)
+	assert.EqualError(t, err, `parse error on line 1, column 11: extraneous or missing " in quoted-field`)
 
 	// test with invalid IP
 	cfg.DisallowedNetworks = `127.0.1`

core/goflow/engine.go

@@ -108,8 +108,8 @@ type simulatorTicketService struct {
 	ticketer *flows.Ticketer
 }
 
-func (s *simulatorTicketService) Open(session flows.Session, subject, body string, logHTTP flows.HTTPLogCallback) (*flows.Ticket, error) {
-	return flows.OpenTicket(s.ticketer, subject, body), nil
+func (s *simulatorTicketService) Open(session flows.Session, topic *flows.Topic, body string, assignee *flows.User, logHTTP flows.HTTPLogCallback) (*flows.Ticket, error) {
+	return flows.OpenTicket(s.ticketer, topic, body, assignee), nil
 }
 
 func simulatorAirtimeServiceFactory(session flows.Session) (flows.AirtimeService, error) {

core/goflow/engine_test.go

@@ -68,10 +68,12 @@ func TestSimulatorTicket(t *testing.T) {
 	svc, err := goflow.Simulator(rt.Config).Services().Ticket(nil, flows.NewTicketer(ticketer))
 	assert.NoError(t, err)
 
-	ticket, err := svc.Open(nil, "New ticket", "Where are my cookies?", nil)
+	oa, err := models.GetOrgAssets(ctx, db, testdata.Org1.ID)
+	require.NoError(t, err)
+
+	ticket, err := svc.Open(nil, oa.SessionAssets().Topics().FindByName("General"), "Where are my cookies?", nil, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, testdata.Mailgun.UUID, ticket.Ticketer().UUID())
-	assert.Equal(t, "New ticket", ticket.Subject())
 	assert.Equal(t, "Where are my cookies?", ticket.Body())
 }
 

core/handlers/airtime_transferred.go

@@ -55,10 +55,12 @@ func handleAirtimeTransferred(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool,
 		transfer.AddLog(models.NewAirtimeTransferredLog(
 			oa.OrgID(),
 			httpLog.URL,
+			httpLog.StatusCode,
 			httpLog.Request,
 			httpLog.Response,
 			httpLog.Status != flows.CallStatusSuccess,
 			time.Duration(httpLog.ElapsedMS)*time.Millisecond,
+			httpLog.Retries,
 			httpLog.CreatedOn,
 		))
 	}

core/handlers/service_called.go

@@ -54,21 +54,25 @@ func handleServiceCalled(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *m
 				oa.OrgID(),
 				classifier.ID(),
 				httpLog.URL,
+				httpLog.StatusCode,
 				httpLog.Request,
 				httpLog.Response,
 				httpLog.Status != flows.CallStatusSuccess,
 				time.Duration(httpLog.ElapsedMS)*time.Millisecond,
+				httpLog.Retries,
 				httpLog.CreatedOn,
 			)
 		} else if event.Service == "ticketer" {
 			log = models.NewTicketerCalledLog(
 				oa.OrgID(),
 				ticketer.ID(),
 				httpLog.URL,
+				httpLog.StatusCode,
 				httpLog.Request,
 				httpLog.Response,
 				httpLog.Status != flows.CallStatusSuccess,
 				time.Duration(httpLog.ElapsedMS)*time.Millisecond,
+				httpLog.Retries,
 				httpLog.CreatedOn,
 			)
 		}

core/handlers/session_triggered_test.go

@@ -95,7 +95,9 @@ func TestSessionTriggered(t *testing.T) {
 }
 
 func TestQuerySessionTriggered(t *testing.T) {
-	ctx, _, db, rp := testsuite.Reset()
+	ctx, _, db, rp := testsuite.Get()
+
+	defer testsuite.Reset()
 
 	oa, err := models.GetOrgAssets(ctx, db, testdata.Org1.ID)
 	assert.NoError(t, err)

core/handlers/ticket_opened.go

@@ -28,6 +28,15 @@ func handleTicketOpened(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *mo
 		return errors.Errorf("unable to find ticketer with UUID: %s", event.Ticket.Ticketer.UUID)
 	}
 
+	var topicID models.TopicID
+	if event.Ticket.Topic != nil {
+		topic := oa.TopicByUUID(event.Ticket.Topic.UUID)
+		if topic == nil {
+			return errors.Errorf("unable to find topic with UUID: %s", event.Ticket.Topic.UUID)
+		}
+		topicID = topic.ID()
+	}
+
 	var assigneeID models.UserID
 	if event.Ticket.Assignee != nil {
 		assignee := oa.UserByEmail(event.Ticket.Assignee.Email)
@@ -43,7 +52,7 @@ func handleTicketOpened(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *mo
 		scene.ContactID(),
 		ticketer.ID(),
 		event.Ticket.ExternalID,
-		event.Ticket.Subject,
+		topicID,
 		event.Ticket.Body,
 		assigneeID,
 		map[string]interface{}{

core/handlers/ticket_opened_test.go

@@ -45,18 +45,32 @@ func TestTicketOpened(t *testing.T) {
 	}))
 
 	// an existing ticket
-	cathyTicket := models.NewTicket(flows.TicketUUID(uuids.New()), testdata.Org1.ID, testdata.Cathy.ID, testdata.Mailgun.ID, "748363", "Old Question", "Who?", models.NilUserID, nil)
+	cathyTicket := models.NewTicket(flows.TicketUUID(uuids.New()), testdata.Org1.ID, testdata.Cathy.ID, testdata.Mailgun.ID, "748363", testdata.DefaultTopic.ID, "Who?", models.NilUserID, nil)
 	err := models.InsertTickets(ctx, db, []*models.Ticket{cathyTicket})
 	require.NoError(t, err)
 
 	tcs := []handlers.TestCase{
 		{
 			Actions: handlers.ContactActionMap{
 				testdata.Cathy: []flows.Action{
-					actions.NewOpenTicket(handlers.NewActionUUID(), assets.NewTicketerReference(testdata.Mailgun.UUID, "Mailgun (IT Support)"), "Need help", "Where are my cookies?", "Email Ticket"),
+					actions.NewOpenTicket(
+						handlers.NewActionUUID(),
+						assets.NewTicketerReference(testdata.Mailgun.UUID, "Mailgun (IT Support)"),
+						assets.NewTopicReference(testdata.SupportTopic.UUID, "Support"),
+						"Where are my cookies?",
+						assets.NewUserReference(testdata.Admin.Email, "Admin"),
+						"Email Ticket",
+					),
 				},
 				testdata.Bob: []flows.Action{
-					actions.NewOpenTicket(handlers.NewActionUUID(), assets.NewTicketerReference(testdata.Zendesk.UUID, "Zendesk (Nyaruka)"), "Interesting", "I've found some cookies", "Zen Ticket"),
+					actions.NewOpenTicket(
+						handlers.NewActionUUID(),
+						assets.NewTicketerReference(testdata.Zendesk.UUID, "Zendesk (Nyaruka)"),
+						nil,
+						"I've found some cookies",
+						nil,
+						"Zen Ticket",
+					),
 				},
 			},
 			SQLAssertions: []handlers.SQLAssertion{
@@ -94,6 +108,25 @@ func TestTicketOpened(t *testing.T) {
 					SQL:   "select count(*) from tickets_ticketevent where event_type = 'O'",
 					Count: 2,
 				},
+				{ // both of our tickets have a topic (one without an explicit topic get's the default)
+					SQL:   "select count(*) from tickets_ticket where topic_id is null",
+					Count: 0,
+				},
+				{ // one of our tickets is assigned to admin
+					SQL:   "select count(*) from tickets_ticket where assignee_id = $1",
+					Args:  []interface{}{testdata.Admin.ID},
+					Count: 1,
+				},
+				{ // admin will have a ticket assigned notification for the ticket directly assigned to them
+					SQL:   "select count(*) from notifications_notification where user_id = $1 and notification_type = 'tickets:activity'",
+					Args:  []interface{}{testdata.Admin.ID},
+					Count: 1,
+				},
+				{ // all assignable users will have a ticket opened notification for the unassigned ticket
+					SQL:   "select count(*) from notifications_notification where notification_type = 'tickets:opened'",
+					Args:  nil,
+					Count: 3,
+				},
 			},
 		},
 	}

core/handlers/webhook_called.go

@@ -41,20 +41,22 @@ func handleWebhookCalled(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *m
 		scene.AppendToEventPreCommitHook(hooks.UnsubscribeResthookHook, unsub)
 	}
 
-	// if this is a connection error, use that as our response
-	response := event.Response
-	if event.Status == flows.CallStatusConnectionError {
-		response = "connection error"
+	run, _ := scene.Session().FindStep(e.StepUUID())
+	flow, _ := oa.Flow(run.FlowReference().UUID)
+
+	// create an HTTP log
+	if flow != nil {
+		httpLog := models.NewWebhookCalledLog(
+			oa.OrgID(),
+			flow.(*models.Flow).ID(),
+			event.URL, event.StatusCode, event.Request, event.Response,
+			event.Status != flows.CallStatusSuccess,
+			time.Millisecond*time.Duration(event.ElapsedMS),
+			event.Retries,
+			event.CreatedOn(),
+		)
+		scene.AppendToEventPreCommitHook(hooks.InsertHTTPLogsHook, httpLog)
 	}
 
-	// create a result for this call
-	result := models.NewWebhookResult(
-		oa.OrgID(), scene.ContactID(),
-		event.URL, event.Request,
-		event.StatusCode, response,
-		time.Millisecond*time.Duration(event.ElapsedMS), event.CreatedOn(),
-	)
-	scene.AppendToEventPreCommitHook(hooks.InsertWebhookResultHook, result)
-
 	return nil
 }

core/handlers/webhook_called_test.go

@@ -43,17 +43,16 @@ func TestWebhookCalled(t *testing.T) {
 		{
 			Actions: handlers.ContactActionMap{
 				testdata.Cathy: []flows.Action{
-					actions.NewCallResthook(handlers.NewActionUUID(), "foo", "foo"),
+					actions.NewCallResthook(handlers.NewActionUUID(), "foo", "foo"), // calls both subscribers
 				},
 				testdata.George: []flows.Action{
-					actions.NewCallResthook(handlers.NewActionUUID(), "foo", "foo"),
+					actions.NewCallResthook(handlers.NewActionUUID(), "foo", "foo"), // calls both subscribers
 					actions.NewCallWebhook(handlers.NewActionUUID(), "GET", "http://rapidpro.io/?unsub=1", nil, "", ""),
 				},
 			},
 			SQLAssertions: []handlers.SQLAssertion{
 				{
 					SQL:   "select count(*) from api_resthooksubscriber where is_active = FALSE",
-					Args:  nil,
 					Count: 1,
 				},
 				{
@@ -63,22 +62,14 @@ func TestWebhookCalled(t *testing.T) {
 				},
 				{
 					SQL:   "select count(*) from api_resthooksubscriber where is_active = TRUE",
-					Args:  nil,
 					Count: 2,
 				},
 				{
-					SQL:   "select count(*) from api_webhookresult where contact_id = $1 AND status_code = 200",
-					Args:  []interface{}{testdata.Cathy.ID},
-					Count: 1,
-				},
-				{
-					SQL:   "select count(*) from api_webhookresult where contact_id = $1 AND status_code = 410",
-					Args:  []interface{}{testdata.Cathy.ID},
-					Count: 1,
+					SQL:   "select count(*) from request_logs_httplog where log_type = 'webhook_called' AND flow_id IS NOT NULL AND status_code = 200",
+					Count: 2,
 				},
 				{
-					SQL:   "select count(*) from api_webhookresult where contact_id = $1",
-					Args:  []interface{}{testdata.George.ID},
+					SQL:   "select count(*) from request_logs_httplog where log_type = 'webhook_called' AND flow_id IS NOT NULL AND status_code = 410",
 					Count: 3,
 				},
 				{

core/hooks/insert_tickets.go

@@ -34,8 +34,11 @@ func (h *insertTicketsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Po
 
 	// generate opened events for each ticket
 	openEvents := make([]*models.TicketEvent, len(tickets))
+	eventsByTicket := make(map[*models.Ticket]*models.TicketEvent, len(tickets))
 	for i, ticket := range tickets {
-		openEvents[i] = models.NewTicketOpenedEvent(ticket, models.NilUserID, ticket.AssigneeID())
+		evt := models.NewTicketOpenedEvent(ticket, models.NilUserID, ticket.AssigneeID())
+		openEvents[i] = evt
+		eventsByTicket[ticket] = evt
 	}
 
 	// and insert those too
@@ -44,5 +47,11 @@ func (h *insertTicketsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Po
 		return errors.Wrapf(err, "error inserting ticket opened events")
 	}
 
+	// and insert logs/notifications for those
+	err = models.NotificationsFromTicketEvents(ctx, tx, oa, eventsByTicket)
+	if err != nil {
+		return errors.Wrapf(err, "error inserting notifications")
+	}
+
 	return nil
 }