Merge pull request #128 from nyaruka/clone_and_inspect

🧰 Clone and inspect endpoints

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/nyaruka/ezconf v0.2.1
 	github.com/nyaruka/gocommon v1.0.0
-	github.com/nyaruka/goflow v0.39.3
+	github.com/nyaruka/goflow v0.40.3
 	github.com/nyaruka/librato v0.0.0-20180827155909-cacc769357b8
 	github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d
 	github.com/nyaruka/null v1.1.1

go.sum

@@ -66,8 +66,8 @@ github.com/nyaruka/gocommon v0.2.0 h1:1Le4Ok0Zp2RYULue0n4/02zL+1MrykN/C79HhirGeR
 github.com/nyaruka/gocommon v0.2.0/go.mod h1:ZrhaOKNc+kK1qWNuCuZivskT+ygLyIwu4KZVgcaC1mw=
 github.com/nyaruka/gocommon v1.0.0 h1:4gdAMOR4BTMHHZjOy5WhfKYGUZVmJ+3LPh1sj011Qzk=
 github.com/nyaruka/gocommon v1.0.0/go.mod h1:QbdU2J9WBsqBmeZRuwndf2f6O7rD7mkC0bGn5UNnwjI=
-github.com/nyaruka/goflow v0.39.3 h1:rNp9htU3D8QZwhMd8TVkbQ4LL9KyfEsM5fdAwFBZR6s=
-github.com/nyaruka/goflow v0.39.3/go.mod h1:QitrAujTi7Mc75aVIoCpAmFsfO794+ljo9QNGt7qSHY=
+github.com/nyaruka/goflow v0.40.3 h1:lQoBdiY1cwg+HVt/Fi+DsH27T/ys1rD6kpnAmpPkh9g=
+github.com/nyaruka/goflow v0.40.3/go.mod h1:QitrAujTi7Mc75aVIoCpAmFsfO794+ljo9QNGt7qSHY=
 github.com/nyaruka/librato v0.0.0-20180827155909-cacc769357b8 h1:TOvxy0u6LNTWP3gwbdNVCiByXJupr9ATFdzBnBJ2TY8=
 github.com/nyaruka/librato v0.0.0-20180827155909-cacc769357b8/go.mod h1:huVocfMEHkttMHD4hSr/wjWNyTx/YMzwwajVzV2bq+0=
 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc=

ivr/nexmo/nexmo_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/nyaruka/goflow/flows/routers/waits"
 	"github.com/nyaruka/goflow/flows/routers/waits/hints"
 	"github.com/nyaruka/goflow/utils"
+	"github.com/nyaruka/goflow/test"
 	"github.com/nyaruka/mailroom/config"
 	"github.com/nyaruka/mailroom/models"
 	"github.com/nyaruka/mailroom/testsuite"
@@ -35,7 +36,7 @@ func TestResponseForSprint(t *testing.T) {
 	db.MustExec(`UPDATE channels_channel SET config = '{"nexmo_app_id": "app_id", "nexmo_app_private_key": "-----BEGIN PRIVATE KEY-----\nMIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKNwapOQ6rQJHetP\nHRlJBIh1OsOsUBiXb3rXXE3xpWAxAha0MH+UPRblOko+5T2JqIb+xKf9Vi3oTM3t\nKvffaOPtzKXZauscjq6NGzA3LgeiMy6q19pvkUUOlGYK6+Xfl+B7Xw6+hBMkQuGE\nnUS8nkpR5mK4ne7djIyfHFfMu4ptAgMBAAECgYA+s0PPtMq1osG9oi4xoxeAGikf\nJB3eMUptP+2DYW7mRibc+ueYKhB9lhcUoKhlQUhL8bUUFVZYakP8xD21thmQqnC4\nf63asad0ycteJMLb3r+z26LHuCyOdPg1pyLk3oQ32lVQHBCYathRMcVznxOG16VK\nI8BFfstJTaJu0lK/wQJBANYFGusBiZsJQ3utrQMVPpKmloO2++4q1v6ZR4puDQHx\nTjLjAIgrkYfwTJBLBRZxec0E7TmuVQ9uJ+wMu/+7zaUCQQDDf2xMnQqYknJoKGq+\noAnyC66UqWC5xAnQS32mlnJ632JXA0pf9pb1SXAYExB1p9Dfqd3VAwQDwBsDDgP6\nHD8pAkEA0lscNQZC2TaGtKZk2hXkdcH1SKru/g3vWTkRHxfCAznJUaza1fx0wzdG\nGcES1Bdez0tbW4llI5By/skZc2eE3QJAFl6fOskBbGHde3Oce0F+wdZ6XIJhEgCP\niukIcKZoZQzoiMJUoVRrA5gqnmaYDI5uRRl/y57zt6YksR3KcLUIuQJAd242M/WF\n6YAZat3q/wEeETeQq1wrooew+8lHl05/Nt0cCpV48RGEhJ83pzBm3mnwHf8lTBJH\nx6XroMXsmbnsEw==\n-----END PRIVATE KEY-----", "callback_domain": "localhost:8090"}', role='SRCA' WHERE id = $1`, models.NexmoChannelID)
 
 	// set our UUID generator
-	utils.SetUUIDGenerator(utils.NewSeededUUID4Generator(0))
+	utils.SetUUIDGenerator(test.NewSeededUUIDGenerator(0))
 
 	// set our attachment domain for testing
 	config.Mailroom.AttachmentDomain = "mailroom.io"

runner/runner.go

@@ -734,7 +734,7 @@ func validateFlow(sa flows.SessionAssets, uuid assets.FlowUUID) error {
 
 	// check for missing assets and log
 	missingDeps := make([]string, 0)
-	err = flow.InspectRecursively(sa, func(r assets.Reference) {
+	err = flow.ValidateRecursive(sa, func(r assets.Reference) {
 		missingDeps = append(missingDeps, r.String())
 	})
 

web/flow/flow.go

@@ -3,8 +3,6 @@ package flow
 import (
 	"context"
 	"encoding/json"
-	"io"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/nyaruka/goflow/flows"
@@ -14,51 +12,38 @@ import (
 	"github.com/nyaruka/mailroom/config"
 	"github.com/nyaruka/mailroom/models"
 	"github.com/nyaruka/mailroom/web"
+
+	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
 )
 
 func init() {
 	web.RegisterJSONRoute(http.MethodPost, "/mr/flow/migrate", web.RequireAuthToken(handleMigrate))
 	web.RegisterJSONRoute(http.MethodPost, "/mr/flow/validate", web.RequireAuthToken(handleValidate))
+	web.RegisterJSONRoute(http.MethodPost, "/mr/flow/inspect", web.RequireAuthToken(handleInspect))
+	web.RegisterJSONRoute(http.MethodPost, "/mr/flow/clone", web.RequireAuthToken(handleClone))
 }
 
 // Migrates a legacy flow to the new flow definition specification
 //
 //   {
-//     "flow": {"uuid": "468621a8-32e6-4cd2-afc1-04416f7151f0", "action_sets": [], ...},
-//     "include_ui": false
+//     "flow": {"uuid": "468621a8-32e6-4cd2-afc1-04416f7151f0", "action_sets": [], ...}
 //   }
 //
 type migrateRequest struct {
 	Flow      json.RawMessage `json:"flow"            validate:"required"`
-	IncludeUI *bool           `json:"include_ui"`
+	IncludeUI *bool           `json:"include_ui"` // ignored
 }
 
 func handleMigrate(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) {
 	request := &migrateRequest{}
-	body, err := ioutil.ReadAll(io.LimitReader(r.Body, web.MaxRequestBytes))
-	if err != nil {
-		return nil, http.StatusBadRequest, err
+	if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil {
+		return nil, http.StatusBadRequest, errors.Wrapf(err, "request failed validation")
 	}
 
-	if err := r.Body.Close(); err != nil {
-		return nil, http.StatusInternalServerError, err
-	}
-
-	if err := utils.UnmarshalAndValidate(body, request); err != nil {
-		return nil, http.StatusBadRequest, errors.Wrapf(err, "error unmarshalling request")
-	}
-
-	legacyFlow, err := legacy.ReadLegacyFlow(request.Flow)
+	flow, err := readFlow(request.Flow)
 	if err != nil {
-		return nil, http.StatusBadRequest, errors.Wrapf(err, "error reading legacy flow")
-	}
-
-	includeUI := request.IncludeUI == nil || *request.IncludeUI
-
-	flow, err := legacyFlow.Migrate(includeUI, "https://"+config.Mailroom.AttachmentDomain)
-	if err != nil {
-		return nil, http.StatusBadRequest, errors.Wrapf(err, "error migrating legacy flow")
+		return nil, http.StatusUnprocessableEntity, err
 	}
 
 	return flow, http.StatusOK, nil
@@ -85,53 +70,140 @@ type validateRequest struct {
 
 func handleValidate(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) {
 	request := &validateRequest{}
-	body, err := ioutil.ReadAll(io.LimitReader(r.Body, web.MaxRequestBytes))
+	if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil {
+		return nil, http.StatusBadRequest, errors.Wrapf(err, "request failed validation")
+	}
+
+	flow, err := readFlow(request.Flow)
 	if err != nil {
-		return nil, http.StatusBadRequest, err
+		return nil, http.StatusUnprocessableEntity, err
 	}
 
-	if err := r.Body.Close(); err != nil {
-		return nil, http.StatusInternalServerError, err
+	// if we have an org ID, do asset validation
+	if request.OrgID != models.NilOrgID {
+		status, err := validate(s.CTX, s.DB, request.OrgID, flow)
+		if err != nil {
+			return nil, status, err
+		}
 	}
 
-	if err := utils.UnmarshalAndValidate(body, request); err != nil {
-		return nil, http.StatusBadRequest, errors.Wrapf(err, "error unmarshalling request")
+	// this endpoint returns inspection results inside the definition
+	result, err := flow.MarshalWithInfo()
+	if err != nil {
+		return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to marshal flow")
 	}
 
-	var flowDef = request.Flow
-	var sa flows.SessionAssets
+	return json.RawMessage(result), http.StatusOK, nil
+}
 
-	// migrate definition if it is in legacy format
-	if legacy.IsLegacyDefinition(flowDef) {
-		flowDef, err = legacy.MigrateLegacyDefinition(flowDef, "https://"+config.Mailroom.AttachmentDomain)
-		if err != nil {
-			return nil, http.StatusUnprocessableEntity, err
-		}
+// Inspects a flow.
+//
+//   {
+//     "flow": { "uuid": "468621a8-32e6-4cd2-afc1-04416f7151f0", "nodes": [...]}
+//   }
+//
+type inspectRequest struct {
+	Flow json.RawMessage `json:"flow" validate:"required"`
+}
+
+func handleInspect(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) {
+	request := &inspectRequest{}
+	if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil {
+		return nil, http.StatusBadRequest, errors.Wrapf(err, "request failed validation")
 	}
 
-	// try to read the flow definition which will fail if it's invalid
-	flow, err := definition.ReadFlow(flowDef)
+	// try to read the flow definition
+	flow, err := definition.ReadFlow(request.Flow)
 	if err != nil {
 		return nil, http.StatusUnprocessableEntity, err
 	}
 
-	// if we have an org ID, build a session assets for it
-	if request.OrgID != models.NilOrgID {
-		org, err := models.NewOrgAssets(s.CTX, s.DB, request.OrgID, nil)
+	return flow.Inspect(), http.StatusOK, nil
+}
+
+// Clones a flow, replacing all UUIDs with either the given mapping or new random UUIDs.
+// If `validate_with_org_id` is specified then the cloned flow will be validated against
+// the assets of that org.
+//
+//   {
+//     "dependency_mapping": {
+//       "4ee4189e-0c06-4b00-b54f-5621329de947": "db31d23f-65b8-4518-b0f6-45638bfbbbf2",
+//       "723e62d8-a544-448f-8590-1dfd0fccfcd4": "f1fd861c-9e75-4376-a829-dcf76db6e721"
+//     },
+//     "flow": { "uuid": "468621a8-32e6-4cd2-afc1-04416f7151f0", "nodes": [...]},
+//     "validate_with_org_id": 1
+//   }
+//
+type cloneRequest struct {
+	DependencyMapping map[utils.UUID]utils.UUID `json:"dependency_mapping"`
+	Flow              json.RawMessage           `json:"flow" validate:"required"`
+	ValidateWithOrgID models.OrgID              `json:"validate_with_org_id"`
+}
+
+func handleClone(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) {
+	request := &cloneRequest{}
+	if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil {
+		return nil, http.StatusBadRequest, errors.Wrapf(err, "request failed validation")
+	}
+
+	// try to read the flow definition
+	flow, err := definition.ReadFlow(request.Flow)
+	if err != nil {
+		return nil, http.StatusUnprocessableEntity, err
+	}
+
+	clone := flow.Clone(request.DependencyMapping)
+
+	// if we have an org ID, do asset validation on the new clone
+	if request.ValidateWithOrgID != models.NilOrgID {
+		status, err := validate(s.CTX, s.DB, request.ValidateWithOrgID, clone)
+		if err != nil {
+			return nil, status, err
+		}
+	}
+
+	return clone, http.StatusOK, nil
+}
+
+func readFlow(flowDef json.RawMessage) (flows.Flow, error) {
+	var flow flows.Flow
+	var err error
+
+	if legacy.IsLegacyDefinition(flowDef) {
+		// migrate definition if it is in legacy format
+		legacyFlow, err := legacy.ReadLegacyFlow(flowDef)
 		if err != nil {
-			return nil, http.StatusBadRequest, err
+			return nil, err
 		}
 
-		sa, err = models.NewSessionAssets(org)
+		flow, err = legacyFlow.Migrate(true, "https://"+config.Mailroom.AttachmentDomain)
 		if err != nil {
-			return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable get session assets")
+			return nil, err
+		}
+	} else {
+		flow, err = definition.ReadFlow(flowDef)
+		if err != nil {
+			return nil, err
 		}
 	}
 
-	// inspect the flow to get dependecies, results etc
-	if err := flow.Inspect(sa); err != nil {
-		return nil, http.StatusUnprocessableEntity, err
+	return flow, nil
+}
+
+func validate(ctx context.Context, db *sqlx.DB, orgID models.OrgID, flow flows.Flow) (int, error) {
+	org, err := models.NewOrgAssets(ctx, db, orgID, nil)
+	if err != nil {
+		return http.StatusBadRequest, err
 	}
 
-	return flow, http.StatusOK, nil
+	sa, err := models.NewSessionAssets(org)
+	if err != nil {
+		return http.StatusInternalServerError, errors.Wrapf(err, "unable build session assets")
+	}
+
+	if err := flow.Validate(sa, nil); err != nil {
+		return http.StatusUnprocessableEntity, err
+	}
+
+	return 0, nil
 }

web/flow/flow_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/nyaruka/goflow/test"
+	"github.com/nyaruka/goflow/utils"
 
 	"github.com/nyaruka/mailroom/config"
 	"github.com/nyaruka/mailroom/testsuite"
@@ -33,42 +34,50 @@ func TestServer(t *testing.T) {
 	time.Sleep(time.Second)
 
 	defer server.Stop()
+	defer utils.SetUUIDGenerator(utils.DefaultUUIDGenerator)
 
 	tcs := []struct {
-		URL          string
-		Method       string
-		BodyFile     string
-		Status       int
-		Response     string
-		ResponseFile string
+		URL             string
+		Method          string
+		BodyFile        string
+		Status          int
+		Response        string
+		ResponseFile    string
+		ResponsePattern string
 	}{
 		{URL: "/mr/flow/migrate", Method: "GET", Status: 405, Response: `{"error": "illegal method: GET"}`},
 		{URL: "/mr/flow/migrate", Method: "POST", BodyFile: "migrate_minimal_legacy.json", Status: 200, ResponseFile: "migrate_minimal_legacy.response.json"},
+
 		{URL: "/mr/flow/validate", Method: "GET", Status: 405, Response: `{"error": "illegal method: GET"}`},
 		{URL: "/mr/flow/validate", Method: "POST", BodyFile: "validate_valid_legacy.json", Status: 200, ResponseFile: "validate_valid_legacy.response.json"},
 		{URL: "/mr/flow/validate", Method: "POST", BodyFile: "validate_invalid_legacy.json", Status: 422, ResponseFile: "validate_invalid_legacy.response.json"},
 		{URL: "/mr/flow/validate", Method: "POST", BodyFile: "validate_valid.json", Status: 200, ResponseFile: "validate_valid.response.json"},
 		{URL: "/mr/flow/validate", Method: "POST", BodyFile: "validate_invalid.json", Status: 422, ResponseFile: "validate_invalid.response.json"},
 		{URL: "/mr/flow/validate", Method: "POST", BodyFile: "validate_valid_without_assets.json", Status: 200, ResponseFile: "validate_valid_without_assets.response.json"},
 		{URL: "/mr/flow/validate", Method: "POST", BodyFile: "validate_legacy_single_msg.json", Status: 200, ResponseFile: "validate_legacy_single_msg.response.json"},
+
+		{URL: "/mr/flow/inspect", Method: "GET", Status: 405, Response: `{"error": "illegal method: GET"}`},
+		{URL: "/mr/flow/inspect", Method: "POST", BodyFile: "inspect_valid.json", Status: 200, ResponseFile: "inspect_valid.response.json"},
+
+		{URL: "/mr/flow/clone", Method: "GET", Status: 405, Response: `{"error": "illegal method: GET"}`},
+		{URL: "/mr/flow/clone", Method: "POST", BodyFile: "clone_valid.json", Status: 200, ResponsePattern: `"uuid": "1cf84575-ee14-4253-88b6-e3675c04a066"`},
+		{URL: "/mr/flow/clone", Method: "POST", BodyFile: "clone_struct_invalid.json", Status: 422, Response: `{"error": "unable to read node: field 'uuid' is required"}`},
+		{URL: "/mr/flow/clone", Method: "POST", BodyFile: "clone_missing_dep_mapping.json", Status: 422, ResponsePattern: `group\[uuid=[-0-9a-f]{36},name=Testers\]`},
+		{URL: "/mr/flow/clone", Method: "POST", BodyFile: "clone_valid_bad_org.json", Status: 400, Response: `{"error": "error loading environment for org 167733: no org with id: 167733"}`},
 	}
 
 	for _, tc := range tcs {
+		utils.SetUUIDGenerator(test.NewSeededUUIDGenerator(123456))
+		time.Sleep(1 * time.Second)
+
 		testID := fmt.Sprintf("%s %s %s", tc.Method, tc.URL, tc.BodyFile)
 		var requestBody io.Reader
-		var expectedRespBody []byte
 		var err error
 
 		if tc.BodyFile != "" {
 			requestBody, err = os.Open("testdata/" + tc.BodyFile)
 			require.NoError(t, err, "unable to open %s", tc.BodyFile)
 		}
-		if tc.ResponseFile != "" {
-			expectedRespBody, err = ioutil.ReadFile("testdata/" + tc.ResponseFile)
-			require.NoError(t, err, "unable to read %s", tc.ResponseFile)
-		} else {
-			expectedRespBody = []byte(tc.Response)
-		}
 
 		req, err := http.NewRequest(tc.Method, "http://localhost:8090"+tc.URL, requestBody)
 		assert.NoError(t, err, "error creating request in %s", testID)
@@ -80,6 +89,16 @@ func TestServer(t *testing.T) {
 		require.NoError(t, err, "error reading body in %s", testID)
 
 		assert.Equal(t, tc.Status, resp.StatusCode, "unexpected status in %s (response=%s)", testID, content)
-		test.AssertEqualJSON(t, expectedRespBody, content, "response mismatch in %s", testID)
+
+		if tc.ResponseFile != "" {
+			expectedRespBody, err := ioutil.ReadFile("testdata/" + tc.ResponseFile)
+			require.NoError(t, err, "unable to read %s", tc.ResponseFile)
+
+			test.AssertEqualJSON(t, expectedRespBody, content, "response mismatch in %s", testID)
+		} else if tc.ResponsePattern != "" {
+			assert.Regexp(t, tc.ResponsePattern, string(content), "response mismatch in %s", testID)
+		} else {
+			test.AssertEqualJSON(t, []byte(tc.Response), content, "response mismatch in %s", testID)
+		}
 	}
 }