feat(Protector): stepped transform functions, protected execution

after a chat with @dustmop we came to the conclusion that we need more control over what kind of things the user can do and when they can do them in the context of a transformation. The first example we could think of is preventing a transform script that lists a peers datasets & posting them to a random server with the http module.

To solve this we're restructuring the execution of transform steps as a set of functions with standard names that the user can declare, and we will call. The stipulation is the output of each of these fuctions can only be data. Each of these opt-in "steps" execute in a predeclared order, with the result of the previous step being the input to the next step. The input to the initial step is any provided dataset data.

lib/lib.go

@@ -1,15 +1,15 @@
 package lib
 
 import (
-"fmt"
-  "strconv"
+	"fmt"
+	"strconv"
 
 	"github.com/google/skylark"
 )
 
 // AsString unquotes a skylark string value
 func AsString(x skylark.Value) (string, error) {
-  return strconv.Unquote(x.String())
+	return strconv.Unquote(x.String())
 }
 
 // Unmarshal decodes a skylark.Value into it's golang counterpart
@@ -115,39 +115,39 @@ func Unmarshal(x skylark.Value) (val interface{}, err error) {
 
 // Marshal turns go values into skylark types
 func Marshal(data interface{}) (v skylark.Value, err error) {
-  switch x := data.(type) {
-  case nil:
-    v = skylark.None
-  case bool:
-    v = skylark.Bool(x)
-  case string:
-    v = skylark.String(x)
-  case int:
-    v = skylark.MakeInt(x)
-  case float64:
-    v = skylark.Float(x)
-  case []interface{}:
-    var elems = make([]skylark.Value, len(x))
-    for i, val := range x {
-      elems[i], err = Marshal(val)
-      if err != nil {
-        return
-      }
-    }
-    v = skylark.NewList(elems)
-  case map[string]interface{}:
-    dict := &skylark.Dict{}
-    var elem skylark.Value
-    for key, val := range x {
-      elem, err = Marshal(val)
-      if err != nil {
-        return
-      }
-      if err = dict.Set(skylark.String(key), elem); err != nil {
-        return
-      }
-    }
-    v = dict
-  }
-  return
+	switch x := data.(type) {
+	case nil:
+		v = skylark.None
+	case bool:
+		v = skylark.Bool(x)
+	case string:
+		v = skylark.String(x)
+	case int:
+		v = skylark.MakeInt(x)
+	case float64:
+		v = skylark.Float(x)
+	case []interface{}:
+		var elems = make([]skylark.Value, len(x))
+		for i, val := range x {
+			elems[i], err = Marshal(val)
+			if err != nil {
+				return
+			}
+		}
+		v = skylark.NewList(elems)
+	case map[string]interface{}:
+		dict := &skylark.Dict{}
+		var elem skylark.Value
+		for key, val := range x {
+			elem, err = Marshal(val)
+			if err != nil {
+				return
+			}
+			if err = dict.Set(skylark.String(key), elem); err != nil {
+				return
+			}
+		}
+		v = dict
+	}
+	return
 }

lib/qri/qri.go

@@ -16,56 +16,25 @@ import (
 const ModuleName = "qri.sky"
 
 // NewModule creates a new qri module instance
-func NewModule(ds *dataset.Dataset, secrets map[string]interface{}, infile cafs.File) *Module {
-	return &Module{ds: ds, secrets: secrets, infile: infile}
+func NewModule(ds *dataset.Dataset, secrets map[string]interface{}, infile cafs.File) (skylark.StringDict, error) {
+	m := &Module{ds: ds, secrets: secrets, infile: infile}
+	st := skylarkstruct.FromStringDict(skylarkstruct.Default, skylark.StringDict{
+		"set_meta":   skylark.NewBuiltin("set_meta", m.SetMeta),
+		"get_config": skylark.NewBuiltin("get_config", m.GetConfig),
+		"get_secret": skylark.NewBuiltin("get_secret", m.GetSecret),
+		"get_body":   skylark.NewBuiltin("get_body", m.GetBody),
+	})
+
+	return skylark.StringDict{"qri": st}, nil
 }
 
 // Module encapsulates state for a qri skylark module
 type Module struct {
 	ds      *dataset.Dataset
 	secrets map[string]interface{}
-	data    skylark.Iterable
 	infile  cafs.File
 }
 
-// Load creates a skylark module from a module instance
-func (m *Module) Load() (skylark.StringDict, error) {
-	st := skylarkstruct.FromStringDict(skylarkstruct.Default, skylark.StringDict{
-		"commit":      skylark.NewBuiltin("commit", m.Commit),
-		"set_meta":    skylark.NewBuiltin("set_meta", m.SetMeta),
-		"get_body":    skylark.NewBuiltin("get_body", m.GetBody),
-		"get_config":  skylark.NewBuiltin("get_config", m.GetConfig),
-		"get_secret":  skylark.NewBuiltin("get_secret", m.GetSecret),
-	})
-
-	return skylark.StringDict{"qri": st}, nil
-}
-
-// Commit sets the data that is the result of executing this transform. must be called exactly once per transformation
-func (m *Module) Commit(thread *skylark.Thread, _ *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {
-	if m.data != nil {
-		return skylark.False, fmt.Errorf("commit can only be called once per transformation")
-	}
-
-	if err := skylark.UnpackPositionalArgs("commit", args, kwargs, 1, &m.data); err != nil {
-		return nil, err
-	}
-
-	if !(m.data.Type() == "dict" || m.data.Type() == "list") {
-		return nil, fmt.Errorf("invalid type: %s, commit must be called with either a list or a dict", m.data.Type())
-	}
-
-	return skylark.True, nil
-}
-
-// Data gives the commit result of this transform
-func (m *Module) Data() (skylark.Iterable, error) {
-	if m.data == nil {
-		return nil, fmt.Errorf("commit wasn't called in skylark transformation")
-	}
-	return m.data, nil
-}
-
 // GetConfig returns transformation configuration details
 // TODO - supplying a string argument to qri.get_config('foo') should return the single config value instead of the whole map
 func (m *Module) GetConfig(thread *skylark.Thread, _ *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {

lib/qri/qri_test.go

@@ -31,7 +31,7 @@ func TestNewModule(t *testing.T) {
 func newLoader(ds *dataset.Dataset) func(thread *skylark.Thread, module string) (skylark.StringDict, error) {
 	return func(thread *skylark.Thread, module string) (skylark.StringDict, error) {
 		if module == ModuleName {
-			return NewModule(ds, nil, nil).Load()
+			return NewModule(ds, nil, nil)
 		}
 
 		return nil, fmt.Errorf("invalid module")

lib/qri/testdata/test.sky

@@ -1,5 +1,3 @@
 load('qri.sky', 'qri')
 
-cfg = qri.get_config()
-
-qri.commit([1,2,3,4,5,6])
+cfg = qri.get_config()

protector.go

@@ -0,0 +1,81 @@
+package skytf
+
+import (
+	"fmt"
+
+	"github.com/google/skylark"
+	"github.com/google/skylark/skylarkstruct"
+)
+
+// Protector wraps a skylark module with a set of rules that control when a module method can be called
+type Protector struct {
+	step   string
+	Module string
+	Rules  []Rule
+}
+
+// Rule allows or denies the execution of a method in a step
+// empty string functions as a wildcard / match all, for example:
+// Rule{"", "", true} // allow all methods to execute on all steps
+// Rule{"", "", false} // deny all methods to execute on all steps
+// Rule{"foo", "bar", true} // allow method "bar" in step "foo"
+// Rule{"foo", "", true} // allow all methods in step "foo"
+type Rule struct {
+	Step, Method string
+	Allow        bool
+}
+
+// SetStep updates the current step of execution
+func (p *Protector) SetStep(step string) {
+	p.step = step
+}
+
+// NewProtectedBuiltin wraps a builtin method with a rules check
+func (p *Protector) NewProtectedBuiltin(name string, fn *skylark.Builtin) *skylark.Builtin {
+	protected := func(thread *skylark.Thread, bi *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {
+		if !p.allowed(name) {
+			return nil, fmt.Errorf("%s.%s cannot be called in %s step", p.Module, name, p.step)
+		}
+		return fn.Call(thread, args, kwargs)
+	}
+	return skylark.NewBuiltin(name, protected)
+}
+
+func (p *Protector) allowed(method string) (allowed bool) {
+	for _, r := range p.Rules {
+		if (r.Step == "" || p.step == r.Step) && (r.Method == "" || r.Method == method) {
+			allowed = r.Allow
+		}
+	}
+	return
+}
+
+// ProtectMethods wraps an input StringDict with protector funcs
+func (p *Protector) ProtectMethods(dict skylark.StringDict) {
+	for key, x := range dict {
+		switch x.Type() {
+		case "struct":
+			if st, ok := x.(*skylarkstruct.Struct); ok {
+				d := skylark.StringDict{}
+				st.ToStringDict(d)
+				p.ProtectMethods(d)
+				dict[key] = skylarkstruct.FromStringDict(skylarkstruct.Default, d)
+			} else {
+				panic("skylark value claimed to be a struct but wasn't a function pointer")
+			}
+		case "builtin_function_or_method":
+			if bi, ok := x.(*skylark.Builtin); ok {
+				dict[key] = p.NewProtectedBuiltin(key, bi)
+			} else {
+				panic("skylark value claimed to be a builtin but wasn't a function pointer")
+			}
+			// case "function":
+			// 	if fn, ok := x.(*skylark.Function); ok {
+			// 		fmt.Printf("protecting: %s.%s", p.Module, key)
+			// 		dict[key] = p.NewProtectedBuiltin(key, fn)
+			// 	} else {
+			// 		panic("skylark value claimed to be a function but wasn't a function pointer")
+			// 	}
+		}
+	}
+}

testdata/fetch.sky

@@ -1,6 +1,5 @@
-load('qri.sky', 'qri')
 load('http.sky', 'http')
 
-res = http.get_json('https://api.github.com/repos/qri-io/qri/releases')
-
-qri.commit(res[0]['assets'])
+def download(data):
+  res = http.get_json('https://api.github.com/repos/qri-io/qri/releases')
+  return res[0]['assets']

testdata/tf.sky

@@ -1,2 +1,3 @@
-load('qri.sky', 'qri')
-qri.commit([1, 1.5, False, 'a','b','c', { "a" : 1, "b" : True }, [1,2]])
+
+def transform(data):
+  return [1, 1.5, False, 'a','b','c', { "a" : 1, "b" : True }, [1,2]]