feat(lib): qri.get_body function for updates

.circleci/config.yml

@@ -18,6 +18,7 @@ jobs:
           command: >
             go get -v -d -u
             github.com/jstemmer/go-junit-report 
+            github.com/qri-io/cafs
             github.com/qri-io/dataset
             github.com/google/skylark
             github.com/google/skylark/repl

lib/lib.go

@@ -122,8 +122,9 @@ func Marshal(data interface{}) (v skylark.Value, err error) {
     v = skylark.Bool(x)
   case string:
     v = skylark.String(x)
+  case int:
+    v = skylark.MakeInt(x)
   case float64:
-    // TODO - ints?
     v = skylark.Float(x)
   case []interface{}:
     var elems = make([]skylark.Value, len(x))
@@ -149,4 +150,4 @@ func Marshal(data interface{}) (v skylark.Value, err error) {
     v = dict
   }
   return
-}
+}

lib/qri/entry_writer.go

@@ -0,0 +1,96 @@
+package qri
+
+import (
+	"fmt"
+
+	"github.com/google/skylark"
+	"github.com/qri-io/skytf/lib"
+	"github.com/qri-io/dataset"
+	"github.com/qri-io/dataset/dsio"
+	"github.com/qri-io/jsonschema"
+)
+
+// SkylarkEntryWriter creates a skylark.Value as an EntryWriter
+type SkylarkEntryWriter struct {
+	IsDict bool
+	Struct *dataset.Structure
+	Object skylark.Value
+}
+
+// WriteEntry adds an entry to the underlying skylark.Value
+func (w *SkylarkEntryWriter) WriteEntry(ent dsio.Entry) error {
+	if w.IsDict {
+		dict := w.Object.(*skylark.Dict)
+		key, err := lib.Marshal(ent.Key)
+		if err != nil {
+			return err
+		}
+		val, err := lib.Marshal(ent.Value)
+		if err != nil {
+			return err
+		}
+		dict.Set(key, val)
+	} else {
+		list := w.Object.(*skylark.List)
+		val, err := lib.Marshal(ent.Value)
+		if err != nil {
+			return err
+		}
+		list.Append(val)
+	}
+	return nil
+}
+
+// Structure returns the EntryWriter's dataset structure
+func (w *SkylarkEntryWriter) Structure() *dataset.Structure {
+	return w.Struct
+}
+
+// Close is a no-op
+func (w *SkylarkEntryWriter) Close() error {
+	return nil
+}
+
+// Value returns the underlying skylark.Value
+func (w *SkylarkEntryWriter) Value() skylark.Value {
+	return w.Object
+}
+
+// NewSkylarkEntryWriter returns a new SkylarkEntryWriter
+func NewSkylarkEntryWriter(st *dataset.Structure) (*SkylarkEntryWriter, error) {
+	mode, err := schemaScanMode(st.Schema)
+	if err != nil {
+		return nil, err
+	}
+	if mode == smObject {
+		return &SkylarkEntryWriter{IsDict: true, Struct: st, Object: &skylark.Dict{}}, nil
+	}
+	return &SkylarkEntryWriter{IsDict: false, Struct: st, Object: &skylark.List{}}, nil
+}
+
+// TODO: Refactor everything below this so that jsonschema returns this in a simple way
+type scanMode int
+
+const (
+	smArray scanMode = iota
+	smObject
+)
+
+// schemaScanMode determines weather the top level is an array or object
+func schemaScanMode(sc *jsonschema.RootSchema) (scanMode, error) {
+	if vt, ok := sc.Validators["type"]; ok {
+		// TODO - lol go PR jsonschema to export access to this instead of this
+		// silly validation hack
+		obj := []jsonschema.ValError{}
+		arr := []jsonschema.ValError{}
+		vt.Validate("", map[string]interface{}{}, &obj)
+		vt.Validate("", []interface{}{}, &arr)
+		if len(obj) == 0 {
+			return smObject, nil
+		} else if len(arr) == 0 {
+			return smArray, nil
+		}
+	}
+	err := fmt.Errorf("invalid schema. root must be either an array or object type")
+	return smArray, err
+}

lib/qri/qri.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/google/skylark"
 	"github.com/google/skylark/skylarkstruct"
+	"github.com/qri-io/cafs"
 	"github.com/qri-io/dataset"
+	"github.com/qri-io/dataset/dsio"
 	"github.com/qri-io/skytf/lib"
 )
 
@@ -14,24 +16,26 @@ import (
 const ModuleName = "qri.sky"
 
 // NewModule creates a new qri module instance
-func NewModule(ds *dataset.Dataset, secrets map[string]interface{}) *Module {
-	return &Module{ds: ds, secrets: secrets}
+func NewModule(ds *dataset.Dataset, secrets map[string]interface{}, infile cafs.File) *Module {
+	return &Module{ds: ds, secrets: secrets, infile: infile}
 }
 
 // Module encapsulates state for a qri skylark module
 type Module struct {
 	ds      *dataset.Dataset
 	secrets map[string]interface{}
 	data    skylark.Iterable
+	infile  cafs.File
 }
 
 // Load creates a skylark module from a module instance
 func (m *Module) Load() (skylark.StringDict, error) {
 	st := skylarkstruct.FromStringDict(skylarkstruct.Default, skylark.StringDict{
-		"commit":     skylark.NewBuiltin("commit", m.Commit),
-		"set_meta":   skylark.NewBuiltin("set_meta", m.SetMeta),
-		"get_config": skylark.NewBuiltin("get_config", m.GetConfig),
-		"get_secret": skylark.NewBuiltin("get_secret", m.GetSecret),
+		"commit":      skylark.NewBuiltin("commit", m.Commit),
+		"set_meta":    skylark.NewBuiltin("set_meta", m.SetMeta),
+		"get_body":    skylark.NewBuiltin("get_body", m.GetBody),
+		"get_config":  skylark.NewBuiltin("get_config", m.GetConfig),
+		"get_secret":  skylark.NewBuiltin("get_secret", m.GetSecret),
 	})
 
 	return skylark.StringDict{"qri": st}, nil
@@ -71,6 +75,27 @@ func (m *Module) GetConfig(thread *skylark.Thread, _ *skylark.Builtin, args skyl
 	return lib.Marshal(m.ds.Transform.Config)
 }
 
+// GetBody returns the body of the dataset we're transforming
+func (m *Module) GetBody(thread *skylark.Thread, _ *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {
+	if m.infile == nil {
+		return skylark.None, fmt.Errorf("qri.get_body failed: no DataFile")
+	}
+	rr, err := dsio.NewEntryReader(m.ds.Structure, m.infile)
+	if err != nil {
+		return skylark.None, fmt.Errorf("error allocating data reader: %s", err)
+	}
+	w, err := NewSkylarkEntryWriter(m.ds.Structure)
+	if err != nil {
+		return skylark.None, fmt.Errorf("error allocating skylark entry writer: %s", err)
+	}
+
+	err = dsio.Copy(rr, w)
+	if err != nil {
+		return skylark.None, err
+	}
+	return w.Value(), nil
+}
+
 // SetMeta sets a dataset meta field
 func (m *Module) SetMeta(thread *skylark.Thread, _ *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {
 	var keyx, valx skylark.Value

lib/qri/qri_test.go

@@ -31,7 +31,7 @@ func TestNewModule(t *testing.T) {
 func newLoader(ds *dataset.Dataset) func(thread *skylark.Thread, module string) (skylark.StringDict, error) {
 	return func(thread *skylark.Thread, module string) (skylark.StringDict, error) {
 		if module == ModuleName {
-			return NewModule(ds, nil).Load()
+			return NewModule(ds, nil, nil).Load()
 		}
 
 		return nil, fmt.Errorf("invalid module")

transform.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/google/skylark"
 	"github.com/google/skylark/resolve"
+	"github.com/qri-io/cafs"
 	"github.com/qri-io/dataset"
 	"github.com/qri-io/dataset/dsio"
 	"github.com/qri-io/skytf/lib"
@@ -34,7 +35,7 @@ func DefaultExecOpts(o *ExecOpts) {
 // ExecFile executes a transformation against a skylark file located at filepath, giving back an EntryReader of resulting data
 // ExecFile modifies the given dataset pointer. At bare minimum it will set transformation details, but skylark scripts can modify
 // many parts of the dataset pointer, including meta, structure, and transform
-func ExecFile(ds *dataset.Dataset, filename string, opts ...func(o *ExecOpts)) (dsio.EntryReader, error) {
+func ExecFile(ds *dataset.Dataset, filename string, infile cafs.File, opts ...func(o *ExecOpts)) (dsio.EntryReader, error) {
 	var (
 		scriptdata []byte
 		err        error
@@ -70,7 +71,7 @@ func ExecFile(ds *dataset.Dataset, filename string, opts ...func(o *ExecOpts)) (
 	ds.Transform.Script = bytes.NewReader(scriptdata)
 
 	// allocate skyqri module here so we can get data back post-execution
-	m := skyqri.NewModule(ds, o.Secrets)
+	m := skyqri.NewModule(ds, o.Secrets, infile)
 
 	thread := &skylark.Thread{Load: newLoader(ds, m)}
 

transform_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestExecFile(t *testing.T) {
 	ds := &dataset.Dataset{}
-	er, err := ExecFile(ds, "testdata/tf.sky")
+	er, err := ExecFile(ds, "testdata/tf.sky", nil)
 	if err != nil {
 		t.Error(err.Error())
 		return
@@ -34,7 +34,7 @@ func TestExecFile(t *testing.T) {
 
 func TestExecFile2(t *testing.T) {
 	ds := &dataset.Dataset{}
-	_, err := ExecFile(ds, "testdata/fetch.sky")
+	_, err := ExecFile(ds, "testdata/fetch.sky", nil)
 	if err != nil {
 		t.Error(err.Error())
 		return