feat(ExecFile): Execute skylark file, get qri dataset. dope.

the firrrrrst commit is the deeeepest
https://www.youtube.com/watch?v=dK9eLe8EQps

.circleci/config.yml

@@ -0,0 +1,35 @@
+version: '2'
+jobs:
+  build:
+    working_directory: /go/src/github.com/qri-io/skytf
+    docker:
+      - image: circleci/golang:1.10.1
+        environment:
+          GOLANG_ENV: test
+          PORT: 3000
+    environment:
+      TEST_RESULTS: /tmp/test-results
+    steps:
+      - checkout
+      - run: mkdir -p $TEST_RESULTS
+      - run: go get github.com/jstemmer/go-junit-report github.com/golang/lint/golint
+      - run:
+          name: Install deps
+          command: >
+            go get -v -d -u
+            github.com/jstemmer/go-junit-report 
+            github.com/qri-io/dataset
+            github.com/google/skylark
+            github.com/google/skylark/repl
+            github.com/google/skylark/resolve
+            github.com/qri-io/dataset
+            github.com/qri-io/dataset/dsio
+      - run: 
+          name: Run Lint Tests
+          command: golint -set_exit_status ./... 
+      - run:
+          name: Run Tests
+          command: go test -v -race -coverprofile=coverage.txt -covermode=atomic
+      - run:
+          name: Publish coverage info to codecov.io
+          command: bash <(curl -s https://codecov.io/bash)

.circleci/cover.test.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+echo "" > coverage.txt
+go test -v -race -coverprofile=profile.out -covermode=atomic github.com/qri-io/dsdiff
+if [ -f profile.out ]; then
+  cat profile.out >> coverage.txt
+  rm profile.out
+fi

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+coverage.txt

LICENSE

@@ -0,0 +1,29 @@
+Copyright (c) 2018 qri, inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,10 @@
+[![Qri](https://img.shields.io/badge/made%20by-qri-magenta.svg?style=flat-square)](https://qri.io)
+[![GoDoc](https://godoc.org/github.com/qri-io/skytf?status.svg)](http://godoc.org/github.com/qri-io/skytf)
+[![License](https://img.shields.io/github/license/qri-io/skytf.svg?style=flat-square)](./LICENSE)
+[![Codecov](https://img.shields.io/codecov/c/github/qri-io/skytf.svg?style=flat-square)](https://codecov.io/gh/qri-io/skytf)
+[![CI](https://img.shields.io/circleci/project/github/qri-io/skytf.svg?style=flat-square)](https://circleci.com/gh/qri-io/skytf)
+[![Go Report Card](https://goreportcard.com/badge/github.com/qri-io/skytf)](https://goreportcard.com/report/github.com/qri-io/skytf)
+
+# skytf
+
+Skylark transformation syntax for qri datasets!

codecov.yml

@@ -0,0 +1,9 @@
+codecov:
+  ci:
+    - "ci/circle-ci"
+  notify:
+    require_ci_to_pass: no
+    after_n_builds: 2
+coverage:
+  range: "80...100"
+comment: off

entry_reader.go

@@ -0,0 +1,71 @@
+package sltf
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/google/skylark"
+	"github.com/qri-io/dataset"
+	"github.com/qri-io/dataset/dsio"
+)
+
+// EntryReader implements the dsio.EntryReader interface for skylark.Iterable's
+type EntryReader struct {
+	i    int
+	st   *dataset.Structure
+	iter skylark.Iterator
+}
+
+// NewEntryReader creates a new Entry Reader
+func NewEntryReader(st *dataset.Structure, data skylark.Iterable) *EntryReader {
+	return &EntryReader{
+		st:   st,
+		iter: data.Iterate(),
+	}
+}
+
+// Structure gives this reader's structure
+func (r *EntryReader) Structure() *dataset.Structure {
+	return r.st
+}
+
+// ReadEntry reads one entry from the reader
+func (r *EntryReader) ReadEntry() (e dsio.Entry, err error) {
+	defer func() { r.i++ }()
+
+	var v skylark.Value
+
+	if !r.iter.Next(&v) {
+		r.iter.Done()
+		return e, io.EOF
+	}
+
+	switch v.Type() {
+	case "NoneType":
+		e.Value = nil
+	case "bool":
+		e.Value = v.Truth() == skylark.True
+	case "float":
+		if f, ok := skylark.AsFloat(v); ok {
+			e.Value = f
+		} else {
+			err = fmt.Errorf("couldn't coerce float")
+		}
+	case "dict":
+		err = fmt.Errorf("dicts aren't yet supported")
+	case "list":
+		err = fmt.Errorf("lists aren't yet supported")
+	case "string":
+		e.Value = v.String()
+	case "tuple":
+		err = fmt.Errorf("tuples aren't yet supported")
+	case "set":
+		err = fmt.Errorf("sets aren't yet supported")
+	case "int":
+		e.Value, err = skylark.AsInt32(v)
+	default:
+		err = fmt.Errorf("unrecognized skylark type: %s", v.Type())
+	}
+
+	return
+}

entry_reader_test.go

@@ -0,0 +1,13 @@
+package sltf
+
+import (
+	"testing"
+
+	"github.com/qri-io/dataset/dsio"
+)
+
+var _ dsio.EntryReader = (*EntryReader)(nil)
+
+func TestEntryReader(t *testing.T) {
+	// TODO
+}

testdata/tf.sky

@@ -0,0 +1,2 @@
+
+commit(['a','b','c'])

transform.go

@@ -0,0 +1,113 @@
+// Package skytf implements dataset transformations using the skylark programming dialect
+// For more info on skylark check github.com/google/skylark
+package skytf
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/google/skylark"
+	"github.com/google/skylark/repl"
+	"github.com/google/skylark/resolve"
+	"github.com/qri-io/dataset"
+	"github.com/qri-io/dataset/dsio"
+)
+
+// ExecOpts defines options for exection
+type ExecOpts struct {
+	AllowFloat     bool // allow floating-point numbers
+	AllowSet       bool // allow set data type
+	AllowLambda    bool // allow lambda expressions
+	AllowNestedDef bool // allow nested def statements
+}
+
+// DefaultExecOpts applies default options to an ExecOpts pointer
+func DefaultExecOpts(o *ExecOpts) {
+	o.AllowFloat = true
+	o.AllowSet = true
+}
+
+// ExecFile executes a transformation against a filepath
+func ExecFile(filename string, opts ...func(o *ExecOpts)) (*dataset.Dataset, dsio.EntryReader, error) {
+	o := &ExecOpts{}
+	DefaultExecOpts(o)
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	resolve.AllowFloat = o.AllowFloat
+	resolve.AllowSet = o.AllowSet
+	resolve.AllowLambda = o.AllowLambda
+	resolve.AllowNestedDef = o.AllowNestedDef
+
+	cm := commit{}
+	skylark.Universe["commit"] = skylark.NewBuiltin("commit", cm.Do)
+
+	thread := &skylark.Thread{Load: repl.MakeLoad()}
+	// globals := make(skylark.StringDict)
+
+	// Execute specified file.
+	var err error
+	_, err = skylark.ExecFile(thread, filename, nil, nil)
+	if err != nil {
+		log.Print(err.Error())
+		return nil, nil, err
+	}
+
+	if !cm.called {
+		return nil, nil, fmt.Errorf("commit must be called once to add data")
+	}
+
+	// Print the global environment.
+	// var names []string
+	// for name := range globals {
+	// 	if !strings.HasPrefix(name, "_") {
+	// 		names = append(names, name)
+	// 	}
+	// }
+	// sort.Strings(names)
+	// for _, name := range names {
+	// 	fmt.Fprintf(os.Stderr, "%s = %s\n", name, globals[name])
+	// }
+
+	ds := &dataset.Dataset{
+		Structure: &dataset.Structure{
+			Format: dataset.UnknownDataFormat,
+			Schema: dataset.BaseSchemaArray,
+		},
+		Transform: &dataset.Transform{
+			Syntax: "skylark",
+		},
+	}
+
+	r := NewEntryReader(ds.Structure, cm.data)
+	return ds, r, nil
+}
+
+type commit struct {
+	called bool
+	data   skylark.Iterable
+}
+
+// Do executes a commit. must be called exactly once per transformation
+func (c *commit) Do(thread *skylark.Thread, _ *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {
+	if c.called {
+		return skylark.False, fmt.Errorf("commit can only be called once per transformation")
+	}
+
+	if err := skylark.UnpackPositionalArgs("foo", args, kwargs, 1, &c.data); err != nil {
+		return nil, err
+	}
+
+	// iter := iterable.Iterate()
+	// defer iter.Done()
+	// var x skylark.Value
+	// for iter.Next(&x) {
+	// 	if x.Truth() {
+	// 		return skylark.True, nil
+	// 	}
+	// }
+
+	c.called = true
+	return skylark.True, nil
+}

transform_test.go

@@ -0,0 +1,31 @@
+package sltf
+
+import (
+	"testing"
+
+	"github.com/qri-io/dataset/dsio"
+)
+
+func TestExecFile(t *testing.T) {
+	ds, er, err := ExecFile("testdata/tf.sky")
+	if err != nil {
+		t.Error(err.Error())
+		return
+	}
+	if ds.Transform == nil {
+		t.Error("expected transform")
+	}
+
+	i := 0
+	dsio.EachEntry(er, func(_ int, x dsio.Entry, e error) error {
+		if e != nil {
+			t.Errorf("entry %d iteration error: %s", i, e.Error())
+		}
+		i++
+		return nil
+	})
+
+	if i != 3 {
+		t.Errorf("expected 3 entries, got: %d", i)
+	}
+}