Support serialization of Go types to arr.ai Values

Sometimes for performance it is preferable to construct data structures in Go, then load them into arr.ai for processing. This is currently possible to an extent with numbers, strings and dictionaries via `rel.NewValue(interface{})`, but custom structs, arrays, pointers, etc. are unsupported. This PR adds support for them.

Changes proposed in this pull request:
- Extend `rel.NewValue` to serialize Go types, such as structs, slices and pointers.
- Change `rel.NewValue` to serialize slices to arrays by default.
- Interpret `` `unordered:"true"` `` tag on struct fields as hint to serialize to a slice to set instead.

Checklist:
- [x] Added related tests
- [x] Made corresponding changes to the documentation

docs/docs/dev/exprs.md

@@ -0,0 +1,23 @@
+---
+id: exprs
+title: Expressions
+---
+
+Arr.ai expressions are combinations of syntax which can be evaluated to produce some value. All arr.ai programs are expressions, and every [value](./values.md) and operation in a program is an expression.
+
+## Implementation
+
+Each expression is a type defined in the `rel` package, and implements the `Expr` interface:
+
+```go
+type Expr interface {
+    // All exprs can be serialized to strings with the String() method.
+	fmt.Stringer
+
+	// Eval evaluates the expr in a given scope.
+	Eval(ctx context.Context, local Scope) (Value, error)
+
+    // Source returns the Scanner that locates the expression in a source file.
+	Source() parser.Scanner
+}
+```

docs/docs/dev/grammar.md

@@ -0,0 +1,8 @@
+---
+id: grammar
+title: Grammar
+---
+
+The grammar specifies which sequences of characters are valid arr.ai programs and which are not. Adding new syntax to arr.ai requires a change to the grammar (but don't forget that [macros](../lang/macros.md) may be able to solve your problem more easily).
+
+The arr.ai grammar is defined using [ωBNF](https://github.com/arr-ai/wbnf) in [arrai.wbnf](https://github.com/arr-ai/arrai/blob/master/syntax/arrai.wbnf).

docs/docs/dev/overview.md

@@ -0,0 +1,6 @@
+---
+id: overview
+title: Developing arr.ai
+---
+
+If you want to make changes to the arr.ai language itself, this section is for your.

docs/docs/dev/values.md

@@ -0,0 +1,12 @@
+---
+id: values
+title: Values
+---
+
+Arr.ai values numbers, tuples, sets, and various other structures built on top of them. Values are [expressions](./exprs.md) that simply evaluate to themselves.
+
+## Implementation
+
+Each value is a type defined in the `rel` package, and implements the `Value` interface (which extends the `Expr` interface).
+
+Constructing new Value instances from Go values can be done with `rel.NewValue(interface{})`.

docs/sidebars.js

@@ -58,5 +58,11 @@ module.exports = {
             'std/str',
             'std/test',
         ],
+        "Developing": [
+            'dev/overview',
+            'dev/grammar',
+            'dev/values',
+            'dev/exprs',
+        ]
     },
 };

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
 	github.com/go-errors/errors v1.1.1
 	github.com/gorilla/websocket v1.4.2
-	github.com/iancoleman/strcase v0.1.1
+	github.com/iancoleman/strcase v0.1.3
 	github.com/mattn/go-isatty v0.0.12
 	github.com/pkg/errors v0.9.1
 	github.com/rjeczalik/notify v0.9.2

rel/value.go

@@ -3,6 +3,10 @@ package rel
 import (
 	"context"
 	"fmt"
+	"reflect"
+	"unsafe"
+
+	"github.com/iancoleman/strcase"
 
 	"github.com/arr-ai/frozen"
 	"github.com/arr-ai/wbnf/parser"
@@ -11,12 +15,13 @@ import (
 
 // Expr represents an arr.ai expression.
 type Expr interface {
-	// Require a String() method.
+	// All exprs can be serialized to strings with the String() method.
 	fmt.Stringer
 
 	// Eval evaluates the expr in a given scope.
 	Eval(ctx context.Context, local Scope) (Value, error)
 
+	// Source returns the Scanner that locates the expression in a source file.
 	Source() parser.Scanner
 }
 
@@ -210,13 +215,98 @@ func NewValue(v interface{}) (Value, error) {
 		return NewBytes(x), nil
 	case map[string]interface{}:
 		return NewTupleFromMap(x)
-	case []interface{}:
-		return NewSetFrom(x...)
 	default:
-		return nil, errors.Errorf("%v (%[1]T) not convertible to Value", v)
+		// Fall back on reflection for custom types.
+		return reflectNewValue(reflect.ValueOf(x))
+	}
+}
+
+// reflectNewValue uses reflection to inspect the type of x and unpack its values.
+func reflectNewValue(x reflect.Value) (Value, error) {
+	if !x.IsValid() {
+		return None, nil
+	}
+	t := x.Type()
+	switch t.Kind() {
+	case reflect.Ptr:
+		if x.IsNil() {
+			return None, nil
+		}
+		return NewValue(x.Elem().Interface())
+	case reflect.Array, reflect.Slice:
+		return reflectToSet(x)
+	case reflect.Map:
+		entries := make([]DictEntryTuple, 0, x.Len())
+		for _, k := range x.MapKeys() {
+			v := x.MapIndex(k)
+			kv, err := NewValue(k.Interface())
+			if err != nil {
+				return nil, err
+			}
+			vv, err := NewValue(v.Interface())
+			if err != nil {
+				return nil, err
+			}
+			entries = append(entries, NewDictEntryTuple(kv, vv))
+		}
+		return NewDict(false, entries...)
+	case reflect.Struct:
+		s := map[string]interface{}{}
+
+		// Ensure x is accessible.
+		xv := reflect.New(t).Elem()
+		xv.Set(x)
+
+		for i := 0; i < t.NumField(); i++ {
+			tf := t.Field(i)
+			// Ensure each field of x is accessible.
+			f := xv.Field(i)
+			f = reflect.NewAt(f.Type(), unsafe.Pointer(f.UnsafeAddr())).Elem()
+
+			var v Value
+			var err error
+			switch f.Type().Kind() {
+			case reflect.Array, reflect.Slice:
+				v, err = reflectToValues(f, tf.Tag.Get("unordered") == "true")
+			default:
+				v, err = NewValue(f.Interface())
+			}
+			if err != nil {
+				return nil, err
+			}
+			// Lowercase the first character in case it's uppercase only for Go exporting.
+			// TODO: Handle a name tag to override behaviour.
+			s[strcase.ToLowerCamel(tf.Name)] = v
+		}
+		return NewTupleFromMap(s)
+	default:
+		return nil, errors.Errorf("%v (%[1]T) not convertible to Value", x)
 	}
 }
 
+// reflectToValues assumed x is a slice or array, and returns x serialized to a collection of Values.
+//
+// If ordered is true, the result will be an Array. If false, it will be a Set.
+// If x is not a slice or array, reflectToValues will panic.
+func reflectToValues(x reflect.Value, unordered bool) (Value, error) {
+	vs := make([]Value, 0, x.Len())
+	for i := 0; i < x.Len(); i++ {
+		v, err := NewValue(x.Index(i).Interface())
+		if err != nil {
+			return nil, err
+		}
+		vs = append(vs, v)
+	}
+	if unordered {
+		return NewSet(vs...)
+	}
+	return NewArray(vs...), nil
+}
+
+func reflectToSet(x reflect.Value) (Value, error) {
+	return reflectToValues(x, true)
+}
+
 // AttrEnumeratorToSlice transcribes its Attrs in a slice.
 func AttrEnumeratorToSlice(e AttrEnumerator) []Attr {
 	attrs := []Attr{}

rel/value_test.go

@@ -33,8 +33,78 @@ func TestSetCall(t *testing.T) {
 
 	result, err = SetCall(ctx, set, NewNumber(1))
 	require.NoError(t, err)
-	assert.True(t, result.Equal(NewNumber(42)))
+	AssertEqualValues(t, result, NewNumber(42))
+
 	result, err = SetCall(ctx, set, NewNumber(2))
 	require.NoError(t, err)
-	assert.True(t, result.Equal(NewNumber(24)))
+	AssertEqualValues(t, result, NewNumber(24))
+}
+
+//nolint:structcheck
+func TestNewValue(t *testing.T) {
+	// Structs are serialized to tuples.
+	type Foo struct {
+		num int
+		str string
+		// Slices without the ordered tag are serialized to arrays.
+		arr []int
+		// Slices with the unordered tag are serialized to sets.
+		set  []int         `unordered:"true"`
+		iset []interface{} `unordered:"true"`
+		none *Foo
+		// All struct field names are serialized to start lowercase.
+		CASE     int
+		children []*Foo
+		// Non-string maps are serialized to dictionaries.
+		mixedMap  map[interface{}]interface{}
+		stringMap map[string]interface{}
+	}
+
+	input := []*Foo{{
+		num:  1,
+		str:  "a",
+		arr:  []int{2, 1},
+		set:  []int{2, 1},
+		iset: []interface{}{3},
+		// Nil values are serialized to empty sets (None).
+		none: nil,
+		CASE: 0,
+		// Unset fields of structs are serialized with default empty values.
+		children:  []*Foo{{num: 2}},
+		mixedMap:  map[interface{}]interface{}{1: 2, "k": nil},
+		stringMap: map[string]interface{}{"a": 1},
+	}}
+
+	actual, err := NewValue(input)
+	require.NoError(t, err)
+
+	expected, err := NewSet(NewTuple(
+		NewIntAttr("num", 1),
+		NewStringAttr("str", []rune("a")),
+		NewAttr("arr", NewArray(NewNumber(2), NewNumber(1))),
+		NewAttr("set", MustNewSet(NewNumber(1), NewNumber(2))),
+		NewAttr("iset", MustNewSet(NewNumber(3))),
+		NewAttr("none", None),
+		NewAttr("cASE", NewNumber(0)),
+		NewAttr("mixedMap", MustNewDict(false,
+			NewDictEntryTuple(NewNumber(1), NewNumber(2)),
+			NewDictEntryTuple(NewString([]rune("k")), None),
+		)),
+		NewAttr("stringMap", NewTuple(NewIntAttr("a", 1))),
+		NewAttr("children", NewArray(NewTuple(
+			NewAttr("num", NewNumber(2)),
+			NewAttr("str", None),
+			NewAttr("arr", None),
+			NewAttr("set", None),
+			NewAttr("iset", None),
+			NewAttr("none", None),
+			NewAttr("cASE", NewNumber(0)),
+			NewAttr("mixedMap", None),
+			NewAttr("stringMap", NewTuple()),
+			NewAttr("children", None),
+		))),
+	))
+	require.NoError(t, err)
+
+	AssertEqualValues(t, expected, actual)
 }