Implement test runner and add Java's test data (#49)

Test cases that fail in 1 or more implementation due to inconsistencies
are commented.

test_suite/BUILD

@@ -16,7 +16,7 @@
             ],
             main = "starlark_test.py",
         )
-        for test_file in glob(["testdata/*"])
+        for test_file in glob(["testdata/java/*"])
     ]
     for impl, binary_rule in [("java", "@io_bazel//src/main/java/com/google/devtools/starlark:Starlark"),
                               ("go", "@net_starlark_go//cmd/starlark:starlark"),

test_suite/starlark_test.py

@@ -1,9 +1,86 @@
 import sys
 import unittest
+import tempfile
+import subprocess
+import os
+import re
 
 import testenv
 
 class StarlarkTest(unittest.TestCase):
+  CHUNK_SEP = "---"
+  ERR_SEP = "###"
+  seen_error = False
+
+  def chunks(self, path):
+    code = []
+    expected_errors = []
+    with open(path, mode="rb") as f:
+      for line in f:
+        line = line.decode("utf-8")
+        if line.strip() == self.CHUNK_SEP:
+          yield code, expected_errors
+          expected_errors = []
+          code = []
+        else:
+          code.append(line)
+          i = line.find(self.ERR_SEP)
+          if i >= 0:
+            expected_errors.append(line[i + len(self.ERR_SEP):].strip())
+    yield code, expected_errors
+
+  def evaluate(self, f):
+    """Execute Starlark file, return stderr."""
+    proc = subprocess.Popen(
+        [binary_path, f], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+    _, stderr = proc.communicate()
+    return stderr
+
+  def check_output(self, output, expected):
+    if expected and not output:
+      self.seen_error = True
+      print("Expected error:", expected)
+
+    if output and not expected:
+      self.seen_error = True
+      print("Unexpected error:", output)
+
+    output_ = output.lower()
+    for exp in expected:
+      exp = exp.lower()
+      # Try both substring and regex matching.
+      if exp not in output_ and not re.search(exp, output_):
+        self.seen_error = True
+        print("Error `{}` not found, got: `{}`".format(exp, output))
+
+  PRELUDE = """
+def assert_eq(x, y):
+  if x != y:
+    print("%r != %r" % (x, y))
+
+def assert_ne(x, y):
+  if x == y:
+    print("%r == %r" % (x, y))
+
+def assert_(cond, msg="assertion failed"):
+  if not cond:
+    print(msg)
+"""
+
+  def testFile(self):
+    f = test_file
+    print("===", f, "===")
+    for chunk, expected in self.chunks(f):
+      with tempfile.NamedTemporaryFile(
+          mode="wb", suffix=".star", delete=False) as tmp:
+        lines = [line.encode("utf-8") for line in
+                 [self.PRELUDE] + chunk]
+        tmp.writelines(lines)
+      output = self.evaluate(tmp.name).decode("utf-8")
+      os.unlink(tmp.name)
+      self.check_output(output, expected)
+    if self.seen_error:
+      raise Exception("Test failed")
     pass
 
 if __name__ == "__main__":

test_suite/testdata/java/all_any.star

@@ -0,0 +1,43 @@
+# All with empty value
+assert_eq(all([]), True)
+
+# All with list
+assert_eq(all(['t', 'e']), True)
+assert_eq(all([False]), False)
+assert_eq(all([True, False]), False)
+assert_eq(all([False, False]), False)
+assert_eq(all([False, True]), False)
+assert_eq(all(['', True]), False)
+assert_eq(all([0, True]), False)
+assert_eq(all([[], True]), False)
+assert_eq(all([True, 't', 1]), True)
+
+# All with dict
+assert_eq(all({1 : None}), True)
+assert_eq(all({None : 1}), False)
+
+# Any with empty value
+assert_eq(any([]), False)
+
+# Any with list
+assert_eq(any([False]), False)
+assert_eq(any([0]), False)
+assert_eq(any(['']), False)
+assert_eq(any([[]]), False)
+assert_eq(any([True, False]), True)
+assert_eq(any([False, False]), False)
+assert_eq(any([False, '', 0]), False)
+assert_eq(any([False, '', 42]), True)
+
+# Any with dict
+assert_eq(any({1 : None, '' : None}), True)
+assert_eq(any({None : 1, '' : 2}), False)
+
+---
+all(None) ### iterable
+---
+any(None) ### iterable
+---
+any(1) ### iterable
+---
+all(1) ### iterable

test_suite/testdata/java/and_or_not.star

@@ -0,0 +1,43 @@
+assert_eq(8 or 9, 8)
+assert_eq(0 or 9, 9)
+assert_eq(8 and 9, 9)
+assert_eq(0 and 9, 0)
+
+assert_eq(1 and 2 or 3, 2)
+assert_eq(0 and 2 or 3, 3)
+assert_eq(1 and 0 or 3, 3)
+
+assert_eq(1 or 2 and 3, 1)
+assert_eq(0 or 2 and 3, 3)
+assert_eq(0 or 0 and 3, 0)
+assert_eq(1 or 0 and 3, 1)
+
+assert_eq(None and 1, None)
+assert_eq("" or 9, 9)
+assert_eq("abc" or 9, "abc")
+
+# check that fail() is not evaluated
+assert_eq(8 or fail("do not execute"), 8)
+assert_eq(0 and fail("do not execute"), 0)
+
+assert_eq(not 1, False)
+assert_eq(not "", True)
+assert_eq(not not 1, True)
+
+assert_eq(not 0 + 0, True)
+assert_eq(not 2 - 1, False)
+
+assert_eq(not (0 and 0), True)
+assert_eq(not (1 or 0), False)
+
+assert_eq(0 and not 0, 0)
+assert_eq(not 0 and 0, 0)
+
+assert_eq(1 and not 0, True)
+assert_eq(not 0 or 0, True)
+
+assert_eq(not 1 or 0, 0)
+assert_eq(not 1 or 1, 1)
+
+assert_eq(not [], True)
+assert_eq(not {"a": 1}, False)

test_suite/testdata/java/dict.star

@@ -0,0 +1,87 @@
+# creation
+
+foo = {'a': 1, 'b': [1, 2]}
+bar = dict(a=1, b=[1, 2])
+baz = dict({'a': 1, 'b': [1, 2]})
+
+assert_eq(foo, bar)
+assert_eq(foo, baz)
+
+# get/setdefault
+
+assert_eq(foo.get('a'), 1)
+assert_eq(bar.get('b'), [1, 2])
+assert_eq(baz.get('c'), None)
+assert_eq(baz.setdefault('c', 15), 15)
+assert_eq(baz.setdefault('c'), 15)
+assert_eq(baz.setdefault('c', 20), 15)
+assert_eq(baz.setdefault('d'), None)
+
+# items
+
+assert_eq(foo.items(), [('a', 1), ('b', [1, 2])])
+
+# keys
+
+assert_eq(bar.keys(), ['a', 'b'])
+
+# values
+
+assert_eq(baz.values(), [1, [1, 2], 15, None])
+
+# pop/popitem
+
+assert_eq(baz.pop('d'), None)
+assert_eq(foo.pop('a'), 1)
+assert_eq(bar.popitem(), ('a', 1))
+assert_eq(foo, bar)
+assert_eq(foo.pop('a', 0), 0)
+assert_eq(foo.popitem(), ('b', [1, 2]))
+
+---
+dict().popitem() ### empty
+---
+dict(a=2).pop('z') ### key
+---
+
+# update
+
+foo = dict()
+baz = dict(a=1, b=[1, 2])
+bar = dict(b=[1, 2])
+
+foo.update(baz)
+bar.update(a=1)
+baz.update({'c': 3})
+foo.update([('c', 3)])
+bar['c'] = 3
+quz = dict()
+quz.update(bar.items())
+
+assert_eq(foo, bar)
+assert_eq(foo, baz)
+assert_eq(foo, quz)
+---
+
+d = {"b": 0}
+d.update({"a": 1}, b = 2)
+d.update({"c": 0}, c = 3, d = 4)
+d.update([("e", 5)])
+
+# _inconsistency_: rust dict.update doesn't accept tuples
+# d.update((["f", 0],), f = 6)
+# expected = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6}
+
+expected = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}
+assert_eq(d, expected)
+
+
+# creation with repeated keys
+
+d1 = dict([('a', 1)], a=2)
+d2 = dict({'a': 1}, a=2)
+d3 = dict([('a', 1), ('a', 2)])
+
+assert_eq(d1['a'], 2)
+assert_eq(d1, d2)
+assert_eq(d1, d3)

test_suite/testdata/java/equality.star

@@ -0,0 +1,69 @@
+# == operator
+assert_eq(1 == 1, True)
+assert_eq(1 == 2, False)
+assert_eq('hello' == 'hel' + 'lo', True)
+assert_eq('hello' == 'bye', False)
+assert_eq(None == None, True)
+assert_eq([1, 2] == [1, 2], True)
+assert_eq([1, 2] == [2, 1], False)
+assert_eq({'a': 1, 'b': 2} == {'b': 2, 'a': 1}, True)
+assert_eq({'a': 1, 'b': 2} == {'a': 1}, False)
+assert_eq({'a': 1, 'b': 2} == {'a': 1, 'b': 2, 'c': 3}, False)
+assert_eq({'a': 1, 'b': 2} == {'a': 1, 'b': 3}, False)
+
+# != operator
+assert_eq(1 != 1, False)
+assert_eq(1 != 2, True)
+assert_eq('hello' != 'hel' + 'lo', False)
+assert_eq('hello' != 'bye', True)
+assert_eq([1, 2] != [1, 2], False)
+assert_eq([1, 2] != [2, 1], True)
+assert_eq({'a': 1, 'b': 2} != {'b': 2, 'a': 1}, False)
+assert_eq({'a': 1, 'b': 2} != {'a': 1}, True)
+assert_eq({'a': 1, 'b': 2} != {'a': 1, 'b': 2, 'c': 3}, True)
+assert_eq({'a': 1, 'b': 2} != {'a': 1, 'b': 3}, True);
+
+# equality precedence
+assert_eq(1 + 3 == 2 + 2, True)
+assert_eq(not 1 == 2, True)
+assert_eq(not 1 != 2, False)
+assert_eq(2 and 3 == 3 or 1, True)
+assert_eq(2 or 3 == 3 and 1, 2);
+
+# < operator
+assert_eq(1 <= 1, True)
+assert_eq(1 < 1, False)
+assert_eq('a' <= 'b', True)
+assert_eq('c' < 'a', False);
+
+# <= and < operators
+assert_eq(1 <= 1, True)
+assert_eq(1 < 1, False)
+assert_eq('a' <= 'b', True)
+assert_eq('c' < 'a', False);
+
+# >= and > operators
+assert_eq(1 >= 1, True)
+assert_eq(1 > 1, False)
+assert_eq('a' >= 'b', False)
+assert_eq('c' > 'a', True);
+
+# list/tuple comparison
+assert_eq([] < [1], True)
+assert_eq([1] < [1, 1], True)
+assert_eq([1, 1] < [1, 2], True)
+assert_eq([1, 2] < [1, 2, 3], True)
+assert_eq([1, 2, 3] <= [1, 2, 3], True)
+
+assert_eq(['a', 'b'] > ['a'], True)
+assert_eq(['a', 'b'] >= ['a'], True)
+assert_eq(['a', 'b'] < ['a'], False)
+assert_eq(['a', 'b'] <= ['a'], False)
+
+assert_eq(('a', 'b') > ('a', 'b'), False)
+assert_eq(('a', 'b') >= ('a', 'b'), True)
+assert_eq(('a', 'b') < ('a', 'b'), False)
+assert_eq(('a', 'b') <= ('a', 'b'), True)
+
+assert_eq([[1, 1]] > [[1, 1], []], False)
+assert_eq([[1, 1]] < [[1, 1], []], True)