Merge pull request #171 from anzx/feature/prettier_test_results

Feature/prettier test results

cmd/arrai/bundle.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/spf13/afero"
+
 	"github.com/arr-ai/arrai/pkg/arraictx"
 	"github.com/arr-ai/arrai/pkg/ctxfs"
 	"github.com/arr-ai/arrai/syntax"
@@ -53,7 +55,7 @@ func bundleFilesTo(ctx context.Context, path string, w io.Writer, out string) (e
 		w = f
 	}
 
-	buf, err := ctxfs.ReadFile(ctxfs.SourceFsFrom(ctx), path)
+	buf, err := afero.ReadFile(ctxfs.SourceFsFrom(ctx), path)
 	if err != nil {
 		return err
 	}

cmd/arrai/compile.go

@@ -151,7 +151,7 @@ func buildBinary(ctx context.Context, bundledScripts []byte, out afero.File) err
 		}
 	}
 
-	file, err := ctxfs.ReadFile(fs, filepath.Join(buildDir, "main"))
+	file, err := afero.ReadFile(fs, filepath.Join(buildDir, "main"))
 	if err != nil {
 		return err
 	}

cmd/arrai/run.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/spf13/afero"
+
 	"github.com/arr-ai/arrai/pkg/arrai"
 	"github.com/arr-ai/arrai/pkg/arraictx"
 	"github.com/arr-ai/arrai/pkg/ctxfs"
@@ -37,7 +39,7 @@ func evalFile(ctx context.Context, path string, w io.Writer, out string) error {
 	if err := fileExists(ctx, path); err != nil {
 		return err
 	}
-	buf, err := ctxfs.ReadFile(ctxfs.SourceFsFrom(ctx), path)
+	buf, err := afero.ReadFile(ctxfs.SourceFsFrom(ctx), path)
 	if err != nil {
 		return err
 	}

cmd/arrai/sync_other.go

@@ -21,6 +21,7 @@ import (
 	"github.com/go-errors/errors"
 	"github.com/rjeczalik/notify"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
 	"github.com/urfave/cli/v2"
 	"google.golang.org/grpc"
 )
@@ -123,7 +124,7 @@ func buildTree(ctx context.Context, root string) (map[string]interface{}, error)
 			if err != nil {
 				log.Printf("ERROR WALKING TO %s: %s", path, err)
 			} else if !info.IsDir() {
-				data, err := ctxfs.ReadFile(ctxfs.SourceFsFrom(ctx), path)
+				data, err := afero.ReadFile(ctxfs.SourceFsFrom(ctx), path)
 				if err != nil {
 					return errors.WrapPrefix(err, "reading "+path, 0)
 				}

cmd/arrai/test.go

@@ -2,10 +2,7 @@ package main
 
 import (
 	"context"
-	"fmt"
-	"io"
 	"os"
-	"strings"
 
 	"github.com/arr-ai/arrai/internal/test"
 	"github.com/arr-ai/arrai/pkg/arraictx"
@@ -20,32 +17,8 @@ var testCommand = &cli.Command{
 }
 
 func doTest(c *cli.Context) error {
-	file := c.Args().Get(0)
+	path := c.Args().Get(0)
 	ctx := arraictx.InitCliCtx(context.Background(), c)
 
-	return testPath(ctx, file, os.Stdout)
-}
-
-// testPath runs and reports on all tests in the subtree of the given path.
-func testPath(ctx context.Context, path string, w io.Writer) error {
-	if path == "" {
-		path = "."
-	}
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		if !strings.Contains(path, string([]rune{os.PathSeparator})) {
-			return fmt.Errorf(`"%s": not a command and not found as a file in the current directory`, path)
-		}
-		return fmt.Errorf(`"%s": file not found`, path)
-	}
-
-	results, err := test.Test(ctx, w, path)
-	if err != nil {
-		return err
-	}
-
-	err = test.Report(w, results)
-	if err != nil {
-		return err
-	}
-	return nil
+	return test.RunTests(ctx, os.Stdout, path)
 }

contrib/table_test.arrai

@@ -10,14 +10,16 @@ let multipleHtml =
     <tr><td>b</td><td>[1, 2, 3]</td></tr>
 </table>';
 
+let remWhitespace = \s //re.compile('\\s').sub('', s);
+
 (
     markdown: (
-        empty: markdown({}) = '|Key|Value|\n',
-        single: markdown(single) = '|Key|Value|\n|a|1|',
-        multiple: markdown(multiple) = '|Key|Value|\n|a|1|\n|b|[1, 2, 3]|',
+        empty: remWhitespace(markdown({})) = '|Key|Value|',
+        single: remWhitespace(markdown(single)) = '|Key|Value||a|1|',
+        multiple: remWhitespace(markdown(multiple)) = '|Key|Value||a|1||b|[1,2,3]|',
     ),
     html: (
-        empty: html({}) = '<table>\n    <tr><th>Key</th><th>Value</th></tr>\n</table>',
-        multiple: html(multiple) = multipleHtml,
+        empty: remWhitespace(html({})) = '<table><tr><th>Key</th><th>Value</th></tr></table>',
+        multiple: remWhitespace(html(multiple)) = remWhitespace(multipleHtml),
     ),
 )

examples/test/README.md

@@ -1,5 +1,5 @@
 # Test case examples
 
-This directory has the following examples of arr.ai test cases:
+This directory has the following examples of arr.ai test cases which uses the new leaf-true structure:
 - **[Single test case in one test file](single_case_test.arrai)**: Only one test case in an arr.ai test file.
 - **[Multiple test cases in one test file](multiple_cases_test.arrai):** Multiple test cases in an arr.ai test file.

examples/test/multiple_cases_test.arrai

@@ -1,9 +1,31 @@
-# 5 test cases
+let str = "Hello, arrai!";
 (
-    # Added expression let to clarify the actual result can be a complex expression
-    testCase1: //test.assert.equal("Hello, arrai!", let words = ["Hello", "arrai!"];//seq.join(", ", words)),
-    testCase2: //test.assert.unequal("Hello, arrai!", //seq.join(", ", ["Nihao", "arrai!"])),
-    testCase3: //test.assert.size(2, {"Hello", "arrai!"}),
-    testCase4: //test.assert.true({"Hello", "arrai!"}),
-    testCase5: //test.assert.false({}),
+    tuple:
+    (
+        testCase1: str = let words = ["Hello", "arrai!"];//seq.join(", ", words),
+        testCase2: str != //seq.join(", ", ["Nihao", "arrai!"]),
+        # notATest1: 5,
+        # notATest2: { 1 = 1, 2 = 1 },
+        #_ignoredTest: 1 = 2,
+    ),
+    #_ignoredContainer:
+    #(
+    #    testCase1: 1 = 2,
+    #    _testCase2: 1 = 2,
+    #),
+    testCase3: 2 = {"Hello", "arrai!"} count,
+    dict:
+    {
+        # "testCase4": 4 = 7,
+        "testCase5": (a: 5 = 5, b: 6 = 6),
+    },
+    array:
+    [
+        7 = 7,
+        [8 = 8, (a: 9 = 9, b: 10 = 10)],
+        let value = {str}; cond {value: true} = true,
+        let value = {}; cond {value: true} = false,
+        #true = false,
+    ],
+    deep: (deeper: (deepest: 9 = 9)),
 )

examples/test/single_case_test.arrai

@@ -1 +1 @@
-//test.assert.equal("Hello", //str.title("hello"))
+"Hello" = //str.title("hello")

examples/test_old/README.md

@@ -0,0 +1,5 @@
+# Test case examples
+
+This directory has the following examples of arr.ai test cases using the obsolete assert functions:
+- **[Single test case in one test file](single_case_test.arrai)**: Only one test case in an arr.ai test file.
+- **[Multiple test cases in one test file](multiple_cases_test.arrai):** Multiple test cases in an arr.ai test file.

examples/test_old/multiple_cases_test.arrai

@@ -0,0 +1,9 @@
+# 5 test cases
+(
+    # Added expression let to clarify the actual result can be a complex expression
+    testCase1: //test.assert.equal("Hello, arrai!", let words = ["Hello", "arrai!"];//seq.join(", ", words)),
+    testCase2: //test.assert.unequal("Hello, arrai!", //seq.join(", ", ["Nihao", "arrai!"])),
+    testCase3: //test.assert.size(2, {"Hello", "arrai!"}),
+    testCase4: //test.assert.true({"Hello", "arrai!"}),
+    testCase5: //test.assert.false({}),
+)

examples/test_old/single_case_test.arrai

@@ -0,0 +1 @@
+//test.assert.equal("Hello", //str.title("hello"))

go.mod

@@ -29,6 +29,7 @@ require (
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73 // indirect
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
 	golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f // indirect
+	golang.org/x/text v0.3.3
 	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
 	google.golang.org/grpc v1.31.1
 	google.golang.org/protobuf v1.25.0

internal/test/entities.go

@@ -0,0 +1,25 @@
+package test
+
+import "time"
+
+type testFile struct {
+	path     string
+	source   string
+	wallTime time.Duration
+	results  []testResult
+}
+
+type testResult struct {
+	name    string
+	outcome testOutcome
+	message string
+}
+
+type testOutcome int
+
+const (
+	Failed testOutcome = iota
+	Invalid
+	Ignored
+	Passed
+)

internal/test/report.go

@@ -3,28 +3,109 @@ package test
 import (
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/text/language"
+	"golang.org/x/text/message"
 )
 
-// Report prints the results of the testing to the IO writer.
-func Report(w io.Writer, rs Results) error {
-	if rs.AllPassed() {
-		_, err := fmt.Fprintln(w, "all tests passed")
-		if err != nil {
-			return err
-		}
-	} else {
-		for _, r := range rs.results {
-			if !r.pass {
-				_, err := fmt.Fprintf(w, "%s failed\n", r.file)
-				if err != nil {
-					return err
-				}
+// Report writes a formatted output of all the test files and their test results, and returns and error if the test run
+// failed.
+func Report(w io.Writer, testFiles []testFile) error {
+	stats := calcStats(testFiles)
+
+	for _, testFile := range testFiles {
+		reportFile(w, testFile, stats.maxNameLen)
+	}
+
+	reportStats(w, stats)
+
+	if stats.runFailed {
+		return fmt.Errorf("test run "+color, 41, "FAILED")
+	}
+
+	return nil
+}
+
+const color = "\033[38;5;255;%d;1m%s\033[0m"
+
+// reportFile writes a formatted output of a file and all its test results, ordered by outcome. The maxName parameter
+// is used to aligned the results, and should contain the length of the longest testResult.name inside testFile.results.
+func reportFile(w io.Writer, testFile testFile, maxName int) {
+	results := testFile.results
+
+	// Sort test results by outcome then by test name
+	sort.Slice(results, func(i, j int) bool {
+		left, right := results[i], results[j]
+		if left.outcome == right.outcome {
+			if left.name == right.name {
+				return true
 			}
+			return left.name < right.name
 		}
-		_, err := fmt.Fprintf(w, "%d/%d tests passed\n", rs.CountPassed(), rs.Count())
-		if err != nil {
-			return err
-		}
+		return left.outcome > right.outcome
+	})
+
+	message.NewPrinter(language.English).
+		Fprintf(w, "\n=======  %s (%dms)\n", relPath(testFile.path), testFile.wallTime.Milliseconds())
+	for _, result := range results {
+		reportTest(w, result, maxName)
 	}
-	return nil
+}
+
+// relPath makes a best-effort attempt to compute the relative path of the given absolute path. If it fails, it returns
+// the absolute path untouched.
+func relPath(absPath string) string {
+	cwd, wdErr := os.Getwd()
+	relPath, relErr := filepath.Rel(cwd, absPath)
+
+	if wdErr != nil || relErr != nil {
+		return absPath
+	}
+
+	return relPath
+}
+
+// reportTest writes a formatted output of a single test result (PASS/FAIL/SKIP/??) with the optional included message.
+func reportTest(w io.Writer, test testResult, maxName int) {
+	switch test.outcome {
+	case Failed:
+		fmt.Fprintf(w, color, 41, "FAIL")
+	case Invalid:
+		fmt.Fprintf(w, color, 31, " ?? ")
+	case Ignored:
+		fmt.Fprintf(w, color, 93, "SKIP")
+	case Passed:
+		fmt.Fprintf(w, color, 32, "PASS")
+	}
+
+	fmt.Fprintf(w, "  %s\n", test.name+strings.Repeat(" ", maxName-utf8.RuneCountInString(test.name)))
+
+	if test.message != "" {
+		fmt.Fprintf(w, "      %s\n", strings.ReplaceAll(test.message, "\n", "\n      "))
+	}
+}
+
+// reportStats writes a formatted single line representation of the aggregated test statistics.
+func reportStats(w io.Writer, stats testStats) {
+	p := message.NewPrinter(language.English)
+	p.Fprintf(w, "\n=======  Summary\n")
+
+	if stats.failed > 0 {
+		p.Fprintf(w, "%d failed, ", stats.failed)
+	}
+
+	if stats.invalid > 0 {
+		p.Fprintf(w, "%d invalid, ", stats.invalid)
+	}
+
+	if stats.ignored > 0 {
+		p.Fprintf(w, "%d ignored, ", stats.ignored)
+	}
+
+	p.Fprintf(w, "%d passed of %d total tests. Took %dms.\n", stats.passed, stats.total, stats.wallTime.Milliseconds())
 }