From 9c60a23e44e293d95dfd241ab342e588175c6739 Mon Sep 17 00:00:00 2001 From: Marc Vertes Date: Thu, 30 May 2024 20:19:07 +0200 Subject: [PATCH] test(gnovm): improve gnovm/pkg/gnolang test coverage (#2143) No new tests have been added yet, but existing tests in gnovm/tests/files are now executed in the context of the package which contains the relevant interpreter virtual machine and parser code. We have now a better baseline to measure and complete the code coverage. This represents more than 800 tests, which takes 10s on my macbook air m1. Those tests are actually run twice, as I haven't yet removed the original execution from gnovm/tests. The testing code is substantially simplified compared to its counterpart in gnovm/tests, but has not yet been deduplicated. This will be done in further commits. In `gnovm/pkg/golang`, `go test -cover` went from 34.2% to 66.7%. Related to #1121, #1145 and probably more
Contributors' checklist... - [*] Added new tests, or not needed, or not feasible - [*] Provided an example (e.g. screenshot) to aid review or the PR is self-explanatory - [*] Updated the official documentation or not needed - [*] No breaking changes were made, or a `BREAKING CHANGE: xxx` message was included in the description - [*] Added references to related issues and PRs - [*] Provided any useful hints for running manual tests - [ ] Added new benchmarks to [generated graphs](https://gnoland.github.io/benchmarks), if any. More info [here](https://github.com/gnolang/gno/blob/master/.benchmarks/README.md).
--------- Co-authored-by: Manfred Touron <94029+moul@users.noreply.github.com> --- gnovm/pkg/gnolang/debugger_test.go | 59 ++++++++++++++++---------- gnovm/pkg/gnolang/eval_test.go | 62 ++++++++++++++++++++++++++++ gnovm/pkg/gnolang/machine.go | 1 - gnovm/tests/file.go | 5 ++- gnovm/tests/files/convert4.gno | 2 +- gnovm/tests/files/convert5.gno | 2 +- gnovm/tests/files/float5_stdlibs.gno | 6 +-- gnovm/tests/files/switch8.gno | 2 +- gnovm/tests/files/switch8b.gno | 2 +- gnovm/tests/files/switch8c.gno | 2 +- gnovm/tests/files/switch9.gno | 2 +- gnovm/tests/imports.go | 4 +- 12 files changed, 114 insertions(+), 35 deletions(-) create mode 100644 gnovm/pkg/gnolang/eval_test.go diff --git a/gnovm/pkg/gnolang/debugger_test.go b/gnovm/pkg/gnolang/debugger_test.go index ca50e243a5c..95146bd48f6 100644 --- a/gnovm/pkg/gnolang/debugger_test.go +++ b/gnovm/pkg/gnolang/debugger_test.go @@ -23,38 +23,54 @@ type writeNopCloser struct{ io.Writer } func (writeNopCloser) Close() error { return nil } -func eval(debugAddr, in, file string) (string, string, error) { - out := bytes.NewBufferString("") - err := bytes.NewBufferString("") +// TODO (Marc): move evalTest to gnovm/tests package and remove code duplicates +func evalTest(debugAddr, in, file string) (out, err string) { + bout := bytes.NewBufferString("") + berr := bytes.NewBufferString("") stdin := bytes.NewBufferString(in) - stdout := writeNopCloser{out} - stderr := writeNopCloser{err} + stdout := writeNopCloser{bout} + stderr := writeNopCloser{berr} + debug := in != "" || debugAddr != "" + mode := tests.ImportModeNativePreferred + if strings.HasSuffix(file, "_stdlibs.gno") { + mode = tests.ImportModeStdlibsPreferred + } + + defer func() { + if r := recover(); r != nil { + err = fmt.Sprintf("%v", r) + } + out = strings.TrimSpace(out) + err = strings.TrimSpace(strings.ReplaceAll(err, "../../tests/files/", "files/")) + }() - testStore := tests.TestStore(gnoenv.RootDir(), "", stdin, stdout, stderr, tests.ImportModeStdlibsPreferred) + testStore := tests.TestStore(gnoenv.RootDir(), "../../tests/files", stdin, stdout, stderr, mode) f := gnolang.MustReadFile(file) m := gnolang.NewMachineWithOptions(gnolang.MachineOptions{ - PkgPath: string(f.PkgName), - Input: stdin, - Output: stdout, - Store: testStore, - Debug: true, - DebugAddr: debugAddr, + PkgPath: string(f.PkgName), + Input: stdin, + Output: stdout, + Store: testStore, + Context: tests.TestContext(string(f.PkgName), nil), + Debug: debug, }) defer m.Release() if debugAddr != "" { - if err := m.Debugger.Serve(debugAddr); err != nil { - return "", "", err + if e := m.Debugger.Serve(debugAddr); e != nil { + err = e.Error() + return } } m.RunFiles(f) ex, _ := gnolang.ParseExpr("main()") m.Eval(ex) - return out.String(), err.String(), nil + out, err = bout.String(), berr.String() + return } func runDebugTest(t *testing.T, targetPath string, tests []dtest) { @@ -62,10 +78,10 @@ func runDebugTest(t *testing.T, targetPath string, tests []dtest) { for _, test := range tests { t.Run("", func(t *testing.T) { - out, err, _ := eval("", test.in, targetPath) + out, err := evalTest("", test.in, targetPath) t.Log("in:", test.in, "out:", out, "err:", err) if !strings.Contains(out, test.out) { - t.Errorf("result does not contain \"%s\", got \"%s\"", test.out, out) + t.Errorf("unexpected output\nwant\"%s\"\n got \"%s\"", test.out, out) } }) } @@ -77,7 +93,7 @@ func TestDebug(t *testing.T) { cont2 := "break 21\ncontinue\n" runDebugTest(t, debugTarget, []dtest{ - {in: "", out: "Welcome to the Gnovm debugger. Type 'help' for list of commands."}, + {in: "\n", out: "Welcome to the Gnovm debugger. Type 'help' for list of commands."}, {in: "help\n", out: "The following commands are available"}, {in: "h\n", out: "The following commands are available"}, {in: "help b\n", out: "Set a breakpoint."}, @@ -154,7 +170,7 @@ func TestRemoteDebug(t *testing.T) { retry int ) - go eval(debugAddress, "", debugTarget) + go evalTest(debugAddress, "", debugTarget) for retry = 100; retry > 0; retry-- { conn, err = net.Dial("tcp", debugAddress) @@ -177,8 +193,9 @@ func TestRemoteDebug(t *testing.T) { } func TestRemoteError(t *testing.T) { - _, _, err := eval(":xxx", "", debugTarget) - if !strings.Contains(err.Error(), "tcp/xxx: unknown port") { + _, err := evalTest(":xxx", "", debugTarget) + t.Log("err:", err) + if !strings.Contains(err, "tcp/xxx: unknown port") { t.Error(err) } } diff --git a/gnovm/pkg/gnolang/eval_test.go b/gnovm/pkg/gnolang/eval_test.go new file mode 100644 index 00000000000..fdd8e0204d1 --- /dev/null +++ b/gnovm/pkg/gnolang/eval_test.go @@ -0,0 +1,62 @@ +package gnolang_test + +import ( + "os" + "path" + "strings" + "testing" +) + +func TestEvalFiles(t *testing.T) { + dir := "../../tests/files" + files, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, f := range files { + wantOut, wantErr, ok := testData(dir, f) + if !ok { + continue + } + t.Run(f.Name(), func(t *testing.T) { + out, err := evalTest("", "", path.Join(dir, f.Name())) + + if wantErr != "" && !strings.Contains(err, wantErr) || + wantErr == "" && err != "" { + t.Fatalf("unexpected error\nWant: %s\n Got: %s", wantErr, err) + } + if wantOut != "" && out != wantOut { + t.Fatalf("unexpected output\nWant: %s\n Got: %s", wantOut, out) + } + }) + } +} + +// testData returns the expected output and error string, and true if entry is valid. +func testData(dir string, f os.DirEntry) (testOut, testErr string, ok bool) { + if f.IsDir() { + return "", "", false + } + name := path.Join(dir, f.Name()) + if !strings.HasSuffix(name, ".gno") || strings.HasSuffix(name, "_long.gno") { + return "", "", false + } + buf, err := os.ReadFile(name) + if err != nil { + return "", "", false + } + str := string(buf) + if strings.Contains(str, "// PKGPATH:") { + return "", "", false + } + return commentFrom(str, "\n// Output:"), commentFrom(str, "\n// Error:"), true +} + +// commentFrom returns the content from a trailing comment block in s starting with delim. +func commentFrom(s, delim string) string { + index := strings.Index(s, delim) + if index < 0 { + return "" + } + return strings.TrimSpace(strings.ReplaceAll(s[index+len(delim):], "\n// ", "\n")) +} diff --git a/gnovm/pkg/gnolang/machine.go b/gnovm/pkg/gnolang/machine.go index e3a202ad8f2..48a2145af3a 100644 --- a/gnovm/pkg/gnolang/machine.go +++ b/gnovm/pkg/gnolang/machine.go @@ -94,7 +94,6 @@ type MachineOptions struct { CheckTypes bool // not yet used ReadOnly bool Debug bool - DebugAddr string // debugger io stream address (stdin/stdout if empty) Input io.Reader // used for default debugger input only Output io.Writer // default os.Stdout Store Store // default NewStore(Alloc, nil, nil) diff --git a/gnovm/tests/file.go b/gnovm/tests/file.go index 4dd62f6f13a..852bb74d198 100644 --- a/gnovm/tests/file.go +++ b/gnovm/tests/file.go @@ -36,7 +36,7 @@ func TestMachine(store gno.Store, stdout io.Writer, pkgPath string) *gno.Machine } func testMachineCustom(store gno.Store, pkgPath string, stdout io.Writer, maxAlloc int64, send std.Coins) *gno.Machine { - ctx := testContext(pkgPath, send) + ctx := TestContext(pkgPath, send) m := gno.NewMachineWithOptions(gno.MachineOptions{ PkgPath: "", // set later. Output: stdout, @@ -47,7 +47,8 @@ func testMachineCustom(store gno.Store, pkgPath string, stdout io.Writer, maxAll return m } -func testContext(pkgPath string, send std.Coins) *teststd.TestExecContext { +// TestContext returns a TestExecContext. Usable for test purpose only. +func TestContext(pkgPath string, send std.Coins) *teststd.TestExecContext { // FIXME: create a better package to manage this, with custom constructors pkgAddr := gno.DerivePkgAddr(pkgPath) // the addr of the pkgPath called. caller := gno.DerivePkgAddr("user1.gno") diff --git a/gnovm/tests/files/convert4.gno b/gnovm/tests/files/convert4.gno index 6ad748499ed..d7ab4905f62 100644 --- a/gnovm/tests/files/convert4.gno +++ b/gnovm/tests/files/convert4.gno @@ -4,4 +4,4 @@ func main() { println(int(nil)) } -// error: cannot convert (undefined) to int +// Error: cannot convert (undefined) to int diff --git a/gnovm/tests/files/convert5.gno b/gnovm/tests/files/convert5.gno index acd33889ff0..74063709110 100644 --- a/gnovm/tests/files/convert5.gno +++ b/gnovm/tests/files/convert5.gno @@ -6,4 +6,4 @@ func main() { println(ints) } -// error: cannot convert (undefined) to int +// Error: cannot convert (undefined) to int diff --git a/gnovm/tests/files/float5_stdlibs.gno b/gnovm/tests/files/float5_stdlibs.gno index b3d8cd84713..564cddb0309 100644 --- a/gnovm/tests/files/float5_stdlibs.gno +++ b/gnovm/tests/files/float5_stdlibs.gno @@ -9,9 +9,9 @@ func main() { println(math.MaxFloat64, math.Float64bits(math.MaxFloat64)) } +// NOTE: 0x7f7fffff is 2139095039 +// NOTE: 0x7fefffffffffffff is 9218868437227405311 + // Output: // 3.4028234663852886e+38 2139095039 // 1.7976931348623157e+308 9218868437227405311 - -// NOTE: 0x7f7fffff is 2139095039 -// NOTE: 0x7fefffffffffffff is 9218868437227405311 diff --git a/gnovm/tests/files/switch8.gno b/gnovm/tests/files/switch8.gno index 7f9269e5c40..c43c72582c0 100644 --- a/gnovm/tests/files/switch8.gno +++ b/gnovm/tests/files/switch8.gno @@ -7,4 +7,4 @@ func main() { } // Error: -// 5:2: fallthrough statement out of place +// fallthrough statement out of place diff --git a/gnovm/tests/files/switch8b.gno b/gnovm/tests/files/switch8b.gno index b1704ec0313..cdf35caf784 100644 --- a/gnovm/tests/files/switch8b.gno +++ b/gnovm/tests/files/switch8b.gno @@ -12,4 +12,4 @@ func main() { } // Error: -// 10:3: cannot fallthrough final case in switch +// cannot fallthrough final case in switch diff --git a/gnovm/tests/files/switch8c.gno b/gnovm/tests/files/switch8c.gno index cd62e5ef874..6897b8a88fd 100644 --- a/gnovm/tests/files/switch8c.gno +++ b/gnovm/tests/files/switch8c.gno @@ -12,4 +12,4 @@ func main() { } // Error: -// 7:3: fallthrough statement out of place +// fallthrough statement out of place diff --git a/gnovm/tests/files/switch9.gno b/gnovm/tests/files/switch9.gno index 2d604438220..5f596de013a 100644 --- a/gnovm/tests/files/switch9.gno +++ b/gnovm/tests/files/switch9.gno @@ -13,4 +13,4 @@ func main() { } // Error: -// 9:3: cannot fallthrough in type switch +// cannot fallthrough in type switch diff --git a/gnovm/tests/imports.go b/gnovm/tests/imports.go index f2349580e63..6a3e6ab2bbb 100644 --- a/gnovm/tests/imports.go +++ b/gnovm/tests/imports.go @@ -79,7 +79,7 @@ func TestStore(rootDir, filesPath string, stdin io.Reader, stdout, stderr io.Wri baseDir := filepath.Join(filesPath, "extern", pkgPath[len(testPath):]) memPkg := gno.ReadMemPackage(baseDir, pkgPath) send := std.Coins{} - ctx := testContext(pkgPath, send) + ctx := TestContext(pkgPath, send) m2 := gno.NewMachineWithOptions(gno.MachineOptions{ PkgPath: "test", Output: stdout, @@ -390,7 +390,7 @@ func TestStore(rootDir, filesPath string, stdin io.Reader, stdout, stderr io.Wri } send := std.Coins{} - ctx := testContext(pkgPath, send) + ctx := TestContext(pkgPath, send) m2 := gno.NewMachineWithOptions(gno.MachineOptions{ PkgPath: "test", Output: stdout,