feat: setup testscripts coverage

.github/workflows/gnovm.yml

@@ -67,35 +67,68 @@ jobs:
           - _test.gnolang.other
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    env:
+      COVERAGE_DIR: "/tmp/coverage"
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.goversion }}
       - name: test
         working-directory: gnovm
+        env:
+          TXTARCOVERDIR: ${{ env.COVERAGE_DIR }}
         run: |
+          mkdir -p $COVERAGE_DIR
+
+          # Setup testing environements variables
           export GOPATH=$HOME/go
-          export GOTEST_FLAGS="-v -p 1 -timeout=30m -coverprofile=coverage.out -covermode=atomic"
+          export GOTEST_FLAGS="-v -p 1 -timeout=30m -covermode=atomic -test.gocoverdir=$COVERAGE_DIR" 
+
+          # Run target test
           make ${{ matrix.args }}
       - uses: actions/upload-artifact@v3
         if: ${{ runner.os == 'Linux' && matrix.goversion == '1.21.x' }}
         with:
           name: ${{runner.os}}-coverage-gnovm-${{ matrix.args}}-${{matrix.goversion}}
-          path: ./gnovm/coverage.out
+          path: ${{ env.COVERAGE_DIR }}
 
   upload-coverage:
     needs: test
     runs-on: ubuntu-latest
+    env:
+      COVERAGE_DATA: /tmp/coverage/coverage-raw
+      COVERAGE_OUTPUT: /tmp/coverage/coverage-out
+      COVERAGE_PROFILE: /tmp/coverage/coverage.txt
     steps:
-      - name: Download all previous coverage artifacts
+      - run: mkdir -p $COVERAGE_DATA $COVERAGE_OUTPUT
+      - name: Download all previous coverage data artifacts
         uses: actions/download-artifact@v3
         with:
-          path: ${{ runner.temp }}/coverage
+          path: ${{ env.COVERAGE_DATA }}
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.x"
+      - name: Merge coverages
+        working-directory: ${{ env.COVERAGE_DATA }}
+        run: |
+          # Create coverage directory list separate by comma
+          export COVERAGE_DIRS="$(ls | tr '\n' ',' | sed s/,$//)"
+
+          # Merge all coverage data directories from previous tests
+          go tool covdata merge -v 1 -i="$COVERAGE_DIRS" -o $COVERAGE_OUTPUT
+
+          # Print coverage percent for debug purpose if needed
+          echo 'coverage results:'
+          go tool covdata percent -i=$COVERAGE_OUTPUT
+
+          # Generate coverage profile
+          go tool covdata textfmt -v 1 -i=$COVERAGE_OUTPUT -o $COVERAGE_PROFILE
+
       - name: Upload combined coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          directory: ${{ runner.temp }}/coverage
+          files: ${{ env.COVERAGE_PROFILE }}
           token: ${{ secrets.CODECOV_TOKEN }} 
           fail_ci_if_error: ${{ github.repository == 'gnolang/gno' }}
 

gnovm/Makefile

@@ -46,15 +46,15 @@ _test.pkg:
 
 .PHONY: _test.gnolang
 _test.gnolang: _test.gnolang.native _test.gnolang.stdlibs _test.gnolang.realm _test.gnolang.pkg0 _test.gnolang.pkg1 _test.gnolang.pkg2 _test.gnolang.other
-_test.gnolang.other:;        go test $(GOTEST_FLAGS) tests/*.go -run "(TestFileStr|TestSelectors)"
-_test.gnolang.realm:;        go test $(GOTEST_FLAGS) tests/*.go -run "TestFiles/^zrealm"
-_test.gnolang.pkg0:;         go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)"
-_test.gnolang.pkg1:;         go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/regexp"
-_test.gnolang.pkg2:;         go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/bytes"
-_test.gnolang.native:;       go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/"
-_test.gnolang.stdlibs:;      go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/'
-_test.gnolang.native.sync:;  go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests
-_test.gnolang.stdlibs.sync:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests
+_test.gnolang.other:;        go test tests/*.go -run "(TestFileStr|TestSelectors)" $(GOTEST_FLAGS)
+_test.gnolang.realm:;        go test tests/*.go -run "TestFiles/^zrealm" $(GOTEST_FLAGS)
+_test.gnolang.pkg0:;         go test tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)" $(GOTEST_FLAGS)
+_test.gnolang.pkg1:;         go test tests/*.go -run "TestPackages/regexp" $(GOTEST_FLAGS)
+_test.gnolang.pkg2:;         go test tests/*.go -run "TestPackages/bytes" $(GOTEST_FLAGS)
+_test.gnolang.native:;       go test tests/*.go -test.short -run "TestFilesNative/" $(GOTEST_FLAGS)
+_test.gnolang.stdlibs:;      go test tests/*.go -test.short -run 'TestFiles$$/' $(GOTEST_FLAGS)
+_test.gnolang.native.sync:;  go test tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests $(GOTEST_FLAGS)
+_test.gnolang.stdlibs.sync:; go test tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests $(GOTEST_FLAGS)
 
 ########################################
 # Code gen

gnovm/cmd/gno/build_test.go

@@ -3,9 +3,23 @@ package main
 import (
 	"testing"
 
+	"github.com/gnolang/gno/gnovm/pkg/integration"
 	"github.com/rogpeppe/go-internal/testscript"
+	"github.com/stretchr/testify/require"
 )
 
-func TestBuild(t *testing.T) {
-	testscript.Run(t, setupTestScript(t, "testdata/gno_build"))
+func Test_ScriptsBuild(t *testing.T) {
+	p := testscript.Params{
+		Dir: "testdata/gno_build",
+	}
+
+	if coverdir, ok := integration.ResolveCoverageDir(); ok {
+		err := integration.SetupTestscriptsCoverage(&p, coverdir)
+		require.NoError(t, err)
+	}
+
+	err := integration.SetupGno(&p, t.TempDir())
+	require.NoError(t, err)
+
+	testscript.Run(t, p)
 }

gnovm/cmd/gno/main_test.go

@@ -5,12 +5,10 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/rogpeppe/go-internal/testscript"
 	"github.com/stretchr/testify/require"
 
 	"github.com/gnolang/gno/tm2/pkg/commands"
@@ -144,45 +142,3 @@ func testMainCaseRun(t *testing.T, tc []testMainCase) {
 		})
 	}
 }
-
-func setupTestScript(t *testing.T, txtarDir string) testscript.Params {
-	t.Helper()
-	// Get root location of github.com/gnolang/gno
-	goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput()
-	require.NoError(t, err)
-	rootDir := filepath.Dir(string(goModPath))
-	// Build a fresh gno binary in a temp directory
-	gnoBin := filepath.Join(t.TempDir(), "gno")
-	err = exec.Command("go", "build", "-o", gnoBin, filepath.Join(rootDir, "gnovm", "cmd", "gno")).Run()
-	require.NoError(t, err)
-	// Define script params
-	return testscript.Params{
-		Setup: func(env *testscript.Env) error {
-			env.Vars = append(env.Vars,
-				"GNOROOT="+rootDir, // thx PR 1014 :)
-				// by default, $HOME=/no-home, but we need an existing $HOME directory
-				// because some commands needs to access $HOME/.cache/go-build
-				"HOME="+t.TempDir(),
-			)
-			return nil
-		},
-		Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
-			// add a custom "gno" command so txtar files can easily execute "gno"
-			// without knowing where is the binary or how it is executed.
-			"gno": func(ts *testscript.TestScript, neg bool, args []string) {
-				err := ts.Exec(gnoBin, args...)
-				if err != nil {
-					ts.Logf("[%v]\n", err)
-					if !neg {
-						ts.Fatalf("unexpected gno command failure")
-					}
-				} else {
-					if neg {
-						ts.Fatalf("unexpected gno command success")
-					}
-				}
-			},
-		},
-		Dir: txtarDir,
-	}
-}

gnovm/cmd/gno/test_test.go

@@ -3,9 +3,23 @@ package main
 import (
 	"testing"
 
+	"github.com/gnolang/gno/gnovm/pkg/integration"
 	"github.com/rogpeppe/go-internal/testscript"
+	"github.com/stretchr/testify/require"
 )
 
-func TestTest(t *testing.T) {
-	testscript.Run(t, setupTestScript(t, "testdata/gno_test"))
+func Test_ScriptsTest(t *testing.T) {
+	p := testscript.Params{
+		Dir: "testdata/gno_test",
+	}
+
+	if coverdir, ok := integration.ResolveCoverageDir(); ok {
+		err := integration.SetupTestscriptsCoverage(&p, coverdir)
+		require.NoError(t, err)
+	}
+
+	err := integration.SetupGno(&p, t.TempDir())
+	require.NoError(t, err)
+
+	testscript.Run(t, p)
 }

gnovm/pkg/integration/coverage.go

@@ -0,0 +1,68 @@
+package integration
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/rogpeppe/go-internal/testscript"
+)
+
+var coverageEnv struct {
+	coverdir string
+}
+
+func init() {
+	flag.StringVar(&coverageEnv.coverdir,
+		"txtarcoverdir", "", "write testscripts coverage intermediate files to this directory")
+}
+
+// ResolveCoverageDir attempts to resolve the coverage directory from the 'TXTARCOVERDIR'
+// environment variable first, and if not set, from the 'test.txtarcoverdir' flag.
+// It returns the resolved directory and a boolean indicating if the resolution was successful.
+func ResolveCoverageDir() (string, bool) {
+	// Attempt to resolve the cover directory from the environment variable or flag
+	coverdir := os.Getenv("TXTARCOVERDIR")
+	if coverdir == "" {
+		coverdir = coverageEnv.coverdir
+	}
+
+	return coverdir, coverdir != ""
+}
+
+// SetupTestscriptsCoverage sets up the given testscripts environment for coverage.
+// It will mostly override `GOCOVERDIR` with the target cover directory
+func SetupTestscriptsCoverage(p *testscript.Params, coverdir string) error {
+	// Check if the given coverage directory exist
+	info, err := os.Stat(coverdir)
+	if err != nil {
+		return fmt.Errorf("output directory %q inaccessible: %w", coverdir, err)
+	} else if !info.IsDir() {
+		return fmt.Errorf("output %q not a directory", coverdir)
+	}
+
+	// We need to have an absolute path here, because current directory
+	// context will change while executing testscripts.
+	if !filepath.IsAbs(coverdir) {
+		var err error
+		if coverdir, err = filepath.Abs(coverdir); err != nil {
+			return fmt.Errorf("unable to determine absolute path of %q: %w", coverdir, err)
+		}
+	}
+
+	// Backup the original setup function
+	origSetup := p.Setup
+	p.Setup = func(env *testscript.Env) error {
+		if origSetup != nil {
+			// Call previous setup first
+			origSetup(env)
+		}
+
+		// Override `GOCOVEDIR` directory for sub-execution
+		env.Setenv("GOCOVERDIR", coverdir)
+		return nil
+	}
+
+	return nil
+}

gnovm/pkg/integration/gno.go

@@ -0,0 +1,108 @@
+package integration
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	osm "github.com/gnolang/gno/tm2/pkg/os"
+	"github.com/rogpeppe/go-internal/testscript"
+)
+
+// SetupGno prepares the given testscript environment for tests that utilize the gno command.
+// If the `gno` binary doesn't exist, it's built using the `go build` command into the specified buildDir.
+// The function also include the `gno` command into `p.Cmds` to and wrap environment into p.Setup
+// to correctly set up the environment variables needed for the `gno` command.
+func SetupGno(p *testscript.Params, buildDir string) error {
+	// Try to fetch `GNOROOT` from the environment variables
+	gnoroot := os.Getenv("GNOROOT")
+	if gnoroot == "" {
+		// If `GNOROOT` isn't set, determine the root directory of github.com/gnolang/gno
+		goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("unable to determine gno root directory")
+		}
+
+		gnoroot = filepath.Dir(string(goModPath))
+	}
+
+	if !osm.DirExists(buildDir) {
+		return fmt.Errorf("%q does not exist or is not a directory", buildDir)
+	}
+
+	// Determine the path to the gno binary within the build directory
+	gnoBin := filepath.Join(buildDir, "gno")
+	if _, err := os.Stat(gnoBin); err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			// Handle other potential errors from os.Stat
+			return err
+		}
+
+		// Build a fresh gno binary in a temp directory
+		gnoArgsBuilder := []string{"build", "-o", gnoBin}
+
+		// Forward `-covermode` settings if set
+		if coverMode := testing.CoverMode(); coverMode != "" {
+			gnoArgsBuilder = append(gnoArgsBuilder, "-covermode", coverMode)
+		}
+
+		// Append the path to the gno command source
+		gnoArgsBuilder = append(gnoArgsBuilder, filepath.Join(gnoroot, "gnovm", "cmd", "gno"))
+
+		if err = exec.Command("go", gnoArgsBuilder...).Run(); err != nil {
+			return fmt.Errorf("unable to build gno binary: %w", err)
+		}
+	}
+
+	// Store the original setup scripts for potential wrapping
+	origSetup := p.Setup
+	p.Setup = func(env *testscript.Env) error {
+		// If there's an original setup, execute it
+		if origSetup != nil {
+			if err := origSetup(env); err != nil {
+				return err
+			}
+		}
+
+		// Set the GNOROOT environment variable
+		env.Setenv("GNOROOT", gnoroot)
+
+		// Create a temporary home directory because certain commands require access to $HOME/.cache/go-build
+		home, err := os.MkdirTemp("", "gno")
+		if err != nil {
+			return fmt.Errorf("unable to create temporary home directory: %w", err)
+		}
+		env.Setenv("HOME", home)
+
+		// Cleanup home folder
+		env.Defer(func() { os.RemoveAll(home) })
+
+		return nil
+	}
+
+	// Initialize cmds map if needed
+	if p.Cmds == nil {
+		p.Cmds = make(map[string]func(ts *testscript.TestScript, neg bool, args []string))
+	}
+
+	// Register the gno command for testscripts
+	p.Cmds["gno"] = func(ts *testscript.TestScript, neg bool, args []string) {
+		err := ts.Exec(gnoBin, args...)
+		if err != nil {
+			ts.Logf("gno command error: %v", err)
+		}
+
+		commandSucceeded := (err == nil)
+		successExpected := !neg
+
+		// Compare the command's success status with the expected outcome.
+		if commandSucceeded != successExpected {
+			ts.Fatalf("unexpected gno command outcome (err=%t expected=%t)", commandSucceeded, successExpected)
+		}
+	}
+
+	return nil
+}