feat: setup testscripts coverage

.github/workflows/gnovm.yml

@@ -67,35 +67,68 @@ jobs:
           - _test.gnolang.other
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    env:
+      COVERAGE_DIR: "/tmp/coverage"
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.goversion }}
       - name: test
         working-directory: gnovm
+        env:
+          GOCOVERDIR_TXTAR: ${{ env.COVERAGE_DIR }}
         run: |
+          mkdir -p $COVERAGE_DIR
+
+          # Setup testing environements variables
           export GOPATH=$HOME/go
-          export GOTEST_FLAGS="-v -p 1 -timeout=30m -coverprofile=coverage.out -covermode=atomic"
+          export GOTEST_FLAGS="-v -p 1 -timeout=30m -covermode=atomic -args -test.gocoverdir=$COVERAGE_DIR"
+
+          # Run target test
           make ${{ matrix.args }}
       - uses: actions/upload-artifact@v3
         if: ${{ runner.os == 'Linux' && matrix.goversion == '1.21.x' }}
         with:
           name: ${{runner.os}}-coverage-gnovm-${{ matrix.args}}-${{matrix.goversion}}
-          path: ./gnovm/coverage.out
+          path: ${{ env.COVERAGE_DIR }}
 
   upload-coverage:
     needs: test
     runs-on: ubuntu-latest
+    env:
+      COVERAGE_DATA: /tmp/coverage/coverage-raw
+      COVERAGE_OUTPUT: /tmp/coverage/coverage-out
+      COVERAGE_PROFILE: /tmp/coverage/coverage.txt
     steps:
-      - name: Download all previous coverage artifacts
+      - run: mkdir -p $COVERAGE_DATA $COVERAGE_OUTPUT
+      - name: Download all previous coverage data artifacts
         uses: actions/download-artifact@v3
         with:
-          path: ${{ runner.temp }}/coverage
+          path: ${{ env.COVERAGE_DATA }}
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.x"
+      - name: Merge coverages
+        working-directory: ${{ env.COVERAGE_DATA }}
+        run: |
+          # Create coverage directory list separate by comma
+          export COVERAGE_DIRS="$(ls | tr '\n' ',' | sed s/,$//)"
+
+          # Merge all coverage data directories from previous tests
+          go tool covdata merge -v 1 -i="$COVERAGE_DIRS" -o $COVERAGE_OUTPUT
+
+          # Print coverage percent for debug purpose if needed
+          echo 'coverage results:'
+          go tool covdata percent -i=$COVERAGE_OUTPUT
+
+          # Generating coverage profile
+          go tool covdata textfmt -v 1 -i=$COVERAGE_OUTPUT -o $COVERAGE_PROFILE
+
       - name: Upload combined coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          directory: ${{ runner.temp }}/coverage
+          files: ${{ env.COVERAGE_PROFILE }}
           token: ${{ secrets.CODECOV_TOKEN }} 
           fail_ci_if_error: ${{ github.repository == 'gnolang/gno' }}
 

gnovm/Makefile

@@ -46,15 +46,15 @@ _test.pkg:
 
 .PHONY: _test.gnolang
 _test.gnolang: _test.gnolang.native _test.gnolang.stdlibs _test.gnolang.realm _test.gnolang.pkg0 _test.gnolang.pkg1 _test.gnolang.pkg2 _test.gnolang.other
-_test.gnolang.other:;        go test $(GOTEST_FLAGS) tests/*.go -run "(TestFileStr|TestSelectors)"
-_test.gnolang.realm:;        go test $(GOTEST_FLAGS) tests/*.go -run "TestFiles/^zrealm"
-_test.gnolang.pkg0:;         go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)"
-_test.gnolang.pkg1:;         go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/regexp"
-_test.gnolang.pkg2:;         go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/bytes"
-_test.gnolang.native:;       go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/"
-_test.gnolang.stdlibs:;      go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/'
-_test.gnolang.native.sync:;  go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests
-_test.gnolang.stdlibs.sync:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests
+_test.gnolang.other:;        go test tests/*.go -run "(TestFileStr|TestSelectors)" $(GOTEST_FLAGS)
+_test.gnolang.realm:;        go test tests/*.go -run "TestFiles/^zrealm" $(GOTEST_FLAGS)
+_test.gnolang.pkg0:;         go test tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)" $(GOTEST_FLAGS)
+_test.gnolang.pkg1:;         go test tests/*.go -run "TestPackages/regexp" $(GOTEST_FLAGS)
+_test.gnolang.pkg2:;         go test tests/*.go -run "TestPackages/bytes" $(GOTEST_FLAGS)
+_test.gnolang.native:;       go test tests/*.go -test.short -run "TestFilesNative/" $(GOTEST_FLAGS)
+_test.gnolang.stdlibs:;      go test tests/*.go -test.short -run 'TestFiles$$/' $(GOTEST_FLAGS)
+_test.gnolang.native.sync:;  go test tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests $(GOTEST_FLAGS)
+_test.gnolang.stdlibs.sync:; go test tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests $(GOTEST_FLAGS)
 
 ########################################
 # Code gen

gnovm/cmd/gno/build_test.go

@@ -3,9 +3,20 @@ package main
 import (
 	"testing"
 
+	"github.com/gnolang/gno/gnovm/pkg/integration"
 	"github.com/rogpeppe/go-internal/testscript"
+	"github.com/stretchr/testify/require"
 )
 
-func TestBuild(t *testing.T) {
-	testscript.Run(t, setupTestScript(t, "testdata/gno_build"))
+func Test_ScriptsBuild(t *testing.T) {
+	p := testscript.Params{
+		Dir: "testdata/gno_build",
+	}
+
+	err := integration.SetupCoverage(&p)
+	require.NoError(t, err)
+	err = integration.SetupGno(&p, t.TempDir())
+	require.NoError(t, err)
+
+	testscript.Run(t, p)
 }

gnovm/cmd/gno/main_test.go

@@ -5,12 +5,10 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/rogpeppe/go-internal/testscript"
 	"github.com/stretchr/testify/require"
 
 	"github.com/gnolang/gno/tm2/pkg/commands"
@@ -144,45 +142,3 @@ func testMainCaseRun(t *testing.T, tc []testMainCase) {
 		})
 	}
 }
-
-func setupTestScript(t *testing.T, txtarDir string) testscript.Params {
-	t.Helper()
-	// Get root location of github.com/gnolang/gno
-	goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput()
-	require.NoError(t, err)
-	rootDir := filepath.Dir(string(goModPath))
-	// Build a fresh gno binary in a temp directory
-	gnoBin := filepath.Join(t.TempDir(), "gno")
-	err = exec.Command("go", "build", "-o", gnoBin, filepath.Join(rootDir, "gnovm", "cmd", "gno")).Run()
-	require.NoError(t, err)
-	// Define script params
-	return testscript.Params{
-		Setup: func(env *testscript.Env) error {
-			env.Vars = append(env.Vars,
-				"GNOROOT="+rootDir, // thx PR 1014 :)
-				// by default, $HOME=/no-home, but we need an existing $HOME directory
-				// because some commands needs to access $HOME/.cache/go-build
-				"HOME="+t.TempDir(),
-			)
-			return nil
-		},
-		Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
-			// add a custom "gno" command so txtar files can easily execute "gno"
-			// without knowing where is the binary or how it is executed.
-			"gno": func(ts *testscript.TestScript, neg bool, args []string) {
-				err := ts.Exec(gnoBin, args...)
-				if err != nil {
-					ts.Logf("[%v]\n", err)
-					if !neg {
-						ts.Fatalf("unexpected gno command failure")
-					}
-				} else {
-					if neg {
-						ts.Fatalf("unexpected gno command success")
-					}
-				}
-			},
-		},
-		Dir: txtarDir,
-	}
-}

gnovm/cmd/gno/test_test.go

@@ -3,9 +3,20 @@ package main
 import (
 	"testing"
 
+	"github.com/gnolang/gno/gnovm/pkg/integration"
 	"github.com/rogpeppe/go-internal/testscript"
+	"github.com/stretchr/testify/require"
 )
 
-func TestTest(t *testing.T) {
-	testscript.Run(t, setupTestScript(t, "testdata/gno_test"))
+func Test_ScriptsTest(t *testing.T) {
+	p := testscript.Params{
+		Dir: "testdata/gno_test",
+	}
+
+	err := integration.SetupCoverage(&p)
+	require.NoError(t, err)
+	err = integration.SetupGno(&p, t.TempDir())
+	require.NoError(t, err)
+
+	testscript.Run(t, p)
 }

gnovm/pkg/integration/coverage.go

@@ -0,0 +1,66 @@
+package integration
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/rogpeppe/go-internal/testscript"
+)
+
+var coverageEnv struct {
+	coverdir string
+	once     sync.Once
+}
+
+// SetupCoverage sets up the given testscripts environment for coverage.
+// It will need both, covermode and a target directory defined with
+// `GOCOVERDIR_TXTAR` environements variable to be set to be effective
+func SetupCoverage(p *testscript.Params) error {
+	coverdir := os.Getenv("GOCOVERDIR_TXTAR")
+	if testing.CoverMode() == "" || coverdir == "" {
+		return nil
+	}
+
+	var err error
+
+	// We need to have an absolute path here, because current directory
+	// context will change while execution test scripts.
+	if !filepath.IsAbs(coverdir) {
+		coverdir, err = filepath.Abs(coverdir)
+		if err != nil {
+			return fmt.Errorf("unable to determine absolute path of %q: %w", coverdir, err)
+		}
+	}
+
+	// If the given coverage directory doesn't exist, create it
+	info, err := os.Stat(coverdir)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			if mkErr := os.Mkdir(coverdir, 0o755); mkErr != nil {
+				return fmt.Errorf("failed to testscripts coverage dir %q: %w", coverdir, mkErr)
+			}
+		} else {
+			// Handle other potential errors from os.Stat
+			return fmt.Errorf("failed to stat %q: %w", coverdir, err)
+		}
+	} else if !info.IsDir() {
+		return fmt.Errorf("coverage: %q is not a directory", coverdir)
+	}
+
+	origSetup := p.Setup
+	p.Setup = func(env *testscript.Env) error {
+		if origSetup != nil {
+			origSetup(env)
+		}
+
+		// Override `GOCOVEDIR` directory for sub-execution
+		env.Setenv("GOCOVERDIR", coverdir)
+		return nil
+	}
+
+	return nil
+}

gnovm/pkg/integration/gno.go

@@ -0,0 +1,104 @@
+package integration
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/rogpeppe/go-internal/testscript"
+)
+
+// SetupGno sets up the given testscripts environment for tests that use the gno
+// command. It build `gno` using `go build` command into the given buildDir if
+// not existing already.
+// It will add `gno` command to p.Cmds. It also wraps p.Setup to set up the environment
+// variables for running the go command appropriately.
+func SetupGno(p *testscript.Params, buildDir string) error {
+	gnoroot := os.Getenv("GNOROOT")
+	if gnoroot == "" {
+		// Get root location of github.com/gnolang/gno
+		goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("unable to determine gno root directory")
+		}
+
+		gnoroot = filepath.Dir(string(goModPath))
+	}
+
+	info, err := os.Stat(buildDir)
+	if err != nil {
+		return fmt.Errorf("unable to stat: %q", buildDir)
+	}
+
+	if !info.IsDir() {
+		return fmt.Errorf("given buildDir is not a directory: %q", buildDir)
+	}
+
+	gnoBin := filepath.Join(buildDir, "gno")
+	if _, err = os.Stat(gnoBin); errors.Is(err, os.ErrNotExist) {
+		// Build a fresh gno binary in a temp directory
+		gnoArgsBuilder := []string{"build", "-o", gnoBin}
+
+		// Add coverage if needed
+		if coverMode := testing.CoverMode(); coverMode != "" {
+			gnoArgsBuilder = append(gnoArgsBuilder, "-covermode", coverMode)
+		}
+
+		// Add target command
+		gnoArgsBuilder = append(gnoArgsBuilder, filepath.Join(gnoroot, "gnovm", "cmd", "gno"))
+
+		if err = exec.Command("go", gnoArgsBuilder...).Run(); err != nil {
+			return fmt.Errorf("uanble to build gno binary: %w", err)
+		}
+	} else if err != nil {
+		// Return any other errors
+		return err
+	}
+
+	// Wrap setup scripts
+	origSetup := p.Setup
+	p.Setup = func(env *testscript.Env) error {
+		if origSetup != nil {
+			if err := origSetup(env); err != nil {
+				return err
+			}
+		}
+
+		env.Setenv("GNOROOT", gnoroot) // thx PR 1014 :)
+
+		// by default, $HOME=/no-home, but we need an existing $HOME directory
+		// because some commands needs to access $HOME/.cache/go-build
+		home, err := os.MkdirTemp("", "gno")
+		if err != nil {
+			return fmt.Errorf("unable to create temporary home directory: %w", err)
+		}
+		env.Setenv("HOME", home)
+		env.Defer(func() { os.RemoveAll(home) }) // cleanup
+
+		return nil
+	}
+
+	if p.Cmds == nil {
+		p.Cmds = make(map[string]func(ts *testscript.TestScript, neg bool, args []string))
+	}
+
+	// Set gno command
+	p.Cmds["gno"] = func(ts *testscript.TestScript, neg bool, args []string) {
+		err := ts.Exec(gnoBin, args...)
+		if err != nil {
+			ts.Logf("[%v]\n", err)
+			if !neg {
+				ts.Fatalf("unexpected gno command failure")
+			}
+		} else {
+			if neg {
+				ts.Fatalf("unexpected gno command success")
+			}
+		}
+	}
+
+	return nil
+}