diff --git a/.github/workflows/gnovm.yml b/.github/workflows/gnovm.yml index 71b03b5ca05..5fa68ac305a 100644 --- a/.github/workflows/gnovm.yml +++ b/.github/workflows/gnovm.yml @@ -67,6 +67,8 @@ jobs: - _test.gnolang.other runs-on: ubuntu-latest timeout-minutes: 15 + env: + COVERAGE_DIR: "/tmp/coverage" steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 @@ -74,28 +76,59 @@ jobs: go-version: ${{ matrix.goversion }} - name: test working-directory: gnovm + env: + TXTARCOVERDIR: ${{ env.COVERAGE_DIR }} run: | + mkdir -p $COVERAGE_DIR + + # Setup testing environements variables export GOPATH=$HOME/go - export GOTEST_FLAGS="-v -p 1 -timeout=30m -coverprofile=coverage.out -covermode=atomic" + export GOTEST_FLAGS="-v -p 1 -timeout=30m -covermode=atomic -test.gocoverdir=$COVERAGE_DIR" + + # Run target test make ${{ matrix.args }} - uses: actions/upload-artifact@v3 if: ${{ runner.os == 'Linux' && matrix.goversion == '1.21.x' }} with: name: ${{runner.os}}-coverage-gnovm-${{ matrix.args}}-${{matrix.goversion}} - path: ./gnovm/coverage.out + path: ${{ env.COVERAGE_DIR }} upload-coverage: needs: test runs-on: ubuntu-latest + env: + COVERAGE_DATA: /tmp/coverage/coverage-raw + COVERAGE_OUTPUT: /tmp/coverage/coverage-out + COVERAGE_PROFILE: /tmp/coverage/coverage.txt steps: - - name: Download all previous coverage artifacts + - run: mkdir -p $COVERAGE_DATA $COVERAGE_OUTPUT + - name: Download all previous coverage data artifacts uses: actions/download-artifact@v3 with: - path: ${{ runner.temp }}/coverage + path: ${{ env.COVERAGE_DATA }} + - uses: actions/setup-go@v4 + with: + go-version: "1.21.x" + - name: Merge coverages + working-directory: ${{ env.COVERAGE_DATA }} + run: | + # Create coverage directory list separate by comma + export COVERAGE_DIRS="$(ls | tr '\n' ',' | sed s/,$//)" + + # Merge all coverage data directories from previous tests + go tool covdata merge -v 1 -i="$COVERAGE_DIRS" -o $COVERAGE_OUTPUT + + # Print coverage percent for debug purpose if needed + echo 'coverage results:' + go tool covdata percent -i=$COVERAGE_OUTPUT + + # Generate coverage profile + go tool covdata textfmt -v 1 -i=$COVERAGE_OUTPUT -o $COVERAGE_PROFILE + - name: Upload combined coverage to Codecov uses: codecov/codecov-action@v3 with: - directory: ${{ runner.temp }}/coverage + files: ${{ env.COVERAGE_PROFILE }} token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: ${{ github.repository == 'gnolang/gno' }} diff --git a/gnovm/Makefile b/gnovm/Makefile index 5fcacf94f62..34e94f88633 100644 --- a/gnovm/Makefile +++ b/gnovm/Makefile @@ -46,15 +46,15 @@ _test.pkg: .PHONY: _test.gnolang _test.gnolang: _test.gnolang.native _test.gnolang.stdlibs _test.gnolang.realm _test.gnolang.pkg0 _test.gnolang.pkg1 _test.gnolang.pkg2 _test.gnolang.other -_test.gnolang.other:; go test $(GOTEST_FLAGS) tests/*.go -run "(TestFileStr|TestSelectors)" -_test.gnolang.realm:; go test $(GOTEST_FLAGS) tests/*.go -run "TestFiles/^zrealm" -_test.gnolang.pkg0:; go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)" -_test.gnolang.pkg1:; go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/regexp" -_test.gnolang.pkg2:; go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/bytes" -_test.gnolang.native:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/" -_test.gnolang.stdlibs:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/' -_test.gnolang.native.sync:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests -_test.gnolang.stdlibs.sync:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests +_test.gnolang.other:; go test tests/*.go -run "(TestFileStr|TestSelectors)" $(GOTEST_FLAGS) +_test.gnolang.realm:; go test tests/*.go -run "TestFiles/^zrealm" $(GOTEST_FLAGS) +_test.gnolang.pkg0:; go test tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)" $(GOTEST_FLAGS) +_test.gnolang.pkg1:; go test tests/*.go -run "TestPackages/regexp" $(GOTEST_FLAGS) +_test.gnolang.pkg2:; go test tests/*.go -run "TestPackages/bytes" $(GOTEST_FLAGS) +_test.gnolang.native:; go test tests/*.go -test.short -run "TestFilesNative/" $(GOTEST_FLAGS) +_test.gnolang.stdlibs:; go test tests/*.go -test.short -run 'TestFiles$$/' $(GOTEST_FLAGS) +_test.gnolang.native.sync:; go test tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests $(GOTEST_FLAGS) +_test.gnolang.stdlibs.sync:; go test tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests $(GOTEST_FLAGS) ######################################## # Code gen diff --git a/gnovm/cmd/gno/build_test.go b/gnovm/cmd/gno/build_test.go index 5bb03ef0d35..81aed7d1c79 100644 --- a/gnovm/cmd/gno/build_test.go +++ b/gnovm/cmd/gno/build_test.go @@ -3,9 +3,23 @@ package main import ( "testing" + "github.com/gnolang/gno/gnovm/pkg/integration" "github.com/rogpeppe/go-internal/testscript" + "github.com/stretchr/testify/require" ) -func TestBuild(t *testing.T) { - testscript.Run(t, setupTestScript(t, "testdata/gno_build")) +func Test_ScriptsBuild(t *testing.T) { + p := testscript.Params{ + Dir: "testdata/gno_build", + } + + if coverdir, ok := integration.ResolveCoverageDir(); ok { + err := integration.SetupTestscriptsCoverage(&p, coverdir) + require.NoError(t, err) + } + + err := integration.SetupGno(&p, t.TempDir()) + require.NoError(t, err) + + testscript.Run(t, p) } diff --git a/gnovm/cmd/gno/main_test.go b/gnovm/cmd/gno/main_test.go index 8d19a50e814..371cdf913e8 100644 --- a/gnovm/cmd/gno/main_test.go +++ b/gnovm/cmd/gno/main_test.go @@ -5,12 +5,10 @@ import ( "context" "fmt" "os" - "os/exec" "path/filepath" "strings" "testing" - "github.com/rogpeppe/go-internal/testscript" "github.com/stretchr/testify/require" "github.com/gnolang/gno/tm2/pkg/commands" @@ -144,45 +142,3 @@ func testMainCaseRun(t *testing.T, tc []testMainCase) { }) } } - -func setupTestScript(t *testing.T, txtarDir string) testscript.Params { - t.Helper() - // Get root location of github.com/gnolang/gno - goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput() - require.NoError(t, err) - rootDir := filepath.Dir(string(goModPath)) - // Build a fresh gno binary in a temp directory - gnoBin := filepath.Join(t.TempDir(), "gno") - err = exec.Command("go", "build", "-o", gnoBin, filepath.Join(rootDir, "gnovm", "cmd", "gno")).Run() - require.NoError(t, err) - // Define script params - return testscript.Params{ - Setup: func(env *testscript.Env) error { - env.Vars = append(env.Vars, - "GNOROOT="+rootDir, // thx PR 1014 :) - // by default, $HOME=/no-home, but we need an existing $HOME directory - // because some commands needs to access $HOME/.cache/go-build - "HOME="+t.TempDir(), - ) - return nil - }, - Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){ - // add a custom "gno" command so txtar files can easily execute "gno" - // without knowing where is the binary or how it is executed. - "gno": func(ts *testscript.TestScript, neg bool, args []string) { - err := ts.Exec(gnoBin, args...) - if err != nil { - ts.Logf("[%v]\n", err) - if !neg { - ts.Fatalf("unexpected gno command failure") - } - } else { - if neg { - ts.Fatalf("unexpected gno command success") - } - } - }, - }, - Dir: txtarDir, - } -} diff --git a/gnovm/cmd/gno/test_test.go b/gnovm/cmd/gno/test_test.go index f5b069a5f03..b1dcfb21d29 100644 --- a/gnovm/cmd/gno/test_test.go +++ b/gnovm/cmd/gno/test_test.go @@ -3,9 +3,23 @@ package main import ( "testing" + "github.com/gnolang/gno/gnovm/pkg/integration" "github.com/rogpeppe/go-internal/testscript" + "github.com/stretchr/testify/require" ) -func TestTest(t *testing.T) { - testscript.Run(t, setupTestScript(t, "testdata/gno_test")) +func Test_ScriptsTest(t *testing.T) { + p := testscript.Params{ + Dir: "testdata/gno_test", + } + + if coverdir, ok := integration.ResolveCoverageDir(); ok { + err := integration.SetupTestscriptsCoverage(&p, coverdir) + require.NoError(t, err) + } + + err := integration.SetupGno(&p, t.TempDir()) + require.NoError(t, err) + + testscript.Run(t, p) } diff --git a/gnovm/pkg/integration/coverage.go b/gnovm/pkg/integration/coverage.go new file mode 100644 index 00000000000..017f5f9de88 --- /dev/null +++ b/gnovm/pkg/integration/coverage.go @@ -0,0 +1,68 @@ +package integration + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/rogpeppe/go-internal/testscript" +) + +var coverageEnv struct { + coverdir string +} + +func init() { + flag.StringVar(&coverageEnv.coverdir, + "txtarcoverdir", "", "write testscripts coverage intermediate files to this directory") +} + +// ResolveCoverageDir attempts to resolve the coverage directory from the 'TXTARCOVERDIR' +// environment variable first, and if not set, from the 'test.txtarcoverdir' flag. +// It returns the resolved directory and a boolean indicating if the resolution was successful. +func ResolveCoverageDir() (string, bool) { + // Attempt to resolve the cover directory from the environment variable or flag + coverdir := os.Getenv("TXTARCOVERDIR") + if coverdir == "" { + coverdir = coverageEnv.coverdir + } + + return coverdir, coverdir != "" +} + +// SetupTestscriptsCoverage sets up the given testscripts environment for coverage. +// It will mostly override `GOCOVERDIR` with the target cover directory +func SetupTestscriptsCoverage(p *testscript.Params, coverdir string) error { + // Check if the given coverage directory exist + info, err := os.Stat(coverdir) + if err != nil { + return fmt.Errorf("output directory %q inaccessible: %w", coverdir, err) + } else if !info.IsDir() { + return fmt.Errorf("output %q not a directory", coverdir) + } + + // We need to have an absolute path here, because current directory + // context will change while executing testscripts. + if !filepath.IsAbs(coverdir) { + var err error + if coverdir, err = filepath.Abs(coverdir); err != nil { + return fmt.Errorf("unable to determine absolute path of %q: %w", coverdir, err) + } + } + + // Backup the original setup function + origSetup := p.Setup + p.Setup = func(env *testscript.Env) error { + if origSetup != nil { + // Call previous setup first + origSetup(env) + } + + // Override `GOCOVEDIR` directory for sub-execution + env.Setenv("GOCOVERDIR", coverdir) + return nil + } + + return nil +} diff --git a/gnovm/pkg/integration/gno.go b/gnovm/pkg/integration/gno.go new file mode 100644 index 00000000000..56e18bbb5db --- /dev/null +++ b/gnovm/pkg/integration/gno.go @@ -0,0 +1,108 @@ +package integration + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + osm "github.com/gnolang/gno/tm2/pkg/os" + "github.com/rogpeppe/go-internal/testscript" +) + +// SetupGno prepares the given testscript environment for tests that utilize the gno command. +// If the `gno` binary doesn't exist, it's built using the `go build` command into the specified buildDir. +// The function also include the `gno` command into `p.Cmds` to and wrap environment into p.Setup +// to correctly set up the environment variables needed for the `gno` command. +func SetupGno(p *testscript.Params, buildDir string) error { + // Try to fetch `GNOROOT` from the environment variables + gnoroot := os.Getenv("GNOROOT") + if gnoroot == "" { + // If `GNOROOT` isn't set, determine the root directory of github.com/gnolang/gno + goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput() + if err != nil { + return fmt.Errorf("unable to determine gno root directory") + } + + gnoroot = filepath.Dir(string(goModPath)) + } + + if !osm.DirExists(buildDir) { + return fmt.Errorf("%q does not exist or is not a directory", buildDir) + } + + // Determine the path to the gno binary within the build directory + gnoBin := filepath.Join(buildDir, "gno") + if _, err := os.Stat(gnoBin); err != nil { + if !errors.Is(err, os.ErrNotExist) { + // Handle other potential errors from os.Stat + return err + } + + // Build a fresh gno binary in a temp directory + gnoArgsBuilder := []string{"build", "-o", gnoBin} + + // Forward `-covermode` settings if set + if coverMode := testing.CoverMode(); coverMode != "" { + gnoArgsBuilder = append(gnoArgsBuilder, "-covermode", coverMode) + } + + // Append the path to the gno command source + gnoArgsBuilder = append(gnoArgsBuilder, filepath.Join(gnoroot, "gnovm", "cmd", "gno")) + + if err = exec.Command("go", gnoArgsBuilder...).Run(); err != nil { + return fmt.Errorf("unable to build gno binary: %w", err) + } + } + + // Store the original setup scripts for potential wrapping + origSetup := p.Setup + p.Setup = func(env *testscript.Env) error { + // If there's an original setup, execute it + if origSetup != nil { + if err := origSetup(env); err != nil { + return err + } + } + + // Set the GNOROOT environment variable + env.Setenv("GNOROOT", gnoroot) + + // Create a temporary home directory because certain commands require access to $HOME/.cache/go-build + home, err := os.MkdirTemp("", "gno") + if err != nil { + return fmt.Errorf("unable to create temporary home directory: %w", err) + } + env.Setenv("HOME", home) + + // Cleanup home folder + env.Defer(func() { os.RemoveAll(home) }) + + return nil + } + + // Initialize cmds map if needed + if p.Cmds == nil { + p.Cmds = make(map[string]func(ts *testscript.TestScript, neg bool, args []string)) + } + + // Register the gno command for testscripts + p.Cmds["gno"] = func(ts *testscript.TestScript, neg bool, args []string) { + err := ts.Exec(gnoBin, args...) + if err != nil { + ts.Logf("gno command error: %v", err) + } + + commandSucceeded := (err == nil) + successExpected := !neg + + // Compare the command's success status with the expected outcome. + if commandSucceeded != successExpected { + ts.Fatalf("unexpected gno command outcome (err=%t expected=%t)", commandSucceeded, successExpected) + } + } + + return nil +}