Skip to content

Commit

Permalink
feat: setup testscripts coverage (gnolang#1249)
Browse files Browse the repository at this point in the history
Co-authored-by: Manfred Touron <94029+moul@users.noreply.github.com>
  • Loading branch information
gfanton and moul committed Nov 14, 2023
1 parent eecd0ff commit 29a4459
Show file tree
Hide file tree
Showing 7 changed files with 255 additions and 62 deletions.
43 changes: 38 additions & 5 deletions .github/workflows/gnovm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,35 +67,68 @@ jobs:
- _test.gnolang.other
runs-on: ubuntu-latest
timeout-minutes: 15
env:
COVERAGE_DIR: "/tmp/coverage"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
with:
go-version: ${{ matrix.goversion }}
- name: test
working-directory: gnovm
env:
TXTARCOVERDIR: ${{ env.COVERAGE_DIR }}
run: |
mkdir -p $COVERAGE_DIR
# Setup testing environements variables
export GOPATH=$HOME/go
export GOTEST_FLAGS="-v -p 1 -timeout=30m -coverprofile=coverage.out -covermode=atomic"
export GOTEST_FLAGS="-v -p 1 -timeout=30m -covermode=atomic -test.gocoverdir=$COVERAGE_DIR"
# Run target test
make ${{ matrix.args }}
- uses: actions/upload-artifact@v3
if: ${{ runner.os == 'Linux' && matrix.goversion == '1.21.x' }}
with:
name: ${{runner.os}}-coverage-gnovm-${{ matrix.args}}-${{matrix.goversion}}
path: ./gnovm/coverage.out
path: ${{ env.COVERAGE_DIR }}

upload-coverage:
needs: test
runs-on: ubuntu-latest
env:
COVERAGE_DATA: /tmp/coverage/coverage-raw
COVERAGE_OUTPUT: /tmp/coverage/coverage-out
COVERAGE_PROFILE: /tmp/coverage/coverage.txt
steps:
- name: Download all previous coverage artifacts
- run: mkdir -p $COVERAGE_DATA $COVERAGE_OUTPUT
- name: Download all previous coverage data artifacts
uses: actions/download-artifact@v3
with:
path: ${{ runner.temp }}/coverage
path: ${{ env.COVERAGE_DATA }}
- uses: actions/setup-go@v4
with:
go-version: "1.21.x"
- name: Merge coverages
working-directory: ${{ env.COVERAGE_DATA }}
run: |
# Create coverage directory list separate by comma
export COVERAGE_DIRS="$(ls | tr '\n' ',' | sed s/,$//)"
# Merge all coverage data directories from previous tests
go tool covdata merge -v 1 -i="$COVERAGE_DIRS" -o $COVERAGE_OUTPUT
# Print coverage percent for debug purpose if needed
echo 'coverage results:'
go tool covdata percent -i=$COVERAGE_OUTPUT
# Generate coverage profile
go tool covdata textfmt -v 1 -i=$COVERAGE_OUTPUT -o $COVERAGE_PROFILE
- name: Upload combined coverage to Codecov
uses: codecov/codecov-action@v3
with:
directory: ${{ runner.temp }}/coverage
files: ${{ env.COVERAGE_PROFILE }}
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: ${{ github.repository == 'gnolang/gno' }}

18 changes: 9 additions & 9 deletions gnovm/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ _test.pkg:

.PHONY: _test.gnolang
_test.gnolang: _test.gnolang.native _test.gnolang.stdlibs _test.gnolang.realm _test.gnolang.pkg0 _test.gnolang.pkg1 _test.gnolang.pkg2 _test.gnolang.other
_test.gnolang.other:; go test $(GOTEST_FLAGS) tests/*.go -run "(TestFileStr|TestSelectors)"
_test.gnolang.realm:; go test $(GOTEST_FLAGS) tests/*.go -run "TestFiles/^zrealm"
_test.gnolang.pkg0:; go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)"
_test.gnolang.pkg1:; go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/regexp"
_test.gnolang.pkg2:; go test $(GOTEST_FLAGS) tests/*.go -run "TestPackages/bytes"
_test.gnolang.native:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/"
_test.gnolang.stdlibs:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/'
_test.gnolang.native.sync:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests
_test.gnolang.stdlibs.sync:; go test $(GOTEST_FLAGS) tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests
_test.gnolang.other:; go test tests/*.go -run "(TestFileStr|TestSelectors)" $(GOTEST_FLAGS)
_test.gnolang.realm:; go test tests/*.go -run "TestFiles/^zrealm" $(GOTEST_FLAGS)
_test.gnolang.pkg0:; go test tests/*.go -run "TestPackages/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)" $(GOTEST_FLAGS)
_test.gnolang.pkg1:; go test tests/*.go -run "TestPackages/regexp" $(GOTEST_FLAGS)
_test.gnolang.pkg2:; go test tests/*.go -run "TestPackages/bytes" $(GOTEST_FLAGS)
_test.gnolang.native:; go test tests/*.go -test.short -run "TestFilesNative/" $(GOTEST_FLAGS)
_test.gnolang.stdlibs:; go test tests/*.go -test.short -run 'TestFiles$$/' $(GOTEST_FLAGS)
_test.gnolang.native.sync:; go test tests/*.go -test.short -run "TestFilesNative/" --update-golden-tests $(GOTEST_FLAGS)
_test.gnolang.stdlibs.sync:; go test tests/*.go -test.short -run 'TestFiles$$/' --update-golden-tests $(GOTEST_FLAGS)

########################################
# Code gen
Expand Down
18 changes: 16 additions & 2 deletions gnovm/cmd/gno/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,23 @@ package main
import (
"testing"

"github.com/gnolang/gno/gnovm/pkg/integration"
"github.com/rogpeppe/go-internal/testscript"
"github.com/stretchr/testify/require"
)

func TestBuild(t *testing.T) {
testscript.Run(t, setupTestScript(t, "testdata/gno_build"))
func Test_ScriptsBuild(t *testing.T) {
p := testscript.Params{
Dir: "testdata/gno_build",
}

if coverdir, ok := integration.ResolveCoverageDir(); ok {
err := integration.SetupTestscriptsCoverage(&p, coverdir)
require.NoError(t, err)
}

err := integration.SetupGno(&p, t.TempDir())
require.NoError(t, err)

testscript.Run(t, p)
}
44 changes: 0 additions & 44 deletions gnovm/cmd/gno/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,10 @@ import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"

"github.com/rogpeppe/go-internal/testscript"
"github.com/stretchr/testify/require"

"github.com/gnolang/gno/tm2/pkg/commands"
Expand Down Expand Up @@ -144,45 +142,3 @@ func testMainCaseRun(t *testing.T, tc []testMainCase) {
})
}
}

func setupTestScript(t *testing.T, txtarDir string) testscript.Params {
t.Helper()
// Get root location of github.com/gnolang/gno
goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput()
require.NoError(t, err)
rootDir := filepath.Dir(string(goModPath))
// Build a fresh gno binary in a temp directory
gnoBin := filepath.Join(t.TempDir(), "gno")
err = exec.Command("go", "build", "-o", gnoBin, filepath.Join(rootDir, "gnovm", "cmd", "gno")).Run()
require.NoError(t, err)
// Define script params
return testscript.Params{
Setup: func(env *testscript.Env) error {
env.Vars = append(env.Vars,
"GNOROOT="+rootDir, // thx PR 1014 :)
// by default, $HOME=/no-home, but we need an existing $HOME directory
// because some commands needs to access $HOME/.cache/go-build
"HOME="+t.TempDir(),
)
return nil
},
Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
// add a custom "gno" command so txtar files can easily execute "gno"
// without knowing where is the binary or how it is executed.
"gno": func(ts *testscript.TestScript, neg bool, args []string) {
err := ts.Exec(gnoBin, args...)
if err != nil {
ts.Logf("[%v]\n", err)
if !neg {
ts.Fatalf("unexpected gno command failure")
}
} else {
if neg {
ts.Fatalf("unexpected gno command success")
}
}
},
},
Dir: txtarDir,
}
}
18 changes: 16 additions & 2 deletions gnovm/cmd/gno/test_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,23 @@ package main
import (
"testing"

"github.com/gnolang/gno/gnovm/pkg/integration"
"github.com/rogpeppe/go-internal/testscript"
"github.com/stretchr/testify/require"
)

func TestTest(t *testing.T) {
testscript.Run(t, setupTestScript(t, "testdata/gno_test"))
func Test_ScriptsTest(t *testing.T) {
p := testscript.Params{
Dir: "testdata/gno_test",
}

if coverdir, ok := integration.ResolveCoverageDir(); ok {
err := integration.SetupTestscriptsCoverage(&p, coverdir)
require.NoError(t, err)
}

err := integration.SetupGno(&p, t.TempDir())
require.NoError(t, err)

testscript.Run(t, p)
}
68 changes: 68 additions & 0 deletions gnovm/pkg/integration/coverage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package integration

import (
"flag"
"fmt"
"os"
"path/filepath"

"github.com/rogpeppe/go-internal/testscript"
)

var coverageEnv struct {
coverdir string
}

func init() {
flag.StringVar(&coverageEnv.coverdir,
"txtarcoverdir", "", "write testscripts coverage intermediate files to this directory")
}

// ResolveCoverageDir attempts to resolve the coverage directory from the 'TXTARCOVERDIR'
// environment variable first, and if not set, from the 'test.txtarcoverdir' flag.
// It returns the resolved directory and a boolean indicating if the resolution was successful.
func ResolveCoverageDir() (string, bool) {
// Attempt to resolve the cover directory from the environment variable or flag
coverdir := os.Getenv("TXTARCOVERDIR")
if coverdir == "" {
coverdir = coverageEnv.coverdir
}

return coverdir, coverdir != ""
}

// SetupTestscriptsCoverage sets up the given testscripts environment for coverage.
// It will mostly override `GOCOVERDIR` with the target cover directory
func SetupTestscriptsCoverage(p *testscript.Params, coverdir string) error {
// Check if the given coverage directory exist
info, err := os.Stat(coverdir)
if err != nil {
return fmt.Errorf("output directory %q inaccessible: %w", coverdir, err)
} else if !info.IsDir() {
return fmt.Errorf("output %q not a directory", coverdir)
}

// We need to have an absolute path here, because current directory
// context will change while executing testscripts.
if !filepath.IsAbs(coverdir) {
var err error
if coverdir, err = filepath.Abs(coverdir); err != nil {
return fmt.Errorf("unable to determine absolute path of %q: %w", coverdir, err)
}
}

// Backup the original setup function
origSetup := p.Setup
p.Setup = func(env *testscript.Env) error {
if origSetup != nil {
// Call previous setup first
origSetup(env)
}

// Override `GOCOVEDIR` directory for sub-execution
env.Setenv("GOCOVERDIR", coverdir)
return nil
}

return nil
}
108 changes: 108 additions & 0 deletions gnovm/pkg/integration/gno.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package integration

import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"testing"

osm "github.com/gnolang/gno/tm2/pkg/os"
"github.com/rogpeppe/go-internal/testscript"
)

// SetupGno prepares the given testscript environment for tests that utilize the gno command.
// If the `gno` binary doesn't exist, it's built using the `go build` command into the specified buildDir.
// The function also include the `gno` command into `p.Cmds` to and wrap environment into p.Setup
// to correctly set up the environment variables needed for the `gno` command.
func SetupGno(p *testscript.Params, buildDir string) error {
// Try to fetch `GNOROOT` from the environment variables
gnoroot := os.Getenv("GNOROOT")
if gnoroot == "" {
// If `GNOROOT` isn't set, determine the root directory of github.com/gnolang/gno
goModPath, err := exec.Command("go", "env", "GOMOD").CombinedOutput()
if err != nil {
return fmt.Errorf("unable to determine gno root directory")
}

gnoroot = filepath.Dir(string(goModPath))
}

if !osm.DirExists(buildDir) {
return fmt.Errorf("%q does not exist or is not a directory", buildDir)
}

// Determine the path to the gno binary within the build directory
gnoBin := filepath.Join(buildDir, "gno")
if _, err := os.Stat(gnoBin); err != nil {
if !errors.Is(err, os.ErrNotExist) {
// Handle other potential errors from os.Stat
return err
}

// Build a fresh gno binary in a temp directory
gnoArgsBuilder := []string{"build", "-o", gnoBin}

// Forward `-covermode` settings if set
if coverMode := testing.CoverMode(); coverMode != "" {
gnoArgsBuilder = append(gnoArgsBuilder, "-covermode", coverMode)
}

// Append the path to the gno command source
gnoArgsBuilder = append(gnoArgsBuilder, filepath.Join(gnoroot, "gnovm", "cmd", "gno"))

if err = exec.Command("go", gnoArgsBuilder...).Run(); err != nil {
return fmt.Errorf("unable to build gno binary: %w", err)
}
}

// Store the original setup scripts for potential wrapping
origSetup := p.Setup
p.Setup = func(env *testscript.Env) error {
// If there's an original setup, execute it
if origSetup != nil {
if err := origSetup(env); err != nil {
return err
}
}

// Set the GNOROOT environment variable
env.Setenv("GNOROOT", gnoroot)

// Create a temporary home directory because certain commands require access to $HOME/.cache/go-build
home, err := os.MkdirTemp("", "gno")
if err != nil {
return fmt.Errorf("unable to create temporary home directory: %w", err)
}
env.Setenv("HOME", home)

// Cleanup home folder
env.Defer(func() { os.RemoveAll(home) })

return nil
}

// Initialize cmds map if needed
if p.Cmds == nil {
p.Cmds = make(map[string]func(ts *testscript.TestScript, neg bool, args []string))
}

// Register the gno command for testscripts
p.Cmds["gno"] = func(ts *testscript.TestScript, neg bool, args []string) {
err := ts.Exec(gnoBin, args...)
if err != nil {
ts.Logf("gno command error: %v", err)
}

commandSucceeded := (err == nil)
successExpected := !neg

// Compare the command's success status with the expected outcome.
if commandSucceeded != successExpected {
ts.Fatalf("unexpected gno command outcome (err=%t expected=%t)", commandSucceeded, successExpected)
}
}

return nil
}

0 comments on commit 29a4459

Please sign in to comment.