diff --git a/cmd/integration_test.go b/cmd/tests/cmd_test.go similarity index 87% rename from cmd/integration_test.go rename to cmd/tests/cmd_test.go index 2f37c2d7474d..de93b4e4daf8 100644 --- a/cmd/integration_test.go +++ b/cmd/tests/cmd_test.go @@ -1,4 +1,4 @@ -package cmd +package tests import ( "bytes" @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tidwall/gjson" "go.k6.io/k6/cloudapi" + "go.k6.io/k6/cmd" "go.k6.io/k6/cmd/state" "go.k6.io/k6/errext/exitcodes" "go.k6.io/k6/lib" @@ -30,19 +31,43 @@ import ( "go.k6.io/k6/lib/testutils/httpmultibin" ) +func TestDeprecatedOptionWarning(t *testing.T) { + t.Parallel() + + ts := state.NewGlobalTestState(t) + ts.CmdArgs = []string{"k6", "--logformat", "json", "run", "-"} + ts.Stdin = bytes.NewBuffer([]byte(` + console.log('foo'); + export default function() { console.log('bar'); }; + `)) + + cmd.Execute(ts.GlobalState) + + logMsgs := ts.LoggerHook.Drain() + assert.True(t, testutils.LogContains(logMsgs, logrus.InfoLevel, "foo")) + assert.True(t, testutils.LogContains(logMsgs, logrus.InfoLevel, "bar")) + assert.Contains(t, ts.Stderr.String(), `"level":"info","msg":"foo","source":"console"`) + assert.Contains(t, ts.Stderr.String(), `"level":"info","msg":"bar","source":"console"`) + + // TODO: after we get rid of cobra, actually emit this message to stderr + // and, ideally, through the log, not just print it... + assert.False(t, testutils.LogContains(logMsgs, logrus.InfoLevel, "logformat")) + assert.Contains(t, ts.Stdout.String(), `--logformat has been deprecated`) +} + func TestVersion(t *testing.T) { t.Parallel() ts := state.NewGlobalTestState(t) ts.CmdArgs = []string{"k6", "version"} - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) - stdOut := ts.Stdout.String() - assert.Contains(t, stdOut, "k6 v"+consts.Version) - assert.Contains(t, stdOut, runtime.Version()) - assert.Contains(t, stdOut, runtime.GOOS) - assert.Contains(t, stdOut, runtime.GOARCH) - assert.Contains(t, stdOut, "k6/x/alarmist") + stdout := ts.Stdout.String() + assert.Contains(t, stdout, "k6 v"+consts.Version) + assert.Contains(t, stdout, runtime.Version()) + assert.Contains(t, stdout, runtime.GOOS) + assert.Contains(t, stdout, runtime.GOARCH) + assert.NotContains(t, stdout[:len(stdout)-1], "\n") assert.Empty(t, ts.Stderr.Bytes()) assert.Empty(t, ts.LoggerHook.Drain()) @@ -54,15 +79,16 @@ func TestSimpleTestStdin(t *testing.T) { ts := state.NewGlobalTestState(t) ts.CmdArgs = []string{"k6", "run", "-"} ts.Stdin = bytes.NewBufferString(`export default function() {};`) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) - stdOut := ts.Stdout.String() - assert.Contains(t, stdOut, "default: 1 iterations for each of 1 VUs") - assert.Contains(t, stdOut, "1 complete and 0 interrupted iterations") + stdout := ts.Stdout.String() + assert.Contains(t, stdout, "default: 1 iterations for each of 1 VUs") + assert.Contains(t, stdout, "1 complete and 0 interrupted iterations") assert.Empty(t, ts.Stderr.Bytes()) assert.Empty(t, ts.LoggerHook.Drain()) } +// TODO: Remove this? It doesn't test anything AFAICT... func TestStdoutAndStderrAreEmptyWithQuietAndHandleSummary(t *testing.T) { t.Parallel() @@ -74,7 +100,7 @@ func TestStdoutAndStderrAreEmptyWithQuietAndHandleSummary(t *testing.T) { return {}; // silence the end of test summary }; `) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) assert.Empty(t, ts.Stderr.Bytes()) assert.Empty(t, ts.Stdout.Bytes()) @@ -97,7 +123,7 @@ func TestStdoutAndStderrAreEmptyWithQuietAndLogsForwarded(t *testing.T) { console.log('init'); export default function() { console.log('foo'); }; `) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) // The test state hook still catches this message assert.True(t, testutils.LogContains(ts.LoggerHook.Drain(), logrus.InfoLevel, `foo`)) @@ -124,7 +150,7 @@ func TestRelativeLogPathWithSetupAndTeardown(t *testing.T) { export function setup() { console.log('bar'); }; export function teardown() { console.log('baz'); }; `) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) // The test state hook still catches these messages logEntries := ts.LoggerHook.Drain() @@ -146,7 +172,7 @@ func TestWrongCliFlagIterations(t *testing.T) { ts.Stdin = bytes.NewBufferString(`export default function() {};`) // TODO: check for exitcodes.InvalidConfig after https://github.com/loadimpact/k6/issues/883 is done... ts.ExpectedExitCode = -1 - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) assert.True(t, testutils.LogContains(ts.LoggerHook.Drain(), logrus.ErrorLevel, `invalid argument "foo"`)) } @@ -158,12 +184,12 @@ func TestWrongEnvVarIterations(t *testing.T) { ts.Env["K6_ITERATIONS"] = "4" ts.Stdin = bytes.NewBufferString(`export default function() {};`) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, "4 iterations shared among 2 VUs") - assert.Contains(t, stdOut, "4 complete and 0 interrupted iterations") + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, "4 iterations shared among 2 VUs") + assert.Contains(t, stdout, "4 complete and 0 interrupted iterations") assert.Empty(t, ts.Stderr.Bytes()) assert.Empty(t, ts.LoggerHook.Drain()) } @@ -242,7 +268,7 @@ func TestMetricsAndThresholds(t *testing.T) { } ` ts := getSingleFileTestState(t, script, []string{"--quiet", "--log-format=raw"}, 0) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) expLogLines := []string{ `setup() start`, `setup() end`, `default({"foo":"bar"})`, @@ -306,7 +332,7 @@ func testSSLKEYLOGFILE(t *testing.T, ts *state.GlobalTestState, filePath string) } `))) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) assert.True(t, testutils.LogContains(ts.LoggerHook.Drain(), logrus.WarnLevel, "SSLKEYLOGFILE was specified")) @@ -334,7 +360,7 @@ func TestThresholdDeprecationWarnings(t *testing.T) { export default function () { }`, )) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) logs := ts.LoggerHook.Drain() @@ -367,7 +393,7 @@ func TestExecutionTestOptionsDefaultValues(t *testing.T) { ` ts := getSingleFileTestState(t, script, []string{"--iterations", "1"}, 0) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) loglines := ts.LoggerHook.Drain() require.Len(t, loglines, 1) @@ -395,7 +421,7 @@ func TestSubMetricThresholdNoData(t *testing.T) { } ` ts := getSingleFileTestState(t, script, []string{"--quiet"}, 0) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) assert.Len(t, ts.LoggerHook.Drain(), 0) assert.Contains(t, ts.Stdout.String(), ` @@ -516,7 +542,7 @@ func TestSetupTeardownThresholds(t *testing.T) { `) ts := getSimpleCloudOutputTestState(t, script, nil, lib.RunStatusFinished, cloudapi.ResultStatusPassed, 0) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) stdOut := ts.Stdout.String() assert.Contains(t, stdOut, `✓ http_reqs......................: 7`) @@ -563,15 +589,15 @@ func TestThresholdsFailed(t *testing.T) { ts := getSimpleCloudOutputTestState( t, script, nil, lib.RunStatusFinished, cloudapi.ResultStatusFailed, exitcodes.ThresholdsHaveFailed, ) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) assert.True(t, testutils.LogContains(ts.LoggerHook.Drain(), logrus.ErrorLevel, `some thresholds have failed`)) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, ` ✓ iterations...........: 3`) - assert.Contains(t, stdOut, ` ✗ { scenario:sc1 }...: 1`) - assert.Contains(t, stdOut, ` ✗ { scenario:sc2 }...: 2`) - assert.Contains(t, stdOut, ` ✓ { scenario:sc3 }...: 0 0/s`) + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, ` ✓ iterations...........: 3`) + assert.Contains(t, stdout, ` ✗ { scenario:sc1 }...: 1`) + assert.Contains(t, stdout, ` ✗ { scenario:sc2 }...: 2`) + assert.Contains(t, stdout, ` ✓ { scenario:sc3 }...: 0 0/s`) } func TestAbortedByThreshold(t *testing.T) { @@ -604,7 +630,7 @@ func TestAbortedByThreshold(t *testing.T) { ts := getSimpleCloudOutputTestState( t, script, nil, lib.RunStatusAbortedThreshold, cloudapi.ResultStatusFailed, exitcodes.ThresholdsHaveFailed, ) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) assert.True(t, testutils.LogContains(ts.LoggerHook.Drain(), logrus.ErrorLevel, `test run aborted by failed thresholds`)) stdOut := ts.Stdout.String() @@ -653,20 +679,20 @@ func TestAbortedByUserWithGoodThresholds(t *testing.T) { asyncWaitForStdoutAndStopTestWithInterruptSignal(t, ts, 15, time.Second, "simple iter 2") - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) logs := ts.LoggerHook.Drain() assert.False(t, testutils.LogContains(logs, logrus.ErrorLevel, `some thresholds have failed`)) assert.True(t, testutils.LogContains(logs, logrus.ErrorLevel, `test run aborted by signal`)) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, `✓ iterations`) - assert.Contains(t, stdOut, `✓ tc`) - assert.Contains(t, stdOut, `✓ { group:::teardown }`) - assert.Contains(t, stdOut, `Stopping k6 in response to signal`) - assert.Contains(t, stdOut, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) - assert.Contains(t, stdOut, `level=debug msg="Metrics processing finished!"`) - assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `✓ iterations`) + assert.Contains(t, stdout, `✓ tc`) + assert.Contains(t, stdout, `✓ { group:::teardown }`) + assert.Contains(t, stdout, `Stopping k6 in response to signal`) + assert.Contains(t, stdout, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) + assert.Contains(t, stdout, `level=debug msg="Metrics processing finished!"`) + assert.Contains(t, stdout, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) } func asyncWaitForStdoutAndRun( @@ -782,17 +808,17 @@ func TestAbortedByUserWithRestAPI(t *testing.T) { asyncWaitForStdoutAndStopTestFromRESTAPI(t, ts, 15, time.Second, "a simple iteration") - newRootCommand(ts.GlobalState).execute() - - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, `a simple iteration`) - assert.Contains(t, stdOut, `teardown() called`) - assert.Contains(t, stdOut, `PATCH /v1/status`) - assert.Contains(t, stdOut, `run: stopped by user via REST API; exiting...`) - assert.Contains(t, stdOut, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) - assert.Contains(t, stdOut, `level=debug msg="Metrics processing finished!"`) - assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) + cmd.Execute(ts.GlobalState) + + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `a simple iteration`) + assert.Contains(t, stdout, `teardown() called`) + assert.Contains(t, stdout, `PATCH /v1/status`) + assert.Contains(t, stdout, `run: stopped by user via REST API; exiting...`) + assert.Contains(t, stdout, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) + assert.Contains(t, stdout, `level=debug msg="Metrics processing finished!"`) + assert.Contains(t, stdout, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) } func TestAbortedByScriptSetupErrorWithDependency(t *testing.T) { @@ -827,30 +853,30 @@ func TestAbortedByScriptSetupErrorWithDependency(t *testing.T) { ts.CmdArgs = []string{"k6", "run", "-v", "--out", "cloud", "--log-output=stdout", "test.js"} ts.ExpectedExitCode = int(exitcodes.ScriptException) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, `wonky setup`) + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `wonky setup`) rootPath := "file:///" if runtime.GOOS == "windows" { rootPath += "c:/" } - assert.Contains(t, stdOut, `level=error msg="Error: baz\n\tat baz (`+rootPath+`test/bar.js:6:9(3))\n\tat `+ + assert.Contains(t, stdout, `level=error msg="Error: baz\n\tat baz (`+rootPath+`test/bar.js:6:9(3))\n\tat `+ rootPath+`test/bar.js:3:3(3)\n\tat setup (`+rootPath+`test/test.js:5:3(9))\n" hint="script exception"`) - assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) - assert.Contains(t, stdOut, "bogus summary") + assert.Contains(t, stdout, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) + assert.Contains(t, stdout, "bogus summary") } func runTestWithNoLinger(t *testing.T, ts *state.GlobalTestState) { - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) } func runTestWithLinger(t *testing.T, ts *state.GlobalTestState) { ts.CmdArgs = append(ts.CmdArgs, "--linger") asyncWaitForStdoutAndStopTestWithInterruptSignal(t, ts, 15, time.Second, "Linger set; waiting for Ctrl+C") - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) } func TestAbortedByScriptSetupError(t *testing.T) { @@ -871,11 +897,11 @@ func TestAbortedByScriptSetupError(t *testing.T) { ` doChecks := func(t *testing.T, ts *state.GlobalTestState) { - stdOut := ts.Stdout.String() - assert.Contains(t, stdOut, "Error: foo") - assert.Contains(t, stdOut, "wonky setup") - assert.NotContains(t, stdOut, "nice teardown") // do not execute teardown if setup failed - assert.Contains(t, stdOut, "bogus summary") + stdout := ts.Stdout.String() + assert.Contains(t, stdout, "Error: foo") + assert.Contains(t, stdout, "wonky setup") + assert.NotContains(t, stdout, "nice teardown") // do not execute teardown if setup failed + assert.Contains(t, stdout, "bogus summary") } t.Run("noLinger", func(t *testing.T) { @@ -910,11 +936,11 @@ func TestAbortedByScriptTeardownError(t *testing.T) { ` doChecks := func(t *testing.T, ts *state.GlobalTestState) { - stdOut := ts.Stdout.String() - assert.Contains(t, stdOut, "Error: foo") - assert.Contains(t, stdOut, "nice setup") - assert.Contains(t, stdOut, "wonky teardown") - assert.Contains(t, stdOut, "bogus summary") + stdout := ts.Stdout.String() + assert.Contains(t, stdout, "Error: foo") + assert.Contains(t, stdout, "nice setup") + assert.Contains(t, stdout, "wonky teardown") + assert.Contains(t, stdout, "bogus summary") } t.Run("noLinger", func(t *testing.T) { @@ -936,12 +962,12 @@ func testAbortedByScriptError(t *testing.T, script string, runTest func(*testing ) runTest(t, ts) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) - assert.Contains(t, stdOut, `level=debug msg="Metrics processing finished!"`) - assert.Contains(t, stdOut, `level=debug msg="Everything has finished, exiting k6!"`) - assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) + assert.Contains(t, stdout, `level=debug msg="Metrics processing finished!"`) + assert.Contains(t, stdout, `level=debug msg="Everything has finished, exiting k6!"`) + assert.Contains(t, stdout, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) return ts } @@ -957,11 +983,11 @@ func TestAbortedByTestAbortFirstInitCode(t *testing.T) { ` ts := getSingleFileTestState(t, script, nil, exitcodes.ScriptAborted) - newRootCommand(ts.GlobalState).execute() - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, "test aborted: foo") - assert.NotContains(t, stdOut, "bogus summary") + cmd.Execute(ts.GlobalState) + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, "test aborted: foo") + assert.NotContains(t, stdout, "bogus summary") } func TestAbortedByTestAbortInNonFirstInitCode(t *testing.T) { @@ -1057,16 +1083,16 @@ func testAbortedByScriptTestAbort( ) runTest(t, ts) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, "test aborted: foo") - assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) - assert.Contains(t, stdOut, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, "test aborted: foo") + assert.Contains(t, stdout, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=5 tainted=false`) + assert.Contains(t, stdout, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) if shouldHaveMetrics { - assert.Contains(t, stdOut, `level=debug msg="Metrics processing finished!"`) - assert.Contains(t, stdOut, "bogus summary") + assert.Contains(t, stdout, `level=debug msg="Metrics processing finished!"`) + assert.Contains(t, stdout, "bogus summary") } else { - assert.NotContains(t, stdOut, "bogus summary") + assert.NotContains(t, stdout, "bogus summary") } return ts } @@ -1097,7 +1123,7 @@ func TestAbortedByInterruptDuringVUInit(t *testing.T) { t, script, nil, lib.RunStatusAbortedSystem, cloudapi.ResultStatusPassed, exitcodes.GenericEngine, ) asyncWaitForStdoutAndStopTestWithInterruptSignal(t, ts, 15, time.Second, "VU init sleeping for a while") - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) stdOut := ts.Stdout.String() t.Log(stdOut) @@ -1128,14 +1154,14 @@ func TestAbortedByScriptInitError(t *testing.T) { ts := getSimpleCloudOutputTestState( t, script, nil, lib.RunStatusAbortedScriptError, cloudapi.ResultStatusPassed, exitcodes.ScriptException, ) - newRootCommand(ts.GlobalState).execute() - - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, `level=error msg="Error: oops in 2\n\tat file:///`) - assert.Contains(t, stdOut, `hint="error while initializing VU #2 (script exception)"`) - assert.Contains(t, stdOut, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) - assert.Contains(t, stdOut, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) + cmd.Execute(ts.GlobalState) + + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `level=error msg="Error: oops in 2\n\tat file:///`) + assert.Contains(t, stdout, `hint="error while initializing VU #2 (script exception)"`) + assert.Contains(t, stdout, `level=debug msg="Metrics emission of VUs and VUsMax metrics stopped"`) + assert.Contains(t, stdout, `level=debug msg="Sending test finished" output=cloud ref=111 run_status=7 tainted=false`) } func TestMetricTagAndSetupDataIsolation(t *testing.T) { @@ -1226,11 +1252,11 @@ func TestMetricTagAndSetupDataIsolation(t *testing.T) { t, script, []string{"--quiet", "--log-output", "stdout"}, lib.RunStatusFinished, cloudapi.ResultStatusPassed, 0, ) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Equal(t, 12, strings.Count(stdOut, "✓")) + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Equal(t, 12, strings.Count(stdout, "✓")) } func getSampleValues(t *testing.T, jsonOutput []byte, metric string, tags map[string]string) []float64 { @@ -1349,10 +1375,10 @@ func TestActiveVUsCount(t *testing.T) { ` ts := getSingleFileTestState(t, script, []string{"--compatibility-mode", "base", "--out", "json=results.json"}, 0) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) - stdOut := ts.Stdout.String() - t.Log(stdOut) + stdout := ts.Stdout.String() + t.Log(stdout) jsonResults, err := afero.ReadFile(ts.FS, "results.json") require.NoError(t, err) @@ -1397,7 +1423,7 @@ func TestMinIterationDuration(t *testing.T) { ts := getSimpleCloudOutputTestState(t, script, nil, lib.RunStatusFinished, cloudapi.ResultStatusPassed, 0) start := time.Now() - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) elapsed := time.Since(start) assert.Greater(t, elapsed, 5*time.Second, "expected more time to have passed because of minIterationDuration") assert.Less( @@ -1405,9 +1431,9 @@ func TestMinIterationDuration(t *testing.T) { "expected less time to have passed because minIterationDuration should not affect setup() and teardown() ", ) - stdOut := ts.Stdout.String() - t.Log(stdOut) - assert.Contains(t, stdOut, "✓ test_counter.........: 3") + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, "✓ test_counter.........: 3") } func TestRunTags(t *testing.T) { @@ -1477,10 +1503,10 @@ func TestRunTags(t *testing.T) { }, 0) ts.Env["K6_ITERATIONS"] = "3" ts.Env["K6_INSECURE_SKIP_TLS_VERIFY"] = "true" - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) - stdOut := ts.Stdout.String() - t.Log(stdOut) + stdout := ts.Stdout.String() + t.Log(stdout) jsonResults, err := afero.ReadFile(ts.FS, "results.json") require.NoError(t, err) @@ -1528,10 +1554,10 @@ func TestPrometheusRemoteWriteOutput(t *testing.T) { export default function () {}; `) - newRootCommand(ts.GlobalState).execute() + cmd.Execute(ts.GlobalState) ts.OutMutex.Lock() - stdOut := ts.Stdout.String() + stdout := ts.Stdout.String() ts.OutMutex.Unlock() - assert.Contains(t, stdOut, "output: Prometheus remote write") + assert.Contains(t, stdout, "output: Prometheus remote write") } diff --git a/cmd/tests/doc.go b/cmd/tests/doc.go new file mode 100644 index 000000000000..1190f210eba2 --- /dev/null +++ b/cmd/tests/doc.go @@ -0,0 +1,8 @@ +// Package tests contains integration tests that run k6 commands, and interact +// with standard I/O streams. They're the highest level tests we have, just +// below E2E tests that execute the k6 binary. Since they initialize all +// internal k6 components similarly to how a user would, they're very useful, +// but also very expensive to run. They're also brittle, as they depend on large +// parts of the codebase. When in doubt, prefer adding lower-level unit tests +// first, and an integration test only if necessary. +package tests diff --git a/cmd/tests/tests_test.go b/cmd/tests/tests_test.go new file mode 100644 index 000000000000..f50d7555d035 --- /dev/null +++ b/cmd/tests/tests_test.go @@ -0,0 +1,53 @@ +// Package tests contains integration tests for multiple packages. +package tests + +import ( + "fmt" + "net/http" + "os" + "sync/atomic" + "testing" +) + +type blockingTransport struct { + fallback http.RoundTripper + forbiddenHosts map[string]bool + counter uint32 +} + +func (bt *blockingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + host := req.URL.Hostname() + if bt.forbiddenHosts[host] { + atomic.AddUint32(&bt.counter, 1) + panic(fmt.Errorf("trying to make forbidden request to %s during test", host)) + } + return bt.fallback.RoundTrip(req) +} + +func TestMain(m *testing.M) { + exitCode := 1 // error out by default + defer func() { + os.Exit(exitCode) + }() + + bt := &blockingTransport{ + fallback: http.DefaultTransport, + forbiddenHosts: map[string]bool{ + "ingest.k6.io": true, + "cloudlogs.k6.io": true, + "app.k6.io": true, + "reports.k6.io": true, + }, + } + http.DefaultTransport = bt + defer func() { + if bt.counter > 0 { + fmt.Printf("Expected blocking transport count to be 0 but was %d\n", bt.counter) //nolint:forbidigo + exitCode = 2 + } + }() + + // TODO: add https://github.com/uber-go/goleak + + exitCode = m.Run() +}