diff --git a/.golangci.yml b/.golangci.yml
index ed38e6f2a27..361c5f6977f 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -38,7 +38,7 @@ linters-settings:
dupl:
threshold: 150
goconst:
- min-len: 5
+ min-len: 10
min-occurrences: 4
linters:
diff --git a/api/v1/setup_teardown_routes_test.go b/api/v1/setup_teardown_routes_test.go
index ef352d6a260..14476da7122 100644
--- a/api/v1/setup_teardown_routes_test.go
+++ b/api/v1/setup_teardown_routes_test.go
@@ -26,6 +26,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
+ "net/url"
"testing"
"time"
@@ -34,8 +35,8 @@ import (
"github.com/loadimpact/k6/js"
"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/lib/types"
+ "github.com/loadimpact/k6/loader"
"github.com/manyminds/api2go/jsonapi"
- "github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
null "gopkg.in/guregu/null.v3"
@@ -130,10 +131,11 @@ func TestSetupData(t *testing.T) {
},
}
for _, testCase := range testCases {
+ testCase := testCase
t.Run(testCase.name, func(t *testing.T) {
runner, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: testCase.script},
- afero.NewMemMapFs(),
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
+ nil,
lib.RuntimeOptions{},
)
require.NoError(t, err)
diff --git a/cmd/archive.go b/cmd/archive.go
index edb4b65e6d3..adb8ad6cb7e 100644
--- a/cmd/archive.go
+++ b/cmd/archive.go
@@ -23,6 +23,7 @@ package cmd
import (
"os"
+ "github.com/loadimpact/k6/loader"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@@ -51,8 +52,8 @@ An archive is a fully self-contained test run, and can be executed identically e
return err
}
filename := args[0]
- fs := afero.NewOsFs()
- src, err := readSource(filename, pwd, fs, os.Stdin)
+ filesystems := loader.CreateFilesystems()
+ src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
if err != nil {
return err
}
@@ -62,7 +63,7 @@ An archive is a fully self-contained test run, and can be executed identically e
return err
}
- r, err := newRunner(src, runType, fs, runtimeOptions)
+ r, err := newRunner(src, runType, filesystems, runtimeOptions)
if err != nil {
return err
}
@@ -71,7 +72,7 @@ An archive is a fully self-contained test run, and can be executed identically e
if err != nil {
return err
}
- conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+ conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
if err != nil {
return err
}
diff --git a/cmd/cloud.go b/cmd/cloud.go
index 3e913994ddd..7fdcd48686c 100644
--- a/cmd/cloud.go
+++ b/cmd/cloud.go
@@ -32,6 +32,7 @@ import (
"github.com/kelseyhightower/envconfig"
"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/lib/consts"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/stats/cloud"
"github.com/loadimpact/k6/ui"
"github.com/pkg/errors"
@@ -71,8 +72,8 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
}
filename := args[0]
- fs := afero.NewOsFs()
- src, err := readSource(filename, pwd, fs, os.Stdin)
+ filesystems := loader.CreateFilesystems()
+ src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
if err != nil {
return err
}
@@ -82,7 +83,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
return err
}
- r, err := newRunner(src, runType, fs, runtimeOptions)
+ r, err := newRunner(src, runType, filesystems, runtimeOptions)
if err != nil {
return err
}
@@ -91,7 +92,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud
if err != nil {
return err
}
- conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r)
+ conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r)
if err != nil {
return err
}
diff --git a/cmd/collectors.go b/cmd/collectors.go
index 93979b9bb64..f92dc7dcb6f 100644
--- a/cmd/collectors.go
+++ b/cmd/collectors.go
@@ -29,6 +29,7 @@ import (
"github.com/kelseyhightower/envconfig"
"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/lib/consts"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/stats/cloud"
"github.com/loadimpact/k6/stats/datadog"
"github.com/loadimpact/k6/stats/influxdb"
@@ -61,7 +62,7 @@ func parseCollector(s string) (t, arg string) {
}
}
-func newCollector(collectorName, arg string, src *lib.SourceData, conf Config) (lib.Collector, error) {
+func newCollector(collectorName, arg string, src *loader.SourceData, conf Config) (lib.Collector, error) {
getCollector := func() (lib.Collector, error) {
switch collectorName {
case collectorJSON:
diff --git a/cmd/inspect.go b/cmd/inspect.go
index efab0afbeb7..c48221d5cf1 100644
--- a/cmd/inspect.go
+++ b/cmd/inspect.go
@@ -28,7 +28,7 @@ import (
"github.com/loadimpact/k6/js"
"github.com/loadimpact/k6/lib"
- "github.com/spf13/afero"
+ "github.com/loadimpact/k6/loader"
"github.com/spf13/cobra"
)
@@ -43,8 +43,8 @@ var inspectCmd = &cobra.Command{
if err != nil {
return err
}
- fs := afero.NewOsFs()
- src, err := readSource(args[0], pwd, fs, os.Stdin)
+ filesystems := loader.CreateFilesystems()
+ src, err := loader.ReadSource(args[0], pwd, filesystems, os.Stdin)
if err != nil {
return err
}
@@ -59,20 +59,24 @@ var inspectCmd = &cobra.Command{
return err
}
- var opts lib.Options
+ var (
+ opts lib.Options
+ b *js.Bundle
+ )
switch typ {
case typeArchive:
- arc, err := lib.ReadArchive(bytes.NewBuffer(src.Data))
+ var arc *lib.Archive
+ arc, err = lib.ReadArchive(bytes.NewBuffer(src.Data))
if err != nil {
return err
}
- b, err := js.NewBundleFromArchive(arc, runtimeOptions)
+ b, err = js.NewBundleFromArchive(arc, runtimeOptions)
if err != nil {
return err
}
opts = b.Options
case typeJS:
- b, err := js.NewBundle(src, fs, runtimeOptions)
+ b, err = js.NewBundle(src, filesystems, runtimeOptions)
if err != nil {
return err
}
diff --git a/cmd/run.go b/cmd/run.go
index 9c2bd2c94ee..caf9df0e3dc 100644
--- a/cmd/run.go
+++ b/cmd/run.go
@@ -26,12 +26,9 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
- "io/ioutil"
"net/http"
"os"
"os/signal"
- "path/filepath"
"runtime"
"strings"
"syscall"
@@ -116,8 +113,8 @@ a commandline interface for interacting with it.`,
return err
}
filename := args[0]
- fs := afero.NewOsFs()
- src, err := readSource(filename, pwd, fs, os.Stdin)
+ filesystems := loader.CreateFilesystems()
+ src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin)
if err != nil {
return err
}
@@ -127,7 +124,7 @@ a commandline interface for interacting with it.`,
return err
}
- r, err := newRunner(src, runType, fs, runtimeOptions)
+ r, err := newRunner(src, runType, filesystems, runtimeOptions)
if err != nil {
return err
}
@@ -138,7 +135,7 @@ a commandline interface for interacting with it.`,
if err != nil {
return err
}
- conf, err := getConsolidatedConfig(fs, cliConf, r)
+ conf, err := getConsolidatedConfig(afero.NewOsFs(), cliConf, r)
if err != nil {
return err
}
@@ -503,29 +500,15 @@ func init() {
runCmd.Flags().AddFlagSet(runCmdFlagSet())
}
-// Reads a source file from any supported destination.
-func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, error) {
- if src == "-" {
- data, err := ioutil.ReadAll(stdin)
- if err != nil {
- return nil, err
- }
- return &lib.SourceData{Filename: "-", Data: data}, nil
- }
- abspath := filepath.Join(pwd, src)
- if ok, _ := afero.Exists(fs, abspath); ok {
- src = abspath
- }
- return loader.Load(fs, pwd, src)
-}
-
// Creates a new runner.
-func newRunner(src *lib.SourceData, typ string, fs afero.Fs, rtOpts lib.RuntimeOptions) (lib.Runner, error) {
+func newRunner(
+ src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions,
+) (lib.Runner, error) {
switch typ {
case "":
- return newRunner(src, detectType(src.Data), fs, rtOpts)
+ return newRunner(src, detectType(src.Data), filesystems, rtOpts)
case typeJS:
- return js.New(src, fs, rtOpts)
+ return js.New(src, filesystems, rtOpts)
case typeArchive:
arc, err := lib.ReadArchive(bytes.NewReader(src.Data))
if err != nil {
diff --git a/cmd/runtime_options_test.go b/cmd/runtime_options_test.go
index 7fbf43b443a..2321f691cb6 100644
--- a/cmd/runtime_options_test.go
+++ b/cmd/runtime_options_test.go
@@ -23,12 +23,14 @@ package cmd
import (
"bytes"
"fmt"
+ "net/url"
"os"
"runtime"
"strings"
"testing"
"github.com/loadimpact/k6/lib"
+ "github.com/loadimpact/k6/loader"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -226,13 +228,15 @@ func TestEnvVars(t *testing.T) {
}
}
+ fs := afero.NewMemMapFs()
+ require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(jsCode), 0644))
runner, err := newRunner(
- &lib.SourceData{
- Data: []byte(jsCode),
- Filename: "/script.js",
+ &loader.SourceData{
+ Data: []byte(jsCode),
+ URL: &url.URL{Path: "/script.js", Scheme: "file"},
},
typeJS,
- afero.NewOsFs(),
+ map[string]afero.Fs{"file": fs},
rtOpts,
)
require.NoError(t, err)
@@ -242,16 +246,15 @@ func TestEnvVars(t *testing.T) {
assert.NoError(t, archive.Write(archiveBuf))
getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) {
- r, err := newRunner(
- &lib.SourceData{
- Data: []byte(archiveBuf.Bytes()),
- Filename: "/script.tar",
+ return newRunner(
+ &loader.SourceData{
+ Data: archiveBuf.Bytes(),
+ URL: &url.URL{Path: "/script.js"},
},
typeArchive,
- afero.NewOsFs(),
+ nil,
rtOpts,
)
- return r, err
}
_, err = getRunnerErr(lib.RuntimeOptions{})
diff --git a/converter/har/converter_test.go b/converter/har/converter_test.go
index 575044f9b4d..fdf92cce8cc 100644
--- a/converter/har/converter_test.go
+++ b/converter/har/converter_test.go
@@ -27,7 +27,7 @@ import (
"github.com/loadimpact/k6/js"
"github.com/loadimpact/k6/lib"
- "github.com/spf13/afero"
+ "github.com/loadimpact/k6/loader"
"github.com/stretchr/testify/assert"
)
@@ -56,10 +56,10 @@ func TestBuildK6RequestObject(t *testing.T) {
}
v, err := buildK6RequestObject(req)
assert.NoError(t, err)
- _, err = js.New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ _, err = js.New(&loader.SourceData{
+ URL: &url.URL{Path: "/script.js"},
+ Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)),
+ }, nil, lib.RuntimeOptions{})
assert.NoError(t, err)
}
diff --git a/core/engine_test.go b/core/engine_test.go
index fe81be047a1..784915594ad 100644
--- a/core/engine_test.go
+++ b/core/engine_test.go
@@ -23,6 +23,7 @@ package core
import (
"context"
"fmt"
+ "net/url"
"testing"
"time"
@@ -32,11 +33,11 @@ import (
"github.com/loadimpact/k6/lib/metrics"
"github.com/loadimpact/k6/lib/testutils"
"github.com/loadimpact/k6/lib/types"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/stats"
"github.com/loadimpact/k6/stats/dummy"
log "github.com/sirupsen/logrus"
logtest "github.com/sirupsen/logrus/hooks/test"
- "github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
null "gopkg.in/guregu/null.v3"
@@ -556,8 +557,8 @@ func TestSentReceivedMetrics(t *testing.T) {
runTest := func(t *testing.T, ts testScript, tc testCase, noConnReuse bool) (float64, float64) {
r, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: []byte(ts.Code)},
- afero.NewMemMapFs(),
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)},
+ nil,
lib.RuntimeOptions{},
)
require.NoError(t, err)
@@ -697,8 +698,8 @@ func TestRunTags(t *testing.T) {
`))
r, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: script},
- afero.NewMemMapFs(),
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
+ nil,
lib.RuntimeOptions{},
)
require.NoError(t, err)
@@ -797,8 +798,8 @@ func TestSetupTeardownThresholds(t *testing.T) {
`))
runner, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: script},
- afero.NewMemMapFs(),
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
+ nil,
lib.RuntimeOptions{},
)
require.NoError(t, err)
@@ -860,8 +861,8 @@ func TestEmittedMetricsWhenScalingDown(t *testing.T) {
`))
runner, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: script},
- afero.NewMemMapFs(),
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
+ nil,
lib.RuntimeOptions{},
)
require.NoError(t, err)
@@ -920,7 +921,7 @@ func TestMinIterationDuration(t *testing.T) {
t.Parallel()
runner, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: []byte(`
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(`
import { Counter } from "k6/metrics";
let testCounter = new Counter("testcounter");
@@ -935,7 +936,7 @@ func TestMinIterationDuration(t *testing.T) {
export default function () {
testCounter.add(1);
};`)},
- afero.NewMemMapFs(),
+ nil,
lib.RuntimeOptions{},
)
require.NoError(t, err)
diff --git a/core/local/local_test.go b/core/local/local_test.go
index 23c27d88179..4af9be82b0c 100644
--- a/core/local/local_test.go
+++ b/core/local/local_test.go
@@ -23,12 +23,14 @@ package local
import (
"context"
"net"
+ "net/url"
"runtime"
"sync/atomic"
"testing"
"time"
"github.com/loadimpact/k6/lib/netext"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/js"
"github.com/loadimpact/k6/lib"
@@ -37,7 +39,6 @@ import (
"github.com/loadimpact/k6/stats"
"github.com/pkg/errors"
logtest "github.com/sirupsen/logrus/hooks/test"
- "github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
null "gopkg.in/guregu/null.v3"
@@ -481,8 +482,8 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) {
}`)
runner, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: script},
- afero.NewMemMapFs(),
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
+ nil,
lib.RuntimeOptions{},
)
require.NoError(t, err)
diff --git a/js/bundle.go b/js/bundle.go
index a2e9d47e47d..9214d6a1bec 100644
--- a/js/bundle.go
+++ b/js/bundle.go
@@ -23,7 +23,8 @@ package js
import (
"context"
"encoding/json"
- "os"
+ "net/url"
+ "runtime"
"github.com/loadimpact/k6/lib/consts"
@@ -40,7 +41,7 @@ import (
// A Bundle is a self-contained bundle of scripts and resources.
// You can use this to produce identical BundleInstance objects.
type Bundle struct {
- Filename string
+ Filename *url.URL
Source string
Program *goja.Program
Options lib.Options
@@ -58,7 +59,7 @@ type BundleInstance struct {
}
// NewBundle creates a new bundle from a source file and a filesystem.
-func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) {
+func NewBundle(src *loader.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) {
compiler, err := compiler.New()
if err != nil {
return nil, err
@@ -66,24 +67,17 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu
// Compile sources, both ES5 and ES6 are supported.
code := string(src.Data)
- pgm, _, err := compiler.Compile(code, src.Filename, "", "", true)
+ pgm, _, err := compiler.Compile(code, src.URL.String(), "", "", true)
if err != nil {
return nil, err
}
-
- // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's
- // written every time something is read from the real filesystem. This cache is then used for
- // successive spawns to read from (they have no access to the real disk).
- mirrorFS := afero.NewMemMapFs()
- cachedFS := afero.NewCacheOnReadFs(fs, mirrorFS, 0)
-
// Make a bundle, instantiate it into a throwaway VM to populate caches.
rt := goja.New()
bundle := Bundle{
- Filename: src.Filename,
+ Filename: src.URL,
Source: code,
Program: pgm,
- BaseInitContext: NewInitContext(rt, compiler, new(context.Context), cachedFS, loader.Dir(src.Filename)),
+ BaseInitContext: NewInitContext(rt, compiler, new(context.Context), filesystems, loader.Dir(src.URL)),
Env: rtOpts.Env,
}
if err := bundle.instantiate(rt, bundle.BaseInitContext); err != nil {
@@ -144,12 +138,12 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
return nil, errors.Errorf("expected bundle type 'js', got '%s'", arc.Type)
}
- pgm, _, err := compiler.Compile(string(arc.Data), arc.Filename, "", "", true)
+ pgm, _, err := compiler.Compile(string(arc.Data), arc.FilenameURL.String(), "", "", true)
if err != nil {
return nil, err
}
- initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FS, arc.Pwd)
- initctx.files = arc.Files
+
+ initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.Filesystems, arc.PwdURL)
env := arc.Env
if env == nil {
@@ -161,7 +155,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
}
bundle := &Bundle{
- Filename: arc.Filename,
+ Filename: arc.FilenameURL,
Source: string(arc.Data),
Program: pgm,
Options: arc.Options,
@@ -176,30 +170,21 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle,
func (b *Bundle) makeArchive() *lib.Archive {
arc := &lib.Archive{
- Type: "js",
- FS: afero.NewMemMapFs(),
- Options: b.Options,
- Filename: b.Filename,
- Data: []byte(b.Source),
- Pwd: b.BaseInitContext.pwd,
- Env: make(map[string]string, len(b.Env)),
- K6Version: consts.Version,
+ Type: "js",
+ Filesystems: b.BaseInitContext.filesystems,
+ Options: b.Options,
+ FilenameURL: b.Filename,
+ Data: []byte(b.Source),
+ PwdURL: b.BaseInitContext.pwd,
+ Env: make(map[string]string, len(b.Env)),
+ K6Version: consts.Version,
+ Goos: runtime.GOOS,
}
// Copy env so changes in the archive are not reflected in the source Bundle
for k, v := range b.Env {
arc.Env[k] = v
}
- arc.Scripts = make(map[string][]byte, len(b.BaseInitContext.programs))
- for name, pgm := range b.BaseInitContext.programs {
- arc.Scripts[name] = []byte(pgm.src)
- err := afero.WriteFile(arc.FS, name, []byte(pgm.src), os.ModePerm)
- if err != nil {
- return nil
- }
- }
- arc.Files = b.BaseInitContext.files
-
return arc
}
diff --git a/js/bundle_test.go b/js/bundle_test.go
index 745c27a452d..971922f9b28 100644
--- a/js/bundle_test.go
+++ b/js/bundle_test.go
@@ -24,6 +24,7 @@ import (
"crypto/tls"
"fmt"
"io/ioutil"
+ "net/url"
"os"
"path/filepath"
"runtime"
@@ -31,24 +32,42 @@ import (
"testing"
"time"
- "github.com/loadimpact/k6/lib/consts"
-
"github.com/dop251/goja"
"github.com/loadimpact/k6/lib"
+ "github.com/loadimpact/k6/lib/consts"
+ "github.com/loadimpact/k6/lib/fsext"
"github.com/loadimpact/k6/lib/types"
+ "github.com/loadimpact/k6/loader"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/guregu/null.v3"
)
+const isWindows = runtime.GOOS == "windows"
+
func getSimpleBundle(filename, data string) (*Bundle, error) {
+ return getSimpleBundleWithFs(filename, data, afero.NewMemMapFs())
+}
+
+func getSimpleBundleWithOptions(filename, data string, options lib.RuntimeOptions) (*Bundle, error) {
+ return NewBundle(
+ &loader.SourceData{
+ URL: &url.URL{Path: filename, Scheme: "file"},
+ Data: []byte(data),
+ },
+ map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": afero.NewMemMapFs()},
+ options,
+ )
+}
+
+func getSimpleBundleWithFs(filename, data string, fs afero.Fs) (*Bundle, error) {
return NewBundle(
- &lib.SourceData{
- Filename: filename,
- Data: []byte(data),
+ &loader.SourceData{
+ URL: &url.URL{Path: filename, Scheme: "file"},
+ Data: []byte(data),
},
- afero.NewMemMapFs(),
+ map[string]afero.Fs{"file": fs, "https": afero.NewMemMapFs()},
lib.RuntimeOptions{},
)
}
@@ -60,11 +79,11 @@ func TestNewBundle(t *testing.T) {
})
t.Run("Invalid", func(t *testing.T) {
_, err := getSimpleBundle("/script.js", "\x00")
- assert.Contains(t, err.Error(), "SyntaxError: /script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n")
+ assert.Contains(t, err.Error(), "SyntaxError: file:///script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n")
})
t.Run("Error", func(t *testing.T) {
_, err := getSimpleBundle("/script.js", `throw new Error("aaaa");`)
- assert.EqualError(t, err, "Error: aaaa at /script.js:1:7(3)")
+ assert.EqualError(t, err, "Error: aaaa at file:///script.js:1:7(3)")
})
t.Run("InvalidExports", func(t *testing.T) {
_, err := getSimpleBundle("/script.js", `exports = null`)
@@ -89,8 +108,8 @@ func TestNewBundle(t *testing.T) {
t.Run("stdin", func(t *testing.T) {
b, err := getSimpleBundle("-", `export default function() {};`)
if assert.NoError(t, err) {
- assert.Equal(t, "-", b.Filename)
- assert.Equal(t, "/", b.BaseInitContext.pwd)
+ assert.Equal(t, "file://-", b.Filename.String())
+ assert.Equal(t, "file:///", b.BaseInitContext.pwd.String())
}
})
t.Run("Options", func(t *testing.T) {
@@ -359,16 +378,13 @@ func TestNewBundleFromArchive(t *testing.T) {
assert.NoError(t, afero.WriteFile(fs, "/path/to/file.txt", []byte(`hi`), 0644))
assert.NoError(t, afero.WriteFile(fs, "/path/to/exclaim.js", []byte(`export default function(s) { return s + "!" };`), 0644))
- src := &lib.SourceData{
- Filename: "/path/to/script.js",
- Data: []byte(`
+ data := `
import exclaim from "./exclaim.js";
export let options = { vus: 12345 };
export let file = open("./file.txt");
export default function() { return exclaim(file); };
- `),
- }
- b, err := NewBundle(src, fs, lib.RuntimeOptions{})
+ `
+ b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs)
if !assert.NoError(t, err) {
return
}
@@ -387,13 +403,17 @@ func TestNewBundleFromArchive(t *testing.T) {
arc := b.makeArchive()
assert.Equal(t, "js", arc.Type)
assert.Equal(t, lib.Options{VUs: null.IntFrom(12345)}, arc.Options)
- assert.Equal(t, "/path/to/script.js", arc.Filename)
- assert.Equal(t, string(src.Data), string(arc.Data))
- assert.Equal(t, "/path/to", arc.Pwd)
- assert.Len(t, arc.Scripts, 1)
- assert.Equal(t, `export default function(s) { return s + "!" };`, string(arc.Scripts["/path/to/exclaim.js"]))
- assert.Len(t, arc.Files, 1)
- assert.Equal(t, `hi`, string(arc.Files["/path/to/file.txt"]))
+ assert.Equal(t, "file:///path/to/script.js", arc.FilenameURL.String())
+ assert.Equal(t, data, string(arc.Data))
+ assert.Equal(t, "file:///path/to/", arc.PwdURL.String())
+
+ exclaimData, err := afero.ReadFile(arc.Filesystems["file"], "/path/to/exclaim.js")
+ assert.NoError(t, err)
+ assert.Equal(t, `export default function(s) { return s + "!" };`, string(exclaimData))
+
+ fileData, err := afero.ReadFile(arc.Filesystems["file"], "/path/to/file.txt")
+ assert.NoError(t, err)
+ assert.Equal(t, `hi`, string(fileData))
assert.Equal(t, consts.Version, arc.K6Version)
b2, err := NewBundleFromArchive(arc, lib.RuntimeOptions{})
@@ -496,8 +516,12 @@ func TestOpen(t *testing.T) {
prefix, err := ioutil.TempDir("", "k6_open_test")
require.NoError(t, err)
fs := afero.NewOsFs()
+ filePath := filepath.Join(prefix, "/path/to/file.txt")
require.NoError(t, fs.MkdirAll(filepath.Join(prefix, "/path/to"), 0755))
- require.NoError(t, afero.WriteFile(fs, filepath.Join(prefix, "/path/to/file.txt"), []byte(`hi`), 0644))
+ require.NoError(t, afero.WriteFile(fs, filePath, []byte(`hi`), 0644))
+ if isWindows {
+ fs = fsext.NewTrimFilePathSeparatorFs(fs)
+ }
return fs, prefix, func() { require.NoError(t, os.RemoveAll(prefix)) }
},
}
@@ -516,21 +540,18 @@ func TestOpen(t *testing.T) {
if openPath != "" && (openPath[0] == '/' || openPath[0] == '\\') {
openPath = filepath.Join(prefix, openPath)
}
- if runtime.GOOS == "windows" {
+ if isWindows {
openPath = strings.Replace(openPath, `\`, `\\`, -1)
}
var pwd = tCase.pwd
if pwd == "" {
pwd = "/path/to/"
}
- src := &lib.SourceData{
- Filename: filepath.Join(prefix, filepath.Join(pwd, "script.js")),
- Data: []byte(`
- export let file = open("` + openPath + `");
- export default function() { return file };
- `),
- }
- sourceBundle, err := NewBundle(src, fs, lib.RuntimeOptions{})
+ data := `
+ export let file = open("` + openPath + `");
+ export default function() { return file };`
+
+ sourceBundle, err := getSimpleBundleWithFs(filepath.ToSlash(filepath.Join(prefix, pwd, "script.js")), data, fs)
if tCase.isError {
assert.Error(t, err)
return
@@ -554,7 +575,7 @@ func TestOpen(t *testing.T) {
}
t.Run(tCase.name, testFunc)
- if runtime.GOOS == "windows" {
+ if isWindows {
// windowsify the testcase
tCase.openPath = strings.Replace(tCase.openPath, `/`, `\`, -1)
tCase.pwd = strings.Replace(tCase.pwd, `/`, `\`, -1)
@@ -600,19 +621,13 @@ func TestBundleEnv(t *testing.T) {
"TEST_A": "1",
"TEST_B": "",
}}
-
- b1, err := NewBundle(
- &lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
- export default function() {
- if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); }
- if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); }
- }
- `),
- },
- afero.NewMemMapFs(), rtOpts,
- )
+ data := `
+ export default function() {
+ if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); }
+ if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); }
+ }
+ `
+ b1, err := getSimpleBundleWithOptions("/script.js", data, rtOpts)
if !assert.NoError(t, err) {
return
}
diff --git a/js/console_test.go b/js/console_test.go
index b85bbbdacd2..8b0448d5fd1 100644
--- a/js/console_test.go
+++ b/js/console_test.go
@@ -24,6 +24,7 @@ import (
"context"
"fmt"
"io/ioutil"
+ "net/url"
"os"
"testing"
@@ -32,6 +33,7 @@ import (
"github.com/dop251/goja"
"github.com/loadimpact/k6/js/common"
"github.com/loadimpact/k6/lib"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/stats"
log "github.com/sirupsen/logrus"
logtest "github.com/sirupsen/logrus/hooks/test"
@@ -68,7 +70,29 @@ func TestConsoleContext(t *testing.T) {
assert.Equal(t, "b", entry.Message)
}
}
+func getSimpleRunner(path, data string) (*Runner, error) {
+ return getSimpleRunnerWithFileFs(path, data, afero.NewMemMapFs())
+}
+
+func getSimpleRunnerWithOptions(path, data string, options lib.RuntimeOptions) (*Runner, error) {
+ return New(&loader.SourceData{
+ URL: &url.URL{Path: path, Scheme: "file"},
+ Data: []byte(data),
+ }, map[string]afero.Fs{
+ "file": afero.NewMemMapFs(),
+ "https": afero.NewMemMapFs()},
+ options)
+}
+func getSimpleRunnerWithFileFs(path, data string, fileFs afero.Fs) (*Runner, error) {
+ return New(&loader.SourceData{
+ URL: &url.URL{Path: path, Scheme: "file"},
+ Data: []byte(data),
+ }, map[string]afero.Fs{
+ "file": fileFs,
+ "https": afero.NewMemMapFs()},
+ lib.RuntimeOptions{})
+}
func TestConsole(t *testing.T) {
levels := map[string]log.Level{
"log": log.InfoLevel,
@@ -87,16 +111,15 @@ func TestConsole(t *testing.T) {
`{}`: {Message: "[object Object]"},
}
for name, level := range levels {
+ name, level := name, level
t.Run(name, func(t *testing.T) {
for args, result := range argsets {
+ args, result := args, result
t.Run(args, func(t *testing.T) {
- r, err := New(&lib.SourceData{
- Filename: "/script",
- Data: []byte(fmt.Sprintf(
- `export default function() { console.%s(%s); }`,
- name, args,
- )),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ r, err := getSimpleRunner("/script.js", fmt.Sprintf(
+ `export default function() { console.%s(%s); }`,
+ name, args,
+ ))
assert.NoError(t, err)
samples := make(chan stats.SampleContainer, 100)
@@ -179,13 +202,11 @@ func TestFileConsole(t *testing.T) {
}
}
- r, err := New(&lib.SourceData{
- Filename: "/script",
- Data: []byte(fmt.Sprintf(
+ r, err := getSimpleRunner("/script",
+ fmt.Sprintf(
`export default function() { console.%s(%s); }`,
name, args,
- )),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ ))
assert.NoError(t, err)
err = r.SetOptions(lib.Options{
diff --git a/js/http_bench_test.go b/js/http_bench_test.go
index 17f45fb0914..4fdfdd10f6a 100644
--- a/js/http_bench_test.go
+++ b/js/http_bench_test.go
@@ -7,7 +7,6 @@ import (
"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/lib/testutils"
"github.com/loadimpact/k6/stats"
- "github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"gopkg.in/guregu/null.v3"
)
@@ -17,17 +16,14 @@ func BenchmarkHTTPRequests(b *testing.B) {
tb := testutils.NewHTTPMultiBin(b)
defer tb.Cleanup()
- r, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ r, err := getSimpleRunner("/script.js", tb.Replacer.Replace(`
import http from "k6/http";
export default function() {
let url = "HTTPBIN_URL";
let res = http.get(url + "/cookies/set?k2=v2&k1=v1");
if (res.status != 200) { throw new Error("wrong status: " + res.status) }
}
- `)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `))
if !assert.NoError(b, err) {
return
}
diff --git a/js/initcontext.go b/js/initcontext.go
index 9842a0f2396..f43c3b14cf6 100644
--- a/js/initcontext.go
+++ b/js/initcontext.go
@@ -22,6 +22,7 @@ package js
import (
"context"
+ "net/url"
"path/filepath"
"strings"
@@ -49,28 +50,26 @@ type InitContext struct {
// Pointer to a context that bridged modules are invoked with.
ctxPtr *context.Context
- // Filesystem to load files and scripts from.
- fs afero.Fs
- pwd string
+ // Filesystem to load files and scripts from with the map key being the scheme
+ filesystems map[string]afero.Fs
+ pwd *url.URL
// Cache of loaded programs and files.
programs map[string]programWithSource
- files map[string][]byte
}
// NewInitContext creates a new initcontext with the provided arguments
func NewInitContext(
- rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string,
+ rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, filesystems map[string]afero.Fs, pwd *url.URL,
) *InitContext {
return &InitContext{
- runtime: rt,
- compiler: compiler,
- ctxPtr: ctxPtr,
- fs: fs,
- pwd: filepath.ToSlash(pwd),
+ runtime: rt,
+ compiler: compiler,
+ ctxPtr: ctxPtr,
+ filesystems: filesystems,
+ pwd: pwd,
programs: make(map[string]programWithSource),
- files: make(map[string][]byte),
}
}
@@ -89,12 +88,11 @@ func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Ru
runtime: rt,
ctxPtr: ctxPtr,
- fs: base.fs,
- pwd: base.pwd,
- compiler: base.compiler,
+ filesystems: base.filesystems,
+ pwd: base.pwd,
+ compiler: base.compiler,
programs: programs,
- files: base.files,
}
}
@@ -130,12 +128,15 @@ func (i *InitContext) requireModule(name string) (goja.Value, error) {
func (i *InitContext) requireFile(name string) (goja.Value, error) {
// Resolve the file path, push the target directory as pwd to make relative imports work.
pwd := i.pwd
- filename := loader.Resolve(pwd, name)
+ fileURL, err := loader.Resolve(pwd, name)
+ if err != nil {
+ return nil, err
+ }
// First, check if we have a cached program already.
- pgm, ok := i.programs[filename]
+ pgm, ok := i.programs[fileURL.String()]
if !ok || pgm.exports == nil {
- i.pwd = loader.Dir(filename)
+ i.pwd = loader.Dir(fileURL)
defer func() { i.pwd = pwd }()
// Swap the importing scope's exports out, then put it back again.
@@ -150,21 +151,22 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) {
i.runtime.Set("module", module)
if pgm.pgm == nil {
// Load the sources; the loader takes care of remote loading, etc.
- data, err := loader.Load(i.fs, pwd, name)
+ data, err := loader.Load(i.filesystems, fileURL, name)
if err != nil {
return goja.Undefined(), err
}
+
pgm.src = string(data.Data)
// Compile the sources; this handles ES5 vs ES6 automatically.
- pgm.pgm, err = i.compileImport(pgm.src, data.Filename)
+ pgm.pgm, err = i.compileImport(pgm.src, data.URL.String())
if err != nil {
return goja.Undefined(), err
}
}
pgm.exports = module.Get("exports")
- i.programs[filename] = pgm
+ i.programs[fileURL.String()] = pgm
// Run the program.
if _, err := i.runtime.RunProgram(pgm.pgm); err != nil {
@@ -193,28 +195,22 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error)
// will probably be need for archive execution under windows if always consider '/...' as an
// absolute path.
if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) {
- filename = filepath.Join(i.pwd, filename)
+ filename = filepath.Join(i.pwd.Path, filename)
}
- filename = filepath.ToSlash(filename)
-
- data, ok := i.files[filename]
- if !ok {
- var (
- err error
- isDir bool
- )
-
- // Workaround for https://github.com/spf13/afero/issues/201
- if isDir, err = afero.IsDir(i.fs, filename); err != nil {
- return nil, err
- } else if isDir {
- return nil, errors.New("open() can't be used with directories")
- }
- data, err = afero.ReadFile(i.fs, filename)
- if err != nil {
- return nil, err
- }
- i.files[filename] = data
+ filename = filepath.Clean(filename)
+ fs := i.filesystems["file"]
+ if filename[0:1] != afero.FilePathSeparator {
+ filename = afero.FilePathSeparator + filename
+ }
+ // Workaround for https://github.com/spf13/afero/issues/201
+ if isDir, err := afero.IsDir(fs, filename); err != nil {
+ return nil, err
+ } else if isDir {
+ return nil, errors.New("open() can't be used with directories")
+ }
+ data, err := afero.ReadFile(fs, filename)
+ if err != nil {
+ return nil, err
}
if len(args) > 0 && args[0] == "b" {
diff --git a/js/initcontext_test.go b/js/initcontext_test.go
index f127e3b0194..de5a7012dc5 100644
--- a/js/initcontext_test.go
+++ b/js/initcontext_test.go
@@ -40,6 +40,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestInitContextRequire(t *testing.T) {
@@ -111,25 +112,20 @@ func TestInitContextRequire(t *testing.T) {
t.Run("Nonexistent", func(t *testing.T) {
path := filepath.FromSlash("/nonexistent.js")
_, err := getSimpleBundle("/script.js", `import "/nonexistent.js"; export default function() {}`)
- assert.EqualError(t, err, fmt.Sprintf("GoError: open %s: file does not exist", path))
+ assert.Contains(t, err.Error(), fmt.Sprintf(`"file://%s" couldn't be found on local disk`, filepath.ToSlash(path)))
})
t.Run("Invalid", func(t *testing.T) {
fs := afero.NewMemMapFs()
assert.NoError(t, afero.WriteFile(fs, "/file.js", []byte{0x00}, 0755))
- _, err := NewBundle(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`import "/file.js"; export default function() {}`),
- }, fs, lib.RuntimeOptions{})
- assert.Contains(t, err.Error(), "SyntaxError: /file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n")
+ _, err := getSimpleBundleWithFs("/script.js", `import "/file.js"; export default function() {}`, fs)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "SyntaxError: file:///file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n")
})
t.Run("Error", func(t *testing.T) {
fs := afero.NewMemMapFs()
assert.NoError(t, afero.WriteFile(fs, "/file.js", []byte(`throw new Error("aaaa")`), 0755))
- _, err := NewBundle(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`import "/file.js"; export default function() {}`),
- }, fs, lib.RuntimeOptions{})
- assert.EqualError(t, err, "Error: aaaa at /file.js:2:7(4)")
+ _, err := getSimpleBundleWithFs("/script.js", `import "/file.js"; export default function() {}`, fs)
+ assert.EqualError(t, err, "Error: aaaa at file:///file.js:2:7(4)")
})
imports := map[string]struct {
@@ -162,23 +158,16 @@ func TestInitContextRequire(t *testing.T) {
}},
}
for libName, data := range imports {
+ libName, data := libName, data
t.Run("lib=\""+libName+"\"", func(t *testing.T) {
for constName, constPath := range data.ConstPaths {
+ constName, constPath := constName, constPath
name := "inline"
if constName != "" {
name = "const=\"" + constName + "\""
}
t.Run(name, func(t *testing.T) {
fs := afero.NewMemMapFs()
- src := &lib.SourceData{
- Filename: `/path/to/script.js`,
- Data: []byte(fmt.Sprintf(`
- import fn from "%s";
- let v = fn();
- export default function() {
- };
- `, libName)),
- }
jsLib := `export default function() { return 12345; }`
if constName != "" {
@@ -195,12 +184,17 @@ func TestInitContextRequire(t *testing.T) {
assert.NoError(t, fs.MkdirAll(filepath.Dir(data.LibPath), 0755))
assert.NoError(t, afero.WriteFile(fs, data.LibPath, []byte(jsLib), 0644))
- b, err := NewBundle(src, fs, lib.RuntimeOptions{})
+ data := fmt.Sprintf(`
+ import fn from "%s";
+ let v = fn();
+ export default function() {};`,
+ libName)
+ b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs)
if !assert.NoError(t, err) {
return
}
if constPath != "" {
- assert.Contains(t, b.BaseInitContext.programs, constPath)
+ assert.Contains(t, b.BaseInitContext.programs, "file://"+constPath)
}
_, err = b.Instantiate()
@@ -216,18 +210,15 @@ func TestInitContextRequire(t *testing.T) {
fs := afero.NewMemMapFs()
assert.NoError(t, afero.WriteFile(fs, "/a.js", []byte(`const myvar = "a";`), 0644))
assert.NoError(t, afero.WriteFile(fs, "/b.js", []byte(`const myvar = "b";`), 0644))
- b, err := NewBundle(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ data := `
import "./a.js";
import "./b.js";
export default function() {
if (typeof myvar != "undefined") {
throw new Error("myvar is set in global scope");
}
- };
- `),
- }, fs, lib.RuntimeOptions{})
+ };`
+ b, err := getSimpleBundleWithFs("/script.js", data, fs)
if !assert.NoError(t, err) {
return
}
@@ -253,17 +244,15 @@ func createAndReadFile(t *testing.T, file string, content []byte, expectedLength
binaryArg = ",\"b\""
}
- b, err := NewBundle(&lib.SourceData{
- Filename: "/path/to/script.js",
- Data: []byte(fmt.Sprintf(`
- export let data = open("/path/to/%s"%s);
- var expectedLength = %d;
- if (data.length != expectedLength) {
- throw new Error("Length not equal, expected: " + expectedLength + ", actual: " + data.length);
- }
- export default function() {}
- `, file, binaryArg, expectedLength)),
- }, fs, lib.RuntimeOptions{})
+ data := fmt.Sprintf(`
+ export let data = open("/path/to/%s"%s);
+ var expectedLength = %d;
+ if (data.length != expectedLength) {
+ throw new Error("Length not equal, expected: " + expectedLength + ", actual: " + data.length);
+ }
+ export default function() {}
+ `, file, binaryArg, expectedLength)
+ b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs)
if !assert.NoError(t, err) {
return nil, err
@@ -322,12 +311,8 @@ func TestInitContextOpen(t *testing.T) {
}
t.Run("Nonexistent", func(t *testing.T) {
- fs := afero.NewMemMapFs()
path := filepath.FromSlash("/nonexistent.txt")
- _, err := NewBundle(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`open("/nonexistent.txt"); export default function() {}`),
- }, fs, lib.RuntimeOptions{})
+ _, err := getSimpleBundle("/script.js", `open("/nonexistent.txt"); export default function() {}`)
assert.EqualError(t, err, fmt.Sprintf("GoError: open %s: file does not exist", path))
})
@@ -363,9 +348,8 @@ func TestRequestWithBinaryFile(t *testing.T) {
assert.NoError(t, fs.MkdirAll("/path/to", 0755))
assert.NoError(t, afero.WriteFile(fs, "/path/to/file.bin", []byte("hi!"), 0644))
- b, err := NewBundle(&lib.SourceData{
- Filename: "/path/to/script.js",
- Data: []byte(fmt.Sprintf(`
+ b, err := getSimpleBundleWithFs("/path/to/script.js",
+ fmt.Sprintf(`
import http from "k6/http";
let binFile = open("/path/to/file.bin", "b");
export default function() {
@@ -376,9 +360,8 @@ func TestRequestWithBinaryFile(t *testing.T) {
var res = http.post("%s", data);
return true;
}
- `, srv.URL)),
- }, fs, lib.RuntimeOptions{})
- assert.NoError(t, err)
+ `, srv.URL), fs)
+ require.NoError(t, err)
bi, err := b.Instantiate()
assert.NoError(t, err)
diff --git a/js/module_loading_test.go b/js/module_loading_test.go
index df94a096ca9..c59426e4ed4 100644
--- a/js/module_loading_test.go
+++ b/js/module_loading_test.go
@@ -65,9 +65,7 @@ func TestLoadOnceGlobalVars(t *testing.T) {
return C();
}
`), os.ModePerm))
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunnerWithFileFs("/script.js", `
import { A } from "./A.js";
import { B } from "./B.js";
@@ -79,12 +77,10 @@ func TestLoadOnceGlobalVars(t *testing.T) {
throw new Error("A() != B() (" + A() + ") != (" + B() + ")");
}
}
- `),
- }, fs, lib.RuntimeOptions{})
+ `, fs)
require.NoError(t, err)
arc := r1.MakeArchive()
- arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)
@@ -115,9 +111,7 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) {
return http.get("HTTPBIN_URL/get");
}
`)), os.ModePerm))
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunnerWithFileFs("/script.js", `
import { A } from "./A.js";
export default function(data) {
@@ -126,13 +120,11 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) {
throw new Error("wrong status "+ resp.status);
}
}
- `),
- }, fs, lib.RuntimeOptions{})
+ `, fs)
require.NoError(t, err)
require.NoError(t, r1.SetOptions(lib.Options{Hosts: tb.Dialer.Hosts}))
arc := r1.MakeArchive()
- arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)
@@ -159,9 +151,7 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) {
return globalVar;
}
`), os.ModePerm))
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunnerWithFileFs("/script.js", `
import { A } from "./A.js";
export default function(data) {
@@ -172,12 +162,10 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) {
throw new Error("wrong value of a " + a);
}
}
- `),
- }, fs, lib.RuntimeOptions{})
+ `, fs)
require.NoError(t, err)
arc := r1.MakeArchive()
- arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)
@@ -231,14 +219,10 @@ func TestLoadCycle(t *testing.T) {
`), os.ModePerm))
data, err := afero.ReadFile(fs, "/main.js")
require.NoError(t, err)
- r1, err := New(&lib.SourceData{
- Filename: "/main.js",
- Data: data,
- }, fs, lib.RuntimeOptions{})
+ r1, err := getSimpleRunnerWithFileFs("/main.js", string(data), fs)
require.NoError(t, err)
arc := r1.MakeArchive()
- arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)
@@ -281,9 +265,7 @@ func TestLoadCycleBinding(t *testing.T) {
}
`), os.ModePerm))
- r1, err := New(&lib.SourceData{
- Filename: "/main.js",
- Data: []byte(`
+ r1, err := getSimpleRunnerWithFileFs("/main.js", `
import {foo} from './a.js';
import {bar} from './b.js';
export default function() {
@@ -296,12 +278,10 @@ func TestLoadCycleBinding(t *testing.T) {
throw new Error("Wrong value of bar() "+ barMessage);
}
}
- `),
- }, fs, lib.RuntimeOptions{})
+ `, fs)
require.NoError(t, err)
arc := r1.MakeArchive()
- arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)
@@ -341,9 +321,7 @@ func TestBrowserified(t *testing.T) {
});
`), os.ModePerm))
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunnerWithFileFs("/script.js", `
import {alpha, bravo } from "./browserified.js";
export default function(data) {
@@ -361,12 +339,10 @@ func TestBrowserified(t *testing.T) {
throw new Error("bravo.B() != 'b' (" + bravo.B() + ") != 'b'");
}
}
- `),
- }, fs, lib.RuntimeOptions{})
+ `, fs)
require.NoError(t, err)
arc := r1.MakeArchive()
- arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)
diff --git a/js/modules/k6/marshalling_test.go b/js/modules/k6/marshalling_test.go
index 22e552a34f8..8c4ca3ae3a8 100644
--- a/js/modules/k6/marshalling_test.go
+++ b/js/modules/k6/marshalling_test.go
@@ -22,6 +22,7 @@ package k6_test
import (
"context"
+ "net/url"
"testing"
"time"
@@ -29,8 +30,8 @@ import (
"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/lib/testutils"
"github.com/loadimpact/k6/lib/types"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/stats"
- "github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -114,8 +115,8 @@ func TestSetupDataMarshalling(t *testing.T) {
`))
runner, err := js.New(
- &lib.SourceData{Filename: "/script.js", Data: script},
- afero.NewMemMapFs(),
+ &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
+ nil,
lib.RuntimeOptions{},
)
diff --git a/js/runner.go b/js/runner.go
index 603161369b4..4d3f97635bc 100644
--- a/js/runner.go
+++ b/js/runner.go
@@ -34,6 +34,7 @@ import (
"github.com/loadimpact/k6/js/common"
"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/lib/netext"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/stats"
"github.com/oxtoacart/bpool"
"github.com/pkg/errors"
@@ -62,8 +63,9 @@ type Runner struct {
setupData []byte
}
-func New(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) {
- bundle, err := NewBundle(src, fs, rtOpts)
+// New returns a new Runner for the provide source
+func New(src *loader.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) {
+ bundle, err := NewBundle(src, filesystems, rtOpts)
if err != nil {
return nil, err
}
diff --git a/js/runner_test.go b/js/runner_test.go
index cbd77ce187c..ddcb80fc2f9 100644
--- a/js/runner_test.go
+++ b/js/runner_test.go
@@ -32,7 +32,6 @@ import (
"net"
"net/http"
"os"
- "runtime"
"strings"
"testing"
"time"
@@ -59,13 +58,10 @@ import (
func TestRunnerNew(t *testing.T) {
t.Run("Valid", func(t *testing.T) {
- r, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r, err := getSimpleRunner("/script.js", `
let counter = 0;
export default function() { counter++; }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `)
assert.NoError(t, err)
t.Run("NewVU", func(t *testing.T) {
@@ -84,19 +80,13 @@ func TestRunnerNew(t *testing.T) {
})
t.Run("Invalid", func(t *testing.T) {
- _, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`blarg`),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
- assert.EqualError(t, err, "ReferenceError: blarg is not defined at /script.js:1:1(0)")
+ _, err := getSimpleRunner("/script.js", `blarg`)
+ assert.EqualError(t, err, "ReferenceError: blarg is not defined at file:///script.js:1:1(0)")
})
}
func TestRunnerGetDefaultGroup(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`export default function() {};`),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ r1, err := getSimpleRunner("/script.js", `export default function() {};`)
if assert.NoError(t, err) {
assert.NotNil(t, r1.GetDefaultGroup())
}
@@ -108,10 +98,7 @@ func TestRunnerGetDefaultGroup(t *testing.T) {
}
func TestRunnerOptions(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`export default function() {};`),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ r1, err := getSimpleRunner("/script.js", `export default function() {};`)
if !assert.NoError(t, err) {
return
}
@@ -151,9 +138,7 @@ func TestOptionsSettingToScript(t *testing.T) {
variant := variant
t.Run(fmt.Sprintf("Variant#%d", i), func(t *testing.T) {
t.Parallel()
- src := &lib.SourceData{
- Filename: "/script.js",
- Data: []byte(variant + `
+ data := variant + `
export default function() {
if (!options) {
throw new Error("Expected options to be defined!");
@@ -161,10 +146,9 @@ func TestOptionsSettingToScript(t *testing.T) {
if (options.teardownTimeout != __ENV.expectedTeardownTimeout) {
throw new Error("expected teardownTimeout to be " + __ENV.expectedTeardownTimeout + " but it was " + options.teardownTimeout);
}
- };
- `),
- }
- r, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{Env: map[string]string{"expectedTeardownTimeout": "4s"}})
+ };`
+ r, err := getSimpleRunnerWithOptions("/script.js", data,
+ lib.RuntimeOptions{Env: map[string]string{"expectedTeardownTimeout": "4s"}})
require.NoError(t, err)
newOptions := lib.Options{TeardownTimeout: types.NullDurationFrom(4 * time.Second)}
@@ -183,9 +167,7 @@ func TestOptionsSettingToScript(t *testing.T) {
func TestOptionsPropagationToScript(t *testing.T) {
t.Parallel()
- src := &lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ data := `
export let options = { setupTimeout: "1s", myOption: "test" };
export default function() {
if (options.external) {
@@ -197,12 +179,11 @@ func TestOptionsPropagationToScript(t *testing.T) {
if (options.setupTimeout != __ENV.expectedSetupTimeout) {
throw new Error("expected setupTimeout to be " + __ENV.expectedSetupTimeout + " but it was " + options.setupTimeout);
}
- };
- `),
- }
+ };`
expScriptOptions := lib.Options{SetupTimeout: types.NullDurationFrom(1 * time.Second)}
- r1, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{Env: map[string]string{"expectedSetupTimeout": "1s"}})
+ r1, err := getSimpleRunnerWithOptions("/script.js", data,
+ lib.RuntimeOptions{Env: map[string]string{"expectedSetupTimeout": "1s"}})
require.NoError(t, err)
require.Equal(t, expScriptOptions, r1.GetOptions())
@@ -233,7 +214,7 @@ func TestMetricName(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
- script := []byte(tb.Replacer.Replace(`
+ script := tb.Replacer.Replace(`
import { Counter } from "k6/metrics";
let myCounter = new Counter("not ok name @");
@@ -241,13 +222,9 @@ func TestMetricName(t *testing.T) {
export default function(data) {
myCounter.add(1);
}
- `))
+ `)
- _, err := New(
- &lib.SourceData{Filename: "/script.js", Data: script},
- afero.NewMemMapFs(),
- lib.RuntimeOptions{},
- )
+ _, err := getSimpleRunner("/script.js", script)
require.Error(t, err)
}
@@ -255,7 +232,7 @@ func TestSetupDataIsolation(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
- script := []byte(tb.Replacer.Replace(`
+ script := tb.Replacer.Replace(`
import { Counter } from "k6/metrics";
export let options = {
@@ -285,13 +262,9 @@ func TestSetupDataIsolation(t *testing.T) {
}
myCounter.add(1);
}
- `))
+ `)
- runner, err := New(
- &lib.SourceData{Filename: "/script.js", Data: script},
- afero.NewMemMapFs(),
- lib.RuntimeOptions{},
- )
+ runner, err := getSimpleRunner("/script.js", script)
require.NoError(t, err)
engine, err := core.NewEngine(local.New(runner), runner.GetOptions())
@@ -322,13 +295,13 @@ func TestSetupDataIsolation(t *testing.T) {
require.Equal(t, 501, count, "mycounter should be the number of iterations + 1 for the teardown")
}
-func testSetupDataHelper(t *testing.T, src *lib.SourceData) {
+func testSetupDataHelper(t *testing.T, data string) {
t.Helper()
expScriptOptions := lib.Options{
SetupTimeout: types.NullDurationFrom(1 * time.Second),
TeardownTimeout: types.NullDurationFrom(1 * time.Second),
}
- r1, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ r1, err := getSimpleRunner("/script.js", data) // TODO fix this
require.NoError(t, err)
require.Equal(t, expScriptOptions, r1.GetOptions())
@@ -349,71 +322,56 @@ func testSetupDataHelper(t *testing.T, src *lib.SourceData) {
}
}
func TestSetupDataReturnValue(t *testing.T) {
- src := &lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
- export let options = { setupTimeout: "1s", teardownTimeout: "1s" };
- export function setup() {
- return 42;
- }
- export default function(data) {
- if (data != 42) {
- throw new Error("default: wrong data: " + JSON.stringify(data))
- }
- };
-
- export function teardown(data) {
- if (data != 42) {
- throw new Error("teardown: wrong data: " + JSON.stringify(data))
- }
- };
- `),
+ testSetupDataHelper(t, `
+ export let options = { setupTimeout: "1s", teardownTimeout: "1s" };
+ export function setup() {
+ return 42;
}
- testSetupDataHelper(t, src)
+ export default function(data) {
+ if (data != 42) {
+ throw new Error("default: wrong data: " + JSON.stringify(data))
+ }
+ };
+
+ export function teardown(data) {
+ if (data != 42) {
+ throw new Error("teardown: wrong data: " + JSON.stringify(data))
+ }
+ };`)
}
func TestSetupDataNoSetup(t *testing.T) {
- src := &lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
- export let options = { setupTimeout: "1s", teardownTimeout: "1s" };
- export default function(data) {
- if (data !== undefined) {
- throw new Error("default: wrong data: " + JSON.stringify(data))
- }
- };
+ testSetupDataHelper(t, `
+ export let options = { setupTimeout: "1s", teardownTimeout: "1s" };
+ export default function(data) {
+ if (data !== undefined) {
+ throw new Error("default: wrong data: " + JSON.stringify(data))
+ }
+ };
- export function teardown(data) {
- if (data !== undefined) {
- console.log(data);
- throw new Error("teardown: wrong data: " + JSON.stringify(data))
- }
- };
- `),
- }
- testSetupDataHelper(t, src)
+ export function teardown(data) {
+ if (data !== undefined) {
+ console.log(data);
+ throw new Error("teardown: wrong data: " + JSON.stringify(data))
+ }
+ };`)
}
func TestSetupDataNoReturn(t *testing.T) {
- src := &lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
- export let options = { setupTimeout: "1s", teardownTimeout: "1s" };
- export function setup() { }
- export default function(data) {
- if (data !== undefined) {
- throw new Error("default: wrong data: " + JSON.stringify(data))
- }
- };
+ testSetupDataHelper(t, `
+ export let options = { setupTimeout: "1s", teardownTimeout: "1s" };
+ export function setup() { }
+ export default function(data) {
+ if (data !== undefined) {
+ throw new Error("default: wrong data: " + JSON.stringify(data))
+ }
+ };
- export function teardown(data) {
- if (data !== undefined) {
- throw new Error("teardown: wrong data: " + JSON.stringify(data))
- }
- };
- `),
- }
- testSetupDataHelper(t, src)
+ export function teardown(data) {
+ if (data !== undefined) {
+ throw new Error("teardown: wrong data: " + JSON.stringify(data))
+ }
+ };`)
}
func TestRunnerIntegrationImports(t *testing.T) {
t.Run("Modules", func(t *testing.T) {
@@ -424,12 +382,10 @@ func TestRunnerIntegrationImports(t *testing.T) {
"k6/html",
}
for _, mod := range modules {
+ mod := mod
t.Run(mod, func(t *testing.T) {
t.Run("Source", func(t *testing.T) {
- _, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(fmt.Sprintf(`import "%s"; export default function() {}`, mod)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ _, err := getSimpleRunner("/script.js", fmt.Sprintf(`import "%s"; export default function() {}`, mod))
assert.NoError(t, err)
})
})
@@ -438,8 +394,8 @@ func TestRunnerIntegrationImports(t *testing.T) {
t.Run("Files", func(t *testing.T) {
fs := afero.NewMemMapFs()
- assert.NoError(t, fs.MkdirAll("/path/to", 0755))
- assert.NoError(t, afero.WriteFile(fs, "/path/to/lib.js", []byte(`export default "hi!";`), 0644))
+ require.NoError(t, fs.MkdirAll("/path/to", 0755))
+ require.NoError(t, afero.WriteFile(fs, "/path/to/lib.js", []byte(`export default "hi!";`), 0644))
testdata := map[string]struct{ filename, path string }{
"Absolute": {"/path/script.js", "/path/to/lib.js"},
@@ -449,33 +405,26 @@ func TestRunnerIntegrationImports(t *testing.T) {
"STDIN-Relative": {"-", "./path/to/lib.js"},
}
for name, data := range testdata {
+ name, data := name, data
t.Run(name, func(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: data.filename,
- Data: []byte(fmt.Sprintf(`
+ r1, err := getSimpleRunnerWithFileFs(data.filename, fmt.Sprintf(`
import hi from "%s";
export default function() {
if (hi != "hi!") { throw new Error("incorrect value"); }
- }`, data.path)),
- }, fs, lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ }`, data.path), fs)
+ require.NoError(t, err)
r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ require.NoError(t, err)
testdata := map[string]*Runner{"Source": r1, "Archive": r2}
for name, r := range testdata {
+ r := r
t.Run(name, func(t *testing.T) {
vu, err := r.NewVU(make(chan stats.SampleContainer, 100))
- if !assert.NoError(t, err) {
- return
- }
+ require.NoError(t, err)
err = vu.RunOnce(context.Background())
- assert.NoError(t, err)
+ require.NoError(t, err)
})
}
})
@@ -484,16 +433,11 @@ func TestRunnerIntegrationImports(t *testing.T) {
}
func TestVURunContext(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
export let options = { vus: 10 };
export default function() { fn(); }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ `)
+ require.NoError(t, err)
r1.SetOptions(r1.GetOptions().Apply(lib.Options{Throw: null.BoolFrom(true)}))
r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{})
@@ -533,16 +477,13 @@ func TestVURunContext(t *testing.T) {
func TestVURunInterrupt(t *testing.T) {
//TODO: figure out why interrupt sometimes fails... data race in goja?
- if runtime.GOOS == "windows" {
+ if isWindows {
t.Skip()
}
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
export default function() { while(true) {} }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `)
require.NoError(t, err)
require.NoError(t, r1.SetOptions(lib.Options{Throw: null.BoolFrom(true)}))
@@ -572,9 +513,7 @@ func TestVURunInterrupt(t *testing.T) {
}
func TestVUIntegrationGroups(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
import { group } from "k6";
export default function() {
fnOuter();
@@ -585,16 +524,11 @@ func TestVUIntegrationGroups(t *testing.T) {
})
});
}
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ `)
+ require.NoError(t, err)
r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ require.NoError(t, err)
testdata := map[string]*Runner{"Source": r1, "Archive": r2}
for name, r := range testdata {
@@ -635,23 +569,16 @@ func TestVUIntegrationGroups(t *testing.T) {
}
func TestVUIntegrationMetrics(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
import { group } from "k6";
import { Trend } from "k6/metrics";
let myMetric = new Trend("my_metric");
export default function() { myMetric.add(5); }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ `)
+ require.NoError(t, err)
r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ require.NoError(t, err)
testdata := map[string]*Runner{"Source": r1, "Archive": r2}
for name, r := range testdata {
@@ -709,23 +636,15 @@ func TestVUIntegrationInsecureRequests(t *testing.T) {
}
for name, data := range testdata {
t.Run(name, func(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
import http from "k6/http";
export default function() { http.get("https://expired.badssl.com/"); }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ `)
+ require.NoError(t, err)
r1.SetOptions(lib.Options{Throw: null.BoolFrom(true)}.Apply(data.opts))
r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
-
+ require.NoError(t, err)
runners := map[string]*Runner{"Source": r1, "Archive": r2}
for name, r := range runners {
t.Run(name, func(t *testing.T) {
@@ -748,18 +667,14 @@ func TestVUIntegrationInsecureRequests(t *testing.T) {
}
func TestVUIntegrationBlacklistOption(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
import http from "k6/http";
export default function() { http.get("http://10.1.2.3/"); }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
- if !assert.NoError(t, err) {
- return
- }
+ `)
+ require.NoError(t, err)
cidr, err := lib.ParseCIDR("10.0.0.0/8")
+
if !assert.NoError(t, err) {
return
}
@@ -787,9 +702,7 @@ func TestVUIntegrationBlacklistOption(t *testing.T) {
}
func TestVUIntegrationBlacklistScript(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
import http from "k6/http";
export let options = {
@@ -798,8 +711,7 @@ func TestVUIntegrationBlacklistScript(t *testing.T) {
};
export default function() { http.get("http://10.1.2.3/"); }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `)
if !assert.NoError(t, err) {
return
}
@@ -828,9 +740,8 @@ func TestVUIntegrationHosts(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ r1, err := getSimpleRunner("/script.js",
+ tb.Replacer.Replace(`
import { check, fail } from "k6";
import http from "k6/http";
export default function() {
@@ -839,8 +750,7 @@ func TestVUIntegrationHosts(t *testing.T) {
"is correct IP": (r) => r.remote_ip === "127.0.0.1"
}) || fail("failed to override dns");
}
- `)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `))
if !assert.NoError(t, err) {
return
}
@@ -912,13 +822,10 @@ func TestVUIntegrationTLSConfig(t *testing.T) {
}
for name, data := range testdata {
t.Run(name, func(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
import http from "k6/http";
export default function() { http.get("https://sha256.badssl.com/"); }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `)
if !assert.NoError(t, err) {
return
}
@@ -951,17 +858,14 @@ func TestVUIntegrationTLSConfig(t *testing.T) {
}
func TestVUIntegrationHTTP2(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
import http from "k6/http";
export default function() {
let res = http.request("GET", "https://http2.akamai.com/demo");
if (res.status != 200) { throw new Error("wrong status: " + res.status) }
if (res.proto != "HTTP/2.0") { throw new Error("wrong proto: " + res.proto) }
}
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `)
if !assert.NoError(t, err) {
return
}
@@ -1001,12 +905,9 @@ func TestVUIntegrationHTTP2(t *testing.T) {
}
func TestVUIntegrationOpenFunctionError(t *testing.T) {
- r, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r, err := getSimpleRunner("/script.js", `
export default function() { open("/tmp/foo") }
- `),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `)
assert.NoError(t, err)
vu, err := r.NewVU(make(chan stats.SampleContainer, 100))
@@ -1020,9 +921,7 @@ func TestVUIntegrationCookiesReset(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ r1, err := getSimpleRunner("/script.js", tb.Replacer.Replace(`
import http from "k6/http";
export default function() {
let url = "HTTPBIN_URL";
@@ -1038,8 +937,7 @@ func TestVUIntegrationCookiesReset(t *testing.T) {
throw new Error("wrong cookies: " + res.body);
}
}
- `)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `))
if !assert.NoError(t, err) {
return
}
@@ -1073,9 +971,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ r1, err := getSimpleRunner("/script.js", tb.Replacer.Replace(`
import http from "k6/http";
export default function() {
let url = "HTTPBIN_URL";
@@ -1095,8 +991,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) {
}
}
}
- `)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `))
if !assert.NoError(t, err) {
return
}
@@ -1130,14 +1025,11 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) {
}
func TestVUIntegrationVUID(t *testing.T) {
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(`
+ r1, err := getSimpleRunner("/script.js", `
export default function() {
if (__VU != 1234) { throw new Error("wrong __VU: " + __VU); }
}`,
- ),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ )
if !assert.NoError(t, err) {
return
}
@@ -1226,13 +1118,10 @@ func TestVUIntegrationClientCerts(t *testing.T) {
}
go func() { _ = srv.Serve(listener) }()
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(fmt.Sprintf(`
+ r1, err := getSimpleRunner("/script.js", fmt.Sprintf(`
import http from "k6/http";
export default function() { http.get("https://%s")}
- `, listener.Addr().String())),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `, listener.Addr().String()))
if !assert.NoError(t, err) {
return
}
@@ -1309,17 +1198,14 @@ func TestHTTPRequestInInitContext(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
- _, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ _, err := getSimpleRunner("/script.js", tb.Replacer.Replace(`
import { check, fail } from "k6";
import http from "k6/http";
let res = http.get("HTTPBIN_URL/");
export default function() {
console.log(test);
}
- `)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `))
if assert.Error(t, err) {
assert.Equal(
t,
@@ -1392,11 +1278,9 @@ func TestInitContextForbidden(t *testing.T) {
defer tb.Cleanup()
for _, test := range table {
+ test := test
t.Run(test[0], func(t *testing.T) {
- _, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(test[1])),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ _, err := getSimpleRunner("/script.js", tb.Replacer.Replace(test[1]))
if assert.Error(t, err) {
assert.Equal(
t,
@@ -1412,10 +1296,7 @@ func TestArchiveRunningIntegraty(t *testing.T) {
defer tb.Cleanup()
fs := afero.NewMemMapFs()
- require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm))
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ data := tb.Replacer.Replace(`
let fput = open("/home/somebody/test.json");
export let options = { setupTimeout: "10s", teardownTimeout: "10s" };
export function setup() {
@@ -1426,8 +1307,10 @@ func TestArchiveRunningIntegraty(t *testing.T) {
throw new Error("incorrect answer " + data);
}
}
- `)),
- }, fs, lib.RuntimeOptions{})
+ `)
+ require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm))
+ require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(data), os.ModePerm))
+ r1, err := getSimpleRunnerWithFileFs("/script.js", data, fs)
require.NoError(t, err)
buf := bytes.NewBuffer(nil)
@@ -1458,18 +1341,15 @@ func TestArchiveNotPanicking(t *testing.T) {
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/non/existent", []byte(`42`), os.ModePerm))
- r1, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ r1, err := getSimpleRunnerWithFileFs("/script.js", tb.Replacer.Replace(`
let fput = open("/non/existent");
export default function(data) {
}
- `)),
- }, fs, lib.RuntimeOptions{})
+ `), fs)
require.NoError(t, err)
arc := r1.MakeArchive()
- arc.Files = make(map[string][]byte)
+ arc.Filesystems = map[string]afero.Fs{"file": afero.NewMemMapFs()}
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
// we do want this to error here as this is where we find out that a given file is not in the
// archive
@@ -1481,9 +1361,7 @@ func TestStuffNotPanicking(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
- r, err := New(&lib.SourceData{
- Filename: "/script.js",
- Data: []byte(tb.Replacer.Replace(`
+ r, err := getSimpleRunner("/script.js", tb.Replacer.Replace(`
import http from "k6/http";
import ws from "k6/ws";
import { group } from "k6";
@@ -1521,8 +1399,7 @@ func TestStuffNotPanicking(t *testing.T) {
}
});
}
- `)),
- }, afero.NewMemMapFs(), lib.RuntimeOptions{})
+ `))
require.NoError(t, err)
ch := make(chan stats.SampleContainer, 1000)
diff --git a/lib/archive.go b/lib/archive.go
index db9f0325593..f18db2d2c72 100644
--- a/lib/archive.go
+++ b/lib/archive.go
@@ -24,8 +24,10 @@ import (
"archive/tar"
"bytes"
"encoding/json"
+ "fmt"
"io"
"io/ioutil"
+ "net/url"
"os"
"path"
"path/filepath"
@@ -34,12 +36,17 @@ import (
"strings"
"time"
+ "github.com/loadimpact/k6/lib/fsext"
+ "github.com/loadimpact/k6/loader"
"github.com/spf13/afero"
)
-var volumeRE = regexp.MustCompile(`^([a-zA-Z]):(.*)`)
-var sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js
-var homeDirRE = regexp.MustCompile(`^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`)
+//nolint: gochecknoglobals, lll
+var (
+ volumeRE = regexp.MustCompile(`^[/\\]?([a-zA-Z]):(.*)`)
+ sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js
+ homeDirRE = regexp.MustCompile(`(?i)^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`)
+)
// NormalizeAndAnonymizePath Normalizes (to use a / path separator) and anonymizes a file path, by scrubbing usernames from home directories.
func NormalizeAndAnonymizePath(path string) string {
@@ -51,20 +58,10 @@ func NormalizeAndAnonymizePath(path string) string {
return homeDirRE.ReplaceAllString(p, `$1/$2/nobody`)
}
-type normalizedFS struct {
- afero.Fs
-}
-
-func (m *normalizedFS) Open(name string) (afero.File, error) {
- return m.Fs.Open(NormalizeAndAnonymizePath(name))
-}
-
-func (m *normalizedFS) OpenFile(name string, flag int, mode os.FileMode) (afero.File, error) {
- return m.Fs.OpenFile(NormalizeAndAnonymizePath(name), flag, mode)
-}
-
-func (m *normalizedFS) Stat(name string) (os.FileInfo, error) {
- return m.Fs.Stat(NormalizeAndAnonymizePath(name))
+func newNormalizedFs(fs afero.Fs) afero.Fs {
+ return fsext.NewChangePathFs(fs, fsext.ChangePathFunc(func(name string) (string, error) {
+ return NormalizeAndAnonymizePath(name), nil
+ }))
}
// An Archive is a rollup of all resources and options needed to reproduce a test identically elsewhere.
@@ -75,34 +72,43 @@ type Archive struct {
// Options to use.
Options Options `json:"options"`
+ // TODO: rewrite the encoding, decoding of json to use another type with only the fields it
+ // needs in order to remove Filename and Pwd from this
// Filename and contents of the main file being executed.
- Filename string `json:"filename"`
- Data []byte `json:"-"`
+ Filename string `json:"filename"` // only for json
+ FilenameURL *url.URL `json:"-"`
+ Data []byte `json:"-"`
// Working directory for resolving relative paths.
- Pwd string `json:"pwd"`
+ Pwd string `json:"pwd"` // only for json
+ PwdURL *url.URL `json:"-"`
- // Archived filesystem.
- Scripts map[string][]byte `json:"-"` // included scripts
- Files map[string][]byte `json:"-"` // non-script resources
-
- FS afero.Fs `json:"-"`
+ Filesystems map[string]afero.Fs `json:"-"`
// Environment variables
Env map[string]string `json:"env"`
K6Version string `json:"k6version"`
+ Goos string `json:"goos"`
}
-// Reads an archive created by Archive.Write from a reader.
-func ReadArchive(in io.Reader) (*Archive, error) {
- r := tar.NewReader(in)
- arc := &Archive{
- Scripts: make(map[string][]byte),
- Files: make(map[string][]byte),
- FS: &normalizedFS{Fs: afero.NewMemMapFs()},
+func (arc *Archive) getFs(name string) afero.Fs {
+ fs, ok := arc.Filesystems[name]
+ if !ok {
+ fs = afero.NewMemMapFs()
+ if name == "file" {
+ fs = newNormalizedFs(fs)
+ }
+ arc.Filesystems[name] = fs
}
+ return fs
+}
+
+// ReadArchive reads an archive created by Archive.Write from a reader.
+func ReadArchive(in io.Reader) (*Archive, error) {
+ r := tar.NewReader(in)
+ arc := &Archive{Filesystems: make(map[string]afero.Fs, 2)}
for {
hdr, err := r.Next()
if err != nil {
@@ -122,15 +128,27 @@ func ReadArchive(in io.Reader) (*Archive, error) {
switch hdr.Name {
case "metadata.json":
- if err := json.Unmarshal(data, &arc); err != nil {
+ if err = json.Unmarshal(data, &arc); err != nil {
return nil, err
}
// Path separator normalization for older archives (<=0.20.0)
- arc.Filename = NormalizeAndAnonymizePath(arc.Filename)
- arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd)
+ if arc.K6Version == "" {
+ arc.Filename = NormalizeAndAnonymizePath(arc.Filename)
+ arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd)
+ }
+ arc.PwdURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd)
+ if err != nil {
+ return nil, err
+ }
+ arc.FilenameURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Filename)
+ if err != nil {
+ return nil, err
+ }
+
continue
case "data":
arc.Data = data
+ continue
}
// Path separator normalization for older archives (<=0.20.0)
@@ -140,30 +158,70 @@ func ReadArchive(in io.Reader) (*Archive, error) {
continue
}
pfx := normPath[:idx]
- name := normPath[idx+1:]
- if name != "" && name[0] == '_' {
- name = name[1:]
- }
+ name := normPath[idx:]
- var dst map[string][]byte
switch pfx {
- case "files":
- dst = arc.Files
- case "scripts":
- dst = arc.Scripts
+ case "files", "scripts": // old archives
+ // in old archives (pre 0.25.0) names without "_" at the beginning were https, the ones with "_" are local files
+ pfx = "https"
+ if len(name) >= 2 && name[0:2] == "/_" {
+ pfx = "file"
+ name = name[2:]
+ }
+ fallthrough
+ case "https", "file":
+ fs := arc.getFs(pfx)
+ name = filepath.FromSlash(name)
+ err = afero.WriteFile(fs, name, data, os.FileMode(hdr.Mode))
+ if err != nil {
+ return nil, err
+ }
+ err = fs.Chtimes(name, hdr.AccessTime, hdr.ModTime)
+ if err != nil {
+ return nil, err
+ }
default:
- continue
+ return nil, fmt.Errorf("unknown file prefix `%s` for file `%s`", pfx, normPath)
}
+ }
+ scheme, pathOnFs := getURLPathOnFs(arc.FilenameURL)
+ var err error
+ pathOnFs, err = url.PathUnescape(pathOnFs)
+ if err != nil {
+ return nil, err
+ }
+ err = afero.WriteFile(arc.getFs(scheme), pathOnFs, arc.Data, 0644) // TODO fix the mode ?
+ if err != nil {
+ return nil, err
+ }
- dst[name] = data
+ return arc, nil
+}
- err = afero.WriteFile(arc.FS, name, data, os.ModePerm)
- if err != nil {
- return nil, err
- }
+func normalizeAndAnonymizeURL(u *url.URL) {
+ if u.Scheme == "file" {
+ u.Path = NormalizeAndAnonymizePath(u.Path)
}
+}
- return arc, nil
+func getURLPathOnFs(u *url.URL) (scheme string, pathOnFs string) {
+ scheme = "https"
+ switch {
+ case u.Opaque != "":
+ return scheme, "/" + u.Opaque
+ case u.Scheme == "":
+ return scheme, path.Clean(u.String()[len("//"):])
+ default:
+ scheme = u.Scheme
+ }
+ return scheme, path.Clean(u.String()[len(u.Scheme)+len(":/"):])
+}
+
+func getURLtoString(u *url.URL) string {
+ if u.Opaque == "" && u.Scheme == "" {
+ return u.String()[len("//"):] // https url without a scheme
+ }
+ return u.String()
}
// Write serialises the archive to a writer.
@@ -173,11 +231,18 @@ func ReadArchive(in io.Reader) (*Archive, error) {
// the current one.
func (arc *Archive) Write(out io.Writer) error {
w := tar.NewWriter(out)
- t := time.Now()
+ now := time.Now()
metaArc := *arc
- metaArc.Filename = NormalizeAndAnonymizePath(metaArc.Filename)
- metaArc.Pwd = NormalizeAndAnonymizePath(metaArc.Pwd)
+ normalizeAndAnonymizeURL(metaArc.FilenameURL)
+ normalizeAndAnonymizeURL(metaArc.PwdURL)
+ metaArc.Filename = getURLtoString(metaArc.FilenameURL)
+ metaArc.Pwd = getURLtoString(metaArc.PwdURL)
+ var actualDataPath, err = url.PathUnescape(path.Join(getURLPathOnFs(metaArc.FilenameURL)))
+ if err != nil {
+ return err
+ }
+ var madeLinkToData bool
metadata, err := metaArc.json()
if err != nil {
return err
@@ -186,10 +251,10 @@ func (arc *Archive) Write(out io.Writer) error {
Name: "metadata.json",
Mode: 0644,
Size: int64(len(metadata)),
- ModTime: t,
+ ModTime: now,
Typeflag: tar.TypeReg,
})
- if _, err := w.Write(metadata); err != nil {
+ if _, err = w.Write(metadata); err != nil {
return err
}
@@ -197,27 +262,20 @@ func (arc *Archive) Write(out io.Writer) error {
Name: "data",
Mode: 0644,
Size: int64(len(arc.Data)),
- ModTime: t,
+ ModTime: now,
Typeflag: tar.TypeReg,
})
- if _, err := w.Write(arc.Data); err != nil {
+ if _, err = w.Write(arc.Data); err != nil {
return err
}
-
- arcfs := []struct {
- name string
- files map[string][]byte
- }{
- {"scripts", arc.Scripts},
- {"files", arc.Files},
- }
- for _, entry := range arcfs {
- _ = w.WriteHeader(&tar.Header{
- Name: entry.name,
- Mode: 0755,
- ModTime: t,
- Typeflag: tar.TypeDir,
- })
+ for _, name := range [...]string{"file", "https"} {
+ filesystem, ok := arc.Filesystems[name]
+ if !ok {
+ continue
+ }
+ if cachedfs, ok := filesystem.(fsext.CacheOnReadFs); ok {
+ filesystem = cachedfs.GetCachingFs()
+ }
// A couple of things going on here:
// - You can't just create file entries, you need to create directory entries too.
@@ -227,21 +285,32 @@ func (arc *Archive) Write(out io.Writer) error {
// - We don't want to leak private information (eg. usernames) in archives, so make sure to
// anonymize paths before stuffing them in a shareable archive.
foundDirs := make(map[string]bool)
- paths := make([]string, 0, len(entry.files))
- files := make(map[string][]byte, len(entry.files))
- for filePath, data := range entry.files {
- filePath = NormalizeAndAnonymizePath(filePath)
- files[filePath] = data
- paths = append(paths, filePath)
- dir := path.Dir(filePath)
- for {
- foundDirs[dir] = true
- idx := strings.LastIndexByte(dir, os.PathSeparator)
- if idx == -1 {
- break
- }
- dir = dir[:idx]
+ paths := make([]string, 0, 10)
+ infos := make(map[string]os.FileInfo) // ... fix this ?
+ files := make(map[string][]byte)
+
+ walkFunc := filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ normalizedPath := NormalizeAndAnonymizePath(filePath)
+
+ infos[normalizedPath] = info
+ if info.IsDir() {
+ foundDirs[normalizedPath] = true
+ return nil
}
+
+ paths = append(paths, normalizedPath)
+ files[normalizedPath], err = afero.ReadFile(filesystem, filePath)
+ return err
+ })
+
+ if err = fsext.Walk(filesystem, afero.FilePathSeparator, walkFunc); err != nil {
+ return err
+ }
+ if len(files) == 0 {
+ continue // we don't need to write anything for this fs, if this is not done the root will be written
}
dirs := make([]string, 0, len(foundDirs))
for dirpath := range foundDirs {
@@ -250,35 +319,51 @@ func (arc *Archive) Write(out io.Writer) error {
sort.Strings(paths)
sort.Strings(dirs)
- for _, dirpath := range dirs {
- if dirpath == "" || dirpath[0] == '/' {
- dirpath = "_" + dirpath
- }
+ for _, dirPath := range dirs {
_ = w.WriteHeader(&tar.Header{
- Name: path.Clean(entry.name + "/" + dirpath),
- Mode: 0755,
- ModTime: t,
- Typeflag: tar.TypeDir,
+ Name: path.Clean(path.Join(name, dirPath)),
+ Mode: 0755, // MemMapFs is buggy
+ AccessTime: now, // MemMapFs is buggy
+ ChangeTime: now, // MemMapFs is buggy
+ ModTime: now, // MemMapFs is buggy
+ Typeflag: tar.TypeDir,
})
}
for _, filePath := range paths {
- data := files[filePath]
- if filePath[0] == '/' {
- filePath = "_" + filePath
+ var fullFilePath = path.Clean(path.Join(name, filePath))
+ // we either have opaque
+ if fullFilePath == actualDataPath {
+ madeLinkToData = true
+ err = w.WriteHeader(&tar.Header{
+ Name: fullFilePath,
+ Size: 0,
+ Typeflag: tar.TypeLink,
+ Linkname: "data",
+ })
+ } else {
+ err = w.WriteHeader(&tar.Header{
+ Name: fullFilePath,
+ Mode: 0644, // MemMapFs is buggy
+ Size: int64(len(files[filePath])),
+ AccessTime: infos[filePath].ModTime(),
+ ChangeTime: infos[filePath].ModTime(),
+ ModTime: infos[filePath].ModTime(),
+ Typeflag: tar.TypeReg,
+ })
+ if err == nil {
+ _, err = w.Write(files[filePath])
+ }
}
- _ = w.WriteHeader(&tar.Header{
- Name: path.Clean(entry.name + "/" + filePath),
- Mode: 0644,
- Size: int64(len(data)),
- ModTime: t,
- Typeflag: tar.TypeReg,
- })
- if _, err := w.Write(data); err != nil {
+ if err != nil {
return err
}
}
}
+ if !madeLinkToData {
+ // This should never happen we should always link to `data` from inside the file/https directories
+ return fmt.Errorf("archive creation failed because the main script wasn't present in the cached filesystem")
+ }
return w.Close()
}
diff --git a/lib/archive_test.go b/lib/archive_test.go
index 48ea6588ba0..bc1ffbb043f 100644
--- a/lib/archive_test.go
+++ b/lib/archive_test.go
@@ -23,10 +23,18 @@ package lib
import (
"bytes"
"fmt"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
"runtime"
"testing"
+ "github.com/loadimpact/k6/lib/consts"
+ "github.com/loadimpact/k6/lib/fsext"
+ "github.com/spf13/afero"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
null "gopkg.in/guregu/null.v3"
)
@@ -42,6 +50,8 @@ func TestNormalizeAndAnonymizePath(t *testing.T) {
"\\NOTSHARED\\dir\\dir\\myfile.txt": "/NOTSHARED/dir/dir/myfile.txt",
"C:\\Users\\myname\\dir\\myfile.txt": "/C/Users/nobody/dir/myfile.txt",
"D:\\Documents and Settings\\myname\\dir\\myfile.txt": "/D/Documents and Settings/nobody/dir/myfile.txt",
+ "C:\\uSers\\myname\\dir\\myfile.txt": "/C/uSers/nobody/dir/myfile.txt",
+ "D:\\doCUMENts aND Settings\\myname\\dir\\myfile.txt": "/D/doCUMENts aND Settings/nobody/dir/myfile.txt",
}
// TODO: fix this - the issue is that filepath.Clean replaces `/` with whatever the path
// separator is on the current OS and as such this gets confused for shared folder on
@@ -50,6 +60,7 @@ func TestNormalizeAndAnonymizePath(t *testing.T) {
testdata["//etc/hosts"] = "/etc/hosts"
}
for from, to := range testdata {
+ from, to := from, to
t.Run("path="+from, func(t *testing.T) {
res := NormalizeAndAnonymizePath(from)
assert.Equal(t, to, res)
@@ -58,36 +69,113 @@ func TestNormalizeAndAnonymizePath(t *testing.T) {
}
}
+func makeMemMapFs(t *testing.T, input map[string][]byte) afero.Fs {
+ fs := afero.NewMemMapFs()
+ for path, data := range input {
+ require.NoError(t, afero.WriteFile(fs, path, data, 0644))
+ }
+ return fs
+}
+
+func getMapKeys(m map[string]afero.Fs) []string {
+ keys := make([]string, 0, len(m))
+ for key := range m {
+ keys = append(keys, key)
+ }
+
+ return keys
+}
+
+func diffMapFilesystems(t *testing.T, first, second map[string]afero.Fs) bool {
+ require.ElementsMatch(t, getMapKeys(first), getMapKeys(second),
+ "fs map keys don't match %s, %s", getMapKeys(first), getMapKeys(second))
+ for key, fs := range first {
+ secondFs := second[key]
+ diffFilesystems(t, fs, secondFs)
+ }
+
+ return true
+}
+
+func diffFilesystems(t *testing.T, first, second afero.Fs) {
+ diffFilesystemsDir(t, first, second, "/")
+}
+
+func getInfoNames(infos []os.FileInfo) []string {
+ var names = make([]string, len(infos))
+ for i, info := range infos {
+ names[i] = info.Name()
+ }
+ return names
+}
+
+func diffFilesystemsDir(t *testing.T, first, second afero.Fs, dirname string) {
+ firstInfos, err := afero.ReadDir(first, dirname)
+ require.NoError(t, err, dirname)
+
+ secondInfos, err := afero.ReadDir(first, dirname)
+ require.NoError(t, err, dirname)
+
+ require.ElementsMatch(t, getInfoNames(firstInfos), getInfoNames(secondInfos), "directory: "+dirname)
+ for _, info := range firstInfos {
+ path := filepath.Join(dirname, info.Name())
+ if info.IsDir() {
+ diffFilesystemsDir(t, first, second, path)
+ continue
+ }
+ firstData, err := afero.ReadFile(first, path)
+ require.NoError(t, err, path)
+
+ secondData, err := afero.ReadFile(second, path)
+ require.NoError(t, err, path)
+
+ assert.Equal(t, firstData, secondData, path)
+ }
+}
+
func TestArchiveReadWrite(t *testing.T) {
t.Run("Roundtrip", func(t *testing.T) {
arc1 := &Archive{
- Type: "js",
+ Type: "js",
+ K6Version: consts.Version,
Options: Options{
VUs: null.IntFrom(12345),
SystemTags: GetTagSet(DefaultSystemTagList...),
},
- Filename: "/path/to/script.js",
- Data: []byte(`// contents...`),
- Pwd: "/path/to",
- Scripts: map[string][]byte{
- "/path/to/a.js": []byte(`// a contents`),
- "/path/to/b.js": []byte(`// b contents`),
- "cdnjs.com/libraries/Faker": []byte(`// faker contents`),
- },
- Files: map[string][]byte{
- "/path/to/file1.txt": []byte(`hi!`),
- "/path/to/file2.txt": []byte(`bye!`),
- "github.com/loadimpact/k6/README.md": []byte(`README`),
+ FilenameURL: &url.URL{Scheme: "file", Path: "/path/to/a.js"},
+ Data: []byte(`// a contents`),
+ PwdURL: &url.URL{Scheme: "file", Path: "/path/to"},
+ Filesystems: map[string]afero.Fs{
+ "file": makeMemMapFs(t, map[string][]byte{
+ "/path/to/a.js": []byte(`// a contents`),
+ "/path/to/b.js": []byte(`// b contents`),
+ "/path/to/file1.txt": []byte(`hi!`),
+ "/path/to/file2.txt": []byte(`bye!`),
+ }),
+ "https": makeMemMapFs(t, map[string][]byte{
+ "/cdnjs.com/libraries/Faker": []byte(`// faker contents`),
+ "/github.com/loadimpact/k6/README.md": []byte(`README`),
+ }),
},
}
buf := bytes.NewBuffer(nil)
- assert.NoError(t, arc1.Write(buf))
+ require.NoError(t, arc1.Write(buf))
+
+ arc1Filesystems := arc1.Filesystems
+ arc1.Filesystems = nil
arc2, err := ReadArchive(buf)
- arc2.FS = nil
- assert.NoError(t, err)
+ require.NoError(t, err)
+
+ arc2Filesystems := arc2.Filesystems
+ arc2.Filesystems = nil
+ arc2.Filename = ""
+ arc2.Pwd = ""
+
assert.Equal(t, arc1, arc2)
+
+ diffMapFilesystems(t, arc1Filesystems, arc2Filesystems)
})
t.Run("Anonymized", func(t *testing.T) {
@@ -95,7 +183,7 @@ func TestArchiveReadWrite(t *testing.T) {
Pwd, PwdNormAnon string
}{
{"/home/myname", "/home/nobody"},
- {"C:\\Users\\Administrator", "/C/Users/nobody"},
+ {filepath.FromSlash("/C:/Users/Administrator"), "/C/Users/nobody"},
}
for _, entry := range testdata {
arc1 := &Archive{
@@ -104,18 +192,21 @@ func TestArchiveReadWrite(t *testing.T) {
VUs: null.IntFrom(12345),
SystemTags: GetTagSet(DefaultSystemTagList...),
},
- Filename: fmt.Sprintf("%s/script.js", entry.Pwd),
- Data: []byte(`// contents...`),
- Pwd: entry.Pwd,
- Scripts: map[string][]byte{
- fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`),
- fmt.Sprintf("%s/b.js", entry.Pwd): []byte(`// b contents`),
- "cdnjs.com/libraries/Faker": []byte(`// faker contents`),
- },
- Files: map[string][]byte{
- fmt.Sprintf("%s/file1.txt", entry.Pwd): []byte(`hi!`),
- fmt.Sprintf("%s/file2.txt", entry.Pwd): []byte(`bye!`),
- "github.com/loadimpact/k6/README.md": []byte(`README`),
+ FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/a.js", entry.Pwd)},
+ K6Version: consts.Version,
+ Data: []byte(`// a contents`),
+ PwdURL: &url.URL{Scheme: "file", Path: entry.Pwd},
+ Filesystems: map[string]afero.Fs{
+ "file": makeMemMapFs(t, map[string][]byte{
+ fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`),
+ fmt.Sprintf("%s/b.js", entry.Pwd): []byte(`// b contents`),
+ fmt.Sprintf("%s/file1.txt", entry.Pwd): []byte(`hi!`),
+ fmt.Sprintf("%s/file2.txt", entry.Pwd): []byte(`bye!`),
+ }),
+ "https": makeMemMapFs(t, map[string][]byte{
+ "/cdnjs.com/libraries/Faker": []byte(`// faker contents`),
+ "/github.com/loadimpact/k6/README.md": []byte(`README`),
+ }),
},
}
arc1Anon := &Archive{
@@ -124,28 +215,41 @@ func TestArchiveReadWrite(t *testing.T) {
VUs: null.IntFrom(12345),
SystemTags: GetTagSet(DefaultSystemTagList...),
},
- Filename: fmt.Sprintf("%s/script.js", entry.PwdNormAnon),
- Data: []byte(`// contents...`),
- Pwd: entry.PwdNormAnon,
- Scripts: map[string][]byte{
- fmt.Sprintf("%s/a.js", entry.PwdNormAnon): []byte(`// a contents`),
- fmt.Sprintf("%s/b.js", entry.PwdNormAnon): []byte(`// b contents`),
- "cdnjs.com/libraries/Faker": []byte(`// faker contents`),
- },
- Files: map[string][]byte{
- fmt.Sprintf("%s/file1.txt", entry.PwdNormAnon): []byte(`hi!`),
- fmt.Sprintf("%s/file2.txt", entry.PwdNormAnon): []byte(`bye!`),
- "github.com/loadimpact/k6/README.md": []byte(`README`),
+ FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/a.js", entry.PwdNormAnon)},
+ K6Version: consts.Version,
+ Data: []byte(`// a contents`),
+ PwdURL: &url.URL{Scheme: "file", Path: entry.PwdNormAnon},
+
+ Filesystems: map[string]afero.Fs{
+ "file": makeMemMapFs(t, map[string][]byte{
+ fmt.Sprintf("%s/a.js", entry.PwdNormAnon): []byte(`// a contents`),
+ fmt.Sprintf("%s/b.js", entry.PwdNormAnon): []byte(`// b contents`),
+ fmt.Sprintf("%s/file1.txt", entry.PwdNormAnon): []byte(`hi!`),
+ fmt.Sprintf("%s/file2.txt", entry.PwdNormAnon): []byte(`bye!`),
+ }),
+ "https": makeMemMapFs(t, map[string][]byte{
+ "/cdnjs.com/libraries/Faker": []byte(`// faker contents`),
+ "/github.com/loadimpact/k6/README.md": []byte(`README`),
+ }),
},
}
buf := bytes.NewBuffer(nil)
- assert.NoError(t, arc1.Write(buf))
+ require.NoError(t, arc1.Write(buf))
+
+ arc1Filesystems := arc1Anon.Filesystems
+ arc1Anon.Filesystems = nil
arc2, err := ReadArchive(buf)
- arc2.FS = nil
assert.NoError(t, err)
+ arc2.Filename = ""
+ arc2.Pwd = ""
+
+ arc2Filesystems := arc2.Filesystems
+ arc2.Filesystems = nil
+
assert.Equal(t, arc1Anon, arc2)
+ diffMapFilesystems(t, arc1Filesystems, arc2Filesystems)
}
})
}
@@ -159,3 +263,139 @@ func TestArchiveJSONEscape(t *testing.T) {
assert.NoError(t, err)
assert.Contains(t, string(b), "test<.js")
}
+
+func TestUsingCacheFromCacheOnReadFs(t *testing.T) {
+ var base = afero.NewMemMapFs()
+ var cached = afero.NewMemMapFs()
+ // we specifically have different contents in both places
+ require.NoError(t, afero.WriteFile(base, "/wrong", []byte(`ooops`), 0644))
+ require.NoError(t, afero.WriteFile(cached, "/correct", []byte(`test`), 0644))
+
+ arc := &Archive{
+ Type: "js",
+ FilenameURL: &url.URL{Scheme: "file", Path: "/correct"},
+ K6Version: consts.Version,
+ Data: []byte(`test`),
+ PwdURL: &url.URL{Scheme: "file", Path: "/"},
+ Filesystems: map[string]afero.Fs{
+ "file": fsext.NewCacheOnReadFs(base, cached, 0),
+ },
+ }
+
+ buf := bytes.NewBuffer(nil)
+ require.NoError(t, arc.Write(buf))
+
+ newArc, err := ReadArchive(buf)
+ require.NoError(t, err)
+
+ data, err := afero.ReadFile(newArc.Filesystems["file"], "/correct")
+ require.NoError(t, err)
+ require.Equal(t, string(data), "test")
+
+ data, err = afero.ReadFile(newArc.Filesystems["file"], "/wrong")
+ require.Error(t, err)
+ require.Nil(t, data)
+}
+
+func TestArchiveWithDataNotInFS(t *testing.T) {
+ t.Parallel()
+
+ arc := &Archive{
+ Type: "js",
+ FilenameURL: &url.URL{Scheme: "file", Path: "/script"},
+ K6Version: consts.Version,
+ Data: []byte(`test`),
+ PwdURL: &url.URL{Scheme: "file", Path: "/"},
+ Filesystems: nil,
+ }
+
+ buf := bytes.NewBuffer(nil)
+ err := arc.Write(buf)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "the main script wasn't present in the cached filesystem")
+}
+
+func TestMalformedMetadata(t *testing.T) {
+ var fs = afero.NewMemMapFs()
+ require.NoError(t, afero.WriteFile(fs, "/metadata.json", []byte("{,}"), 0644))
+ var b, err = dumpMemMapFsToBuf(fs)
+ require.NoError(t, err)
+ _, err = ReadArchive(b)
+ require.Error(t, err)
+ require.Equal(t, err.Error(), `invalid character ',' looking for beginning of object key string`)
+}
+
+func TestStrangePaths(t *testing.T) {
+ var pathsToChange = []string{
+ `/path/with spaces/a.js`,
+ `/path/with spaces/a.js`,
+ `/path/with日本語/b.js`,
+ `/path/with spaces and 日本語/file1.txt`,
+ }
+ for _, pathToChange := range pathsToChange {
+ otherMap := make(map[string][]byte, len(pathsToChange))
+ for _, other := range pathsToChange {
+ otherMap[other] = []byte(`// ` + other + ` contents`)
+ }
+ arc1 := &Archive{
+ Type: "js",
+ K6Version: consts.Version,
+ Options: Options{
+ VUs: null.IntFrom(12345),
+ SystemTags: GetTagSet(DefaultSystemTagList...),
+ },
+ FilenameURL: &url.URL{Scheme: "file", Path: pathToChange},
+ Data: []byte(`// ` + pathToChange + ` contents`),
+ PwdURL: &url.URL{Scheme: "file", Path: path.Dir(pathToChange)},
+ Filesystems: map[string]afero.Fs{
+ "file": makeMemMapFs(t, otherMap),
+ },
+ }
+
+ buf := bytes.NewBuffer(nil)
+ require.NoError(t, arc1.Write(buf), pathToChange)
+
+ arc1Filesystems := arc1.Filesystems
+ arc1.Filesystems = nil
+
+ arc2, err := ReadArchive(buf)
+ require.NoError(t, err, pathToChange)
+
+ arc2Filesystems := arc2.Filesystems
+ arc2.Filesystems = nil
+ arc2.Filename = ""
+ arc2.Pwd = ""
+
+ assert.Equal(t, arc1, arc2, pathToChange)
+
+ diffMapFilesystems(t, arc1Filesystems, arc2Filesystems)
+ }
+}
+
+func TestStdinArchive(t *testing.T) {
+ var fs = afero.NewMemMapFs()
+ // we specifically have different contents in both places
+ require.NoError(t, afero.WriteFile(fs, "/-", []byte(`test`), 0644))
+
+ arc := &Archive{
+ Type: "js",
+ FilenameURL: &url.URL{Scheme: "file", Path: "/-"},
+ K6Version: consts.Version,
+ Data: []byte(`test`),
+ PwdURL: &url.URL{Scheme: "file", Path: "/"},
+ Filesystems: map[string]afero.Fs{
+ "file": fs,
+ },
+ }
+
+ buf := bytes.NewBuffer(nil)
+ require.NoError(t, arc.Write(buf))
+
+ newArc, err := ReadArchive(buf)
+ require.NoError(t, err)
+
+ data, err := afero.ReadFile(newArc.Filesystems["file"], "/-")
+ require.NoError(t, err)
+ require.Equal(t, string(data), "test")
+
+}
diff --git a/lib/fsext/cacheonread.go b/lib/fsext/cacheonread.go
new file mode 100644
index 00000000000..9a534f3f664
--- /dev/null
+++ b/lib/fsext/cacheonread.go
@@ -0,0 +1,27 @@
+package fsext
+
+import (
+ "time"
+
+ "github.com/spf13/afero"
+)
+
+// CacheOnReadFs is wrapper around afero.CacheOnReadFs with the ability to return the filesystem
+// that is used as cache
+type CacheOnReadFs struct {
+ afero.Fs
+ cache afero.Fs
+}
+
+// NewCacheOnReadFs returns a new CacheOnReadFs
+func NewCacheOnReadFs(base, layer afero.Fs, cacheTime time.Duration) afero.Fs {
+ return CacheOnReadFs{
+ Fs: afero.NewCacheOnReadFs(base, layer, cacheTime),
+ cache: layer,
+ }
+}
+
+// GetCachingFs returns the afero.Fs being used for cache
+func (c CacheOnReadFs) GetCachingFs() afero.Fs {
+ return c.cache
+}
diff --git a/lib/fsext/changepathfs.go b/lib/fsext/changepathfs.go
new file mode 100644
index 00000000000..7f7a81d6f2f
--- /dev/null
+++ b/lib/fsext/changepathfs.go
@@ -0,0 +1,190 @@
+package fsext
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/spf13/afero"
+)
+
+var _ afero.Lstater = (*ChangePathFs)(nil)
+
+// ChangePathFs is a filesystem that wraps another afero.Fs and changes all given paths from all
+// file and directory names, with a function, before calling the same method on the wrapped afero.Fs.
+// Heavily based on afero.BasePathFs
+type ChangePathFs struct {
+ source afero.Fs
+ fn ChangePathFunc
+}
+
+// ChangePathFile is a file from ChangePathFs
+type ChangePathFile struct {
+ afero.File
+ originalName string
+}
+
+// NewChangePathFs return a ChangePathFs where all paths will be change with the provided funcs
+func NewChangePathFs(source afero.Fs, fn ChangePathFunc) *ChangePathFs {
+ return &ChangePathFs{source: source, fn: fn}
+}
+
+// ChangePathFunc is the function that will be called by ChangePathFs to change the path
+type ChangePathFunc func(name string) (path string, err error)
+
+// NewTrimFilePathSeparatorFs is ChangePathFs that trims a Afero.FilePathSeparator from all paths
+// Heavily based on afero.BasePathFs
+func NewTrimFilePathSeparatorFs(source afero.Fs) *ChangePathFs {
+ return &ChangePathFs{source: source, fn: ChangePathFunc(func(name string) (path string, err error) {
+ if !strings.HasPrefix(name, afero.FilePathSeparator) {
+ return name, os.ErrNotExist
+ }
+
+ return filepath.Clean(strings.TrimPrefix(name, afero.FilePathSeparator)), nil
+ })}
+}
+
+// Name Returns the name of the file
+func (f *ChangePathFile) Name() string {
+ return f.originalName
+}
+
+//Chtimes changes the access and modification times of the named file
+func (b *ChangePathFs) Chtimes(name string, atime, mtime time.Time) (err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return &os.PathError{Op: "chtimes", Path: name, Err: err}
+ }
+ return b.source.Chtimes(newName, atime, mtime)
+}
+
+// Chmod changes the mode of the named file to mode.
+func (b *ChangePathFs) Chmod(name string, mode os.FileMode) (err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return &os.PathError{Op: "chmod", Path: name, Err: err}
+ }
+ return b.source.Chmod(newName, mode)
+}
+
+// Name return the name of this FileSystem
+func (b *ChangePathFs) Name() string {
+ return "ChangePathFs"
+}
+
+// Stat returns a FileInfo describing the named file, or an error, if any
+// happens.
+func (b *ChangePathFs) Stat(name string) (fi os.FileInfo, err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return nil, &os.PathError{Op: "stat", Path: name, Err: err}
+ }
+ return b.source.Stat(newName)
+}
+
+// Rename renames a file.
+func (b *ChangePathFs) Rename(oldName, newName string) (err error) {
+ var newOldName, newNewName string
+ if newOldName, err = b.fn(oldName); err != nil {
+ return &os.PathError{Op: "rename", Path: oldName, Err: err}
+ }
+ if newNewName, err = b.fn(newName); err != nil {
+ return &os.PathError{Op: "rename", Path: newName, Err: err}
+ }
+ return b.source.Rename(newOldName, newNewName)
+}
+
+// RemoveAll removes a directory path and any children it contains. It
+// does not fail if the path does not exist (return nil).
+func (b *ChangePathFs) RemoveAll(name string) (err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return &os.PathError{Op: "remove_all", Path: name, Err: err}
+ }
+ return b.source.RemoveAll(newName)
+}
+
+// Remove removes a file identified by name, returning an error, if any
+// happens.
+func (b *ChangePathFs) Remove(name string) (err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return &os.PathError{Op: "remove", Path: name, Err: err}
+ }
+ return b.source.Remove(newName)
+}
+
+// OpenFile opens a file using the given flags and the given mode.
+func (b *ChangePathFs) OpenFile(name string, flag int, mode os.FileMode) (f afero.File, err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return nil, &os.PathError{Op: "openfile", Path: name, Err: err}
+ }
+ sourcef, err := b.source.OpenFile(newName, flag, mode)
+ if err != nil {
+ return nil, err
+ }
+ return &ChangePathFile{File: sourcef, originalName: name}, nil
+}
+
+// Open opens a file, returning it or an error, if any happens.
+func (b *ChangePathFs) Open(name string) (f afero.File, err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return nil, &os.PathError{Op: "open", Path: name, Err: err}
+ }
+ sourcef, err := b.source.Open(newName)
+ if err != nil {
+ return nil, err
+ }
+ return &ChangePathFile{File: sourcef, originalName: name}, nil
+}
+
+// Mkdir creates a directory in the filesystem, return an error if any
+// happens.
+func (b *ChangePathFs) Mkdir(name string, mode os.FileMode) (err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return &os.PathError{Op: "mkdir", Path: name, Err: err}
+ }
+ return b.source.Mkdir(newName, mode)
+}
+
+// MkdirAll creates a directory path and all parents that does not exist
+// yet.
+func (b *ChangePathFs) MkdirAll(name string, mode os.FileMode) (err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return &os.PathError{Op: "mkdir", Path: name, Err: err}
+ }
+ return b.source.MkdirAll(newName, mode)
+}
+
+// Create creates a file in the filesystem, returning the file and an
+// error, if any happens
+func (b *ChangePathFs) Create(name string) (f afero.File, err error) {
+ var newName string
+ if newName, err = b.fn(name); err != nil {
+ return nil, &os.PathError{Op: "create", Path: name, Err: err}
+ }
+ sourcef, err := b.source.Create(newName)
+ if err != nil {
+ return nil, err
+ }
+ return &ChangePathFile{File: sourcef, originalName: name}, nil
+}
+
+// LstatIfPossible implements the afero.Lstater interface
+func (b *ChangePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+ var newName string
+ newName, err := b.fn(name)
+ if err != nil {
+ return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err}
+ }
+ if lstater, ok := b.source.(afero.Lstater); ok {
+ return lstater.LstatIfPossible(newName)
+ }
+ fi, err := b.source.Stat(newName)
+ return fi, false, err
+}
diff --git a/lib/fsext/changepathfs_test.go b/lib/fsext/changepathfs_test.go
new file mode 100644
index 00000000000..5e91ea77f97
--- /dev/null
+++ b/lib/fsext/changepathfs_test.go
@@ -0,0 +1,167 @@
+package fsext
+
+import (
+ "fmt"
+ "os"
+ "path"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/require"
+)
+
+func TestChangePathFs(t *testing.T) {
+ var m = afero.NewMemMapFs()
+ var prefix = "/another/"
+ var c = NewChangePathFs(m, ChangePathFunc(func(name string) (string, error) {
+ if !strings.HasPrefix(name, prefix) {
+ return "", fmt.Errorf("path %s doesn't start with `%s`", name, prefix)
+ }
+ return name[len(prefix):], nil
+ }))
+
+ var filePath = "/another/path/to/file.txt"
+
+ require.Equal(t, c.Name(), "ChangePathFs")
+ t.Run("Create", func(t *testing.T) {
+ f, err := c.Create(filePath)
+ require.NoError(t, err)
+ require.Equal(t, filePath, f.Name())
+
+ /** TODO figure out if this is error in MemMapFs
+ _, err = c.Create(filePath)
+ require.Error(t, err)
+ require.True(t, os.IsExist(err))
+ */
+
+ _, err = c.Create("/notanother/path/to/file.txt")
+ checkErrorPath(t, err, "/notanother/path/to/file.txt")
+ })
+
+ t.Run("Mkdir", func(t *testing.T) {
+ require.NoError(t, c.Mkdir("/another/path/too", 0644))
+ checkErrorPath(t, c.Mkdir("/notanother/path/too", 0644), "/notanother/path/too")
+ })
+
+ t.Run("MkdirAll", func(t *testing.T) {
+ require.NoError(t, c.MkdirAll("/another/pattth/too", 0644))
+ checkErrorPath(t, c.MkdirAll("/notanother/pattth/too", 0644), "/notanother/pattth/too")
+ })
+
+ t.Run("Open", func(t *testing.T) {
+ f, err := c.Open(filePath)
+ require.NoError(t, err)
+ require.Equal(t, filePath, f.Name())
+
+ _, err = c.Open("/notanother/path/to/file.txt")
+ checkErrorPath(t, err, "/notanother/path/to/file.txt")
+ })
+
+ t.Run("OpenFile", func(t *testing.T) {
+ f, err := c.OpenFile(filePath, os.O_RDWR, 0644)
+ require.NoError(t, err)
+ require.Equal(t, filePath, f.Name())
+
+ _, err = c.OpenFile("/notanother/path/to/file.txt", os.O_RDWR, 0644)
+ checkErrorPath(t, err, "/notanother/path/to/file.txt")
+
+ _, err = c.OpenFile("/another/nonexistant", os.O_RDWR, 0644)
+ require.True(t, os.IsNotExist(err))
+ })
+
+ t.Run("Stat Chmod Chtimes", func(t *testing.T) {
+ info, err := c.Stat(filePath)
+ require.NoError(t, err)
+ require.Equal(t, "file.txt", info.Name())
+
+ sometime := time.Unix(10000, 13)
+ require.NotEqual(t, sometime, info.ModTime())
+ require.NoError(t, c.Chtimes(filePath, time.Now(), sometime))
+ require.Equal(t, sometime, info.ModTime())
+
+ mode := os.FileMode(0007)
+ require.NotEqual(t, mode, info.Mode())
+ require.NoError(t, c.Chmod(filePath, mode))
+ require.Equal(t, mode, info.Mode())
+
+ _, err = c.Stat("/notanother/path/to/file.txt")
+ checkErrorPath(t, err, "/notanother/path/to/file.txt")
+
+ checkErrorPath(t, c.Chtimes("/notanother/path/to/file.txt", time.Now(), time.Now()), "/notanother/path/to/file.txt")
+
+ checkErrorPath(t, c.Chmod("/notanother/path/to/file.txt", mode), "/notanother/path/to/file.txt")
+ })
+
+ t.Run("LstatIfPossible", func(t *testing.T) {
+ info, ok, err := c.LstatIfPossible(filePath)
+ require.NoError(t, err)
+ require.False(t, ok)
+ require.Equal(t, "file.txt", info.Name())
+
+ _, _, err = c.LstatIfPossible("/notanother/path/to/file.txt")
+ checkErrorPath(t, err, "/notanother/path/to/file.txt")
+ })
+
+ t.Run("Rename", func(t *testing.T) {
+ info, err := c.Stat(filePath)
+ require.NoError(t, err)
+ require.False(t, info.IsDir())
+
+ require.NoError(t, c.Rename(filePath, "/another/path/to/file.doc"))
+
+ _, err = c.Stat(filePath)
+ require.Error(t, err)
+ require.True(t, os.IsNotExist(err))
+
+ info, err = c.Stat("/another/path/to/file.doc")
+ require.NoError(t, err)
+ require.False(t, info.IsDir())
+
+ checkErrorPath(t,
+ c.Rename("/notanother/path/to/file.txt", "/another/path/to/file.doc"),
+ "/notanother/path/to/file.txt")
+
+ checkErrorPath(t,
+ c.Rename(filePath, "/notanother/path/to/file.doc"),
+ "/notanother/path/to/file.doc")
+ })
+
+ t.Run("Remove", func(t *testing.T) {
+ var removeFilePath = "/another/file/to/remove.txt"
+ _, err := c.Create(removeFilePath)
+ require.NoError(t, err)
+
+ require.NoError(t, c.Remove(removeFilePath))
+
+ _, err = c.Stat(removeFilePath)
+ require.Error(t, err)
+ require.True(t, os.IsNotExist(err))
+
+ _, err = c.Create(removeFilePath)
+ require.NoError(t, err)
+
+ require.NoError(t, c.RemoveAll(path.Dir(removeFilePath)))
+
+ _, err = c.Stat(removeFilePath)
+ require.Error(t, err)
+ require.True(t, os.IsNotExist(err))
+
+ checkErrorPath(t,
+ c.Remove("/notanother/path/to/file.txt"),
+ "/notanother/path/to/file.txt")
+
+ checkErrorPath(t,
+ c.RemoveAll("/notanother/path/to"),
+ "/notanother/path/to")
+ })
+}
+
+func checkErrorPath(t *testing.T, err error, path string) {
+ require.Error(t, err)
+ p, ok := err.(*os.PathError)
+ require.True(t, ok)
+ require.Equal(t, p.Path, path)
+
+}
diff --git a/lib/fsext/trimpathseparator_test.go b/lib/fsext/trimpathseparator_test.go
new file mode 100644
index 00000000000..678b0c1b758
--- /dev/null
+++ b/lib/fsext/trimpathseparator_test.go
@@ -0,0 +1,30 @@
+package fsext
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/require"
+)
+
+func TestTrimAferoPathSeparatorFs(t *testing.T) {
+ m := afero.NewMemMapFs()
+ fs := NewTrimFilePathSeparatorFs(m)
+ expecteData := []byte("something")
+ err := afero.WriteFile(fs, filepath.FromSlash("/path/to/somewhere"), expecteData, 0644)
+ require.NoError(t, err)
+ data, err := afero.ReadFile(m, "/path/to/somewhere")
+ require.Error(t, err)
+ require.True(t, os.IsNotExist(err))
+ require.Nil(t, data)
+
+ data, err = afero.ReadFile(m, "path/to/somewhere")
+ require.NoError(t, err)
+ require.Equal(t, expecteData, data)
+
+ err = afero.WriteFile(fs, filepath.FromSlash("path/without/separtor"), expecteData, 0644)
+ require.Error(t, err)
+ require.True(t, os.IsNotExist(err))
+}
diff --git a/lib/fsext/walk.go b/lib/fsext/walk.go
new file mode 100644
index 00000000000..2869f022c82
--- /dev/null
+++ b/lib/fsext/walk.go
@@ -0,0 +1,84 @@
+package fsext
+
+import (
+ "os"
+ "path/filepath"
+ "sort"
+
+ "github.com/spf13/afero"
+)
+
+// Walk implements afero.Walk, but in a way that it doesn't loop to infinity and doesn't have
+// problems if a given path part looks like a windows volume name
+func Walk(fs afero.Fs, root string, walkFn filepath.WalkFunc) error {
+ info, err := fs.Stat(root)
+ if err != nil {
+ return walkFn(root, nil, err)
+ }
+ return walk(fs, root, info, walkFn)
+}
+
+// readDirNames reads the directory named by dirname and returns
+// a sorted list of directory entries.
+// adapted from https://github.com/spf13/afero/blob/master/path.go#L27
+func readDirNames(fs afero.Fs, dirname string) ([]string, error) {
+ f, err := fs.Open(dirname)
+ if err != nil {
+ return nil, err
+ }
+ infos, err := f.Readdir(-1)
+ if err != nil {
+ return nil, err
+ }
+ err = f.Close()
+
+ if err != nil {
+ return nil, err
+ }
+
+ var names = make([]string, len(infos))
+ for i, info := range infos {
+ names[i] = info.Name()
+ }
+ sort.Strings(names)
+ return names, nil
+}
+
+// walk recursively descends path, calling walkFn
+// adapted from https://github.com/spf13/afero/blob/master/path.go#L27
+func walk(fs afero.Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
+ err := walkFn(path, info, nil)
+ if err != nil {
+ if info.IsDir() && err == filepath.SkipDir {
+ return nil
+ }
+ return err
+ }
+
+ if !info.IsDir() {
+ return nil
+ }
+
+ names, err := readDirNames(fs, path)
+ if err != nil {
+ return walkFn(path, info, err)
+ }
+
+ for _, name := range names {
+ filename := filepath.Join(path, name)
+ fileInfo, err := fs.Stat(filename)
+ if err != nil {
+ if err = walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
+ return err
+ }
+ } else {
+ err = walk(fs, filename, fileInfo, walkFn)
+ if err != nil {
+ if !fileInfo.IsDir() || err != filepath.SkipDir {
+ return err
+ }
+ }
+ }
+ }
+ return nil
+}
diff --git a/lib/models.go b/lib/models.go
index c187b5b2382..3944eeecba6 100644
--- a/lib/models.go
+++ b/lib/models.go
@@ -40,12 +40,6 @@ const GroupSeparator = "::"
// Error emitted if you attempt to instantiate a Group or Check that contains the separator.
var ErrNameContainsGroupSeparator = errors.New("group and check names may not contain '::'")
-// Wraps a source file; data and filename.
-type SourceData struct {
- Data []byte
- Filename string
-}
-
// StageFields defines the fields used for a Stage; this is a dumb hack to make the JSON code
// cleaner. pls fix.
type StageFields struct {
diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go
new file mode 100644
index 00000000000..a72b1d975ce
--- /dev/null
+++ b/lib/old_archive_test.go
@@ -0,0 +1,208 @@
+package lib
+
+import (
+ "archive/tar"
+ "bytes"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "testing"
+
+ "github.com/loadimpact/k6/lib/fsext"
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/require"
+)
+
+func dumpMemMapFsToBuf(fs afero.Fs) (*bytes.Buffer, error) {
+ var b = bytes.NewBuffer(nil)
+ var w = tar.NewWriter(b)
+ err := fsext.Walk(fs, afero.FilePathSeparator,
+ filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error {
+ if filePath == afero.FilePathSeparator {
+ return nil // skip the root
+ }
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return w.WriteHeader(&tar.Header{
+ Name: path.Clean(filepath.ToSlash(filePath)[1:]),
+ Mode: 0555,
+ Typeflag: tar.TypeDir,
+ })
+ }
+ var data []byte
+ data, err = afero.ReadFile(fs, filePath)
+ if err != nil {
+ return err
+ }
+ err = w.WriteHeader(&tar.Header{
+ Name: path.Clean(filepath.ToSlash(filePath)[1:]),
+ Mode: 0644,
+ Size: int64(len(data)),
+ Typeflag: tar.TypeReg,
+ })
+ if err != nil {
+ return err
+ }
+ _, err = w.Write(data)
+ if err != nil {
+ return err
+ }
+ return nil
+ }))
+ if err != nil {
+ return nil, err
+ }
+ return b, w.Close()
+}
+
+func TestOldArchive(t *testing.T) {
+ var testCases = map[string]string{
+ // map of filename to data for each main file tested
+ "github.com/loadimpact/k6/samples/example.js": `github file`,
+ "cdnjs.com/packages/Faker": `faker file`,
+ "C:/something/path2": `windows script`,
+ "/absolulte/path2": `unix script`,
+ }
+ for filename, data := range testCases {
+ filename, data := filename, data
+ t.Run(filename, func(t *testing.T) {
+ metadata := `{"filename": "` + filename + `"}`
+ fs := makeMemMapFs(t, map[string][]byte{
+ // files
+ "/files/github.com/loadimpact/k6/samples/example.js": []byte(`github file`),
+ "/files/cdnjs.com/packages/Faker": []byte(`faker file`),
+ "/files/example.com/path/to.js": []byte(`example.com file`),
+ "/files/_/C/something/path": []byte(`windows file`),
+ "/files/_/absolulte/path": []byte(`unix file`),
+
+ // scripts
+ "/scripts/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`),
+ "/scripts/cdnjs.com/packages/Faker2": []byte(`faker script`),
+ "/scripts/example.com/path/too.js": []byte(`example.com script`),
+ "/scripts/_/C/something/path2": []byte(`windows script`),
+ "/scripts/_/absolulte/path2": []byte(`unix script`),
+ "/data": []byte(data),
+ "/metadata.json": []byte(metadata),
+ })
+
+ buf, err := dumpMemMapFsToBuf(fs)
+ require.NoError(t, err)
+
+ var (
+ expectedFilesystems = map[string]afero.Fs{
+ "file": makeMemMapFs(t, map[string][]byte{
+ "/C:/something/path": []byte(`windows file`),
+ "/absolulte/path": []byte(`unix file`),
+ "/C:/something/path2": []byte(`windows script`),
+ "/absolulte/path2": []byte(`unix script`),
+ }),
+ "https": makeMemMapFs(t, map[string][]byte{
+ "/example.com/path/to.js": []byte(`example.com file`),
+ "/example.com/path/too.js": []byte(`example.com script`),
+ "/github.com/loadimpact/k6/samples/example.js": []byte(`github file`),
+ "/cdnjs.com/packages/Faker": []byte(`faker file`),
+ "/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`),
+ "/cdnjs.com/packages/Faker2": []byte(`faker script`),
+ }),
+ }
+ )
+
+ arc, err := ReadArchive(buf)
+ require.NoError(t, err)
+
+ diffMapFilesystems(t, expectedFilesystems, arc.Filesystems)
+ })
+ }
+}
+
+func TestUnknownPrefix(t *testing.T) {
+ fs := makeMemMapFs(t, map[string][]byte{
+ "/strange/something": []byte(`github file`),
+ })
+ buf, err := dumpMemMapFsToBuf(fs)
+ require.NoError(t, err)
+
+ _, err = ReadArchive(buf)
+ require.Error(t, err)
+ require.Equal(t, err.Error(),
+ "unknown file prefix `strange` for file `strange/something`")
+}
+
+func TestFilenamePwdResolve(t *testing.T) {
+ var tests = []struct {
+ Filename, Pwd, version string
+ expectedFilenameURL, expectedPwdURL *url.URL
+ expectedError string
+ }{
+ {
+ Filename: "/home/nobody/something.js",
+ Pwd: "/home/nobody",
+ expectedFilenameURL: &url.URL{Scheme: "file", Path: "/home/nobody/something.js"},
+ expectedPwdURL: &url.URL{Scheme: "file", Path: "/home/nobody"},
+ },
+ {
+ Filename: "github.com/loadimpact/k6/samples/http2.js",
+ Pwd: "github.com/loadimpact/k6/samples",
+ expectedFilenameURL: &url.URL{Opaque: "github.com/loadimpact/k6/samples/http2.js"},
+ expectedPwdURL: &url.URL{Opaque: "github.com/loadimpact/k6/samples"},
+ },
+ {
+ Filename: "cdnjs.com/libraries/Faker",
+ Pwd: "/home/nobody",
+ expectedFilenameURL: &url.URL{Opaque: "cdnjs.com/libraries/Faker"},
+ expectedPwdURL: &url.URL{Scheme: "file", Path: "/home/nobody"},
+ },
+ {
+ Filename: "example.com/something/dot.js",
+ Pwd: "example.com/something/",
+ expectedFilenameURL: &url.URL{Host: "example.com", Scheme: "", Path: "/something/dot.js"},
+ expectedPwdURL: &url.URL{Host: "example.com", Scheme: "", Path: "/something"},
+ },
+ {
+ Filename: "https://example.com/something/dot.js",
+ Pwd: "https://example.com/something",
+ expectedFilenameURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something/dot.js"},
+ expectedPwdURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something"},
+ version: "0.25.0",
+ },
+ {
+ Filename: "ftps://example.com/something/dot.js",
+ Pwd: "https://example.com/something",
+ expectedError: "only supported schemes for imports are file and https",
+ version: "0.25.0",
+ },
+
+ {
+ Filename: "https://example.com/something/dot.js",
+ Pwd: "ftps://example.com/something",
+ expectedError: "only supported schemes for imports are file and https",
+ version: "0.25.0",
+ },
+ }
+
+ for _, test := range tests {
+ metadata := `{
+ "filename": "` + test.Filename + `",
+ "pwd": "` + test.Pwd + `",
+ "k6version": "` + test.version + `"
+ }`
+
+ buf, err := dumpMemMapFsToBuf(makeMemMapFs(t, map[string][]byte{
+ "/metadata.json": []byte(metadata),
+ }))
+ require.NoError(t, err)
+
+ arc, err := ReadArchive(buf)
+ if test.expectedError != "" {
+ require.Error(t, err)
+ require.Contains(t, err.Error(), test.expectedError)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, test.expectedFilenameURL, arc.FilenameURL)
+ require.Equal(t, test.expectedPwdURL, arc.PwdURL)
+ }
+ }
+}
diff --git a/loader/cdnjs_test.go b/loader/cdnjs_test.go
index 78f3c69ad73..55de95eadba 100644
--- a/loader/cdnjs_test.go
+++ b/loader/cdnjs_test.go
@@ -21,10 +21,12 @@
package loader
import (
+ "net/url"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestCDNJS(t *testing.T) {
@@ -61,19 +63,28 @@ func TestCDNJS(t *testing.T) {
`^https://cdnjs.cloudflare.com/ajax/libs/Faker/0.7.2/MinFaker.js$`,
},
}
+
+ var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"}
for path, expected := range paths {
+ path, expected := path, expected
t.Run(path, func(t *testing.T) {
name, loader, parts := pickLoader(path)
assert.Equal(t, "cdnjs", name)
assert.Equal(t, expected.parts, parts)
+
src, err := loader(path, parts)
- assert.NoError(t, err)
+ require.NoError(t, err)
assert.Regexp(t, expected.src, src)
- data, err := Load(afero.NewMemMapFs(), "/", path)
- if assert.NoError(t, err) {
- assert.Equal(t, path, data.Filename)
- assert.NotEmpty(t, data.Data)
- }
+
+ resolvedURL, err := Resolve(root, path)
+ require.NoError(t, err)
+ require.Empty(t, resolvedURL.Scheme)
+ require.Equal(t, path, resolvedURL.Opaque)
+
+ data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path)
+ require.NoError(t, err)
+ assert.Equal(t, resolvedURL, data.URL)
+ assert.NotEmpty(t, data.Data)
})
}
@@ -92,9 +103,14 @@ func TestCDNJS(t *testing.T) {
assert.Equal(t, "cdnjs", name)
assert.Equal(t, []string{"Faker", "3.1.0", "nonexistent.js"}, parts)
src, err := loader(path, parts)
- assert.NoError(t, err)
+ require.NoError(t, err)
assert.Equal(t, "https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js", src)
- _, err = Load(afero.NewMemMapFs(), "/", path)
- assert.EqualError(t, err, "cdnjs: not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js")
+
+ pathURL, err := url.Parse(src)
+ require.NoError(t, err)
+
+ _, err = Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js")
})
}
diff --git a/loader/filesystems.go b/loader/filesystems.go
new file mode 100644
index 00000000000..2bfde5c1205
--- /dev/null
+++ b/loader/filesystems.go
@@ -0,0 +1,27 @@
+package loader
+
+import (
+ "runtime"
+
+ "github.com/loadimpact/k6/lib/fsext"
+ "github.com/spf13/afero"
+)
+
+// CreateFilesystems creates the correct filesystem map for the current OS
+func CreateFilesystems() map[string]afero.Fs {
+ // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's
+ // written every time something is read from the real filesystem. This cache is then used for
+ // successive spawns to read from (they have no access to the real disk).
+ // Also initialize the same for `https` but the caching is handled manually in the loader package
+ osfs := afero.NewOsFs()
+ if runtime.GOOS == "windows" {
+ // This is done so that we can continue to use paths with /|"\" through the code but also to
+ // be easier to traverse the cachedFs later as it doesn't work very well if you have windows
+ // volumes
+ osfs = fsext.NewTrimFilePathSeparatorFs(osfs)
+ }
+ return map[string]afero.Fs{
+ "file": fsext.NewCacheOnReadFs(osfs, afero.NewMemMapFs(), 0),
+ "https": afero.NewMemMapFs(),
+ }
+}
diff --git a/loader/github_test.go b/loader/github_test.go
index 2d6d2daa2f4..bd110185b44 100644
--- a/loader/github_test.go
+++ b/loader/github_test.go
@@ -21,23 +21,64 @@
package loader
import (
+ "net/url"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestGithub(t *testing.T) {
path := "github.com/github/gitignore/Go.gitignore"
+ expectedEndSrc := "https://raw.githubusercontent.com/github/gitignore/master/Go.gitignore"
name, loader, parts := pickLoader(path)
assert.Equal(t, "github", name)
assert.Equal(t, []string{"github", "gitignore", "Go.gitignore"}, parts)
src, err := loader(path, parts)
assert.NoError(t, err)
- assert.Equal(t, "https://raw.githubusercontent.com/github/gitignore/master/Go.gitignore", src)
- data, err := Load(afero.NewMemMapFs(), "/", path)
- if assert.NoError(t, err) {
- assert.Equal(t, path, data.Filename)
+ assert.Equal(t, expectedEndSrc, src)
+
+ var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"}
+ resolvedURL, err := Resolve(root, path)
+ require.NoError(t, err)
+ require.Empty(t, resolvedURL.Scheme)
+ require.Equal(t, path, resolvedURL.Opaque)
+ t.Run("not cached", func(t *testing.T) {
+ data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path)
+ require.NoError(t, err)
+ assert.Equal(t, data.URL, resolvedURL)
+ assert.Equal(t, path, data.URL.String())
assert.NotEmpty(t, data.Data)
- }
+ })
+
+ t.Run("cached", func(t *testing.T) {
+ fs := afero.NewMemMapFs()
+ testData := []byte("test data")
+
+ err := afero.WriteFile(fs, "/github.com/github/gitignore/Go.gitignore", testData, 0644)
+ require.NoError(t, err)
+
+ data, err := Load(map[string]afero.Fs{"https": fs}, resolvedURL, path)
+ require.NoError(t, err)
+ assert.Equal(t, path, data.URL.String())
+ assert.Equal(t, data.Data, testData)
+ })
+
+ t.Run("relative", func(t *testing.T) {
+ var tests = map[string]string{
+ "./something.else": "github.com/github/gitignore/something.else",
+ "../something.else": "github.com/github/something.else",
+ "/something.else": "github.com/something.else",
+ }
+ for relative, expected := range tests {
+ relativeURL, err := Resolve(Dir(resolvedURL), relative)
+ require.NoError(t, err)
+ assert.Equal(t, expected, relativeURL.String())
+ }
+ })
+
+ t.Run("dir", func(t *testing.T) {
+ require.Equal(t, &url.URL{Opaque: "github.com/github/gitignore"}, Dir(resolvedURL))
+ })
}
diff --git a/loader/loader.go b/loader/loader.go
index 083ba8739c9..eb94568a24f 100644
--- a/loader/loader.go
+++ b/loader/loader.go
@@ -1,7 +1,7 @@
/*
*
* k6 - a next-generation load testing tool
- * Copyright (C) 2016 Load Impact
+ * Copyright (C) 2019 Load Impact
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,22 +22,29 @@ package loader
import (
"io/ioutil"
- "net"
"net/http"
"net/url"
+ "os"
+ "path"
"path/filepath"
"regexp"
"strings"
"time"
- "github.com/loadimpact/k6/lib"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
+// SourceData wraps a source file; data and filename.
+type SourceData struct {
+ Data []byte
+ URL *url.URL
+}
+
type loaderFunc func(path string, parts []string) (string, error)
+//nolint: gochecknoglobals
var (
loaders = []struct {
name string
@@ -47,114 +54,200 @@ var (
{"cdnjs", cdnjs, regexp.MustCompile(`^cdnjs.com/libraries/([^/]+)(?:/([(\d\.)]+-?[^/]*))?(?:/(.*))?$`)},
{"github", github, regexp.MustCompile(`^github.com/([^/]+)/([^/]+)/(.*)$`)},
}
- invalidScriptErrMsg = `The file "%[1]s" couldn't be found on local disk, ` +
- `and trying to retrieve it from https://%[1]s failed as well. Make ` +
- `sure that you've specified the right path to the file. If you're ` +
+ httpsSchemeCouldntBeLoadedMsg = `The moduleSpecifier "%s" couldn't be retrieved from` +
+ ` the resolved url "%s". Error : "%s"`
+ fileSchemeCouldntBeLoadedMsg = `The moduleSpecifier "%s" couldn't be found on ` +
+ `local disk. Make sure that you've specified the right path to the file. If you're ` +
`running k6 using the Docker image make sure you have mounted the ` +
`local directory (-v /local/path/:/inside/docker/path) containing ` +
`your script and modules so that they're accessible by k6 from ` +
`inside of the container, see ` +
`https://docs.k6.io/v1.0/docs/modules#section-using-local-modules-with-docker.`
+ errNoLoaderMatched = errors.New("no loader matched")
)
-// Resolves a relative path to an absolute one.
-func Resolve(pwd, name string) string {
- if name[0] == '.' {
- return filepath.ToSlash(filepath.Join(pwd, name))
+// Resolve a relative path to an absolute one.
+func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) {
+ if moduleSpecifier == "" {
+ return nil, errors.New("local or remote path required")
+ }
+
+ if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' || filepath.IsAbs(moduleSpecifier) {
+ if pwd.Opaque != "" { // this is a loader reference
+ parts := strings.SplitN(pwd.Opaque, "/", 2)
+ if moduleSpecifier[0] == '/' {
+ return &url.URL{Opaque: path.Join(parts[0], moduleSpecifier)}, nil
+ }
+ return &url.URL{Opaque: path.Join(parts[0], path.Join(path.Dir(parts[1]+"/"), moduleSpecifier))}, nil
+ }
+
+ // The file is in format like C:/something/path.js. But this will be decoded as scheme `C`
+ // ... which is not what we want we want it to be decode as file:///C:/something/path.js
+ if filepath.VolumeName(moduleSpecifier) != "" {
+ moduleSpecifier = "/" + moduleSpecifier
+ }
+
+ // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we read
+ // it if it's missing
+ var finalPwd = pwd
+ if pwd.Opaque != "" {
+ if !strings.HasSuffix(pwd.Opaque, "/") {
+ finalPwd = &url.URL{Opaque: pwd.Opaque + "/"}
+ }
+ } else if !strings.HasSuffix(pwd.Path, "/") {
+ finalPwd = &url.URL{}
+ *finalPwd = *pwd
+ finalPwd.Path += "/"
+ }
+ return finalPwd.Parse(moduleSpecifier)
+ }
+
+ if strings.Contains(moduleSpecifier, "://") {
+ u, err := url.Parse(moduleSpecifier)
+ if err != nil {
+ return nil, err
+ }
+ if u.Scheme != "file" && u.Scheme != "https" {
+ return nil,
+ errors.Errorf("only supported schemes for imports are file and https, %s has `%s`",
+ moduleSpecifier, u.Scheme)
+ }
+ if u.Scheme == "file" && pwd.Scheme == "https" {
+ return nil, errors.Errorf("origin (%s) not allowed to load local file: %s", pwd, moduleSpecifier)
+ }
+ return u, err
+ }
+ // here we only care if a loader is pickable, if it is and later there is an error in the loading
+ // from it we don't want to try another resolve
+ _, loader, _ := pickLoader(moduleSpecifier)
+ if loader == nil {
+ u, err := url.Parse("https://" + moduleSpecifier)
+ if err != nil {
+ return nil, err
+ }
+ u.Scheme = ""
+ return u, nil
}
- return name
+ return &url.URL{Opaque: moduleSpecifier}, nil
}
-// Returns the directory for the path.
-func Dir(name string) string {
- if name == "-" {
- return "/"
+// Dir returns the directory for the path.
+func Dir(old *url.URL) *url.URL {
+ if old.Opaque != "" { // loader
+ return &url.URL{Opaque: path.Join(old.Opaque, "../")}
}
- return filepath.Dir(name)
+ return old.ResolveReference(&url.URL{Path: "./"})
}
-func Load(fs afero.Fs, pwd, name string) (*lib.SourceData, error) {
- log.WithFields(log.Fields{"pwd": pwd, "name": name}).Debug("Loading...")
+// Load loads the provided moduleSpecifier from the given filesystems which are map of afero.Fs
+// for a given scheme which is they key of the map. If the scheme is https then a request will
+// be made if the files is not found in the map and written to the map.
+func Load(
+ filesystems map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string,
+) (*SourceData, error) {
+ log.WithFields(
+ log.Fields{
+ "moduleSpecifier": moduleSpecifier,
+ "original moduleSpecifier": originalModuleSpecifier,
+ }).Debug("Loading...")
- // We just need to make sure `import ""` doesn't crash the loader.
- if name == "" {
- return nil, errors.New("local or remote path required")
+ var pathOnFs string
+ switch {
+ case moduleSpecifier.Opaque != "": // This is loader
+ pathOnFs = filepath.Join(afero.FilePathSeparator, moduleSpecifier.Opaque)
+ case moduleSpecifier.Scheme == "":
+ pathOnFs = path.Clean(moduleSpecifier.String())
+ default:
+ pathOnFs = path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):])
}
-
- // Do not allow the protocol to be specified, it messes everything up.
- if strings.Contains(name, "://") {
- return nil, errors.New("imports should not contain a protocol")
+ scheme := moduleSpecifier.Scheme
+ if scheme == "" {
+ scheme = "https"
}
- // Do not allow remote-loaded scripts to lift arbitrary files off the user's machine.
- if (name[0] == '/' && pwd[0] != '/') || (filepath.VolumeName(name) != "" && filepath.VolumeName(pwd) == "") {
- return nil, errors.Errorf("origin (%s) not allowed to load local file: %s", pwd, name)
+ pathOnFs, err := url.PathUnescape(filepath.FromSlash(pathOnFs))
+ if err != nil {
+ return nil, err
}
- // If the file starts with ".", resolve it as a relative path.
- name = Resolve(pwd, name)
- log.WithField("name", name).Debug("Resolved...")
+ data, err := afero.ReadFile(filesystems[scheme], pathOnFs)
- // If the resolved path starts with a "/" or has a volume, it's a local file.
- if name[0] == '/' || filepath.VolumeName(name) != "" {
- data, err := afero.ReadFile(fs, name)
- if err != nil {
- return nil, err
+ if err != nil {
+ if os.IsNotExist(err) {
+ if scheme == "https" {
+ var finalModuleSpecifierURL = &url.URL{}
+
+ switch {
+ case moduleSpecifier.Opaque != "": // This is loader
+ finalModuleSpecifierURL, err = resolveUsingLoaders(moduleSpecifier.Opaque)
+ if err != nil {
+ return nil, err
+ }
+ case moduleSpecifier.Scheme == "":
+ log.WithField("url", moduleSpecifier).Warning(
+ "A url was resolved but it didn't have scheme. " +
+ "This will be deprecated in the future and all remote modules will " +
+ "need to explicitly use `https` as scheme")
+ *finalModuleSpecifierURL = *moduleSpecifier
+ finalModuleSpecifierURL.Scheme = scheme
+ default:
+ finalModuleSpecifierURL = moduleSpecifier
+ }
+ var result *SourceData
+ result, err = loadRemoteURL(finalModuleSpecifierURL)
+ if err != nil {
+ return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, finalModuleSpecifierURL, err)
+ }
+ result.URL = moduleSpecifier
+ // TODO maybe make an afero.Fs which makes request directly and than use CacheOnReadFs
+ // on top of as with the `file` scheme fs
+ _ = afero.WriteFile(filesystems[scheme], pathOnFs, result.Data, 0644)
+ return result, nil
+ }
+ return nil, errors.Errorf(fileSchemeCouldntBeLoadedMsg, moduleSpecifier)
}
- return &lib.SourceData{Filename: name, Data: data}, nil
+ return nil, err
}
- // If the file is from a known service, try loading from there.
- loaderName, loader, loaderArgs := pickLoader(name)
+ return &SourceData{URL: moduleSpecifier, Data: data}, nil
+}
+
+func resolveUsingLoaders(name string) (*url.URL, error) {
+ _, loader, loaderArgs := pickLoader(name)
if loader != nil {
- u, err := loader(name, loaderArgs)
+ urlString, err := loader(name, loaderArgs)
if err != nil {
return nil, err
}
- data, err := fetch(u)
- if err != nil {
- return nil, errors.Wrap(err, loaderName)
- }
- return &lib.SourceData{Filename: name, Data: data}, nil
+ return url.Parse(urlString)
}
- // If it's not a file, check is it a remote location. HTTPS is enforced, because it's 2017, HTTPS is easy,
- // running arbitrary, trivially MitM'd code (even sandboxed) is very, very bad.
- origURL := "https://" + name
- parsedURL, err := url.Parse(origURL)
-
- if err != nil {
- return nil, errors.Errorf(invalidScriptErrMsg, name)
- }
+ return nil, errNoLoaderMatched
+}
- if _, err = net.LookupHost(parsedURL.Hostname()); err != nil {
- return nil, errors.Errorf(invalidScriptErrMsg, name)
+func loadRemoteURL(u *url.URL) (*SourceData, error) {
+ var oldQuery = u.RawQuery
+ if u.RawQuery != "" {
+ u.RawQuery += "&"
}
+ u.RawQuery += "_k6=1"
- // Load it and have a look.
- url := origURL
- if !strings.ContainsRune(url, '?') {
- url += "?"
- } else {
- url += "&"
- }
- url += "_k6=1"
- data, err := fetch(url)
+ data, err := fetch(u.String())
+ u.RawQuery = oldQuery
// If this fails, try to fetch without ?_k6=1 - some sources act weird around unknown GET args.
if err != nil {
- data2, err2 := fetch(origURL)
- if err2 != nil {
- return nil, errors.Errorf(invalidScriptErrMsg, name)
+ data, err = fetch(u.String())
+ if err != nil {
+ return nil, err
}
- data = data2
}
// TODO: Parse the HTML, look for meta tags!!
//
//
- return &lib.SourceData{Filename: name, Data: data}, nil
+ return &SourceData{URL: u, Data: data}, nil
}
func pickLoader(path string) (string, loaderFunc, []string) {
diff --git a/loader/loader_test.go b/loader/loader_test.go
index 33e0f8c640c..745d6007509 100644
--- a/loader/loader_test.go
+++ b/loader/loader_test.go
@@ -18,31 +18,87 @@
*
*/
-package loader
+package loader_test
import (
"fmt"
"net/http"
+ "net/url"
"path/filepath"
"testing"
"github.com/loadimpact/k6/lib/testutils"
+ "github.com/loadimpact/k6/loader"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestDir(t *testing.T) {
testdata := map[string]string{
- "/path/to/file.txt": filepath.FromSlash("/path/to"),
+ "/path/to/file.txt": filepath.FromSlash("/path/to/"),
"-": "/",
}
for name, dir := range testdata {
+ nameURL := &url.URL{Scheme: "file", Path: name}
+ dirURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(dir)}
t.Run("path="+name, func(t *testing.T) {
- assert.Equal(t, dir, Dir(name))
+ assert.Equal(t, dirURL, loader.Dir(nameURL))
})
}
}
+func TestResolve(t *testing.T) {
+
+ t.Run("Blank", func(t *testing.T) {
+ _, err := loader.Resolve(nil, "")
+ assert.EqualError(t, err, "local or remote path required")
+ })
+
+ t.Run("Protocol", func(t *testing.T) {
+ root, err := url.Parse("file:///")
+ require.NoError(t, err)
+
+ t.Run("Missing", func(t *testing.T) {
+ u, err := loader.Resolve(root, "example.com/html")
+ require.NoError(t, err)
+ assert.Equal(t, u.String(), "//example.com/html")
+ // TODO: check that warning will be emitted if Loaded
+ })
+ t.Run("WS", func(t *testing.T) {
+ moduleSpecifier := "ws://example.com/html"
+ _, err := loader.Resolve(root, moduleSpecifier)
+ assert.EqualError(t, err,
+ "only supported schemes for imports are file and https, "+moduleSpecifier+" has `ws`")
+ })
+
+ t.Run("HTTP", func(t *testing.T) {
+ moduleSpecifier := "http://example.com/html"
+ _, err := loader.Resolve(root, moduleSpecifier)
+ assert.EqualError(t, err,
+ "only supported schemes for imports are file and https, "+moduleSpecifier+" has `http`")
+ })
+ })
+
+ t.Run("Remote Lifting Denied", func(t *testing.T) {
+ pwdURL, err := url.Parse("https://example.com/")
+ require.NoError(t, err)
+
+ _, err = loader.Resolve(pwdURL, "file:///etc/shadow")
+ assert.EqualError(t, err, "origin (https://example.com/) not allowed to load local file: file:///etc/shadow")
+ })
+
+ t.Run("Fixes missing slash in pwd", func(t *testing.T) {
+ pwdURL, err := url.Parse("https://example.com/path/to")
+ require.NoError(t, err)
+
+ moduleURL, err := loader.Resolve(pwdURL, "./something")
+ require.NoError(t, err)
+ require.Equal(t, "https://example.com/path/to/something", moduleURL.String())
+ require.Equal(t, "https://example.com/path/to", pwdURL.String())
+ })
+
+}
func TestLoad(t *testing.T) {
tb := testutils.NewHTTPMultiBin(t)
sr := tb.Replacer.Replace
@@ -55,69 +111,93 @@ func TestLoad(t *testing.T) {
http.DefaultTransport = oldHTTPTransport
}()
- t.Run("Blank", func(t *testing.T) {
- _, err := Load(nil, "/", "")
- assert.EqualError(t, err, "local or remote path required")
- })
-
- t.Run("Protocol", func(t *testing.T) {
- _, err := Load(nil, "/", sr("HTTPSBIN_URL/html"))
- assert.EqualError(t, err, "imports should not contain a protocol")
- })
-
t.Run("Local", func(t *testing.T) {
- fs := afero.NewMemMapFs()
- assert.NoError(t, fs.MkdirAll("/path/to", 0755))
- assert.NoError(t, afero.WriteFile(fs, "/path/to/file.txt", []byte("hi"), 0644))
+ filesystems := make(map[string]afero.Fs)
+ filesystems["file"] = afero.NewMemMapFs()
+ assert.NoError(t, filesystems["file"].MkdirAll("/path/to", 0755))
+ assert.NoError(t, afero.WriteFile(filesystems["file"], "/path/to/file.txt", []byte("hi"), 0644))
testdata := map[string]struct{ pwd, path string }{
- "Absolute": {"/path", "/path/to/file.txt"},
- "Relative": {"/path", "./to/file.txt"},
- "Adjacent": {"/path/to", "./file.txt"},
+ "Absolute": {"/path/", "/path/to/file.txt"},
+ "Relative": {"/path/", "./to/file.txt"},
+ "Adjacent": {"/path/to/", "./file.txt"},
}
for name, data := range testdata {
+ data := data
t.Run(name, func(t *testing.T) {
- src, err := Load(fs, data.pwd, data.path)
- if assert.NoError(t, err) {
- assert.Equal(t, "/path/to/file.txt", src.Filename)
- assert.Equal(t, "hi", string(src.Data))
- }
+ pwdURL, err := url.Parse("file://" + data.pwd)
+ require.NoError(t, err)
+
+ moduleURL, err := loader.Resolve(pwdURL, data.path)
+ require.NoError(t, err)
+
+ src, err := loader.Load(filesystems, moduleURL, data.path)
+ require.NoError(t, err)
+
+ assert.Equal(t, "file:///path/to/file.txt", src.URL.String())
+ assert.Equal(t, "hi", string(src.Data))
})
}
t.Run("Nonexistent", func(t *testing.T) {
+ root, err := url.Parse("file:///")
+ require.NoError(t, err)
+
path := filepath.FromSlash("/nonexistent")
- _, err := Load(fs, "/", "/nonexistent")
- assert.EqualError(t, err, fmt.Sprintf("open %s: file does not exist", path))
- })
+ pathURL, err := loader.Resolve(root, "/nonexistent")
+ require.NoError(t, err)
- t.Run("Remote Lifting Denied", func(t *testing.T) {
- _, err := Load(fs, "example.com", "/etc/shadow")
- assert.EqualError(t, err, "origin (example.com) not allowed to load local file: /etc/shadow")
+ _, err = loader.Load(filesystems, pathURL, path)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(),
+ fmt.Sprintf(`The moduleSpecifier "file://%s" couldn't be found on local disk. `,
+ filepath.ToSlash(path)))
})
+
})
t.Run("Remote", func(t *testing.T) {
- src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/html"))
- if assert.NoError(t, err) {
- assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/html"))
+ filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()}
+ t.Run("From local", func(t *testing.T) {
+ root, err := url.Parse("file:///")
+ require.NoError(t, err)
+
+ moduleSpecifier := sr("HTTPSBIN_URL/html")
+ moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier)
+ require.NoError(t, err)
+
+ src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier)
+ require.NoError(t, err)
+ assert.Equal(t, src.URL, moduleSpecifierURL)
assert.Contains(t, string(src.Data), "Herman Melville - Moby-Dick")
- }
+ })
t.Run("Absolute", func(t *testing.T) {
- src, err := Load(nil, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT"), sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt"))
- if assert.NoError(t, err) {
- assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt"))
- assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n")
- }
+ pwdURL, err := url.Parse(sr("HTTPSBIN_URL"))
+ require.NoError(t, err)
+
+ moduleSpecifier := sr("HTTPSBIN_URL/robots.txt")
+ moduleSpecifierURL, err := loader.Resolve(pwdURL, moduleSpecifier)
+ require.NoError(t, err)
+
+ src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier)
+ require.NoError(t, err)
+ assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/robots.txt"))
+ assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n")
})
t.Run("Relative", func(t *testing.T) {
- src, err := Load(nil, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT"), "./robots.txt")
- if assert.NoError(t, err) {
- assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt"))
- assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n")
- }
+ pwdURL, err := url.Parse(sr("HTTPSBIN_URL"))
+ require.NoError(t, err)
+
+ moduleSpecifier := ("./robots.txt")
+ moduleSpecifierURL, err := loader.Resolve(pwdURL, moduleSpecifier)
+ require.NoError(t, err)
+
+ src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier)
+ require.NoError(t, err)
+ assert.Equal(t, sr("HTTPSBIN_URL/robots.txt"), src.URL.String())
+ assert.Equal(t, "User-agent: *\nDisallow: /deny\n", string(src.Data))
})
})
@@ -132,11 +212,19 @@ func TestLoad(t *testing.T) {
})
t.Run("No _k6=1 Fallback", func(t *testing.T) {
- src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/raw/something"))
- if assert.NoError(t, err) {
- assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/raw/something"))
- assert.Equal(t, responseStr, string(src.Data))
- }
+ root, err := url.Parse("file:///")
+ require.NoError(t, err)
+
+ moduleSpecifier := sr("HTTPSBIN_URL/raw/something")
+ moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier)
+ require.NoError(t, err)
+
+ filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()}
+ src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier)
+
+ require.NoError(t, err)
+ assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/raw/something"))
+ assert.Equal(t, responseStr, string(src.Data))
})
tb.Mux.HandleFunc("/invalid", func(w http.ResponseWriter, r *http.Request) {
@@ -144,19 +232,32 @@ func TestLoad(t *testing.T) {
})
t.Run("Invalid", func(t *testing.T) {
- src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/invalid"))
- assert.Nil(t, src)
- assert.Error(t, err)
-
- t.Run("Host", func(t *testing.T) {
- src, err := Load(nil, "/", "some-path-that-doesnt-exist.js")
- assert.Nil(t, src)
- assert.Error(t, err)
- })
- t.Run("URL", func(t *testing.T) {
- src, err := Load(nil, "/", "192.168.0.%31")
- assert.Nil(t, src)
- assert.Error(t, err)
+ root, err := url.Parse("file:///")
+ require.NoError(t, err)
+
+ t.Run("IP URL", func(t *testing.T) {
+ _, err := loader.Resolve(root, "192.168.0.%31")
+ require.Error(t, err)
+ require.Contains(t, err.Error(), `invalid URL escape "%31"`)
})
+
+ var testData = [...]struct {
+ name, moduleSpecifier string
+ }{
+ {"URL", sr("HTTPSBIN_URL/invalid")},
+ {"HOST", "some-path-that-doesnt-exist.js"},
+ }
+
+ filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()}
+ for _, data := range testData {
+ moduleSpecifier := data.moduleSpecifier
+ t.Run(data.name, func(t *testing.T) {
+ moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier)
+ require.NoError(t, err)
+
+ _, err = loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier)
+ require.Error(t, err)
+ })
+ }
})
}
diff --git a/loader/readsource.go b/loader/readsource.go
new file mode 100644
index 00000000000..3efd4ce85a5
--- /dev/null
+++ b/loader/readsource.go
@@ -0,0 +1,48 @@
+package loader
+
+import (
+ "io"
+ "io/ioutil"
+ "net/url"
+ "path/filepath"
+
+ "github.com/loadimpact/k6/lib/fsext"
+ "github.com/pkg/errors"
+ "github.com/spf13/afero"
+)
+
+// ReadSource Reads a source file from any supported destination.
+func ReadSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*SourceData, error) {
+ if src == "-" {
+ data, err := ioutil.ReadAll(stdin)
+ if err != nil {
+ return nil, err
+ }
+ // TODO: don't do it in this way ...
+ err = afero.WriteFile(filesystems["file"].(fsext.CacheOnReadFs).GetCachingFs(), "/-", data, 0644)
+ if err != nil {
+ return nil, errors.Wrap(err, "caching data read from -")
+ }
+ return &SourceData{URL: &url.URL{Path: "/-", Scheme: "file"}, Data: data}, err
+ }
+ var srcLocalPath string
+ if filepath.IsAbs(src) {
+ srcLocalPath = src
+ } else {
+ srcLocalPath = filepath.Join(pwd, src)
+ }
+ // All paths should start with a / in all fses. This is mostly for windows where it will start
+ // with a volume name : C:\something.js
+ srcLocalPath = filepath.Clean(afero.FilePathSeparator + srcLocalPath)
+ if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok {
+ // there is file on the local disk ... lets use it :)
+ return Load(filesystems, &url.URL{Scheme: "file", Path: filepath.ToSlash(srcLocalPath)}, src)
+ }
+
+ pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"}
+ srcURL, err := Resolve(pwdURL, filepath.ToSlash(src))
+ if err != nil {
+ return nil, err
+ }
+ return Load(filesystems, srcURL, src)
+}
diff --git a/loader/readsource_test.go b/loader/readsource_test.go
new file mode 100644
index 00000000000..e962f304f87
--- /dev/null
+++ b/loader/readsource_test.go
@@ -0,0 +1,88 @@
+package loader
+
+import (
+ "bytes"
+ "io"
+ "net/url"
+ "testing"
+
+ "github.com/loadimpact/k6/lib/fsext"
+ "github.com/pkg/errors"
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/require"
+)
+
+type errorReader string
+
+func (e errorReader) Read(_ []byte) (int, error) {
+ return 0, errors.New((string)(e))
+}
+
+var _ io.Reader = errorReader("")
+
+func TestReadSourceSTDINError(t *testing.T) {
+ _, err := ReadSource("-", "", nil, errorReader("1234"))
+ require.Error(t, err)
+ require.Equal(t, "1234", err.Error())
+}
+
+func TestReadSourceSTDINCache(t *testing.T) {
+ var data = []byte(`test contents`)
+ var r = bytes.NewReader(data)
+ var fs = afero.NewMemMapFs()
+ sourceData, err := ReadSource("-", "/path/to/pwd",
+ map[string]afero.Fs{"file": fsext.NewCacheOnReadFs(nil, fs, 0)}, r)
+ require.NoError(t, err)
+ require.Equal(t, &SourceData{
+ URL: &url.URL{Scheme: "file", Path: "/-"},
+ Data: data}, sourceData)
+ fileData, err := afero.ReadFile(fs, "/-")
+ require.NoError(t, err)
+ require.Equal(t, data, fileData)
+}
+
+func TestReadSourceRelative(t *testing.T) {
+ var data = []byte(`test contents`)
+ var fs = afero.NewMemMapFs()
+ require.NoError(t, afero.WriteFile(fs, "/path/to/somewhere/script.js", data, 0644))
+ sourceData, err := ReadSource("../somewhere/script.js", "/path/to/pwd", map[string]afero.Fs{"file": fs}, nil)
+ require.NoError(t, err)
+ require.Equal(t, &SourceData{
+ URL: &url.URL{Scheme: "file", Path: "/path/to/somewhere/script.js"},
+ Data: data}, sourceData)
+}
+
+func TestReadSourceAbsolute(t *testing.T) {
+ var data = []byte(`test contents`)
+ var r = bytes.NewReader(data)
+ var fs = afero.NewMemMapFs()
+ require.NoError(t, afero.WriteFile(fs, "/a/b", data, 0644))
+ require.NoError(t, afero.WriteFile(fs, "/c/a/b", []byte("wrong"), 0644))
+ sourceData, err := ReadSource("/a/b", "/c", map[string]afero.Fs{"file": fs}, r)
+ require.NoError(t, err)
+ require.Equal(t, &SourceData{
+ URL: &url.URL{Scheme: "file", Path: "/a/b"},
+ Data: data}, sourceData)
+}
+
+func TestReadSourceHttps(t *testing.T) {
+ var data = []byte(`test contents`)
+ var fs = afero.NewMemMapFs()
+ require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644))
+ sourceData, err := ReadSource("https://github.com/something", "/c",
+ map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil)
+ require.NoError(t, err)
+ require.Equal(t, &SourceData{
+ URL: &url.URL{Scheme: "https", Host: "github.com", Path: "/something"},
+ Data: data}, sourceData)
+}
+
+func TestReadSourceHttpError(t *testing.T) {
+ var data = []byte(`test contents`)
+ var fs = afero.NewMemMapFs()
+ require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644))
+ _, err := ReadSource("http://github.com/something", "/c",
+ map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), `only supported schemes for imports are file and https`)
+}
diff --git a/stats/cloud/collector.go b/stats/cloud/collector.go
index d5108a11f40..8fa8ec07453 100644
--- a/stats/cloud/collector.go
+++ b/stats/cloud/collector.go
@@ -30,6 +30,7 @@ import (
"github.com/loadimpact/k6/lib/metrics"
"github.com/loadimpact/k6/lib/netext"
"github.com/loadimpact/k6/lib/netext/httpext"
+ "github.com/loadimpact/k6/loader"
"github.com/pkg/errors"
"gopkg.in/guregu/null.v3"
@@ -97,7 +98,7 @@ func MergeFromExternal(external map[string]json.RawMessage, conf *Config) error
}
// New creates a new cloud collector
-func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*Collector, error) {
+func New(conf Config, src *loader.SourceData, opts lib.Options, version string) (*Collector, error) {
if err := MergeFromExternal(opts.External, &conf); err != nil {
return nil, err
}
@@ -107,7 +108,7 @@ func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*C
}
if !conf.Name.Valid || conf.Name.String == "" {
- conf.Name = null.StringFrom(filepath.Base(src.Filename))
+ conf.Name = null.StringFrom(filepath.Base(src.URL.Path))
}
if conf.Name.String == "-" {
conf.Name = null.StringFrom(TestName)
diff --git a/stats/cloud/collector_test.go b/stats/cloud/collector_test.go
index ccdec01b936..a0f926271c0 100644
--- a/stats/cloud/collector_test.go
+++ b/stats/cloud/collector_test.go
@@ -27,6 +27,7 @@ import (
"io/ioutil"
"math/rand"
"net/http"
+ "net/url"
"sync"
"testing"
"time"
@@ -42,6 +43,7 @@ import (
"github.com/loadimpact/k6/lib/netext/httpext"
"github.com/loadimpact/k6/lib/testutils"
"github.com/loadimpact/k6/lib/types"
+ "github.com/loadimpact/k6/loader"
"github.com/loadimpact/k6/stats"
)
@@ -134,9 +136,9 @@ func TestCloudCollector(t *testing.T) {
}))
defer tb.Cleanup()
- script := &lib.SourceData{
- Data: []byte(""),
- Filename: "/script.js",
+ script := &loader.SourceData{
+ Data: []byte(""),
+ URL: &url.URL{Path: "/script.js"},
}
options := lib.Options{
@@ -280,9 +282,9 @@ func TestCloudCollectorMaxPerPacket(t *testing.T) {
}))
defer tb.Cleanup()
- script := &lib.SourceData{
- Data: []byte(""),
- Filename: "/script.js",
+ script := &loader.SourceData{
+ Data: []byte(""),
+ URL: &url.URL{Path: "/script.js"},
}
options := lib.Options{