From aa4b0907c44878f923d3ea77e29f13ffba5730a3 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 7 Jun 2019 18:35:47 +0300 Subject: [PATCH 01/45] Rewrite script/files loading to be be url based This includes also change to the archive file structure. Now instead of separating files by whether they are scripts or no, they are separated based on whether their URI scheme. Through the (majority) of k6 now instead of simple strings a url.URL is used to idenitify files(scripts or otherwise). This also means that all imports or `open` can have schemes. Previously remote modules were supported specifically without scheme. With this change we specifically prefer if they are with a scheme. The old variant is supported but logs a warning. Additionally if remote module requires a relative/absolute path that doesn't have a scheme it is relative/absolute given the remote module url. Because of some of the changes, now caching is done through a afero.Fs instead of additional map. This also fixes not laoding remotely imported files from an archive, but instead requesting them again. fixes #1037, closes #838, fixes #887 and fixes #1051 --- api/v1/setup_teardown_routes_test.go | 4 +- cmd/run.go | 13 +- cmd/runtime_options_test.go | 12 +- converter/har/converter_test.go | 4 +- core/engine_test.go | 11 +- core/local/local_test.go | 3 +- js/bundle.go | 66 ++++++--- js/bundle_test.go | 40 +++-- js/console_test.go | 5 +- js/http_bench_test.go | 3 +- js/initcontext.go | 64 ++++---- js/initcontext_test.go | 34 ++--- js/module_loading_test.go | 21 +-- js/modules/k6/marshalling_test.go | 3 +- js/runner_test.go | 101 +++++++------ lib/archive.go | 188 +++++++++++++----------- lib/archive_test.go | 155 ++++++++++++++++---- lib/models.go | 7 +- loader/cdnjs_test.go | 30 ++-- loader/github_test.go | 30 +++- loader/loader.go | 167 +++++++++++---------- loader/loader_test.go | 209 +++++++++++++++++++-------- stats/cloud/collector.go | 2 +- stats/cloud/collector_test.go | 9 +- 24 files changed, 725 insertions(+), 456 deletions(-) diff --git a/api/v1/setup_teardown_routes_test.go b/api/v1/setup_teardown_routes_test.go index ef352d6a260..4663c9cb34e 100644 --- a/api/v1/setup_teardown_routes_test.go +++ b/api/v1/setup_teardown_routes_test.go @@ -26,6 +26,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "net/url" "testing" "time" @@ -130,9 +131,10 @@ func TestSetupData(t *testing.T) { }, } for _, testCase := range testCases { + testCase := testCase t.Run(testCase.name, func(t *testing.T) { runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: testCase.script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) diff --git a/cmd/run.go b/cmd/run.go index 7376a0d16ab..3a3ebdca354 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -29,9 +29,9 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "os" "os/signal" - "path/filepath" "runtime" "strings" "syscall" @@ -508,13 +508,14 @@ func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, if err != nil { return nil, err } - return &lib.SourceData{Filename: "-", Data: data}, nil + return &lib.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil } - abspath := filepath.Join(pwd, src) - if ok, _ := afero.Exists(fs, abspath); ok { - src = abspath + pwdURL := &url.URL{Scheme: "file", Path: pwd} + srcURL, err := loader.Resolve(pwdURL, src) + if err != nil { + return nil, err } - return loader.Load(fs, pwd, src) + return loader.Load(map[string]afero.Fs{"file": fs, "https": afero.NewMemMapFs()}, srcURL, src) } // Creates a new runner. diff --git a/cmd/runtime_options_test.go b/cmd/runtime_options_test.go index c279fe39b93..c8c7b25b2bf 100644 --- a/cmd/runtime_options_test.go +++ b/cmd/runtime_options_test.go @@ -23,6 +23,7 @@ package cmd import ( "bytes" "fmt" + "net/url" "os" "runtime" "strings" @@ -220,8 +221,8 @@ func TestEnvVars(t *testing.T) { runner, err := newRunner( &lib.SourceData{ - Data: []byte(jsCode), - Filename: "/script.js", + Data: []byte(jsCode), + URL: &url.URL{Path: "/script.js"}, }, typeJS, afero.NewOsFs(), @@ -234,16 +235,15 @@ func TestEnvVars(t *testing.T) { assert.NoError(t, archive.Write(archiveBuf)) getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) { - r, err := newRunner( + return newRunner( &lib.SourceData{ - Data: []byte(archiveBuf.Bytes()), - Filename: "/script.tar", + Data: archiveBuf.Bytes(), + URL: &url.URL{Path: "/script.js"}, }, typeArchive, afero.NewOsFs(), rtOpts, ) - return r, err } _, err = getRunnerErr(lib.RuntimeOptions{}) diff --git a/converter/har/converter_test.go b/converter/har/converter_test.go index 575044f9b4d..3d94aee6fa5 100644 --- a/converter/har/converter_test.go +++ b/converter/har/converter_test.go @@ -57,8 +57,8 @@ func TestBuildK6RequestObject(t *testing.T) { v, err := buildK6RequestObject(req) assert.NoError(t, err) _, err = js.New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)), + URL: &url.URL{Path: "/script.js"}, + Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)), }, afero.NewMemMapFs(), lib.RuntimeOptions{}) assert.NoError(t, err) } diff --git a/core/engine_test.go b/core/engine_test.go index fe81be047a1..7d1818a94c1 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -23,6 +23,7 @@ package core import ( "context" "fmt" + "net/url" "testing" "time" @@ -556,7 +557,7 @@ func TestSentReceivedMetrics(t *testing.T) { runTest := func(t *testing.T, ts testScript, tc testCase, noConnReuse bool) (float64, float64) { r, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: []byte(ts.Code)}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) @@ -697,7 +698,7 @@ func TestRunTags(t *testing.T) { `)) r, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) @@ -797,7 +798,7 @@ func TestSetupTeardownThresholds(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) @@ -860,7 +861,7 @@ func TestEmittedMetricsWhenScalingDown(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) @@ -920,7 +921,7 @@ func TestMinIterationDuration(t *testing.T) { t.Parallel() runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: []byte(` + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(` import { Counter } from "k6/metrics"; let testCounter = new Counter("testcounter"); diff --git a/core/local/local_test.go b/core/local/local_test.go index 23c27d88179..7360ad1008d 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -23,6 +23,7 @@ package local import ( "context" "net" + "net/url" "runtime" "sync/atomic" "testing" @@ -481,7 +482,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { }`) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) diff --git a/js/bundle.go b/js/bundle.go index df0df2805ae..b1a0791c3d1 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -23,7 +23,8 @@ package js import ( "context" "encoding/json" - "os" + "net/url" + "time" "github.com/dop251/goja" "github.com/loadimpact/k6/js/common" @@ -38,7 +39,7 @@ import ( // A Bundle is a self-contained bundle of scripts and resources. // You can use this to produce identical BundleInstance objects. type Bundle struct { - Filename string + Filename *url.URL Source string Program *goja.Program Options lib.Options @@ -55,8 +56,24 @@ type BundleInstance struct { Default goja.Callable } +type cacheOnReadFs struct { + afero.Fs + cache afero.Fs +} + +func newCacheOnReadFs(base, layer afero.Fs, cacheTime time.Duration) afero.Fs { + return cacheOnReadFs{ + Fs: afero.NewCacheOnReadFs(base, layer, cacheTime), + cache: layer, + } +} + +func (c cacheOnReadFs) GetCachedFs() afero.Fs { + return c.cache +} + // NewBundle creates a new bundle from a source file and a filesystem. -func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { +func NewBundle(src *lib.SourceData, fileFS afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { compiler, err := compiler.New() if err != nil { return nil, err @@ -64,7 +81,7 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu // Compile sources, both ES5 and ES6 are supported. code := string(src.Data) - pgm, _, err := compiler.Compile(code, src.Filename, "", "", true) + pgm, _, err := compiler.Compile(code, src.URL.String(), "", "", true) if err != nil { return nil, err } @@ -73,15 +90,19 @@ func NewBundle(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Bu // written every time something is read from the real filesystem. This cache is then used for // successive spawns to read from (they have no access to the real disk). mirrorFS := afero.NewMemMapFs() - cachedFS := afero.NewCacheOnReadFs(fs, mirrorFS, 0) + cachedFS := newCacheOnReadFs(fileFS, mirrorFS, 0) + fses := map[string]afero.Fs{ + "file": cachedFS, + "https": afero.NewMemMapFs(), + } // Make a bundle, instantiate it into a throwaway VM to populate caches. rt := goja.New() bundle := Bundle{ - Filename: src.Filename, + Filename: src.URL, Source: code, Program: pgm, - BaseInitContext: NewInitContext(rt, compiler, new(context.Context), cachedFS, loader.Dir(src.Filename)), + BaseInitContext: NewInitContext(rt, compiler, new(context.Context), fses, loader.Dir(src.URL)), Env: rtOpts.Env, } if err := bundle.instantiate(rt, bundle.BaseInitContext); err != nil { @@ -146,8 +167,17 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, if err != nil { return nil, err } - initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FS, arc.Pwd) - initctx.files = arc.Files + pwdURL, err := loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd) + if err != nil { + return nil, err + } + + filenameURL, err := loader.Resolve(pwdURL, arc.Filename) + if err != nil { + return nil, err + } + + initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FSes, pwdURL) env := arc.Env if env == nil { @@ -159,7 +189,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, } bundle := &Bundle{ - Filename: arc.Filename, + Filename: filenameURL, Source: string(arc.Data), Program: pgm, Options: arc.Options, @@ -175,11 +205,11 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, func (b *Bundle) makeArchive() *lib.Archive { arc := &lib.Archive{ Type: "js", - FS: afero.NewMemMapFs(), + FSes: b.BaseInitContext.fses, Options: b.Options, - Filename: b.Filename, + Filename: b.Filename.String(), Data: []byte(b.Source), - Pwd: b.BaseInitContext.pwd, + Pwd: b.BaseInitContext.pwd.String(), Env: make(map[string]string, len(b.Env)), } // Copy env so changes in the archive are not reflected in the source Bundle @@ -187,16 +217,6 @@ func (b *Bundle) makeArchive() *lib.Archive { arc.Env[k] = v } - arc.Scripts = make(map[string][]byte, len(b.BaseInitContext.programs)) - for name, pgm := range b.BaseInitContext.programs { - arc.Scripts[name] = []byte(pgm.src) - err := afero.WriteFile(arc.FS, name, []byte(pgm.src), os.ModePerm) - if err != nil { - return nil - } - } - arc.Files = b.BaseInitContext.files - return arc } diff --git a/js/bundle_test.go b/js/bundle_test.go index 603afda382b..23af9d21558 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -24,6 +24,7 @@ import ( "crypto/tls" "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "runtime" @@ -41,12 +42,15 @@ import ( ) func getSimpleBundle(filename, data string) (*Bundle, error) { + return getSimpleBundleWithFs(filename, data, afero.NewMemMapFs()) +} +func getSimpleBundleWithFs(filename, data string, fs afero.Fs) (*Bundle, error) { return NewBundle( &lib.SourceData{ - Filename: filename, - Data: []byte(data), + URL: &url.URL{Path: filename, Scheme: "file"}, + Data: []byte(data), }, - afero.NewMemMapFs(), + fs, lib.RuntimeOptions{}, ) } @@ -58,11 +62,11 @@ func TestNewBundle(t *testing.T) { }) t.Run("Invalid", func(t *testing.T) { _, err := getSimpleBundle("/script.js", "\x00") - assert.Contains(t, err.Error(), "SyntaxError: /script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") + assert.Contains(t, err.Error(), "SyntaxError: file:///script.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") }) t.Run("Error", func(t *testing.T) { _, err := getSimpleBundle("/script.js", `throw new Error("aaaa");`) - assert.EqualError(t, err, "Error: aaaa at /script.js:1:7(3)") + assert.EqualError(t, err, "Error: aaaa at file:///script.js:1:7(3)") }) t.Run("InvalidExports", func(t *testing.T) { _, err := getSimpleBundle("/script.js", `exports = null`) @@ -87,8 +91,8 @@ func TestNewBundle(t *testing.T) { t.Run("stdin", func(t *testing.T) { b, err := getSimpleBundle("-", `export default function() {};`) if assert.NoError(t, err) { - assert.Equal(t, "-", b.Filename) - assert.Equal(t, "/", b.BaseInitContext.pwd) + assert.Equal(t, "file://-", b.Filename.String()) + assert.Equal(t, "file:///", b.BaseInitContext.pwd.String()) } }) t.Run("Options", func(t *testing.T) { @@ -358,7 +362,7 @@ func TestNewBundleFromArchive(t *testing.T) { assert.NoError(t, afero.WriteFile(fs, "/path/to/exclaim.js", []byte(`export default function(s) { return s + "!" };`), 0644)) src := &lib.SourceData{ - Filename: "/path/to/script.js", + URL: &url.URL{Path: "/path/to/script.js", Scheme: "file"}, Data: []byte(` import exclaim from "./exclaim.js"; export let options = { vus: 12345 }; @@ -385,13 +389,17 @@ func TestNewBundleFromArchive(t *testing.T) { arc := b.makeArchive() assert.Equal(t, "js", arc.Type) assert.Equal(t, lib.Options{VUs: null.IntFrom(12345)}, arc.Options) - assert.Equal(t, "/path/to/script.js", arc.Filename) + assert.Equal(t, "file:///path/to/script.js", arc.Filename) assert.Equal(t, string(src.Data), string(arc.Data)) - assert.Equal(t, "/path/to", arc.Pwd) - assert.Len(t, arc.Scripts, 1) - assert.Equal(t, `export default function(s) { return s + "!" };`, string(arc.Scripts["/path/to/exclaim.js"])) - assert.Len(t, arc.Files, 1) - assert.Equal(t, `hi`, string(arc.Files["/path/to/file.txt"])) + assert.Equal(t, "file:///path/to/", arc.Pwd) + + exclaimData, err := afero.ReadFile(arc.FSes["file"], "/path/to/exclaim.js") + assert.NoError(t, err) + assert.Equal(t, `export default function(s) { return s + "!" };`, string(exclaimData)) + + fileData, err := afero.ReadFile(arc.FSes["file"], "/path/to/file.txt") + assert.NoError(t, err) + assert.Equal(t, `hi`, string(fileData)) b2, err := NewBundleFromArchive(arc, lib.RuntimeOptions{}) if !assert.NoError(t, err) { @@ -521,7 +529,7 @@ func TestOpen(t *testing.T) { pwd = "/path/to/" } src := &lib.SourceData{ - Filename: filepath.Join(prefix, filepath.Join(pwd, "script.js")), + URL: &url.URL{Scheme: "file", Path: filepath.Join(prefix, filepath.Join(pwd, "script.js"))}, Data: []byte(` export let file = open("` + openPath + `"); export default function() { return file }; @@ -600,7 +608,7 @@ func TestBundleEnv(t *testing.T) { b1, err := NewBundle( &lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export default function() { if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); } diff --git a/js/console_test.go b/js/console_test.go index b85bbbdacd2..6d47cba4bbf 100644 --- a/js/console_test.go +++ b/js/console_test.go @@ -24,6 +24,7 @@ import ( "context" "fmt" "io/ioutil" + "net/url" "os" "testing" @@ -91,7 +92,7 @@ func TestConsole(t *testing.T) { for args, result := range argsets { t.Run(args, func(t *testing.T) { r, err := New(&lib.SourceData{ - Filename: "/script", + URL: &url.URL{Path: "/script"}, Data: []byte(fmt.Sprintf( `export default function() { console.%s(%s); }`, name, args, @@ -180,7 +181,7 @@ func TestFileConsole(t *testing.T) { } r, err := New(&lib.SourceData{ - Filename: "/script", + URL: &url.URL{Path: "/script"}, Data: []byte(fmt.Sprintf( `export default function() { console.%s(%s); }`, name, args, diff --git a/js/http_bench_test.go b/js/http_bench_test.go index 17f45fb0914..2415a4edc4e 100644 --- a/js/http_bench_test.go +++ b/js/http_bench_test.go @@ -2,6 +2,7 @@ package js import ( "context" + "net/url" "testing" "github.com/loadimpact/k6/lib" @@ -18,7 +19,7 @@ func BenchmarkHTTPRequests(b *testing.B) { defer tb.Cleanup() r, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` import http from "k6/http"; export default function() { diff --git a/js/initcontext.go b/js/initcontext.go index 9842a0f2396..bcbc68fd52e 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -22,6 +22,7 @@ package js import ( "context" + "net/url" "path/filepath" "strings" @@ -49,28 +50,26 @@ type InitContext struct { // Pointer to a context that bridged modules are invoked with. ctxPtr *context.Context - // Filesystem to load files and scripts from. - fs afero.Fs - pwd string + // Filesystem to load files and scripts from with the map key being the scheme + fses map[string]afero.Fs + pwd *url.URL // Cache of loaded programs and files. programs map[string]programWithSource - files map[string][]byte } // NewInitContext creates a new initcontext with the provided arguments func NewInitContext( - rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string, + rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fses map[string]afero.Fs, pwd *url.URL, ) *InitContext { return &InitContext{ runtime: rt, compiler: compiler, ctxPtr: ctxPtr, - fs: fs, - pwd: filepath.ToSlash(pwd), + fses: fses, + pwd: pwd, programs: make(map[string]programWithSource), - files: make(map[string][]byte), } } @@ -89,12 +88,11 @@ func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Ru runtime: rt, ctxPtr: ctxPtr, - fs: base.fs, + fses: base.fses, pwd: base.pwd, compiler: base.compiler, programs: programs, - files: base.files, } } @@ -130,12 +128,15 @@ func (i *InitContext) requireModule(name string) (goja.Value, error) { func (i *InitContext) requireFile(name string) (goja.Value, error) { // Resolve the file path, push the target directory as pwd to make relative imports work. pwd := i.pwd - filename := loader.Resolve(pwd, name) + fileURL, err := loader.Resolve(pwd, name) + if err != nil { + return nil, err + } // First, check if we have a cached program already. - pgm, ok := i.programs[filename] + pgm, ok := i.programs[fileURL.String()] if !ok || pgm.exports == nil { - i.pwd = loader.Dir(filename) + i.pwd = loader.Dir(fileURL) defer func() { i.pwd = pwd }() // Swap the importing scope's exports out, then put it back again. @@ -150,21 +151,22 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) { i.runtime.Set("module", module) if pgm.pgm == nil { // Load the sources; the loader takes care of remote loading, etc. - data, err := loader.Load(i.fs, pwd, name) + data, err := loader.Load(i.fses, fileURL, name) if err != nil { return goja.Undefined(), err } + pgm.src = string(data.Data) // Compile the sources; this handles ES5 vs ES6 automatically. - pgm.pgm, err = i.compileImport(pgm.src, data.Filename) + pgm.pgm, err = i.compileImport(pgm.src, data.URL.String()) if err != nil { return goja.Undefined(), err } } pgm.exports = module.Get("exports") - i.programs[filename] = pgm + i.programs[fileURL.String()] = pgm // Run the program. if _, err := i.runtime.RunProgram(pgm.pgm); err != nil { @@ -193,28 +195,20 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) // will probably be need for archive execution under windows if always consider '/...' as an // absolute path. if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) { - filename = filepath.Join(i.pwd, filename) + filename = filepath.Join(i.pwd.Path, filename) } filename = filepath.ToSlash(filename) + fs := i.fses["file"] - data, ok := i.files[filename] - if !ok { - var ( - err error - isDir bool - ) - - // Workaround for https://github.com/spf13/afero/issues/201 - if isDir, err = afero.IsDir(i.fs, filename); err != nil { - return nil, err - } else if isDir { - return nil, errors.New("open() can't be used with directories") - } - data, err = afero.ReadFile(i.fs, filename) - if err != nil { - return nil, err - } - i.files[filename] = data + // Workaround for https://github.com/spf13/afero/issues/201 + if isDir, err := afero.IsDir(fs, filename); err != nil { + return nil, err + } else if isDir { + return nil, errors.New("open() can't be used with directories") + } + data, err := afero.ReadFile(fs, filename) + if err != nil { + return nil, err } if len(args) > 0 && args[0] == "b" { diff --git a/js/initcontext_test.go b/js/initcontext_test.go index f127e3b0194..2fd558f6699 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -27,6 +27,7 @@ import ( "net" "net/http" "net/http/httptest" + "net/url" "path/filepath" "testing" "time" @@ -40,6 +41,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestInitContextRequire(t *testing.T) { @@ -111,25 +113,20 @@ func TestInitContextRequire(t *testing.T) { t.Run("Nonexistent", func(t *testing.T) { path := filepath.FromSlash("/nonexistent.js") _, err := getSimpleBundle("/script.js", `import "/nonexistent.js"; export default function() {}`) - assert.EqualError(t, err, fmt.Sprintf("GoError: open %s: file does not exist", path)) + assert.Contains(t, err.Error(), fmt.Sprintf(`"file://%s" couldn't be found on local disk`, path)) }) t.Run("Invalid", func(t *testing.T) { fs := afero.NewMemMapFs() assert.NoError(t, afero.WriteFile(fs, "/file.js", []byte{0x00}, 0755)) - _, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`import "/file.js"; export default function() {}`), - }, fs, lib.RuntimeOptions{}) - assert.Contains(t, err.Error(), "SyntaxError: /file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") + _, err := getSimpleBundleWithFs("/script.js", `import "/file.js"; export default function() {}`, fs) + require.Error(t, err) + assert.Contains(t, err.Error(), "SyntaxError: file:///file.js: Unexpected character '\x00' (1:0)\n> 1 | \x00\n") }) t.Run("Error", func(t *testing.T) { fs := afero.NewMemMapFs() assert.NoError(t, afero.WriteFile(fs, "/file.js", []byte(`throw new Error("aaaa")`), 0755)) - _, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`import "/file.js"; export default function() {}`), - }, fs, lib.RuntimeOptions{}) - assert.EqualError(t, err, "Error: aaaa at /file.js:2:7(4)") + _, err := getSimpleBundleWithFs("/script.js", `import "/file.js"; export default function() {}`, fs) + assert.EqualError(t, err, "Error: aaaa at file:///file.js:2:7(4)") }) imports := map[string]struct { @@ -164,6 +161,7 @@ func TestInitContextRequire(t *testing.T) { for libName, data := range imports { t.Run("lib=\""+libName+"\"", func(t *testing.T) { for constName, constPath := range data.ConstPaths { + constName, constPath := constName, constPath name := "inline" if constName != "" { name = "const=\"" + constName + "\"" @@ -171,7 +169,7 @@ func TestInitContextRequire(t *testing.T) { t.Run(name, func(t *testing.T) { fs := afero.NewMemMapFs() src := &lib.SourceData{ - Filename: `/path/to/script.js`, + URL: &url.URL{Path: "/path/to/script.js", Scheme: "file"}, Data: []byte(fmt.Sprintf(` import fn from "%s"; let v = fn(); @@ -200,7 +198,7 @@ func TestInitContextRequire(t *testing.T) { return } if constPath != "" { - assert.Contains(t, b.BaseInitContext.programs, constPath) + assert.Contains(t, b.BaseInitContext.programs, "file://"+constPath) } _, err = b.Instantiate() @@ -217,7 +215,7 @@ func TestInitContextRequire(t *testing.T) { assert.NoError(t, afero.WriteFile(fs, "/a.js", []byte(`const myvar = "a";`), 0644)) assert.NoError(t, afero.WriteFile(fs, "/b.js", []byte(`const myvar = "b";`), 0644)) b, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js", Scheme: "file"}, Data: []byte(` import "./a.js"; import "./b.js"; @@ -254,7 +252,7 @@ func createAndReadFile(t *testing.T, file string, content []byte, expectedLength } b, err := NewBundle(&lib.SourceData{ - Filename: "/path/to/script.js", + URL: &url.URL{Path: "/path/to/script.js"}, Data: []byte(fmt.Sprintf(` export let data = open("/path/to/%s"%s); var expectedLength = %d; @@ -325,8 +323,8 @@ func TestInitContextOpen(t *testing.T) { fs := afero.NewMemMapFs() path := filepath.FromSlash("/nonexistent.txt") _, err := NewBundle(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`open("/nonexistent.txt"); export default function() {}`), + URL: &url.URL{Path: "/script.js"}, + Data: []byte(`open("/nonexistent.txt"); export default function() {}`), }, fs, lib.RuntimeOptions{}) assert.EqualError(t, err, fmt.Sprintf("GoError: open %s: file does not exist", path)) }) @@ -364,7 +362,7 @@ func TestRequestWithBinaryFile(t *testing.T) { assert.NoError(t, afero.WriteFile(fs, "/path/to/file.bin", []byte("hi!"), 0644)) b, err := NewBundle(&lib.SourceData{ - Filename: "/path/to/script.js", + URL: &url.URL{Path: "/path/to/script.js"}, Data: []byte(fmt.Sprintf(` import http from "k6/http"; let binFile = open("/path/to/file.bin", "b"); diff --git a/js/module_loading_test.go b/js/module_loading_test.go index df94a096ca9..2e9080deea6 100644 --- a/js/module_loading_test.go +++ b/js/module_loading_test.go @@ -22,6 +22,7 @@ package js import ( "context" + "net/url" "os" "testing" @@ -66,7 +67,7 @@ func TestLoadOnceGlobalVars(t *testing.T) { } `), os.ModePerm)) r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js", Scheme: "file"}, Data: []byte(` import { A } from "./A.js"; import { B } from "./B.js"; @@ -84,7 +85,6 @@ func TestLoadOnceGlobalVars(t *testing.T) { require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -116,7 +116,7 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) { } `)), os.ModePerm)) r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js", Scheme: "file"}, Data: []byte(` import { A } from "./A.js"; @@ -132,7 +132,6 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) { require.NoError(t, r1.SetOptions(lib.Options{Hosts: tb.Dialer.Hosts})) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -160,7 +159,7 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) { } `), os.ModePerm)) r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js", Scheme: "file"}, Data: []byte(` import { A } from "./A.js"; @@ -177,7 +176,6 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) { require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -232,13 +230,12 @@ func TestLoadCycle(t *testing.T) { data, err := afero.ReadFile(fs, "/main.js") require.NoError(t, err) r1, err := New(&lib.SourceData{ - Filename: "/main.js", - Data: data, + URL: &url.URL{Path: "/main.js", Scheme: "file"}, + Data: data, }, fs, lib.RuntimeOptions{}) require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -282,7 +279,7 @@ func TestLoadCycleBinding(t *testing.T) { `), os.ModePerm)) r1, err := New(&lib.SourceData{ - Filename: "/main.js", + URL: &url.URL{Path: "/main.js", Scheme: "file"}, Data: []byte(` import {foo} from './a.js'; import {bar} from './b.js'; @@ -301,7 +298,6 @@ func TestLoadCycleBinding(t *testing.T) { require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) @@ -342,7 +338,7 @@ func TestBrowserified(t *testing.T) { `), os.ModePerm)) r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js", Scheme: "file"}, Data: []byte(` import {alpha, bravo } from "./browserified.js"; @@ -366,7 +362,6 @@ func TestBrowserified(t *testing.T) { require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) require.NoError(t, err) diff --git a/js/modules/k6/marshalling_test.go b/js/modules/k6/marshalling_test.go index 22e552a34f8..e62f2190038 100644 --- a/js/modules/k6/marshalling_test.go +++ b/js/modules/k6/marshalling_test.go @@ -22,6 +22,7 @@ package k6_test import ( "context" + "net/url" "testing" "time" @@ -114,7 +115,7 @@ func TestSetupDataMarshalling(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{Filename: "/script.js", Data: script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) diff --git a/js/runner_test.go b/js/runner_test.go index cbd77ce187c..96b7c56f809 100644 --- a/js/runner_test.go +++ b/js/runner_test.go @@ -31,6 +31,7 @@ import ( stdlog "log" "net" "net/http" + "net/url" "os" "runtime" "strings" @@ -60,7 +61,7 @@ import ( func TestRunnerNew(t *testing.T) { t.Run("Valid", func(t *testing.T) { r, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` let counter = 0; export default function() { counter++; } @@ -85,8 +86,8 @@ func TestRunnerNew(t *testing.T) { t.Run("Invalid", func(t *testing.T) { _, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`blarg`), + URL: &url.URL{Path: "/script.js"}, + Data: []byte(`blarg`), }, afero.NewMemMapFs(), lib.RuntimeOptions{}) assert.EqualError(t, err, "ReferenceError: blarg is not defined at /script.js:1:1(0)") }) @@ -94,8 +95,8 @@ func TestRunnerNew(t *testing.T) { func TestRunnerGetDefaultGroup(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`export default function() {};`), + URL: &url.URL{Path: "/script.js"}, + Data: []byte(`export default function() {};`), }, afero.NewMemMapFs(), lib.RuntimeOptions{}) if assert.NoError(t, err) { assert.NotNil(t, r1.GetDefaultGroup()) @@ -109,8 +110,8 @@ func TestRunnerGetDefaultGroup(t *testing.T) { func TestRunnerOptions(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(`export default function() {};`), + URL: &url.URL{Path: "/script.js"}, + Data: []byte(`export default function() {};`), }, afero.NewMemMapFs(), lib.RuntimeOptions{}) if !assert.NoError(t, err) { return @@ -152,7 +153,7 @@ func TestOptionsSettingToScript(t *testing.T) { t.Run(fmt.Sprintf("Variant#%d", i), func(t *testing.T) { t.Parallel() src := &lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(variant + ` export default function() { if (!options) { @@ -184,7 +185,7 @@ func TestOptionsSettingToScript(t *testing.T) { func TestOptionsPropagationToScript(t *testing.T) { t.Parallel() src := &lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export let options = { setupTimeout: "1s", myOption: "test" }; export default function() { @@ -244,7 +245,7 @@ func TestMetricName(t *testing.T) { `)) _, err := New( - &lib.SourceData{Filename: "/script.js", Data: script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) @@ -288,7 +289,7 @@ func TestSetupDataIsolation(t *testing.T) { `)) runner, err := New( - &lib.SourceData{Filename: "/script.js", Data: script}, + &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, afero.NewMemMapFs(), lib.RuntimeOptions{}, ) @@ -350,7 +351,7 @@ func testSetupDataHelper(t *testing.T, src *lib.SourceData) { } func TestSetupDataReturnValue(t *testing.T) { src := &lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; export function setup() { @@ -374,7 +375,7 @@ func TestSetupDataReturnValue(t *testing.T) { func TestSetupDataNoSetup(t *testing.T) { src := &lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; export default function(data) { @@ -396,7 +397,7 @@ func TestSetupDataNoSetup(t *testing.T) { func TestSetupDataNoReturn(t *testing.T) { src := &lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; export function setup() { } @@ -424,11 +425,12 @@ func TestRunnerIntegrationImports(t *testing.T) { "k6/html", } for _, mod := range modules { + mod := mod t.Run(mod, func(t *testing.T) { t.Run("Source", func(t *testing.T) { _, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(fmt.Sprintf(`import "%s"; export default function() {}`, mod)), + URL: &url.URL{Path: "/script.js"}, + Data: []byte(fmt.Sprintf(`import "%s"; export default function() {}`, mod)), }, afero.NewMemMapFs(), lib.RuntimeOptions{}) assert.NoError(t, err) }) @@ -438,8 +440,8 @@ func TestRunnerIntegrationImports(t *testing.T) { t.Run("Files", func(t *testing.T) { fs := afero.NewMemMapFs() - assert.NoError(t, fs.MkdirAll("/path/to", 0755)) - assert.NoError(t, afero.WriteFile(fs, "/path/to/lib.js", []byte(`export default "hi!";`), 0644)) + require.NoError(t, fs.MkdirAll("/path/to", 0755)) + require.NoError(t, afero.WriteFile(fs, "/path/to/lib.js", []byte(`export default "hi!";`), 0644)) testdata := map[string]struct{ filename, path string }{ "Absolute": {"/path/script.js", "/path/to/lib.js"}, @@ -449,33 +451,29 @@ func TestRunnerIntegrationImports(t *testing.T) { "STDIN-Relative": {"-", "./path/to/lib.js"}, } for name, data := range testdata { + name, data := name, data t.Run(name, func(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: data.filename, + URL: &url.URL{Path: data.filename, Scheme: "file"}, Data: []byte(fmt.Sprintf(` import hi from "%s"; export default function() { if (hi != "hi!") { throw new Error("incorrect value"); } }`, data.path)), }, fs, lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) testdata := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range testdata { + r := r t.Run(name, func(t *testing.T) { vu, err := r.NewVU(make(chan stats.SampleContainer, 100)) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) err = vu.RunOnce(context.Background()) - assert.NoError(t, err) + require.NoError(t, err) }) } }) @@ -485,7 +483,7 @@ func TestRunnerIntegrationImports(t *testing.T) { func TestVURunContext(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export let options = { vus: 10 }; export default function() { fn(); } @@ -538,7 +536,7 @@ func TestVURunInterrupt(t *testing.T) { } r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export default function() { while(true) {} } `), @@ -573,7 +571,7 @@ func TestVURunInterrupt(t *testing.T) { func TestVUIntegrationGroups(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` import { group } from "k6"; export default function() { @@ -636,7 +634,7 @@ func TestVUIntegrationGroups(t *testing.T) { func TestVUIntegrationMetrics(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` import { group } from "k6"; import { Trend } from "k6/metrics"; @@ -710,7 +708,7 @@ func TestVUIntegrationInsecureRequests(t *testing.T) { for name, data := range testdata { t.Run(name, func(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` import http from "k6/http"; export default function() { http.get("https://expired.badssl.com/"); } @@ -749,7 +747,7 @@ func TestVUIntegrationInsecureRequests(t *testing.T) { func TestVUIntegrationBlacklistOption(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` import http from "k6/http"; export default function() { http.get("http://10.1.2.3/"); } @@ -788,7 +786,7 @@ func TestVUIntegrationBlacklistOption(t *testing.T) { func TestVUIntegrationBlacklistScript(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` import http from "k6/http"; @@ -829,7 +827,7 @@ func TestVUIntegrationHosts(t *testing.T) { defer tb.Cleanup() r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` import { check, fail } from "k6"; import http from "k6/http"; @@ -913,7 +911,7 @@ func TestVUIntegrationTLSConfig(t *testing.T) { for name, data := range testdata { t.Run(name, func(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` import http from "k6/http"; export default function() { http.get("https://sha256.badssl.com/"); } @@ -952,7 +950,7 @@ func TestVUIntegrationTLSConfig(t *testing.T) { func TestVUIntegrationHTTP2(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` import http from "k6/http"; export default function() { @@ -1002,7 +1000,7 @@ func TestVUIntegrationHTTP2(t *testing.T) { func TestVUIntegrationOpenFunctionError(t *testing.T) { r, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export default function() { open("/tmp/foo") } `), @@ -1021,7 +1019,7 @@ func TestVUIntegrationCookiesReset(t *testing.T) { defer tb.Cleanup() r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` import http from "k6/http"; export default function() { @@ -1074,7 +1072,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { defer tb.Cleanup() r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` import http from "k6/http"; export default function() { @@ -1131,7 +1129,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { func TestVUIntegrationVUID(t *testing.T) { r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(` export default function() { if (__VU != 1234) { throw new Error("wrong __VU: " + __VU); } @@ -1227,7 +1225,7 @@ func TestVUIntegrationClientCerts(t *testing.T) { go func() { _ = srv.Serve(listener) }() r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(fmt.Sprintf(` import http from "k6/http"; export default function() { http.get("https://%s")} @@ -1310,7 +1308,7 @@ func TestHTTPRequestInInitContext(t *testing.T) { defer tb.Cleanup() _, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` import { check, fail } from "k6"; import http from "k6/http"; @@ -1392,10 +1390,11 @@ func TestInitContextForbidden(t *testing.T) { defer tb.Cleanup() for _, test := range table { + test := test t.Run(test[0], func(t *testing.T) { _, err := New(&lib.SourceData{ - Filename: "/script.js", - Data: []byte(tb.Replacer.Replace(test[1])), + URL: &url.URL{Path: "/script.js"}, + Data: []byte(tb.Replacer.Replace(test[1])), }, afero.NewMemMapFs(), lib.RuntimeOptions{}) if assert.Error(t, err) { assert.Equal( @@ -1414,7 +1413,7 @@ func TestArchiveRunningIntegraty(t *testing.T) { fs := afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm)) r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` let fput = open("/home/somebody/test.json"); export let options = { setupTimeout: "10s", teardownTimeout: "10s" }; @@ -1459,7 +1458,7 @@ func TestArchiveNotPanicking(t *testing.T) { fs := afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/non/existent", []byte(`42`), os.ModePerm)) r1, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` let fput = open("/non/existent"); export default function(data) { @@ -1469,7 +1468,7 @@ func TestArchiveNotPanicking(t *testing.T) { require.NoError(t, err) arc := r1.MakeArchive() - arc.Files = make(map[string][]byte) + arc.FSes = map[string]afero.Fs{"file": afero.NewMemMapFs()} r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) // we do want this to error here as this is where we find out that a given file is not in the // archive @@ -1482,7 +1481,7 @@ func TestStuffNotPanicking(t *testing.T) { defer tb.Cleanup() r, err := New(&lib.SourceData{ - Filename: "/script.js", + URL: &url.URL{Path: "/script.js"}, Data: []byte(tb.Replacer.Replace(` import http from "k6/http"; import ws from "k6/ws"; diff --git a/lib/archive.go b/lib/archive.go index 187a1cb75f6..43a60deb99c 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -24,6 +24,7 @@ import ( "archive/tar" "bytes" "encoding/json" + "fmt" "io" "io/ioutil" "os" @@ -37,9 +38,12 @@ import ( "github.com/spf13/afero" ) -var volumeRE = regexp.MustCompile(`^([a-zA-Z]):(.*)`) -var sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js -var homeDirRE = regexp.MustCompile(`^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`) +//nolint: gochecknoglobals, lll +var ( + volumeRE = regexp.MustCompile(`^/?([a-zA-Z]):(.*)`) + sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js + homeDirRE = regexp.MustCompile(`^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`) +) // NormalizeAndAnonymizePath Normalizes (to use a / path separator) and anonymizes a file path, by scrubbing usernames from home directories. func NormalizeAndAnonymizePath(path string) string { @@ -82,25 +86,29 @@ type Archive struct { // Working directory for resolving relative paths. Pwd string `json:"pwd"` - // Archived filesystem. - Scripts map[string][]byte `json:"-"` // included scripts - Files map[string][]byte `json:"-"` // non-script resources - - FS afero.Fs `json:"-"` + FSes map[string]afero.Fs `json:"-"` // Environment variables Env map[string]string `json:"env"` } -// Reads an archive created by Archive.Write from a reader. -func ReadArchive(in io.Reader) (*Archive, error) { - r := tar.NewReader(in) - arc := &Archive{ - Scripts: make(map[string][]byte), - Files: make(map[string][]byte), - FS: &normalizedFS{Fs: afero.NewMemMapFs()}, +func (arc *Archive) getFs(name string) afero.Fs { + fs, ok := arc.FSes[name] + if !ok { + fs = afero.NewMemMapFs() + if name == "file" { + fs = &normalizedFS{fs} + } + arc.FSes[name] = fs } + return fs +} + +// ReadArchive reads an archive created by Archive.Write from a reader. +func ReadArchive(in io.Reader) (*Archive, error) { + r := tar.NewReader(in) + arc := &Archive{FSes: make(map[string]afero.Fs, 2)} for { hdr, err := r.Next() if err != nil { @@ -109,7 +117,7 @@ func ReadArchive(in io.Reader) (*Archive, error) { } return nil, err } - if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA && hdr.Typeflag != tar.TypeDir { continue } @@ -129,6 +137,7 @@ func ReadArchive(in io.Reader) (*Archive, error) { continue case "data": arc.Data = data + continue } // Path separator normalization for older archives (<=0.20.0) @@ -138,28 +147,36 @@ func ReadArchive(in io.Reader) (*Archive, error) { continue } pfx := normPath[:idx] - name := normPath[idx+1:] - if name != "" && name[0] == '_' { - name = name[1:] - } + name := normPath[idx:] - var dst map[string][]byte switch pfx { - case "files": - dst = arc.Files - case "scripts": - dst = arc.Scripts + case "files", "scripts": // old archives + // in old archives (pre 0.25.0) names without "_" at the beginning were https, the ones with "_" are local files + pfx = "https" + if len(name) >= 2 && name[0:2] == "/_" { + pfx = "file" + name = name[2:] + } + fallthrough + case "https", "file": + fs := arc.getFs(pfx) + if hdr.Typeflag == tar.TypeDir { + err = fs.Mkdir(name, os.FileMode(hdr.Mode)) + } else { + err = afero.WriteFile(fs, name, data, os.FileMode(hdr.Mode)) + } + if err != nil { + return nil, err + } + err = fs.Chtimes(name, hdr.AccessTime, hdr.ModTime) + if err != nil { + return nil, err + } default: - continue - } - - dst[name] = data - - err = afero.WriteFile(arc.FS, name, data, os.ModePerm) - if err != nil { - return nil, err + return nil, fmt.Errorf("unknown file prefix `%s` for file `%s`", pfx, normPath) } } + // TODO write the data to pwd in the appropriate archive return arc, nil } @@ -171,7 +188,6 @@ func ReadArchive(in io.Reader) (*Archive, error) { // the current one. func (arc *Archive) Write(out io.Writer) error { w := tar.NewWriter(out) - t := time.Now() metaArc := *arc metaArc.Filename = NormalizeAndAnonymizePath(metaArc.Filename) @@ -184,10 +200,10 @@ func (arc *Archive) Write(out io.Writer) error { Name: "metadata.json", Mode: 0644, Size: int64(len(metadata)), - ModTime: t, + ModTime: time.Now(), Typeflag: tar.TypeReg, }) - if _, err := w.Write(metadata); err != nil { + if _, err = w.Write(metadata); err != nil { return err } @@ -195,27 +211,20 @@ func (arc *Archive) Write(out io.Writer) error { Name: "data", Mode: 0644, Size: int64(len(arc.Data)), - ModTime: t, + ModTime: time.Now(), Typeflag: tar.TypeReg, }) - if _, err := w.Write(arc.Data); err != nil { + if _, err = w.Write(arc.Data); err != nil { return err } - - arcfs := []struct { - name string - files map[string][]byte - }{ - {"scripts", arc.Scripts}, - {"files", arc.Files}, - } - for _, entry := range arcfs { - _ = w.WriteHeader(&tar.Header{ - Name: entry.name, - Mode: 0755, - ModTime: t, - Typeflag: tar.TypeDir, - }) + for _, name := range [...]string{"file", "https"} { + fs, ok := arc.FSes[name] + if !ok { + continue + } + if cachedfs, ok := fs.(interface{ GetCachedFs() afero.Fs }); ok { + fs = cachedfs.GetCachedFs() + } // A couple of things going on here: // - You can't just create file entries, you need to create directory entries too. @@ -225,21 +234,33 @@ func (arc *Archive) Write(out io.Writer) error { // - We don't want to leak private information (eg. usernames) in archives, so make sure to // anonymize paths before stuffing them in a shareable archive. foundDirs := make(map[string]bool) - paths := make([]string, 0, len(entry.files)) - files := make(map[string][]byte, len(entry.files)) - for filePath, data := range entry.files { - filePath = NormalizeAndAnonymizePath(filePath) - files[filePath] = data - paths = append(paths, filePath) - dir := path.Dir(filePath) - for { - foundDirs[dir] = true - idx := strings.LastIndexByte(dir, os.PathSeparator) - if idx == -1 { - break + paths := make([]string, 0, 10) + infos := make(map[string]os.FileInfo) // ... fix this ? + files := make(map[string][]byte) + err = afero.Walk(fs, "/", + filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err } - dir = dir[:idx] - } + normalizedPath := NormalizeAndAnonymizePath(filePath) + infos[normalizedPath] = info + if info.IsDir() { + foundDirs[normalizedPath] = true + return nil + } + + files[normalizedPath], err = afero.ReadFile(fs, filePath) + if err != nil { + return err + } + paths = append(paths, normalizedPath) + return nil + })) + if err != nil { + return err + } + if len(files) == 0 { + continue // we don't need to write anything for this fs, if this is not done the root will be written } dirs := make([]string, 0, len(foundDirs)) for dirpath := range foundDirs { @@ -248,31 +269,28 @@ func (arc *Archive) Write(out io.Writer) error { sort.Strings(paths) sort.Strings(dirs) - for _, dirpath := range dirs { - if dirpath == "" || dirpath[0] == '/' { - dirpath = "_" + dirpath - } + for _, dirPath := range dirs { _ = w.WriteHeader(&tar.Header{ - Name: path.Clean(entry.name + "/" + dirpath), - Mode: 0755, - ModTime: t, - Typeflag: tar.TypeDir, + Name: path.Clean(name + "/" + dirPath), + Mode: int64(infos[dirPath].Mode()), + AccessTime: infos[dirPath].ModTime(), + ChangeTime: infos[dirPath].ModTime(), + ModTime: infos[dirPath].ModTime(), + Typeflag: tar.TypeDir, }) } for _, filePath := range paths { - data := files[filePath] - if filePath[0] == '/' { - filePath = "_" + filePath - } _ = w.WriteHeader(&tar.Header{ - Name: path.Clean(entry.name + "/" + filePath), - Mode: 0644, - Size: int64(len(data)), - ModTime: t, - Typeflag: tar.TypeReg, + Name: path.Clean(name + "/" + filePath), + Mode: int64(infos[filePath].Mode()), + Size: int64(len(files[filePath])), + AccessTime: infos[filePath].ModTime(), + ChangeTime: infos[filePath].ModTime(), + ModTime: infos[filePath].ModTime(), + Typeflag: tar.TypeReg, }) - if _, err := w.Write(data); err != nil { + if _, err := w.Write(files[filePath]); err != nil { return err } } diff --git a/lib/archive_test.go b/lib/archive_test.go index 48ea6588ba0..e4d4d3ace52 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -23,10 +23,13 @@ package lib import ( "bytes" "fmt" + "os" "runtime" "testing" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" ) @@ -50,6 +53,7 @@ func TestNormalizeAndAnonymizePath(t *testing.T) { testdata["//etc/hosts"] = "/etc/hosts" } for from, to := range testdata { + from, to := from, to t.Run("path="+from, func(t *testing.T) { res := NormalizeAndAnonymizePath(from) assert.Equal(t, to, res) @@ -58,6 +62,69 @@ func TestNormalizeAndAnonymizePath(t *testing.T) { } } +func makeMemMapFs(t *testing.T, input map[string][]byte) afero.Fs { + fs := afero.NewMemMapFs() + for path, data := range input { + require.NoError(t, afero.WriteFile(fs, path, data, 0644)) + } + return fs +} + +func getMapKeys(m map[string]afero.Fs) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + + return keys +} + +func diffMapFSes(t *testing.T, first, second map[string]afero.Fs) bool { + require.ElementsMatch(t, getMapKeys(first), getMapKeys(second), "fs map keys don't match") + for key, fs := range first { + secondFs := second[key] + diffFSes(t, fs, secondFs) + } + + return true +} + +func diffFSes(t *testing.T, first, second afero.Fs) { + diffFSesDir(t, first, second, "/") +} + +func getInfoNames(infos []os.FileInfo) []string { + var names = make([]string, len(infos)) + for i, info := range infos { + names[i] = info.Name() + } + return names +} + +func diffFSesDir(t *testing.T, first, second afero.Fs, dirname string) { + firstInfos, err := afero.ReadDir(first, dirname) + require.NoError(t, err, dirname) + + secondInfos, err := afero.ReadDir(first, dirname) + require.NoError(t, err, dirname) + + require.ElementsMatch(t, getInfoNames(firstInfos), getInfoNames(secondInfos), "directory: "+dirname) + for _, info := range firstInfos { + path := dirname + "/" + info.Name() + if info.IsDir() { + diffFSesDir(t, first, second, path) + continue + } + firstData, err := afero.ReadFile(first, path) + require.NoError(t, err, path) + + secondData, err := afero.ReadFile(second, path) + require.NoError(t, err, path) + + assert.Equal(t, firstData, secondData, path) + } +} + func TestArchiveReadWrite(t *testing.T) { t.Run("Roundtrip", func(t *testing.T) { arc1 := &Archive{ @@ -69,25 +136,37 @@ func TestArchiveReadWrite(t *testing.T) { Filename: "/path/to/script.js", Data: []byte(`// contents...`), Pwd: "/path/to", - Scripts: map[string][]byte{ - "/path/to/a.js": []byte(`// a contents`), - "/path/to/b.js": []byte(`// b contents`), - "cdnjs.com/libraries/Faker": []byte(`// faker contents`), - }, - Files: map[string][]byte{ - "/path/to/file1.txt": []byte(`hi!`), - "/path/to/file2.txt": []byte(`bye!`), - "github.com/loadimpact/k6/README.md": []byte(`README`), + FSes: map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + "/path/to/a.js": []byte(`// a contents`), + "/path/to/b.js": []byte(`// b contents`), + "/path/to/file1.txt": []byte(`hi!`), + "/path/to/file2.txt": []byte(`bye!`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/cdnjs.com/libraries/Faker": []byte(`// faker contents`), + "/github.com/loadimpact/k6/README.md": []byte(`README`), + }), }, } buf := bytes.NewBuffer(nil) - assert.NoError(t, arc1.Write(buf)) + require.NoError(t, arc1.Write(buf)) + + arc1FSes := arc1.FSes + arc1.FSes = nil arc2, err := ReadArchive(buf) - arc2.FS = nil - assert.NoError(t, err) + require.NoError(t, err) + + // arc2.FSes["file"] = arc2.FSes["file"].(*normalizedFS).Fs + + arc2FSes := arc2.FSes + arc2.FSes = nil + assert.Equal(t, arc1, arc2) + + diffMapFSes(t, arc1FSes, arc2FSes) }) t.Run("Anonymized", func(t *testing.T) { @@ -95,7 +174,7 @@ func TestArchiveReadWrite(t *testing.T) { Pwd, PwdNormAnon string }{ {"/home/myname", "/home/nobody"}, - {"C:\\Users\\Administrator", "/C/Users/nobody"}, + {"/C:/Users/Administrator", "/C/Users/nobody"}, } for _, entry := range testdata { arc1 := &Archive{ @@ -107,15 +186,17 @@ func TestArchiveReadWrite(t *testing.T) { Filename: fmt.Sprintf("%s/script.js", entry.Pwd), Data: []byte(`// contents...`), Pwd: entry.Pwd, - Scripts: map[string][]byte{ - fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`), - fmt.Sprintf("%s/b.js", entry.Pwd): []byte(`// b contents`), - "cdnjs.com/libraries/Faker": []byte(`// faker contents`), - }, - Files: map[string][]byte{ - fmt.Sprintf("%s/file1.txt", entry.Pwd): []byte(`hi!`), - fmt.Sprintf("%s/file2.txt", entry.Pwd): []byte(`bye!`), - "github.com/loadimpact/k6/README.md": []byte(`README`), + FSes: map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`), + fmt.Sprintf("%s/b.js", entry.Pwd): []byte(`// b contents`), + fmt.Sprintf("%s/file1.txt", entry.Pwd): []byte(`hi!`), + fmt.Sprintf("%s/file2.txt", entry.Pwd): []byte(`bye!`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/cdnjs.com/libraries/Faker": []byte(`// faker contents`), + "/github.com/loadimpact/k6/README.md": []byte(`README`), + }), }, } arc1Anon := &Archive{ @@ -127,25 +208,35 @@ func TestArchiveReadWrite(t *testing.T) { Filename: fmt.Sprintf("%s/script.js", entry.PwdNormAnon), Data: []byte(`// contents...`), Pwd: entry.PwdNormAnon, - Scripts: map[string][]byte{ - fmt.Sprintf("%s/a.js", entry.PwdNormAnon): []byte(`// a contents`), - fmt.Sprintf("%s/b.js", entry.PwdNormAnon): []byte(`// b contents`), - "cdnjs.com/libraries/Faker": []byte(`// faker contents`), - }, - Files: map[string][]byte{ - fmt.Sprintf("%s/file1.txt", entry.PwdNormAnon): []byte(`hi!`), - fmt.Sprintf("%s/file2.txt", entry.PwdNormAnon): []byte(`bye!`), - "github.com/loadimpact/k6/README.md": []byte(`README`), + + FSes: map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + fmt.Sprintf("%s/a.js", entry.PwdNormAnon): []byte(`// a contents`), + fmt.Sprintf("%s/b.js", entry.PwdNormAnon): []byte(`// b contents`), + fmt.Sprintf("%s/file1.txt", entry.PwdNormAnon): []byte(`hi!`), + fmt.Sprintf("%s/file2.txt", entry.PwdNormAnon): []byte(`bye!`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/cdnjs.com/libraries/Faker": []byte(`// faker contents`), + "/github.com/loadimpact/k6/README.md": []byte(`README`), + }), }, } buf := bytes.NewBuffer(nil) assert.NoError(t, arc1.Write(buf)) + arc1FSes := arc1Anon.FSes + arc1Anon.FSes = nil + arc2, err := ReadArchive(buf) - arc2.FS = nil assert.NoError(t, err) + + arc2FSes := arc2.FSes + arc2.FSes = nil + assert.Equal(t, arc1Anon, arc2) + diffMapFSes(t, arc1FSes, arc2FSes) } }) } diff --git a/lib/models.go b/lib/models.go index c187b5b2382..567a24a08fc 100644 --- a/lib/models.go +++ b/lib/models.go @@ -24,6 +24,7 @@ import ( "crypto/md5" "encoding/hex" "encoding/json" + "net/url" "strconv" "strings" "sync" @@ -40,10 +41,10 @@ const GroupSeparator = "::" // Error emitted if you attempt to instantiate a Group or Check that contains the separator. var ErrNameContainsGroupSeparator = errors.New("group and check names may not contain '::'") -// Wraps a source file; data and filename. +// SourceData wraps a source file; data and filename. type SourceData struct { - Data []byte - Filename string + Data []byte + URL *url.URL } // StageFields defines the fields used for a Stage; this is a dumb hack to make the JSON code diff --git a/loader/cdnjs_test.go b/loader/cdnjs_test.go index 78f3c69ad73..80bdb87afad 100644 --- a/loader/cdnjs_test.go +++ b/loader/cdnjs_test.go @@ -21,10 +21,12 @@ package loader import ( + "net/url" "testing" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCDNJS(t *testing.T) { @@ -62,18 +64,23 @@ func TestCDNJS(t *testing.T) { }, } for path, expected := range paths { + path, expected := path, expected t.Run(path, func(t *testing.T) { name, loader, parts := pickLoader(path) assert.Equal(t, "cdnjs", name) assert.Equal(t, expected.parts, parts) + src, err := loader(path, parts) - assert.NoError(t, err) + require.NoError(t, err) assert.Regexp(t, expected.src, src) - data, err := Load(afero.NewMemMapFs(), "/", path) - if assert.NoError(t, err) { - assert.Equal(t, path, data.Filename) - assert.NotEmpty(t, data.Data) - } + + pathURL, err := url.Parse(src) + require.NoError(t, err) + + data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path) + require.NoError(t, err) + assert.Equal(t, pathURL, data.URL) + assert.NotEmpty(t, data.Data) }) } @@ -92,9 +99,14 @@ func TestCDNJS(t *testing.T) { assert.Equal(t, "cdnjs", name) assert.Equal(t, []string{"Faker", "3.1.0", "nonexistent.js"}, parts) src, err := loader(path, parts) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js", src) - _, err = Load(afero.NewMemMapFs(), "/", path) - assert.EqualError(t, err, "cdnjs: not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js") + + pathURL, err := url.Parse(src) + require.NoError(t, err) + + _, err = Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path) + require.Error(t, err) + assert.Contains(t, err.Error(), "not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js") }) } diff --git a/loader/github_test.go b/loader/github_test.go index 2d6d2daa2f4..5297d25ca5f 100644 --- a/loader/github_test.go +++ b/loader/github_test.go @@ -21,23 +21,43 @@ package loader import ( + "net/url" "testing" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGithub(t *testing.T) { path := "github.com/github/gitignore/Go.gitignore" + expectedEndSrc := "https://raw.githubusercontent.com/github/gitignore/master/Go.gitignore" name, loader, parts := pickLoader(path) assert.Equal(t, "github", name) assert.Equal(t, []string{"github", "gitignore", "Go.gitignore"}, parts) src, err := loader(path, parts) assert.NoError(t, err) - assert.Equal(t, "https://raw.githubusercontent.com/github/gitignore/master/Go.gitignore", src) - data, err := Load(afero.NewMemMapFs(), "/", path) - if assert.NoError(t, err) { - assert.Equal(t, path, data.Filename) + assert.Equal(t, expectedEndSrc, src) + + pathURL, err := url.Parse(src) + require.NoError(t, err) + t.Run("not cached", func(t *testing.T) { + data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path) + require.NoError(t, err) + assert.Equal(t, expectedEndSrc, data.URL.String()) assert.NotEmpty(t, data.Data) - } + }) + + t.Run("cached", func(t *testing.T) { + fs := afero.NewMemMapFs() + testData := []byte("test data") + + err := afero.WriteFile(fs, "/raw.githubusercontent.com/github/gitignore/master/Go.gitignore", testData, 0644) + require.NoError(t, err) + + data, err := Load(map[string]afero.Fs{"https": fs}, pathURL, path) + require.NoError(t, err) + assert.Equal(t, expectedEndSrc, data.URL.String()) + assert.Equal(t, data.Data, testData) + }) } diff --git a/loader/loader.go b/loader/loader.go index 083ba8739c9..9b21656cabc 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -1,7 +1,7 @@ /* * * k6 - a next-generation load testing tool - * Copyright (C) 2016 Load Impact + * Copyright (C) 2019 Load Impact * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -22,10 +22,10 @@ package loader import ( "io/ioutil" - "net" "net/http" "net/url" - "path/filepath" + "os" + "path" "regexp" "strings" "time" @@ -38,6 +38,7 @@ import ( type loaderFunc func(path string, parts []string) (string, error) +//nolint: gochecknoglobals var ( loaders = []struct { name string @@ -47,114 +48,130 @@ var ( {"cdnjs", cdnjs, regexp.MustCompile(`^cdnjs.com/libraries/([^/]+)(?:/([(\d\.)]+-?[^/]*))?(?:/(.*))?$`)}, {"github", github, regexp.MustCompile(`^github.com/([^/]+)/([^/]+)/(.*)$`)}, } - invalidScriptErrMsg = `The file "%[1]s" couldn't be found on local disk, ` + - `and trying to retrieve it from https://%[1]s failed as well. Make ` + - `sure that you've specified the right path to the file. If you're ` + + httpsSchemeCouldntBeLoadedMsg = `The moduleSpecifier "%s" couldn't be retrieved from` + + ` the resolved url "%s". Error : "%s"` + fileSchemeCouldntBeLoadedMsg = `The moduleSpecifier "%s" couldn't be found on ` + + `local disk. Make sure that you've specified the right path to the file. If you're ` + `running k6 using the Docker image make sure you have mounted the ` + `local directory (-v /local/path/:/inside/docker/path) containing ` + `your script and modules so that they're accessible by k6 from ` + `inside of the container, see ` + `https://docs.k6.io/v1.0/docs/modules#section-using-local-modules-with-docker.` + errNoLoaderMatched = errors.New("no loader matched") ) -// Resolves a relative path to an absolute one. -func Resolve(pwd, name string) string { - if name[0] == '.' { - return filepath.ToSlash(filepath.Join(pwd, name)) - } - return name -} - -// Returns the directory for the path. -func Dir(name string) string { - if name == "-" { - return "/" - } - return filepath.Dir(name) -} - -func Load(fs afero.Fs, pwd, name string) (*lib.SourceData, error) { - log.WithFields(log.Fields{"pwd": pwd, "name": name}).Debug("Loading...") - - // We just need to make sure `import ""` doesn't crash the loader. - if name == "" { +// Resolve a relative path to an absolute one. +func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { + if moduleSpecifier == "" { return nil, errors.New("local or remote path required") } - // Do not allow the protocol to be specified, it messes everything up. - if strings.Contains(name, "://") { - return nil, errors.New("imports should not contain a protocol") - } - - // Do not allow remote-loaded scripts to lift arbitrary files off the user's machine. - if (name[0] == '/' && pwd[0] != '/') || (filepath.VolumeName(name) != "" && filepath.VolumeName(pwd) == "") { - return nil, errors.Errorf("origin (%s) not allowed to load local file: %s", pwd, name) + if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' { + return pwd.Parse(moduleSpecifier) } - // If the file starts with ".", resolve it as a relative path. - name = Resolve(pwd, name) - log.WithField("name", name).Debug("Resolved...") - - // If the resolved path starts with a "/" or has a volume, it's a local file. - if name[0] == '/' || filepath.VolumeName(name) != "" { - data, err := afero.ReadFile(fs, name) + if strings.Contains(moduleSpecifier, "://") { + u, err := url.Parse(moduleSpecifier) if err != nil { return nil, err } - return &lib.SourceData{Filename: name, Data: data}, nil - } - - // If the file is from a known service, try loading from there. - loaderName, loader, loaderArgs := pickLoader(name) - if loader != nil { - u, err := loader(name, loaderArgs) - if err != nil { - return nil, err + if u.Scheme != "file" && u.Scheme != "https" { + return nil, + errors.Errorf("only supported schemes for imports are file and https, %s has `%s`", + moduleSpecifier, u.Scheme) } - data, err := fetch(u) - if err != nil { - return nil, errors.Wrap(err, loaderName) + if u.Scheme == "file" && pwd.Scheme == "https" { + return nil, errors.Errorf("origin (%s) not allowed to load local file: %s", pwd, moduleSpecifier) } - return &lib.SourceData{Filename: name, Data: data}, nil + return u, err + } + + stringURL, err := resolveUsingLoaders(moduleSpecifier) + if err == errNoLoaderMatched { + log.WithField("url", moduleSpecifier).Warning( + "A url was resolved but it didn't have scheme. " + + "This will be deprecated in the future and all remote modules will " + + "need to explicitly use `https` as scheme") + + return url.Parse("https://" + moduleSpecifier) } + if err != nil { + return nil, err + } + return url.Parse(stringURL) +} - // If it's not a file, check is it a remote location. HTTPS is enforced, because it's 2017, HTTPS is easy, - // running arbitrary, trivially MitM'd code (even sandboxed) is very, very bad. - origURL := "https://" + name - parsedURL, err := url.Parse(origURL) +// Dir returns the directory for the path. +func Dir(old *url.URL) *url.URL { + return old.ResolveReference(&url.URL{Path: "./"}) +} + +// Load loads the provided moduleSpecifier from the given fses which are map of fses for a given scheme which +// is they key of the map. If the scheme is https then a request will be made if the files is not +// found in the map and written to the map. +func Load(fses map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string) (*lib.SourceData, error) { + log.WithFields( + log.Fields{ + "moduleSpecifier": moduleSpecifier, + "original moduleSpecifier": originalModuleSpecifier, + }).Debug("Loading...") + + pathOnFs := path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):]) + data, err := afero.ReadFile(fses[moduleSpecifier.Scheme], pathOnFs) if err != nil { - return nil, errors.Errorf(invalidScriptErrMsg, name) + if os.IsNotExist(err) { + if moduleSpecifier.Scheme == "https" { + var result *lib.SourceData + result, err = loadRemoteURL(moduleSpecifier) + if err != nil { + return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, moduleSpecifier, err) + } + // TODO maybe make an afero.Fs which makes request directly and than use CacheOnReadFs + // on top of as with the `file` scheme fs + _ = afero.WriteFile(fses[moduleSpecifier.Scheme], pathOnFs, result.Data, 0644) + return result, nil + } + return nil, errors.Errorf(fileSchemeCouldntBeLoadedMsg, moduleSpecifier) + } + return nil, err } - if _, err = net.LookupHost(parsedURL.Hostname()); err != nil { - return nil, errors.Errorf(invalidScriptErrMsg, name) + return &lib.SourceData{URL: moduleSpecifier, Data: data}, nil +} + +func resolveUsingLoaders(name string) (string, error) { + _, loader, loaderArgs := pickLoader(name) + if loader != nil { + return loader(name, loaderArgs) } - // Load it and have a look. - url := origURL - if !strings.ContainsRune(url, '?') { - url += "?" - } else { - url += "&" + return "", errNoLoaderMatched +} + +func loadRemoteURL(u *url.URL) (*lib.SourceData, error) { + var oldQuery = u.RawQuery + if u.RawQuery != "" { + u.RawQuery += "&" } - url += "_k6=1" - data, err := fetch(url) + u.RawQuery += "_k6=1" + + data, err := fetch(u.String()) + u.RawQuery = oldQuery // If this fails, try to fetch without ?_k6=1 - some sources act weird around unknown GET args. if err != nil { - data2, err2 := fetch(origURL) - if err2 != nil { - return nil, errors.Errorf(invalidScriptErrMsg, name) + data, err = fetch(u.String()) + if err != nil { + return nil, err } - data = data2 } // TODO: Parse the HTML, look for meta tags!! // // - return &lib.SourceData{Filename: name, Data: data}, nil + return &lib.SourceData{URL: u, Data: data}, nil } func pickLoader(path string) (string, loaderFunc, []string) { diff --git a/loader/loader_test.go b/loader/loader_test.go index 33e0f8c640c..c5cc2d606f9 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -23,26 +23,71 @@ package loader import ( "fmt" "net/http" + "net/url" "path/filepath" "testing" "github.com/loadimpact/k6/lib/testutils" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDir(t *testing.T) { testdata := map[string]string{ - "/path/to/file.txt": filepath.FromSlash("/path/to"), + "/path/to/file.txt": filepath.FromSlash("/path/to/"), "-": "/", } for name, dir := range testdata { + nameURL := &url.URL{Scheme: "file", Path: name} + dirURL := &url.URL{Scheme: "file", Path: dir} t.Run("path="+name, func(t *testing.T) { - assert.Equal(t, dir, Dir(name)) + assert.Equal(t, dirURL, Dir(nameURL)) }) } } +func TestResolve(t *testing.T) { + + t.Run("Blank", func(t *testing.T) { + _, err := Resolve(nil, "") + assert.EqualError(t, err, "local or remote path required") + }) + + t.Run("Protocol", func(t *testing.T) { + root, err := url.Parse("file:///") + require.NoError(t, err) + + t.Run("Missing", func(t *testing.T) { + u, err := Resolve(root, "example.com/html") + require.NoError(t, err) + assert.Equal(t, u.String(), "https://example.com/html") + // TODO: check that warning was emitted + }) + t.Run("WS", func(t *testing.T) { + moduleSpecifier := "ws://example.com/html" + _, err := Resolve(root, moduleSpecifier) + assert.EqualError(t, err, + "only supported schemes for imports are file and https, "+moduleSpecifier+" has `ws`") + }) + + t.Run("HTTP", func(t *testing.T) { + moduleSpecifier := "http://example.com/html" + _, err := Resolve(root, moduleSpecifier) + assert.EqualError(t, err, + "only supported schemes for imports are file and https, "+moduleSpecifier+" has `http`") + }) + }) + + t.Run("Remote Lifting Denied", func(t *testing.T) { + pwdURL, err := url.Parse("https://example.com") + require.NoError(t, err) + + _, err = Resolve(pwdURL, "file:///etc/shadow") + assert.EqualError(t, err, "origin (https://example.com) not allowed to load local file: file:///etc/shadow") + }) + +} func TestLoad(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) sr := tb.Replacer.Replace @@ -55,69 +100,90 @@ func TestLoad(t *testing.T) { http.DefaultTransport = oldHTTPTransport }() - t.Run("Blank", func(t *testing.T) { - _, err := Load(nil, "/", "") - assert.EqualError(t, err, "local or remote path required") - }) - - t.Run("Protocol", func(t *testing.T) { - _, err := Load(nil, "/", sr("HTTPSBIN_URL/html")) - assert.EqualError(t, err, "imports should not contain a protocol") - }) - t.Run("Local", func(t *testing.T) { - fs := afero.NewMemMapFs() - assert.NoError(t, fs.MkdirAll("/path/to", 0755)) - assert.NoError(t, afero.WriteFile(fs, "/path/to/file.txt", []byte("hi"), 0644)) + fses := make(map[string]afero.Fs) + fses["file"] = afero.NewMemMapFs() + assert.NoError(t, fses["file"].MkdirAll("/path/to", 0755)) + assert.NoError(t, afero.WriteFile(fses["file"], "/path/to/file.txt", []byte("hi"), 0644)) testdata := map[string]struct{ pwd, path string }{ - "Absolute": {"/path", "/path/to/file.txt"}, - "Relative": {"/path", "./to/file.txt"}, - "Adjacent": {"/path/to", "./file.txt"}, + "Absolute": {"/path/", "/path/to/file.txt"}, + "Relative": {"/path/", "./to/file.txt"}, + "Adjacent": {"/path/to/", "./file.txt"}, } for name, data := range testdata { + data := data t.Run(name, func(t *testing.T) { - src, err := Load(fs, data.pwd, data.path) - if assert.NoError(t, err) { - assert.Equal(t, "/path/to/file.txt", src.Filename) - assert.Equal(t, "hi", string(src.Data)) - } + pwdURL, err := url.Parse("file://" + data.pwd) + require.NoError(t, err) + + moduleURL, err := Resolve(pwdURL, data.path) + require.NoError(t, err) + + src, err := Load(fses, moduleURL, data.path) + require.NoError(t, err) + + assert.Equal(t, "file:///path/to/file.txt", src.URL.String()) + assert.Equal(t, "hi", string(src.Data)) }) } t.Run("Nonexistent", func(t *testing.T) { + root, err := url.Parse("file:///") + require.NoError(t, err) + path := filepath.FromSlash("/nonexistent") - _, err := Load(fs, "/", "/nonexistent") - assert.EqualError(t, err, fmt.Sprintf("open %s: file does not exist", path)) - }) + pathURL, err := Resolve(root, "/nonexistent") + require.NoError(t, err) - t.Run("Remote Lifting Denied", func(t *testing.T) { - _, err := Load(fs, "example.com", "/etc/shadow") - assert.EqualError(t, err, "origin (example.com) not allowed to load local file: /etc/shadow") + _, err = Load(fses, pathURL, path) + assert.EqualError(t, err, fmt.Sprintf(fileSchemeCouldntBeLoadedMsg, "file://"+path)) }) + }) t.Run("Remote", func(t *testing.T) { - src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/html")) - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/html")) + fses := map[string]afero.Fs{"https": afero.NewMemMapFs()} + t.Run("From local", func(t *testing.T) { + root, err := url.Parse("file:///") + require.NoError(t, err) + + moduleSpecifier := sr("HTTPSBIN_URL/html") + moduleSpecifierURL, err := Resolve(root, moduleSpecifier) + require.NoError(t, err) + + src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + require.NoError(t, err) + assert.Equal(t, src.URL, moduleSpecifierURL) assert.Contains(t, string(src.Data), "Herman Melville - Moby-Dick") - } + }) t.Run("Absolute", func(t *testing.T) { - src, err := Load(nil, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT"), sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt")) - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt")) - assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") - } + pwdURL, err := url.Parse(sr("HTTPSBIN_URL")) + require.NoError(t, err) + + moduleSpecifier := sr("HTTPSBIN_URL/robots.txt") + moduleSpecifierURL, err := Resolve(pwdURL, moduleSpecifier) + require.NoError(t, err) + + src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + require.NoError(t, err) + assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/robots.txt")) + assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") }) t.Run("Relative", func(t *testing.T) { - src, err := Load(nil, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT"), "./robots.txt") - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/robots.txt")) - assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") - } + pwdURL, err := url.Parse(sr("HTTPSBIN_URL")) + require.NoError(t, err) + + moduleSpecifier := ("./robots.txt") + moduleSpecifierURL, err := Resolve(pwdURL, moduleSpecifier) + require.NoError(t, err) + + src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + require.NoError(t, err) + assert.Equal(t, sr("HTTPSBIN_URL/robots.txt"), src.URL.String()) + assert.Equal(t, "User-agent: *\nDisallow: /deny\n", string(src.Data)) }) }) @@ -132,11 +198,19 @@ func TestLoad(t *testing.T) { }) t.Run("No _k6=1 Fallback", func(t *testing.T) { - src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/raw/something")) - if assert.NoError(t, err) { - assert.Equal(t, src.Filename, sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/raw/something")) - assert.Equal(t, responseStr, string(src.Data)) - } + root, err := url.Parse("file:///") + require.NoError(t, err) + + moduleSpecifier := sr("HTTPSBIN_URL/raw/something") + moduleSpecifierURL, err := Resolve(root, moduleSpecifier) + require.NoError(t, err) + + fses := map[string]afero.Fs{"https": afero.NewMemMapFs()} + src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + + require.NoError(t, err) + assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/raw/something")) + assert.Equal(t, responseStr, string(src.Data)) }) tb.Mux.HandleFunc("/invalid", func(w http.ResponseWriter, r *http.Request) { @@ -144,19 +218,32 @@ func TestLoad(t *testing.T) { }) t.Run("Invalid", func(t *testing.T) { - src, err := Load(nil, "/", sr("HTTPSBIN_DOMAIN:HTTPSBIN_PORT/invalid")) - assert.Nil(t, src) - assert.Error(t, err) - - t.Run("Host", func(t *testing.T) { - src, err := Load(nil, "/", "some-path-that-doesnt-exist.js") - assert.Nil(t, src) - assert.Error(t, err) - }) - t.Run("URL", func(t *testing.T) { - src, err := Load(nil, "/", "192.168.0.%31") - assert.Nil(t, src) - assert.Error(t, err) + root, err := url.Parse("file:///") + require.NoError(t, err) + + t.Run("IP URL", func(t *testing.T) { + _, err := Resolve(root, "192.168.0.%31") + require.Error(t, err) + require.Contains(t, err.Error(), `invalid URL escape "%31"`) }) + + var testData = [...]struct { + name, moduleSpecifier string + }{ + {"URL", sr("HTTPSBIN_URL/invalid")}, + {"HOST", "some-path-that-doesnt-exist.js"}, + } + + fses := map[string]afero.Fs{"https": afero.NewMemMapFs()} + for _, data := range testData { + moduleSpecifier := data.moduleSpecifier + t.Run(data.name, func(t *testing.T) { + moduleSpecifierURL, err := Resolve(root, moduleSpecifier) + require.NoError(t, err) + + _, err = Load(fses, moduleSpecifierURL, moduleSpecifier) + require.Error(t, err) + }) + } }) } diff --git a/stats/cloud/collector.go b/stats/cloud/collector.go index d5108a11f40..5df5c37f97d 100644 --- a/stats/cloud/collector.go +++ b/stats/cloud/collector.go @@ -107,7 +107,7 @@ func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*C } if !conf.Name.Valid || conf.Name.String == "" { - conf.Name = null.StringFrom(filepath.Base(src.Filename)) + conf.Name = null.StringFrom(filepath.Base(src.URL.Path)) } if conf.Name.String == "-" { conf.Name = null.StringFrom(TestName) diff --git a/stats/cloud/collector_test.go b/stats/cloud/collector_test.go index ccdec01b936..084b2ed5acd 100644 --- a/stats/cloud/collector_test.go +++ b/stats/cloud/collector_test.go @@ -27,6 +27,7 @@ import ( "io/ioutil" "math/rand" "net/http" + "net/url" "sync" "testing" "time" @@ -135,8 +136,8 @@ func TestCloudCollector(t *testing.T) { defer tb.Cleanup() script := &lib.SourceData{ - Data: []byte(""), - Filename: "/script.js", + Data: []byte(""), + URL: &url.URL{Path: "/script.js"}, } options := lib.Options{ @@ -281,8 +282,8 @@ func TestCloudCollectorMaxPerPacket(t *testing.T) { defer tb.Cleanup() script := &lib.SourceData{ - Data: []byte(""), - Filename: "/script.js", + Data: []byte(""), + URL: &url.URL{Path: "/script.js"}, } options := lib.Options{ From 4b91966cdf41b232fb3d62d693e71d033f494359 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 27 Jun 2019 11:36:34 +0300 Subject: [PATCH 02/45] fixup! Rewrite script/files loading to be be url based --- cmd/run.go | 3 ++- lib/archive.go | 5 +++-- lib/archive_test.go | 5 ++--- loader/loader.go | 4 ++-- loader/loader_test.go | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 3a3ebdca354..320c50cbfc8 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -32,6 +32,7 @@ import ( "net/url" "os" "os/signal" + "path/filepath" "runtime" "strings" "syscall" @@ -510,7 +511,7 @@ func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, } return &lib.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil } - pwdURL := &url.URL{Scheme: "file", Path: pwd} + pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd))} srcURL, err := loader.Resolve(pwdURL, src) if err != nil { return nil, err diff --git a/lib/archive.go b/lib/archive.go index 43a60deb99c..dc96c46de41 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -160,6 +160,7 @@ func ReadArchive(in io.Reader) (*Archive, error) { fallthrough case "https", "file": fs := arc.getFs(pfx) + name = filepath.FromSlash(name) if hdr.Typeflag == tar.TypeDir { err = fs.Mkdir(name, os.FileMode(hdr.Mode)) } else { @@ -271,7 +272,7 @@ func (arc *Archive) Write(out io.Writer) error { for _, dirPath := range dirs { _ = w.WriteHeader(&tar.Header{ - Name: path.Clean(name + "/" + dirPath), + Name: path.Clean(path.Join(name, dirPath)), Mode: int64(infos[dirPath].Mode()), AccessTime: infos[dirPath].ModTime(), ChangeTime: infos[dirPath].ModTime(), @@ -282,7 +283,7 @@ func (arc *Archive) Write(out io.Writer) error { for _, filePath := range paths { _ = w.WriteHeader(&tar.Header{ - Name: path.Clean(name + "/" + filePath), + Name: path.Clean(path.Join(name, filePath)), Mode: int64(infos[filePath].Mode()), Size: int64(len(files[filePath])), AccessTime: infos[filePath].ModTime(), diff --git a/lib/archive_test.go b/lib/archive_test.go index e4d4d3ace52..8f946162f10 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -24,6 +24,7 @@ import ( "bytes" "fmt" "os" + "path/filepath" "runtime" "testing" @@ -110,7 +111,7 @@ func diffFSesDir(t *testing.T, first, second afero.Fs, dirname string) { require.ElementsMatch(t, getInfoNames(firstInfos), getInfoNames(secondInfos), "directory: "+dirname) for _, info := range firstInfos { - path := dirname + "/" + info.Name() + path := filepath.Join(dirname, info.Name()) if info.IsDir() { diffFSesDir(t, first, second, path) continue @@ -159,8 +160,6 @@ func TestArchiveReadWrite(t *testing.T) { arc2, err := ReadArchive(buf) require.NoError(t, err) - // arc2.FSes["file"] = arc2.FSes["file"].(*normalizedFS).Fs - arc2FSes := arc2.FSes arc2.FSes = nil diff --git a/loader/loader.go b/loader/loader.go index 9b21656cabc..69ec1e9ad63 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -25,7 +25,7 @@ import ( "net/http" "net/url" "os" - "path" + "path/filepath" "regexp" "strings" "time" @@ -116,7 +116,7 @@ func Load(fses map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpec "original moduleSpecifier": originalModuleSpecifier, }).Debug("Loading...") - pathOnFs := path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):]) + pathOnFs := filepath.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):]) data, err := afero.ReadFile(fses[moduleSpecifier.Scheme], pathOnFs) if err != nil { diff --git a/loader/loader_test.go b/loader/loader_test.go index c5cc2d606f9..fd5438df45a 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -40,7 +40,7 @@ func TestDir(t *testing.T) { } for name, dir := range testdata { nameURL := &url.URL{Scheme: "file", Path: name} - dirURL := &url.URL{Scheme: "file", Path: dir} + dirURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(dir)} t.Run("path="+name, func(t *testing.T) { assert.Equal(t, dirURL, Dir(nameURL)) }) @@ -137,7 +137,7 @@ func TestLoad(t *testing.T) { require.NoError(t, err) _, err = Load(fses, pathURL, path) - assert.EqualError(t, err, fmt.Sprintf(fileSchemeCouldntBeLoadedMsg, "file://"+path)) + assert.EqualError(t, err, fmt.Sprintf(fileSchemeCouldntBeLoadedMsg, "file://"+filepath.ToSlash(path))) }) }) From a894e805d0cc1d28ee216a5bae989e878ca4cd8f Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 4 Jul 2019 09:58:43 +0300 Subject: [PATCH 03/45] fixup! Rewrite script/files loading to be be url based --- api/v1/setup_teardown_routes_test.go | 3 +- cmd/archive.go | 9 +- cmd/cloud.go | 9 +- cmd/inspect.go | 17 +- cmd/run.go | 38 ++- cmd/runtime_options_test.go | 5 +- converter/har/converter_test.go | 3 +- core/engine_test.go | 11 +- core/local/local_test.go | 3 +- js/bundle.go | 32 +-- js/bundle_test.go | 74 +++--- js/console_test.go | 43 +++- js/http_bench_test.go | 9 +- js/initcontext.go | 6 +- js/initcontext_test.go | 65 ++--- js/module_loading_test.go | 41 +-- js/modules/k6/marshalling_test.go | 3 +- js/runner.go | 5 +- js/runner_test.go | 356 +++++++++------------------ lib/archive.go | 16 +- lib/archive_test.go | 7 +- lib/fsext/cacheonread.go | 27 ++ lib/fsext/unprependfs.go | 172 +++++++++++++ lib/fsext/walk.go | 84 +++++++ loader/loader.go | 3 +- 25 files changed, 591 insertions(+), 450 deletions(-) create mode 100644 lib/fsext/cacheonread.go create mode 100644 lib/fsext/unprependfs.go create mode 100644 lib/fsext/walk.go diff --git a/api/v1/setup_teardown_routes_test.go b/api/v1/setup_teardown_routes_test.go index 4663c9cb34e..81a6db30d2c 100644 --- a/api/v1/setup_teardown_routes_test.go +++ b/api/v1/setup_teardown_routes_test.go @@ -36,7 +36,6 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/types" "github.com/manyminds/api2go/jsonapi" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" @@ -135,7 +134,7 @@ func TestSetupData(t *testing.T) { t.Run(testCase.name, func(t *testing.T) { runner, err := js.New( &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) diff --git a/cmd/archive.go b/cmd/archive.go index 3fe9734cd8d..bc9a8c1495a 100644 --- a/cmd/archive.go +++ b/cmd/archive.go @@ -23,7 +23,6 @@ package cmd import ( "os" - "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -51,8 +50,8 @@ An archive is a fully self-contained test run, and can be executed identically e return err } filename := args[0] - fs := afero.NewOsFs() - src, err := readSource(filename, pwd, fs, os.Stdin) + filesystems := createFilesystems() + src, err := readSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } @@ -62,7 +61,7 @@ An archive is a fully self-contained test run, and can be executed identically e return err } - r, err := newRunner(src, runType, fs, runtimeOptions) + r, err := newRunner(src, runType, filesystems, runtimeOptions) if err != nil { return err } @@ -71,7 +70,7 @@ An archive is a fully self-contained test run, and can be executed identically e if err != nil { return err } - conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r) + conf, err := getConsolidatedConfig(filesystems["file"], Config{Options: cliOpts}, r) if err != nil { return err } diff --git a/cmd/cloud.go b/cmd/cloud.go index 89f71aaf0a6..aa4d718b6db 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -34,7 +34,6 @@ import ( "github.com/loadimpact/k6/stats/cloud" "github.com/loadimpact/k6/ui" "github.com/pkg/errors" - "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -70,8 +69,8 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud } filename := args[0] - fs := afero.NewOsFs() - src, err := readSource(filename, pwd, fs, os.Stdin) + filesystems := createFilesystems() + src, err := readSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } @@ -81,7 +80,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud return err } - r, err := newRunner(src, runType, fs, runtimeOptions) + r, err := newRunner(src, runType, filesystems, runtimeOptions) if err != nil { return err } @@ -90,7 +89,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud if err != nil { return err } - conf, err := getConsolidatedConfig(fs, Config{Options: cliOpts}, r) + conf, err := getConsolidatedConfig(filesystems["file"], Config{Options: cliOpts}, r) if err != nil { return err } diff --git a/cmd/inspect.go b/cmd/inspect.go index efab0afbeb7..9a26b49e0f7 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -28,7 +28,6 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" - "github.com/spf13/afero" "github.com/spf13/cobra" ) @@ -43,8 +42,8 @@ var inspectCmd = &cobra.Command{ if err != nil { return err } - fs := afero.NewOsFs() - src, err := readSource(args[0], pwd, fs, os.Stdin) + filesystems := createFilesystems() + src, err := readSource(args[0], pwd, filesystems, os.Stdin) if err != nil { return err } @@ -59,20 +58,24 @@ var inspectCmd = &cobra.Command{ return err } - var opts lib.Options + var ( + opts lib.Options + b *js.Bundle + ) switch typ { case typeArchive: - arc, err := lib.ReadArchive(bytes.NewBuffer(src.Data)) + var arc *lib.Archive + arc, err = lib.ReadArchive(bytes.NewBuffer(src.Data)) if err != nil { return err } - b, err := js.NewBundleFromArchive(arc, runtimeOptions) + b, err = js.NewBundleFromArchive(arc, runtimeOptions) if err != nil { return err } opts = b.Options case typeJS: - b, err := js.NewBundle(src, fs, runtimeOptions) + b, err = js.NewBundle(src, filesystems, runtimeOptions) if err != nil { return err } diff --git a/cmd/run.go b/cmd/run.go index 320c50cbfc8..44843db2797 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -43,6 +43,7 @@ import ( "github.com/loadimpact/k6/core/local" "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/fsext" "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/ui" @@ -116,8 +117,8 @@ a commandline interface for interacting with it.`, return err } filename := args[0] - fs := afero.NewOsFs() - src, err := readSource(filename, pwd, fs, os.Stdin) + filesystems := createFilesystems() + src, err := readSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } @@ -127,7 +128,7 @@ a commandline interface for interacting with it.`, return err } - r, err := newRunner(src, runType, fs, runtimeOptions) + r, err := newRunner(src, runType, filesystems, runtimeOptions) if err != nil { return err } @@ -138,7 +139,7 @@ a commandline interface for interacting with it.`, if err != nil { return err } - conf, err := getConsolidatedConfig(fs, cliConf, r) + conf, err := getConsolidatedConfig(filesystems["file"], cliConf, r) if err != nil { return err } @@ -502,8 +503,21 @@ func init() { runCmd.Flags().AddFlagSet(runCmdFlagSet()) } +func createFilesystems() map[string]afero.Fs { + // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's + // written every time something is read from the real filesystem. This cache is then used for + // successive spawns to read from (they have no access to the real disk). + // Also initialize the same for `https` but the caching is handled manually in the loader package + return map[string]afero.Fs{ + "file": fsext.NewCacheOnReadFs( + fsext.NewUnprependPathFs(afero.NewOsFs(), afero.FilePathSeparator), + afero.NewMemMapFs(), 0), + "https": afero.NewMemMapFs(), + } +} + // Reads a source file from any supported destination. -func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, error) { +func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*lib.SourceData, error) { if src == "-" { data, err := ioutil.ReadAll(stdin) if err != nil { @@ -511,21 +525,23 @@ func readSource(src, pwd string, fs afero.Fs, stdin io.Reader) (*lib.SourceData, } return &lib.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil } - pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd))} - srcURL, err := loader.Resolve(pwdURL, src) + pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} + srcURL, err := loader.Resolve(pwdURL, filepath.ToSlash(src)) if err != nil { return nil, err } - return loader.Load(map[string]afero.Fs{"file": fs, "https": afero.NewMemMapFs()}, srcURL, src) + return loader.Load(filesystems, srcURL, src) } // Creates a new runner. -func newRunner(src *lib.SourceData, typ string, fs afero.Fs, rtOpts lib.RuntimeOptions) (lib.Runner, error) { +func newRunner( + src *lib.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions, +) (lib.Runner, error) { switch typ { case "": - return newRunner(src, detectType(src.Data), fs, rtOpts) + return newRunner(src, detectType(src.Data), filesystems, rtOpts) case typeJS: - return js.New(src, fs, rtOpts) + return js.New(src, filesystems, rtOpts) case typeArchive: arc, err := lib.ReadArchive(bytes.NewReader(src.Data)) if err != nil { diff --git a/cmd/runtime_options_test.go b/cmd/runtime_options_test.go index c8c7b25b2bf..25dcc7ed9bf 100644 --- a/cmd/runtime_options_test.go +++ b/cmd/runtime_options_test.go @@ -30,7 +30,6 @@ import ( "testing" "github.com/loadimpact/k6/lib" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -225,7 +224,7 @@ func TestEnvVars(t *testing.T) { URL: &url.URL{Path: "/script.js"}, }, typeJS, - afero.NewOsFs(), + nil, rtOpts, ) require.NoError(t, err) @@ -241,7 +240,7 @@ func TestEnvVars(t *testing.T) { URL: &url.URL{Path: "/script.js"}, }, typeArchive, - afero.NewOsFs(), + nil, rtOpts, ) } diff --git a/converter/har/converter_test.go b/converter/har/converter_test.go index 3d94aee6fa5..d427dde6b0d 100644 --- a/converter/har/converter_test.go +++ b/converter/har/converter_test.go @@ -27,7 +27,6 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" ) @@ -59,7 +58,7 @@ func TestBuildK6RequestObject(t *testing.T) { _, err = js.New(&lib.SourceData{ URL: &url.URL{Path: "/script.js"}, Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + }, nil, lib.RuntimeOptions{}) assert.NoError(t, err) } diff --git a/core/engine_test.go b/core/engine_test.go index 7d1818a94c1..f4e084d004c 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -37,7 +37,6 @@ import ( "github.com/loadimpact/k6/stats/dummy" log "github.com/sirupsen/logrus" logtest "github.com/sirupsen/logrus/hooks/test" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" @@ -558,7 +557,7 @@ func TestSentReceivedMetrics(t *testing.T) { runTest := func(t *testing.T, ts testScript, tc testCase, noConnReuse bool) (float64, float64) { r, err := js.New( &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -699,7 +698,7 @@ func TestRunTags(t *testing.T) { r, err := js.New( &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -799,7 +798,7 @@ func TestSetupTeardownThresholds(t *testing.T) { runner, err := js.New( &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -862,7 +861,7 @@ func TestEmittedMetricsWhenScalingDown(t *testing.T) { runner, err := js.New( &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) @@ -936,7 +935,7 @@ func TestMinIterationDuration(t *testing.T) { export default function () { testCounter.add(1); };`)}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) diff --git a/core/local/local_test.go b/core/local/local_test.go index 7360ad1008d..787bf05eef5 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -38,7 +38,6 @@ import ( "github.com/loadimpact/k6/stats" "github.com/pkg/errors" logtest "github.com/sirupsen/logrus/hooks/test" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" null "gopkg.in/guregu/null.v3" @@ -483,7 +482,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { runner, err := js.New( &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) require.NoError(t, err) diff --git a/js/bundle.go b/js/bundle.go index b1a0791c3d1..54fb0d702e8 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -24,7 +24,6 @@ import ( "context" "encoding/json" "net/url" - "time" "github.com/dop251/goja" "github.com/loadimpact/k6/js/common" @@ -56,24 +55,8 @@ type BundleInstance struct { Default goja.Callable } -type cacheOnReadFs struct { - afero.Fs - cache afero.Fs -} - -func newCacheOnReadFs(base, layer afero.Fs, cacheTime time.Duration) afero.Fs { - return cacheOnReadFs{ - Fs: afero.NewCacheOnReadFs(base, layer, cacheTime), - cache: layer, - } -} - -func (c cacheOnReadFs) GetCachedFs() afero.Fs { - return c.cache -} - // NewBundle creates a new bundle from a source file and a filesystem. -func NewBundle(src *lib.SourceData, fileFS afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { +func NewBundle(src *lib.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { compiler, err := compiler.New() if err != nil { return nil, err @@ -85,24 +68,13 @@ func NewBundle(src *lib.SourceData, fileFS afero.Fs, rtOpts lib.RuntimeOptions) if err != nil { return nil, err } - - // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's - // written every time something is read from the real filesystem. This cache is then used for - // successive spawns to read from (they have no access to the real disk). - mirrorFS := afero.NewMemMapFs() - cachedFS := newCacheOnReadFs(fileFS, mirrorFS, 0) - fses := map[string]afero.Fs{ - "file": cachedFS, - "https": afero.NewMemMapFs(), - } - // Make a bundle, instantiate it into a throwaway VM to populate caches. rt := goja.New() bundle := Bundle{ Filename: src.URL, Source: code, Program: pgm, - BaseInitContext: NewInitContext(rt, compiler, new(context.Context), fses, loader.Dir(src.URL)), + BaseInitContext: NewInitContext(rt, compiler, new(context.Context), filesystems, loader.Dir(src.URL)), Env: rtOpts.Env, } if err := bundle.instantiate(rt, bundle.BaseInitContext); err != nil { diff --git a/js/bundle_test.go b/js/bundle_test.go index 23af9d21558..928bf98e3e2 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -32,6 +32,9 @@ import ( "testing" "time" + "github.com/davecgh/go-spew/spew" + "github.com/loadimpact/k6/lib/fsext" + "github.com/dop251/goja" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/types" @@ -41,16 +44,30 @@ import ( "gopkg.in/guregu/null.v3" ) +const isWindows = runtime.GOOS == "windows" + func getSimpleBundle(filename, data string) (*Bundle, error) { return getSimpleBundleWithFs(filename, data, afero.NewMemMapFs()) } + +func getSimpleBundleWithOptions(filename, data string, options lib.RuntimeOptions) (*Bundle, error) { + return NewBundle( + &lib.SourceData{ + URL: &url.URL{Path: filename, Scheme: "file"}, + Data: []byte(data), + }, + map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": afero.NewMemMapFs()}, + options, + ) +} + func getSimpleBundleWithFs(filename, data string, fs afero.Fs) (*Bundle, error) { return NewBundle( &lib.SourceData{ URL: &url.URL{Path: filename, Scheme: "file"}, Data: []byte(data), }, - fs, + map[string]afero.Fs{"file": fs, "https": afero.NewMemMapFs()}, lib.RuntimeOptions{}, ) } @@ -361,16 +378,13 @@ func TestNewBundleFromArchive(t *testing.T) { assert.NoError(t, afero.WriteFile(fs, "/path/to/file.txt", []byte(`hi`), 0644)) assert.NoError(t, afero.WriteFile(fs, "/path/to/exclaim.js", []byte(`export default function(s) { return s + "!" };`), 0644)) - src := &lib.SourceData{ - URL: &url.URL{Path: "/path/to/script.js", Scheme: "file"}, - Data: []byte(` + data := ` import exclaim from "./exclaim.js"; export let options = { vus: 12345 }; export let file = open("./file.txt"); export default function() { return exclaim(file); }; - `), - } - b, err := NewBundle(src, fs, lib.RuntimeOptions{}) + ` + b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs) if !assert.NoError(t, err) { return } @@ -390,7 +404,7 @@ func TestNewBundleFromArchive(t *testing.T) { assert.Equal(t, "js", arc.Type) assert.Equal(t, lib.Options{VUs: null.IntFrom(12345)}, arc.Options) assert.Equal(t, "file:///path/to/script.js", arc.Filename) - assert.Equal(t, string(src.Data), string(arc.Data)) + assert.Equal(t, data, string(arc.Data)) assert.Equal(t, "file:///path/to/", arc.Pwd) exclaimData, err := afero.ReadFile(arc.FSes["file"], "/path/to/exclaim.js") @@ -501,8 +515,13 @@ func TestOpen(t *testing.T) { prefix, err := ioutil.TempDir("", "k6_open_test") require.NoError(t, err) fs := afero.NewOsFs() + filePath := filepath.Join(prefix, "/path/to/file.txt") + spew.Dump(filePath) require.NoError(t, fs.MkdirAll(filepath.Join(prefix, "/path/to"), 0755)) - require.NoError(t, afero.WriteFile(fs, filepath.Join(prefix, "/path/to/file.txt"), []byte(`hi`), 0644)) + require.NoError(t, afero.WriteFile(fs, filePath, []byte(`hi`), 0644)) + if isWindows { + fs = fsext.NewUnprependPathFs(fs, afero.FilePathSeparator) + } return fs, prefix, func() { require.NoError(t, os.RemoveAll(prefix)) } }, } @@ -521,21 +540,18 @@ func TestOpen(t *testing.T) { if openPath != "" && (openPath[0] == '/' || openPath[0] == '\\') { openPath = filepath.Join(prefix, openPath) } - if runtime.GOOS == "windows" { + if isWindows { openPath = strings.Replace(openPath, `\`, `\\`, -1) } var pwd = tCase.pwd if pwd == "" { pwd = "/path/to/" } - src := &lib.SourceData{ - URL: &url.URL{Scheme: "file", Path: filepath.Join(prefix, filepath.Join(pwd, "script.js"))}, - Data: []byte(` - export let file = open("` + openPath + `"); - export default function() { return file }; - `), - } - sourceBundle, err := NewBundle(src, fs, lib.RuntimeOptions{}) + data := ` + export let file = open("` + openPath + `"); + export default function() { return file };` + + sourceBundle, err := getSimpleBundleWithFs(filepath.ToSlash(filepath.Join(prefix, pwd, "script.js")), data, fs) if tCase.isError { assert.Error(t, err) return @@ -559,7 +575,7 @@ func TestOpen(t *testing.T) { } t.Run(tCase.name, testFunc) - if runtime.GOOS == "windows" { + if isWindows { // windowsify the testcase tCase.openPath = strings.Replace(tCase.openPath, `/`, `\`, -1) tCase.pwd = strings.Replace(tCase.pwd, `/`, `\`, -1) @@ -605,19 +621,13 @@ func TestBundleEnv(t *testing.T) { "TEST_A": "1", "TEST_B": "", }} - - b1, err := NewBundle( - &lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` - export default function() { - if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); } - if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); } - } - `), - }, - afero.NewMemMapFs(), rtOpts, - ) + data := ` +export default function() { + if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); } + if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); } +} +` + b1, err := getSimpleBundleWithOptions("/script.js", data, rtOpts) if !assert.NoError(t, err) { return } diff --git a/js/console_test.go b/js/console_test.go index 6d47cba4bbf..fc8750e77d4 100644 --- a/js/console_test.go +++ b/js/console_test.go @@ -69,7 +69,29 @@ func TestConsoleContext(t *testing.T) { assert.Equal(t, "b", entry.Message) } } +func getSimpleRunner(path, data string) (*Runner, error) { + return getSimpleRunnerWithFileFs(path, data, afero.NewMemMapFs()) +} + +func getSimpleRunnerWithOptions(path, data string, options lib.RuntimeOptions) (*Runner, error) { + return New(&lib.SourceData{ + URL: &url.URL{Path: path, Scheme: "file"}, + Data: []byte(data), + }, map[string]afero.Fs{ + "file": afero.NewMemMapFs(), + "https": afero.NewMemMapFs()}, + options) +} +func getSimpleRunnerWithFileFs(path, data string, fileFs afero.Fs) (*Runner, error) { + return New(&lib.SourceData{ + URL: &url.URL{Path: path, Scheme: "file"}, + Data: []byte(data), + }, map[string]afero.Fs{ + "file": fileFs, + "https": afero.NewMemMapFs()}, + lib.RuntimeOptions{}) +} func TestConsole(t *testing.T) { levels := map[string]log.Level{ "log": log.InfoLevel, @@ -88,16 +110,15 @@ func TestConsole(t *testing.T) { `{}`: {Message: "[object Object]"}, } for name, level := range levels { + name, level := name, level t.Run(name, func(t *testing.T) { for args, result := range argsets { + args, result := args, result t.Run(args, func(t *testing.T) { - r, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script"}, - Data: []byte(fmt.Sprintf( - `export default function() { console.%s(%s); }`, - name, args, - )), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r, err := getSimpleRunner("/script.js", fmt.Sprintf( + `export default function() { console.%s(%s); }`, + name, args, + )) assert.NoError(t, err) samples := make(chan stats.SampleContainer, 100) @@ -180,13 +201,11 @@ func TestFileConsole(t *testing.T) { } } - r, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script"}, - Data: []byte(fmt.Sprintf( + r, err := getSimpleRunner("/script", + fmt.Sprintf( `export default function() { console.%s(%s); }`, name, args, - )), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + )) assert.NoError(t, err) err = r.SetOptions(lib.Options{ diff --git a/js/http_bench_test.go b/js/http_bench_test.go index 2415a4edc4e..4fdfdd10f6a 100644 --- a/js/http_bench_test.go +++ b/js/http_bench_test.go @@ -2,13 +2,11 @@ package js import ( "context" - "net/url" "testing" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/stats" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "gopkg.in/guregu/null.v3" ) @@ -18,17 +16,14 @@ func BenchmarkHTTPRequests(b *testing.B) { tb := testutils.NewHTTPMultiBin(b) defer tb.Cleanup() - r, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + r, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; export default function() { let url = "HTTPBIN_URL"; let res = http.get(url + "/cookies/set?k2=v2&k1=v1"); if (res.status != 200) { throw new Error("wrong status: " + res.status) } } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(b, err) { return } diff --git a/js/initcontext.go b/js/initcontext.go index bcbc68fd52e..9ef13dd3ada 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -197,9 +197,11 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) { filename = filepath.Join(i.pwd.Path, filename) } - filename = filepath.ToSlash(filename) + filename = filepath.Clean(filename) fs := i.fses["file"] - + if filename[0:1] != afero.FilePathSeparator { + filename = afero.FilePathSeparator + filename + } // Workaround for https://github.com/spf13/afero/issues/201 if isDir, err := afero.IsDir(fs, filename); err != nil { return nil, err diff --git a/js/initcontext_test.go b/js/initcontext_test.go index 2fd558f6699..de5a7012dc5 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -27,7 +27,6 @@ import ( "net" "net/http" "net/http/httptest" - "net/url" "path/filepath" "testing" "time" @@ -113,7 +112,7 @@ func TestInitContextRequire(t *testing.T) { t.Run("Nonexistent", func(t *testing.T) { path := filepath.FromSlash("/nonexistent.js") _, err := getSimpleBundle("/script.js", `import "/nonexistent.js"; export default function() {}`) - assert.Contains(t, err.Error(), fmt.Sprintf(`"file://%s" couldn't be found on local disk`, path)) + assert.Contains(t, err.Error(), fmt.Sprintf(`"file://%s" couldn't be found on local disk`, filepath.ToSlash(path))) }) t.Run("Invalid", func(t *testing.T) { fs := afero.NewMemMapFs() @@ -159,6 +158,7 @@ func TestInitContextRequire(t *testing.T) { }}, } for libName, data := range imports { + libName, data := libName, data t.Run("lib=\""+libName+"\"", func(t *testing.T) { for constName, constPath := range data.ConstPaths { constName, constPath := constName, constPath @@ -168,15 +168,6 @@ func TestInitContextRequire(t *testing.T) { } t.Run(name, func(t *testing.T) { fs := afero.NewMemMapFs() - src := &lib.SourceData{ - URL: &url.URL{Path: "/path/to/script.js", Scheme: "file"}, - Data: []byte(fmt.Sprintf(` - import fn from "%s"; - let v = fn(); - export default function() { - }; - `, libName)), - } jsLib := `export default function() { return 12345; }` if constName != "" { @@ -193,7 +184,12 @@ func TestInitContextRequire(t *testing.T) { assert.NoError(t, fs.MkdirAll(filepath.Dir(data.LibPath), 0755)) assert.NoError(t, afero.WriteFile(fs, data.LibPath, []byte(jsLib), 0644)) - b, err := NewBundle(src, fs, lib.RuntimeOptions{}) + data := fmt.Sprintf(` + import fn from "%s"; + let v = fn(); + export default function() {};`, + libName) + b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs) if !assert.NoError(t, err) { return } @@ -214,18 +210,15 @@ func TestInitContextRequire(t *testing.T) { fs := afero.NewMemMapFs() assert.NoError(t, afero.WriteFile(fs, "/a.js", []byte(`const myvar = "a";`), 0644)) assert.NoError(t, afero.WriteFile(fs, "/b.js", []byte(`const myvar = "b";`), 0644)) - b, err := NewBundle(&lib.SourceData{ - URL: &url.URL{Path: "/script.js", Scheme: "file"}, - Data: []byte(` + data := ` import "./a.js"; import "./b.js"; export default function() { if (typeof myvar != "undefined") { throw new Error("myvar is set in global scope"); } - }; - `), - }, fs, lib.RuntimeOptions{}) + };` + b, err := getSimpleBundleWithFs("/script.js", data, fs) if !assert.NoError(t, err) { return } @@ -251,17 +244,15 @@ func createAndReadFile(t *testing.T, file string, content []byte, expectedLength binaryArg = ",\"b\"" } - b, err := NewBundle(&lib.SourceData{ - URL: &url.URL{Path: "/path/to/script.js"}, - Data: []byte(fmt.Sprintf(` - export let data = open("/path/to/%s"%s); - var expectedLength = %d; - if (data.length != expectedLength) { - throw new Error("Length not equal, expected: " + expectedLength + ", actual: " + data.length); - } - export default function() {} - `, file, binaryArg, expectedLength)), - }, fs, lib.RuntimeOptions{}) + data := fmt.Sprintf(` + export let data = open("/path/to/%s"%s); + var expectedLength = %d; + if (data.length != expectedLength) { + throw new Error("Length not equal, expected: " + expectedLength + ", actual: " + data.length); + } + export default function() {} + `, file, binaryArg, expectedLength) + b, err := getSimpleBundleWithFs("/path/to/script.js", data, fs) if !assert.NoError(t, err) { return nil, err @@ -320,12 +311,8 @@ func TestInitContextOpen(t *testing.T) { } t.Run("Nonexistent", func(t *testing.T) { - fs := afero.NewMemMapFs() path := filepath.FromSlash("/nonexistent.txt") - _, err := NewBundle(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(`open("/nonexistent.txt"); export default function() {}`), - }, fs, lib.RuntimeOptions{}) + _, err := getSimpleBundle("/script.js", `open("/nonexistent.txt"); export default function() {}`) assert.EqualError(t, err, fmt.Sprintf("GoError: open %s: file does not exist", path)) }) @@ -361,9 +348,8 @@ func TestRequestWithBinaryFile(t *testing.T) { assert.NoError(t, fs.MkdirAll("/path/to", 0755)) assert.NoError(t, afero.WriteFile(fs, "/path/to/file.bin", []byte("hi!"), 0644)) - b, err := NewBundle(&lib.SourceData{ - URL: &url.URL{Path: "/path/to/script.js"}, - Data: []byte(fmt.Sprintf(` + b, err := getSimpleBundleWithFs("/path/to/script.js", + fmt.Sprintf(` import http from "k6/http"; let binFile = open("/path/to/file.bin", "b"); export default function() { @@ -374,9 +360,8 @@ func TestRequestWithBinaryFile(t *testing.T) { var res = http.post("%s", data); return true; } - `, srv.URL)), - }, fs, lib.RuntimeOptions{}) - assert.NoError(t, err) + `, srv.URL), fs) + require.NoError(t, err) bi, err := b.Instantiate() assert.NoError(t, err) diff --git a/js/module_loading_test.go b/js/module_loading_test.go index 2e9080deea6..c59426e4ed4 100644 --- a/js/module_loading_test.go +++ b/js/module_loading_test.go @@ -22,7 +22,6 @@ package js import ( "context" - "net/url" "os" "testing" @@ -66,9 +65,7 @@ func TestLoadOnceGlobalVars(t *testing.T) { return C(); } `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js", Scheme: "file"}, - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import { A } from "./A.js"; import { B } from "./B.js"; @@ -80,8 +77,7 @@ func TestLoadOnceGlobalVars(t *testing.T) { throw new Error("A() != B() (" + A() + ") != (" + B() + ")"); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() @@ -115,9 +111,7 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) { return http.get("HTTPBIN_URL/get"); } `)), os.ModePerm)) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js", Scheme: "file"}, - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import { A } from "./A.js"; export default function(data) { @@ -126,8 +120,7 @@ func TestLoadDoesntBreakHTTPGet(t *testing.T) { throw new Error("wrong status "+ resp.status); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) require.NoError(t, r1.SetOptions(lib.Options{Hosts: tb.Dialer.Hosts})) @@ -158,9 +151,7 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) { return globalVar; } `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js", Scheme: "file"}, - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import { A } from "./A.js"; export default function(data) { @@ -171,8 +162,7 @@ func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) { throw new Error("wrong value of a " + a); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() @@ -229,10 +219,7 @@ func TestLoadCycle(t *testing.T) { `), os.ModePerm)) data, err := afero.ReadFile(fs, "/main.js") require.NoError(t, err) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/main.js", Scheme: "file"}, - Data: data, - }, fs, lib.RuntimeOptions{}) + r1, err := getSimpleRunnerWithFileFs("/main.js", string(data), fs) require.NoError(t, err) arc := r1.MakeArchive() @@ -278,9 +265,7 @@ func TestLoadCycleBinding(t *testing.T) { } `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/main.js", Scheme: "file"}, - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/main.js", ` import {foo} from './a.js'; import {bar} from './b.js'; export default function() { @@ -293,8 +278,7 @@ func TestLoadCycleBinding(t *testing.T) { throw new Error("Wrong value of bar() "+ barMessage); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() @@ -337,9 +321,7 @@ func TestBrowserified(t *testing.T) { }); `), os.ModePerm)) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js", Scheme: "file"}, - Data: []byte(` + r1, err := getSimpleRunnerWithFileFs("/script.js", ` import {alpha, bravo } from "./browserified.js"; export default function(data) { @@ -357,8 +339,7 @@ func TestBrowserified(t *testing.T) { throw new Error("bravo.B() != 'b' (" + bravo.B() + ") != 'b'"); } } - `), - }, fs, lib.RuntimeOptions{}) + `, fs) require.NoError(t, err) arc := r1.MakeArchive() diff --git a/js/modules/k6/marshalling_test.go b/js/modules/k6/marshalling_test.go index e62f2190038..d24baf07661 100644 --- a/js/modules/k6/marshalling_test.go +++ b/js/modules/k6/marshalling_test.go @@ -31,7 +31,6 @@ import ( "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/stats" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -116,7 +115,7 @@ func TestSetupDataMarshalling(t *testing.T) { runner, err := js.New( &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, - afero.NewMemMapFs(), + nil, lib.RuntimeOptions{}, ) diff --git a/js/runner.go b/js/runner.go index 603161369b4..c43ff6a96db 100644 --- a/js/runner.go +++ b/js/runner.go @@ -62,8 +62,9 @@ type Runner struct { setupData []byte } -func New(src *lib.SourceData, fs afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) { - bundle, err := NewBundle(src, fs, rtOpts) +// New returns a new Runner for the provide source +func New(src *lib.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) { + bundle, err := NewBundle(src, filesystems, rtOpts) if err != nil { return nil, err } diff --git a/js/runner_test.go b/js/runner_test.go index 96b7c56f809..3d252c55ab0 100644 --- a/js/runner_test.go +++ b/js/runner_test.go @@ -31,9 +31,7 @@ import ( stdlog "log" "net" "net/http" - "net/url" "os" - "runtime" "strings" "testing" "time" @@ -60,13 +58,10 @@ import ( func TestRunnerNew(t *testing.T) { t.Run("Valid", func(t *testing.T) { - r, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r, err := getSimpleRunner("/script.js", ` let counter = 0; export default function() { counter++; } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) assert.NoError(t, err) t.Run("NewVU", func(t *testing.T) { @@ -85,19 +80,13 @@ func TestRunnerNew(t *testing.T) { }) t.Run("Invalid", func(t *testing.T) { - _, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(`blarg`), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - assert.EqualError(t, err, "ReferenceError: blarg is not defined at /script.js:1:1(0)") + _, err := getSimpleRunner("/script.js", `blarg`) + assert.EqualError(t, err, "ReferenceError: blarg is not defined at file:///script.js:1:1(0)") }) } func TestRunnerGetDefaultGroup(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(`export default function() {};`), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r1, err := getSimpleRunner("/script.js", `export default function() {};`) if assert.NoError(t, err) { assert.NotNil(t, r1.GetDefaultGroup()) } @@ -109,10 +98,7 @@ func TestRunnerGetDefaultGroup(t *testing.T) { } func TestRunnerOptions(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(`export default function() {};`), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r1, err := getSimpleRunner("/script.js", `export default function() {};`) if !assert.NoError(t, err) { return } @@ -152,9 +138,7 @@ func TestOptionsSettingToScript(t *testing.T) { variant := variant t.Run(fmt.Sprintf("Variant#%d", i), func(t *testing.T) { t.Parallel() - src := &lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(variant + ` + data := variant + ` export default function() { if (!options) { throw new Error("Expected options to be defined!"); @@ -162,10 +146,9 @@ func TestOptionsSettingToScript(t *testing.T) { if (options.teardownTimeout != __ENV.expectedTeardownTimeout) { throw new Error("expected teardownTimeout to be " + __ENV.expectedTeardownTimeout + " but it was " + options.teardownTimeout); } - }; - `), - } - r, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{Env: map[string]string{"expectedTeardownTimeout": "4s"}}) + };` + r, err := getSimpleRunnerWithOptions("/script.js", data, + lib.RuntimeOptions{Env: map[string]string{"expectedTeardownTimeout": "4s"}}) require.NoError(t, err) newOptions := lib.Options{TeardownTimeout: types.NullDurationFrom(4 * time.Second)} @@ -184,9 +167,7 @@ func TestOptionsSettingToScript(t *testing.T) { func TestOptionsPropagationToScript(t *testing.T) { t.Parallel() - src := &lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + data := ` export let options = { setupTimeout: "1s", myOption: "test" }; export default function() { if (options.external) { @@ -198,12 +179,11 @@ func TestOptionsPropagationToScript(t *testing.T) { if (options.setupTimeout != __ENV.expectedSetupTimeout) { throw new Error("expected setupTimeout to be " + __ENV.expectedSetupTimeout + " but it was " + options.setupTimeout); } - }; - `), - } + };` expScriptOptions := lib.Options{SetupTimeout: types.NullDurationFrom(1 * time.Second)} - r1, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{Env: map[string]string{"expectedSetupTimeout": "1s"}}) + r1, err := getSimpleRunnerWithOptions("/script.js", data, + lib.RuntimeOptions{Env: map[string]string{"expectedSetupTimeout": "1s"}}) require.NoError(t, err) require.Equal(t, expScriptOptions, r1.GetOptions()) @@ -234,7 +214,7 @@ func TestMetricName(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - script := []byte(tb.Replacer.Replace(` + script := tb.Replacer.Replace(` import { Counter } from "k6/metrics"; let myCounter = new Counter("not ok name @"); @@ -242,13 +222,9 @@ func TestMetricName(t *testing.T) { export default function(data) { myCounter.add(1); } - `)) + `) - _, err := New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, - afero.NewMemMapFs(), - lib.RuntimeOptions{}, - ) + _, err := getSimpleRunner("/script.js", script) require.Error(t, err) } @@ -256,7 +232,7 @@ func TestSetupDataIsolation(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - script := []byte(tb.Replacer.Replace(` + script := tb.Replacer.Replace(` import { Counter } from "k6/metrics"; export let options = { @@ -286,13 +262,9 @@ func TestSetupDataIsolation(t *testing.T) { } myCounter.add(1); } - `)) + `) - runner, err := New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, - afero.NewMemMapFs(), - lib.RuntimeOptions{}, - ) + runner, err := getSimpleRunner("/script.js", script) require.NoError(t, err) engine, err := core.NewEngine(local.New(runner), runner.GetOptions()) @@ -323,13 +295,13 @@ func TestSetupDataIsolation(t *testing.T) { require.Equal(t, 501, count, "mycounter should be the number of iterations + 1 for the teardown") } -func testSetupDataHelper(t *testing.T, src *lib.SourceData) { +func testSetupDataHelper(t *testing.T, data string) { t.Helper() expScriptOptions := lib.Options{ SetupTimeout: types.NullDurationFrom(1 * time.Second), TeardownTimeout: types.NullDurationFrom(1 * time.Second), } - r1, err := New(src, afero.NewMemMapFs(), lib.RuntimeOptions{}) + r1, err := getSimpleRunner("/script.js", data) // TODO fix this require.NoError(t, err) require.Equal(t, expScriptOptions, r1.GetOptions()) @@ -350,71 +322,56 @@ func testSetupDataHelper(t *testing.T, src *lib.SourceData) { } } func TestSetupDataReturnValue(t *testing.T) { - src := &lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` - export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; - export function setup() { - return 42; - } - export default function(data) { - if (data != 42) { - throw new Error("default: wrong data: " + JSON.stringify(data)) - } - }; - - export function teardown(data) { - if (data != 42) { - throw new Error("teardown: wrong data: " + JSON.stringify(data)) - } - }; - `), + testSetupDataHelper(t, ` + export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; + export function setup() { + return 42; } - testSetupDataHelper(t, src) + export default function(data) { + if (data != 42) { + throw new Error("default: wrong data: " + JSON.stringify(data)) + } + }; + + export function teardown(data) { + if (data != 42) { + throw new Error("teardown: wrong data: " + JSON.stringify(data)) + } + };`) } func TestSetupDataNoSetup(t *testing.T) { - src := &lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` - export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; - export default function(data) { - if (data !== undefined) { - throw new Error("default: wrong data: " + JSON.stringify(data)) - } - }; + testSetupDataHelper(t, ` + export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; + export default function(data) { + if (data !== undefined) { + throw new Error("default: wrong data: " + JSON.stringify(data)) + } + }; - export function teardown(data) { - if (data !== undefined) { - console.log(data); - throw new Error("teardown: wrong data: " + JSON.stringify(data)) - } - }; - `), - } - testSetupDataHelper(t, src) + export function teardown(data) { + if (data !== undefined) { + console.log(data); + throw new Error("teardown: wrong data: " + JSON.stringify(data)) + } + };`) } func TestSetupDataNoReturn(t *testing.T) { - src := &lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` - export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; - export function setup() { } - export default function(data) { - if (data !== undefined) { - throw new Error("default: wrong data: " + JSON.stringify(data)) - } - }; + testSetupDataHelper(t, ` + export let options = { setupTimeout: "1s", teardownTimeout: "1s" }; + export function setup() { } + export default function(data) { + if (data !== undefined) { + throw new Error("default: wrong data: " + JSON.stringify(data)) + } + }; - export function teardown(data) { - if (data !== undefined) { - throw new Error("teardown: wrong data: " + JSON.stringify(data)) - } - }; - `), - } - testSetupDataHelper(t, src) + export function teardown(data) { + if (data !== undefined) { + throw new Error("teardown: wrong data: " + JSON.stringify(data)) + } + };`) } func TestRunnerIntegrationImports(t *testing.T) { t.Run("Modules", func(t *testing.T) { @@ -428,10 +385,7 @@ func TestRunnerIntegrationImports(t *testing.T) { mod := mod t.Run(mod, func(t *testing.T) { t.Run("Source", func(t *testing.T) { - _, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(fmt.Sprintf(`import "%s"; export default function() {}`, mod)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + _, err := getSimpleRunner("/script.js", fmt.Sprintf(`import "%s"; export default function() {}`, mod)) assert.NoError(t, err) }) }) @@ -453,14 +407,11 @@ func TestRunnerIntegrationImports(t *testing.T) { for name, data := range testdata { name, data := name, data t.Run(name, func(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: data.filename, Scheme: "file"}, - Data: []byte(fmt.Sprintf(` + r1, err := getSimpleRunnerWithFileFs(data.filename, fmt.Sprintf(` import hi from "%s"; export default function() { if (hi != "hi!") { throw new Error("incorrect value"); } - }`, data.path)), - }, fs, lib.RuntimeOptions{}) + }`, data.path), fs) require.NoError(t, err) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) @@ -482,16 +433,11 @@ func TestRunnerIntegrationImports(t *testing.T) { } func TestVURunContext(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` export let options = { vus: 10 }; export default function() { fn(); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r1.SetOptions(r1.GetOptions().Apply(lib.Options{Throw: null.BoolFrom(true)})) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) @@ -531,16 +477,13 @@ func TestVURunContext(t *testing.T) { func TestVURunInterrupt(t *testing.T) { //TODO: figure out why interrupt sometimes fails... data race in goja? - if runtime.GOOS == "windows" { + if isWindows { t.Skip() } - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` export default function() { while(true) {} } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) require.NoError(t, err) require.NoError(t, r1.SetOptions(lib.Options{Throw: null.BoolFrom(true)})) @@ -570,9 +513,7 @@ func TestVURunInterrupt(t *testing.T) { } func TestVUIntegrationGroups(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import { group } from "k6"; export default function() { fnOuter(); @@ -583,16 +524,11 @@ func TestVUIntegrationGroups(t *testing.T) { }) }); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) testdata := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range testdata { @@ -633,23 +569,16 @@ func TestVUIntegrationGroups(t *testing.T) { } func TestVUIntegrationMetrics(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import { group } from "k6"; import { Trend } from "k6/metrics"; let myMetric = new Trend("my_metric"); export default function() { myMetric.add(5); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) testdata := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range testdata { @@ -707,23 +636,15 @@ func TestVUIntegrationInsecureRequests(t *testing.T) { } for name, data := range testdata { t.Run(name, func(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { http.get("https://expired.badssl.com/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) r1.SetOptions(lib.Options{Throw: null.BoolFrom(true)}.Apply(data.opts)) r2, err := NewFromArchive(r1.MakeArchive(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } - + require.NoError(t, err) runners := map[string]*Runner{"Source": r1, "Archive": r2} for name, r := range runners { t.Run(name, func(t *testing.T) { @@ -746,18 +667,14 @@ func TestVUIntegrationInsecureRequests(t *testing.T) { } func TestVUIntegrationBlacklistOption(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { http.get("http://10.1.2.3/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) - if !assert.NoError(t, err) { - return - } + `) + require.NoError(t, err) cidr, err := lib.ParseCIDR("10.0.0.0/8") + if !assert.NoError(t, err) { return } @@ -785,9 +702,7 @@ func TestVUIntegrationBlacklistOption(t *testing.T) { } func TestVUIntegrationBlacklistScript(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export let options = { @@ -796,8 +711,7 @@ func TestVUIntegrationBlacklistScript(t *testing.T) { }; export default function() { http.get("http://10.1.2.3/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) if !assert.NoError(t, err) { return } @@ -826,9 +740,8 @@ func TestVUIntegrationHosts(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunner("/script.js", + tb.Replacer.Replace(` import { check, fail } from "k6"; import http from "k6/http"; export default function() { @@ -837,8 +750,7 @@ func TestVUIntegrationHosts(t *testing.T) { "is correct IP": (r) => r.remote_ip === "127.0.0.1" }) || fail("failed to override dns"); } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(t, err) { return } @@ -910,13 +822,10 @@ func TestVUIntegrationTLSConfig(t *testing.T) { } for name, data := range testdata { t.Run(name, func(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { http.get("https://sha256.badssl.com/"); } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) if !assert.NoError(t, err) { return } @@ -949,17 +858,14 @@ func TestVUIntegrationTLSConfig(t *testing.T) { } func TestVUIntegrationHTTP2(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` import http from "k6/http"; export default function() { let res = http.request("GET", "https://http2.akamai.com/demo"); if (res.status != 200) { throw new Error("wrong status: " + res.status) } if (res.proto != "HTTP/2.0") { throw new Error("wrong proto: " + res.proto) } } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) if !assert.NoError(t, err) { return } @@ -999,12 +905,9 @@ func TestVUIntegrationHTTP2(t *testing.T) { } func TestVUIntegrationOpenFunctionError(t *testing.T) { - r, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r, err := getSimpleRunner("/script.js", ` export default function() { open("/tmp/foo") } - `), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `) assert.NoError(t, err) vu, err := r.NewVU(make(chan stats.SampleContainer, 100)) @@ -1018,9 +921,7 @@ func TestVUIntegrationCookiesReset(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; export default function() { let url = "HTTPBIN_URL"; @@ -1036,8 +937,7 @@ func TestVUIntegrationCookiesReset(t *testing.T) { throw new Error("wrong cookies: " + res.body); } } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(t, err) { return } @@ -1071,9 +971,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; export default function() { let url = "HTTPBIN_URL"; @@ -1093,8 +991,7 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { } } } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if !assert.NoError(t, err) { return } @@ -1128,14 +1025,11 @@ func TestVUIntegrationCookiesNoReset(t *testing.T) { } func TestVUIntegrationVUID(t *testing.T) { - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(` + r1, err := getSimpleRunner("/script.js", ` export default function() { if (__VU != 1234) { throw new Error("wrong __VU: " + __VU); } }`, - ), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + ) if !assert.NoError(t, err) { return } @@ -1224,13 +1118,10 @@ func TestVUIntegrationClientCerts(t *testing.T) { } go func() { _ = srv.Serve(listener) }() - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(fmt.Sprintf(` + r1, err := getSimpleRunner("/script.js", fmt.Sprintf(` import http from "k6/http"; export default function() { http.get("https://%s")} - `, listener.Addr().String())), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `, listener.Addr().String())) if !assert.NoError(t, err) { return } @@ -1307,17 +1198,14 @@ func TestHTTPRequestInInitContext(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - _, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + _, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import { check, fail } from "k6"; import http from "k6/http"; let res = http.get("HTTPBIN_URL/"); export default function() { console.log(test); } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) if assert.Error(t, err) { assert.Equal( t, @@ -1392,10 +1280,7 @@ func TestInitContextForbidden(t *testing.T) { for _, test := range table { test := test t.Run(test[0], func(t *testing.T) { - _, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(test[1])), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + _, err := getSimpleRunner("/script.js", tb.Replacer.Replace(test[1])) if assert.Error(t, err) { assert.Equal( t, @@ -1412,9 +1297,7 @@ func TestArchiveRunningIntegraty(t *testing.T) { fs := afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm)) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunnerWithFileFs("/script.js", tb.Replacer.Replace(` let fput = open("/home/somebody/test.json"); export let options = { setupTimeout: "10s", teardownTimeout: "10s" }; export function setup() { @@ -1425,8 +1308,7 @@ func TestArchiveRunningIntegraty(t *testing.T) { throw new Error("incorrect answer " + data); } } - `)), - }, fs, lib.RuntimeOptions{}) + `), fs) require.NoError(t, err) buf := bytes.NewBuffer(nil) @@ -1457,14 +1339,11 @@ func TestArchiveNotPanicking(t *testing.T) { fs := afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/non/existent", []byte(`42`), os.ModePerm)) - r1, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + r1, err := getSimpleRunnerWithFileFs("/script.js", tb.Replacer.Replace(` let fput = open("/non/existent"); export default function(data) { } - `)), - }, fs, lib.RuntimeOptions{}) + `), fs) require.NoError(t, err) arc := r1.MakeArchive() @@ -1480,9 +1359,7 @@ func TestStuffNotPanicking(t *testing.T) { tb := testutils.NewHTTPMultiBin(t) defer tb.Cleanup() - r, err := New(&lib.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(tb.Replacer.Replace(` + r, err := getSimpleRunner("/script.js", tb.Replacer.Replace(` import http from "k6/http"; import ws from "k6/ws"; import { group } from "k6"; @@ -1520,8 +1397,7 @@ func TestStuffNotPanicking(t *testing.T) { } }); } - `)), - }, afero.NewMemMapFs(), lib.RuntimeOptions{}) + `)) require.NoError(t, err) ch := make(chan stats.SampleContainer, 1000) diff --git a/lib/archive.go b/lib/archive.go index dc96c46de41..a36a642330d 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -35,12 +35,13 @@ import ( "strings" "time" + "github.com/loadimpact/k6/lib/fsext" "github.com/spf13/afero" ) //nolint: gochecknoglobals, lll var ( - volumeRE = regexp.MustCompile(`^/?([a-zA-Z]):(.*)`) + volumeRE = regexp.MustCompile(`^[/\\]?([a-zA-Z]):(.*)`) sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js homeDirRE = regexp.MustCompile(`^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`) ) @@ -219,12 +220,12 @@ func (arc *Archive) Write(out io.Writer) error { return err } for _, name := range [...]string{"file", "https"} { - fs, ok := arc.FSes[name] + filesystem, ok := arc.FSes[name] if !ok { continue } - if cachedfs, ok := fs.(interface{ GetCachedFs() afero.Fs }); ok { - fs = cachedfs.GetCachedFs() + if cachedfs, ok := filesystem.(fsext.CacheOnReadFs); ok { + filesystem = cachedfs.GetCachingFs() } // A couple of things going on here: @@ -238,25 +239,28 @@ func (arc *Archive) Write(out io.Writer) error { paths := make([]string, 0, 10) infos := make(map[string]os.FileInfo) // ... fix this ? files := make(map[string][]byte) - err = afero.Walk(fs, "/", + + err = fsext.Walk(filesystem, afero.FilePathSeparator, filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error { if err != nil { return err } normalizedPath := NormalizeAndAnonymizePath(filePath) + infos[normalizedPath] = info if info.IsDir() { foundDirs[normalizedPath] = true return nil } - files[normalizedPath], err = afero.ReadFile(fs, filePath) + files[normalizedPath], err = afero.ReadFile(filesystem, filePath) if err != nil { return err } paths = append(paths, normalizedPath) return nil })) + if err != nil { return err } diff --git a/lib/archive_test.go b/lib/archive_test.go index 8f946162f10..56122dd5237 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -81,7 +81,8 @@ func getMapKeys(m map[string]afero.Fs) []string { } func diffMapFSes(t *testing.T, first, second map[string]afero.Fs) bool { - require.ElementsMatch(t, getMapKeys(first), getMapKeys(second), "fs map keys don't match") + require.ElementsMatch(t, getMapKeys(first), getMapKeys(second), + "fs map keys don't match %s, %s", getMapKeys(first), getMapKeys(second)) for key, fs := range first { secondFs := second[key] diffFSes(t, fs, secondFs) @@ -173,7 +174,7 @@ func TestArchiveReadWrite(t *testing.T) { Pwd, PwdNormAnon string }{ {"/home/myname", "/home/nobody"}, - {"/C:/Users/Administrator", "/C/Users/nobody"}, + {filepath.FromSlash("/C:/Users/Administrator"), "/C/Users/nobody"}, } for _, entry := range testdata { arc1 := &Archive{ @@ -223,7 +224,7 @@ func TestArchiveReadWrite(t *testing.T) { } buf := bytes.NewBuffer(nil) - assert.NoError(t, arc1.Write(buf)) + require.NoError(t, arc1.Write(buf)) arc1FSes := arc1Anon.FSes arc1Anon.FSes = nil diff --git a/lib/fsext/cacheonread.go b/lib/fsext/cacheonread.go new file mode 100644 index 00000000000..9a534f3f664 --- /dev/null +++ b/lib/fsext/cacheonread.go @@ -0,0 +1,27 @@ +package fsext + +import ( + "time" + + "github.com/spf13/afero" +) + +// CacheOnReadFs is wrapper around afero.CacheOnReadFs with the ability to return the filesystem +// that is used as cache +type CacheOnReadFs struct { + afero.Fs + cache afero.Fs +} + +// NewCacheOnReadFs returns a new CacheOnReadFs +func NewCacheOnReadFs(base, layer afero.Fs, cacheTime time.Duration) afero.Fs { + return CacheOnReadFs{ + Fs: afero.NewCacheOnReadFs(base, layer, cacheTime), + cache: layer, + } +} + +// GetCachingFs returns the afero.Fs being used for cache +func (c CacheOnReadFs) GetCachingFs() afero.Fs { + return c.cache +} diff --git a/lib/fsext/unprependfs.go b/lib/fsext/unprependfs.go new file mode 100644 index 00000000000..bfdab3e53c8 --- /dev/null +++ b/lib/fsext/unprependfs.go @@ -0,0 +1,172 @@ +package fsext + +import ( + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/afero" +) + +var _ afero.Lstater = (*UnprependPathFs)(nil) + +// UnprependPathFs is a filesystem that wraps another afero.Fs and unprepend a given path from all +// file and directory names before calling the same method on the wrapped afero.Fs. +// Heavily based on afero.BasePathFs +type UnprependPathFs struct { + source afero.Fs + path string +} + +// UnprependPathFile is a file from UnprependPathFs +type UnprependPathFile struct { + afero.File + path string +} + +// Name Returns the name of the file +func (f *UnprependPathFile) Name() string { + sourcename := f.File.Name() + return strings.TrimPrefix(sourcename, filepath.Clean(f.path)) +} + +// NewUnprependPathFs returns a new UnprependPathFs that will unprepend +func NewUnprependPathFs(source afero.Fs, path string) *UnprependPathFs { + return &UnprependPathFs{source: source, path: path} +} + +func (b *UnprependPathFs) realPath(name string) (path string, err error) { + if !strings.HasPrefix(name, b.path) { + return name, os.ErrNotExist + } + + return filepath.Clean(strings.TrimPrefix(name, b.path)), nil +} + +//Chtimes changes the access and modification times of the named file +func (b *UnprependPathFs) Chtimes(name string, atime, mtime time.Time) (err error) { + if name, err = b.realPath(name); err != nil { + return &os.PathError{Op: "chtimes", Path: name, Err: err} + } + return b.source.Chtimes(name, atime, mtime) +} + +// Chmod changes the mode of the named file to mode. +func (b *UnprependPathFs) Chmod(name string, mode os.FileMode) (err error) { + if name, err = b.realPath(name); err != nil { + return &os.PathError{Op: "chmod", Path: name, Err: err} + } + return b.source.Chmod(name, mode) +} + +// Name return the name of this FileSystem +func (b *UnprependPathFs) Name() string { + return "UnprependPathFs" +} + +// Stat returns a FileInfo describing the named file, or an error, if any +// happens. +func (b *UnprependPathFs) Stat(name string) (fi os.FileInfo, err error) { + if name, err = b.realPath(name); err != nil { + return nil, &os.PathError{Op: "stat", Path: name, Err: err} + } + return b.source.Stat(name) +} + +// Rename renames a file. +func (b *UnprependPathFs) Rename(oldname, newname string) (err error) { + if oldname, err = b.realPath(oldname); err != nil { + return &os.PathError{Op: "rename", Path: oldname, Err: err} + } + if newname, err = b.realPath(newname); err != nil { + return &os.PathError{Op: "rename", Path: newname, Err: err} + } + return b.source.Rename(oldname, newname) +} + +// RemoveAll removes a directory path and any children it contains. It +// does not fail if the path does not exist (return nil). +func (b *UnprependPathFs) RemoveAll(name string) (err error) { + if name, err = b.realPath(name); err != nil { + return &os.PathError{Op: "remove_all", Path: name, Err: err} + } + return b.source.RemoveAll(name) +} + +// Remove removes a file identified by name, returning an error, if any +// happens. +func (b *UnprependPathFs) Remove(name string) (err error) { + if name, err = b.realPath(name); err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + return b.source.Remove(name) +} + +// OpenFile opens a file using the given flags and the given mode. +func (b *UnprependPathFs) OpenFile(name string, flag int, mode os.FileMode) (f afero.File, err error) { + if name, err = b.realPath(name); err != nil { + return nil, &os.PathError{Op: "openfile", Path: name, Err: err} + } + sourcef, err := b.source.OpenFile(name, flag, mode) + if err != nil { + return nil, err + } + return &UnprependPathFile{sourcef, b.path}, nil +} + +// Open opens a file, returning it or an error, if any happens. +func (b *UnprependPathFs) Open(name string) (f afero.File, err error) { + if name, err = b.realPath(name); err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + sourcef, err := b.source.Open(name) + if err != nil { + return nil, err + } + return &UnprependPathFile{File: sourcef, path: b.path}, nil +} + +// Mkdir creates a directory in the filesystem, return an error if any +// happens. +func (b *UnprependPathFs) Mkdir(name string, mode os.FileMode) (err error) { + if name, err = b.realPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.Mkdir(name, mode) +} + +// MkdirAll creates a directory path and all parents that does not exist +// yet. +func (b *UnprependPathFs) MkdirAll(name string, mode os.FileMode) (err error) { + if name, err = b.realPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.MkdirAll(name, mode) +} + +// Create creates a file in the filesystem, returning the file and an +// error, if any happens +func (b *UnprependPathFs) Create(name string) (f afero.File, err error) { + if name, err = b.realPath(name); err != nil { + return nil, &os.PathError{Op: "create", Path: name, Err: err} + } + sourcef, err := b.source.Create(name) + if err != nil { + return nil, err + } + return &UnprependPathFile{File: sourcef, path: b.path}, nil +} + +// LstatIfPossible implements the afero.Lstater interface +func (b *UnprependPathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + name, err := b.realPath(name) + if err != nil { + return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} + } + if lstater, ok := b.source.(afero.Lstater); ok { + return lstater.LstatIfPossible(name) + } + fi, err := b.source.Stat(name) + return fi, false, err +} diff --git a/lib/fsext/walk.go b/lib/fsext/walk.go new file mode 100644 index 00000000000..2869f022c82 --- /dev/null +++ b/lib/fsext/walk.go @@ -0,0 +1,84 @@ +package fsext + +import ( + "os" + "path/filepath" + "sort" + + "github.com/spf13/afero" +) + +// Walk implements afero.Walk, but in a way that it doesn't loop to infinity and doesn't have +// problems if a given path part looks like a windows volume name +func Walk(fs afero.Fs, root string, walkFn filepath.WalkFunc) error { + info, err := fs.Stat(root) + if err != nil { + return walkFn(root, nil, err) + } + return walk(fs, root, info, walkFn) +} + +// readDirNames reads the directory named by dirname and returns +// a sorted list of directory entries. +// adapted from https://github.com/spf13/afero/blob/master/path.go#L27 +func readDirNames(fs afero.Fs, dirname string) ([]string, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + infos, err := f.Readdir(-1) + if err != nil { + return nil, err + } + err = f.Close() + + if err != nil { + return nil, err + } + + var names = make([]string, len(infos)) + for i, info := range infos { + names[i] = info.Name() + } + sort.Strings(names) + return names, nil +} + +// walk recursively descends path, calling walkFn +// adapted from https://github.com/spf13/afero/blob/master/path.go#L27 +func walk(fs afero.Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + + if !info.IsDir() { + return nil + } + + names, err := readDirNames(fs, path) + if err != nil { + return walkFn(path, info, err) + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := fs.Stat(filename) + if err != nil { + if err = walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(fs, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} diff --git a/loader/loader.go b/loader/loader.go index 69ec1e9ad63..b5d6db918bb 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -25,6 +25,7 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "regexp" "strings" @@ -116,7 +117,7 @@ func Load(fses map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpec "original moduleSpecifier": originalModuleSpecifier, }).Debug("Loading...") - pathOnFs := filepath.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):]) + pathOnFs := filepath.FromSlash(path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):])) data, err := afero.ReadFile(fses[moduleSpecifier.Scheme], pathOnFs) if err != nil { From 3843bc93c79b839f26fbb56fd06d9f97f822082b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 4 Jul 2019 15:11:46 +0300 Subject: [PATCH 04/45] fixup! Rewrite script/files loading to be be url based --- js/bundle_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/js/bundle_test.go b/js/bundle_test.go index 928bf98e3e2..bfe064c2fd3 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -32,7 +32,6 @@ import ( "testing" "time" - "github.com/davecgh/go-spew/spew" "github.com/loadimpact/k6/lib/fsext" "github.com/dop251/goja" @@ -516,7 +515,6 @@ func TestOpen(t *testing.T) { require.NoError(t, err) fs := afero.NewOsFs() filePath := filepath.Join(prefix, "/path/to/file.txt") - spew.Dump(filePath) require.NoError(t, fs.MkdirAll(filepath.Join(prefix, "/path/to"), 0755)) require.NoError(t, afero.WriteFile(fs, filePath, []byte(`hi`), 0644)) if isWindows { @@ -622,11 +620,11 @@ func TestBundleEnv(t *testing.T) { "TEST_B": "", }} data := ` -export default function() { - if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); } - if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); } -} -` + export default function() { + if (__ENV.TEST_A !== "1") { throw new Error("Invalid TEST_A: " + __ENV.TEST_A); } + if (__ENV.TEST_B !== "") { throw new Error("Invalid TEST_B: " + __ENV.TEST_B); } + } + ` b1, err := getSimpleBundleWithOptions("/script.js", data, rtOpts) if !assert.NoError(t, err) { return From 14e49b157f52966d190b03282295f9e49ca02d8a Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 4 Jul 2019 15:44:57 +0300 Subject: [PATCH 05/45] Rename FSes to Filesystems --- js/bundle.go | 18 +++++++++--------- js/bundle_test.go | 4 ++-- js/initcontext.go | 26 +++++++++++++------------- js/runner_test.go | 2 +- lib/archive.go | 10 +++++----- lib/archive_test.go | 38 +++++++++++++++++++------------------- loader/loader.go | 8 ++++---- loader/loader_test.go | 28 ++++++++++++++-------------- 8 files changed, 67 insertions(+), 67 deletions(-) diff --git a/js/bundle.go b/js/bundle.go index d38ab1701e0..f8f9dc97e84 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -151,7 +151,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, return nil, err } - initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.FSes, pwdURL) + initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.Filesystems, pwdURL) env := arc.Env if env == nil { @@ -178,14 +178,14 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, func (b *Bundle) makeArchive() *lib.Archive { arc := &lib.Archive{ - Type: "js", - FSes: b.BaseInitContext.fses, - Options: b.Options, - Filename: b.Filename.String(), - Data: []byte(b.Source), - Pwd: b.BaseInitContext.pwd.String(), - Env: make(map[string]string, len(b.Env)), - K6Version: consts.Version, + Type: "js", + Filesystems: b.BaseInitContext.filesystems, + Options: b.Options, + Filename: b.Filename.String(), + Data: []byte(b.Source), + Pwd: b.BaseInitContext.pwd.String(), + Env: make(map[string]string, len(b.Env)), + K6Version: consts.Version, } // Copy env so changes in the archive are not reflected in the source Bundle for k, v := range b.Env { diff --git a/js/bundle_test.go b/js/bundle_test.go index 09195b12d13..e15928f13b3 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -406,11 +406,11 @@ func TestNewBundleFromArchive(t *testing.T) { assert.Equal(t, data, string(arc.Data)) assert.Equal(t, "file:///path/to/", arc.Pwd) - exclaimData, err := afero.ReadFile(arc.FSes["file"], "/path/to/exclaim.js") + exclaimData, err := afero.ReadFile(arc.Filesystems["file"], "/path/to/exclaim.js") assert.NoError(t, err) assert.Equal(t, `export default function(s) { return s + "!" };`, string(exclaimData)) - fileData, err := afero.ReadFile(arc.FSes["file"], "/path/to/file.txt") + fileData, err := afero.ReadFile(arc.Filesystems["file"], "/path/to/file.txt") assert.NoError(t, err) assert.Equal(t, `hi`, string(fileData)) assert.Equal(t, consts.Version, arc.K6Version) diff --git a/js/initcontext.go b/js/initcontext.go index 9ef13dd3ada..f43c3b14cf6 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -51,8 +51,8 @@ type InitContext struct { ctxPtr *context.Context // Filesystem to load files and scripts from with the map key being the scheme - fses map[string]afero.Fs - pwd *url.URL + filesystems map[string]afero.Fs + pwd *url.URL // Cache of loaded programs and files. programs map[string]programWithSource @@ -60,14 +60,14 @@ type InitContext struct { // NewInitContext creates a new initcontext with the provided arguments func NewInitContext( - rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fses map[string]afero.Fs, pwd *url.URL, + rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, filesystems map[string]afero.Fs, pwd *url.URL, ) *InitContext { return &InitContext{ - runtime: rt, - compiler: compiler, - ctxPtr: ctxPtr, - fses: fses, - pwd: pwd, + runtime: rt, + compiler: compiler, + ctxPtr: ctxPtr, + filesystems: filesystems, + pwd: pwd, programs: make(map[string]programWithSource), } @@ -88,9 +88,9 @@ func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Ru runtime: rt, ctxPtr: ctxPtr, - fses: base.fses, - pwd: base.pwd, - compiler: base.compiler, + filesystems: base.filesystems, + pwd: base.pwd, + compiler: base.compiler, programs: programs, } @@ -151,7 +151,7 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) { i.runtime.Set("module", module) if pgm.pgm == nil { // Load the sources; the loader takes care of remote loading, etc. - data, err := loader.Load(i.fses, fileURL, name) + data, err := loader.Load(i.filesystems, fileURL, name) if err != nil { return goja.Undefined(), err } @@ -198,7 +198,7 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) filename = filepath.Join(i.pwd.Path, filename) } filename = filepath.Clean(filename) - fs := i.fses["file"] + fs := i.filesystems["file"] if filename[0:1] != afero.FilePathSeparator { filename = afero.FilePathSeparator + filename } diff --git a/js/runner_test.go b/js/runner_test.go index 3d252c55ab0..1053f93c6be 100644 --- a/js/runner_test.go +++ b/js/runner_test.go @@ -1347,7 +1347,7 @@ func TestArchiveNotPanicking(t *testing.T) { require.NoError(t, err) arc := r1.MakeArchive() - arc.FSes = map[string]afero.Fs{"file": afero.NewMemMapFs()} + arc.Filesystems = map[string]afero.Fs{"file": afero.NewMemMapFs()} r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) // we do want this to error here as this is where we find out that a given file is not in the // archive diff --git a/lib/archive.go b/lib/archive.go index 661f69a6caf..a6c835a6515 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -87,7 +87,7 @@ type Archive struct { // Working directory for resolving relative paths. Pwd string `json:"pwd"` - FSes map[string]afero.Fs `json:"-"` + Filesystems map[string]afero.Fs `json:"-"` // Environment variables Env map[string]string `json:"env"` @@ -96,13 +96,13 @@ type Archive struct { } func (arc *Archive) getFs(name string) afero.Fs { - fs, ok := arc.FSes[name] + fs, ok := arc.Filesystems[name] if !ok { fs = afero.NewMemMapFs() if name == "file" { fs = &normalizedFS{fs} } - arc.FSes[name] = fs + arc.Filesystems[name] = fs } return fs @@ -111,7 +111,7 @@ func (arc *Archive) getFs(name string) afero.Fs { // ReadArchive reads an archive created by Archive.Write from a reader. func ReadArchive(in io.Reader) (*Archive, error) { r := tar.NewReader(in) - arc := &Archive{FSes: make(map[string]afero.Fs, 2)} + arc := &Archive{Filesystems: make(map[string]afero.Fs, 2)} for { hdr, err := r.Next() if err != nil { @@ -222,7 +222,7 @@ func (arc *Archive) Write(out io.Writer) error { return err } for _, name := range [...]string{"file", "https"} { - filesystem, ok := arc.FSes[name] + filesystem, ok := arc.Filesystems[name] if !ok { continue } diff --git a/lib/archive_test.go b/lib/archive_test.go index 56122dd5237..09c89147067 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -80,19 +80,19 @@ func getMapKeys(m map[string]afero.Fs) []string { return keys } -func diffMapFSes(t *testing.T, first, second map[string]afero.Fs) bool { +func diffMapFilesystems(t *testing.T, first, second map[string]afero.Fs) bool { require.ElementsMatch(t, getMapKeys(first), getMapKeys(second), "fs map keys don't match %s, %s", getMapKeys(first), getMapKeys(second)) for key, fs := range first { secondFs := second[key] - diffFSes(t, fs, secondFs) + diffFilesystems(t, fs, secondFs) } return true } -func diffFSes(t *testing.T, first, second afero.Fs) { - diffFSesDir(t, first, second, "/") +func diffFilesystems(t *testing.T, first, second afero.Fs) { + diffFilesystemsDir(t, first, second, "/") } func getInfoNames(infos []os.FileInfo) []string { @@ -103,7 +103,7 @@ func getInfoNames(infos []os.FileInfo) []string { return names } -func diffFSesDir(t *testing.T, first, second afero.Fs, dirname string) { +func diffFilesystemsDir(t *testing.T, first, second afero.Fs, dirname string) { firstInfos, err := afero.ReadDir(first, dirname) require.NoError(t, err, dirname) @@ -114,7 +114,7 @@ func diffFSesDir(t *testing.T, first, second afero.Fs, dirname string) { for _, info := range firstInfos { path := filepath.Join(dirname, info.Name()) if info.IsDir() { - diffFSesDir(t, first, second, path) + diffFilesystemsDir(t, first, second, path) continue } firstData, err := afero.ReadFile(first, path) @@ -138,7 +138,7 @@ func TestArchiveReadWrite(t *testing.T) { Filename: "/path/to/script.js", Data: []byte(`// contents...`), Pwd: "/path/to", - FSes: map[string]afero.Fs{ + Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ "/path/to/a.js": []byte(`// a contents`), "/path/to/b.js": []byte(`// b contents`), @@ -155,18 +155,18 @@ func TestArchiveReadWrite(t *testing.T) { buf := bytes.NewBuffer(nil) require.NoError(t, arc1.Write(buf)) - arc1FSes := arc1.FSes - arc1.FSes = nil + arc1Filesystems := arc1.Filesystems + arc1.Filesystems = nil arc2, err := ReadArchive(buf) require.NoError(t, err) - arc2FSes := arc2.FSes - arc2.FSes = nil + arc2Filesystems := arc2.Filesystems + arc2.Filesystems = nil assert.Equal(t, arc1, arc2) - diffMapFSes(t, arc1FSes, arc2FSes) + diffMapFilesystems(t, arc1Filesystems, arc2Filesystems) }) t.Run("Anonymized", func(t *testing.T) { @@ -186,7 +186,7 @@ func TestArchiveReadWrite(t *testing.T) { Filename: fmt.Sprintf("%s/script.js", entry.Pwd), Data: []byte(`// contents...`), Pwd: entry.Pwd, - FSes: map[string]afero.Fs{ + Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`), fmt.Sprintf("%s/b.js", entry.Pwd): []byte(`// b contents`), @@ -209,7 +209,7 @@ func TestArchiveReadWrite(t *testing.T) { Data: []byte(`// contents...`), Pwd: entry.PwdNormAnon, - FSes: map[string]afero.Fs{ + Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ fmt.Sprintf("%s/a.js", entry.PwdNormAnon): []byte(`// a contents`), fmt.Sprintf("%s/b.js", entry.PwdNormAnon): []byte(`// b contents`), @@ -226,17 +226,17 @@ func TestArchiveReadWrite(t *testing.T) { buf := bytes.NewBuffer(nil) require.NoError(t, arc1.Write(buf)) - arc1FSes := arc1Anon.FSes - arc1Anon.FSes = nil + arc1Filesystems := arc1Anon.Filesystems + arc1Anon.Filesystems = nil arc2, err := ReadArchive(buf) assert.NoError(t, err) - arc2FSes := arc2.FSes - arc2.FSes = nil + arc2Filesystems := arc2.Filesystems + arc2.Filesystems = nil assert.Equal(t, arc1Anon, arc2) - diffMapFSes(t, arc1FSes, arc2FSes) + diffMapFilesystems(t, arc1Filesystems, arc2Filesystems) } }) } diff --git a/loader/loader.go b/loader/loader.go index b5d6db918bb..6e80aa63694 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -107,10 +107,10 @@ func Dir(old *url.URL) *url.URL { return old.ResolveReference(&url.URL{Path: "./"}) } -// Load loads the provided moduleSpecifier from the given fses which are map of fses for a given scheme which +// Load loads the provided moduleSpecifier from the given filesystems which are map of filesystems for a given scheme which // is they key of the map. If the scheme is https then a request will be made if the files is not // found in the map and written to the map. -func Load(fses map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string) (*lib.SourceData, error) { +func Load(filesystems map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string) (*lib.SourceData, error) { log.WithFields( log.Fields{ "moduleSpecifier": moduleSpecifier, @@ -118,7 +118,7 @@ func Load(fses map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpec }).Debug("Loading...") pathOnFs := filepath.FromSlash(path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):])) - data, err := afero.ReadFile(fses[moduleSpecifier.Scheme], pathOnFs) + data, err := afero.ReadFile(filesystems[moduleSpecifier.Scheme], pathOnFs) if err != nil { if os.IsNotExist(err) { @@ -130,7 +130,7 @@ func Load(fses map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpec } // TODO maybe make an afero.Fs which makes request directly and than use CacheOnReadFs // on top of as with the `file` scheme fs - _ = afero.WriteFile(fses[moduleSpecifier.Scheme], pathOnFs, result.Data, 0644) + _ = afero.WriteFile(filesystems[moduleSpecifier.Scheme], pathOnFs, result.Data, 0644) return result, nil } return nil, errors.Errorf(fileSchemeCouldntBeLoadedMsg, moduleSpecifier) diff --git a/loader/loader_test.go b/loader/loader_test.go index fd5438df45a..2147907981a 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -101,10 +101,10 @@ func TestLoad(t *testing.T) { }() t.Run("Local", func(t *testing.T) { - fses := make(map[string]afero.Fs) - fses["file"] = afero.NewMemMapFs() - assert.NoError(t, fses["file"].MkdirAll("/path/to", 0755)) - assert.NoError(t, afero.WriteFile(fses["file"], "/path/to/file.txt", []byte("hi"), 0644)) + filesystems := make(map[string]afero.Fs) + filesystems["file"] = afero.NewMemMapFs() + assert.NoError(t, filesystems["file"].MkdirAll("/path/to", 0755)) + assert.NoError(t, afero.WriteFile(filesystems["file"], "/path/to/file.txt", []byte("hi"), 0644)) testdata := map[string]struct{ pwd, path string }{ "Absolute": {"/path/", "/path/to/file.txt"}, @@ -120,7 +120,7 @@ func TestLoad(t *testing.T) { moduleURL, err := Resolve(pwdURL, data.path) require.NoError(t, err) - src, err := Load(fses, moduleURL, data.path) + src, err := Load(filesystems, moduleURL, data.path) require.NoError(t, err) assert.Equal(t, "file:///path/to/file.txt", src.URL.String()) @@ -136,14 +136,14 @@ func TestLoad(t *testing.T) { pathURL, err := Resolve(root, "/nonexistent") require.NoError(t, err) - _, err = Load(fses, pathURL, path) + _, err = Load(filesystems, pathURL, path) assert.EqualError(t, err, fmt.Sprintf(fileSchemeCouldntBeLoadedMsg, "file://"+filepath.ToSlash(path))) }) }) t.Run("Remote", func(t *testing.T) { - fses := map[string]afero.Fs{"https": afero.NewMemMapFs()} + filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()} t.Run("From local", func(t *testing.T) { root, err := url.Parse("file:///") require.NoError(t, err) @@ -152,7 +152,7 @@ func TestLoad(t *testing.T) { moduleSpecifierURL, err := Resolve(root, moduleSpecifier) require.NoError(t, err) - src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, src.URL, moduleSpecifierURL) assert.Contains(t, string(src.Data), "Herman Melville - Moby-Dick") @@ -166,7 +166,7 @@ func TestLoad(t *testing.T) { moduleSpecifierURL, err := Resolve(pwdURL, moduleSpecifier) require.NoError(t, err) - src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/robots.txt")) assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") @@ -180,7 +180,7 @@ func TestLoad(t *testing.T) { moduleSpecifierURL, err := Resolve(pwdURL, moduleSpecifier) require.NoError(t, err) - src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, sr("HTTPSBIN_URL/robots.txt"), src.URL.String()) assert.Equal(t, "User-agent: *\nDisallow: /deny\n", string(src.Data)) @@ -205,8 +205,8 @@ func TestLoad(t *testing.T) { moduleSpecifierURL, err := Resolve(root, moduleSpecifier) require.NoError(t, err) - fses := map[string]afero.Fs{"https": afero.NewMemMapFs()} - src, err := Load(fses, moduleSpecifierURL, moduleSpecifier) + filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()} + src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/raw/something")) @@ -234,14 +234,14 @@ func TestLoad(t *testing.T) { {"HOST", "some-path-that-doesnt-exist.js"}, } - fses := map[string]afero.Fs{"https": afero.NewMemMapFs()} + filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()} for _, data := range testData { moduleSpecifier := data.moduleSpecifier t.Run(data.name, func(t *testing.T) { moduleSpecifierURL, err := Resolve(root, moduleSpecifier) require.NoError(t, err) - _, err = Load(fses, moduleSpecifierURL, moduleSpecifier) + _, err = Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.Error(t, err) }) } From 2222235b8ee26a676557b2060efdb479252af18b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 4 Jul 2019 15:50:29 +0300 Subject: [PATCH 06/45] Fix lll error --- loader/loader.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 6e80aa63694..2cd673e64ff 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -107,10 +107,12 @@ func Dir(old *url.URL) *url.URL { return old.ResolveReference(&url.URL{Path: "./"}) } -// Load loads the provided moduleSpecifier from the given filesystems which are map of filesystems for a given scheme which -// is they key of the map. If the scheme is https then a request will be made if the files is not -// found in the map and written to the map. -func Load(filesystems map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string) (*lib.SourceData, error) { +// Load loads the provided moduleSpecifier from the given filesystems which are map of afero.Fs +// for a given scheme which is they key of the map. If the scheme is https then a request will +// be made if the files is not found in the map and written to the map. +func Load( + filesystems map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string, +) (*lib.SourceData, error) { log.WithFields( log.Fields{ "moduleSpecifier": moduleSpecifier, From 53f10f5ecd0b871e9f9917030bfa36c80dc4e67c Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 4 Jul 2019 18:00:48 +0300 Subject: [PATCH 07/45] Add tests for old archive support --- lib/archive.go | 10 ++-- lib/old_archive_test.go | 115 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 lib/old_archive_test.go diff --git a/lib/archive.go b/lib/archive.go index a6c835a6515..faaa8438c17 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -120,7 +120,7 @@ func ReadArchive(in io.Reader) (*Archive, error) { } return nil, err } - if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA && hdr.Typeflag != tar.TypeDir { + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { continue } @@ -131,7 +131,7 @@ func ReadArchive(in io.Reader) (*Archive, error) { switch hdr.Name { case "metadata.json": - if err := json.Unmarshal(data, &arc); err != nil { + if err = json.Unmarshal(data, &arc); err != nil { return nil, err } // Path separator normalization for older archives (<=0.20.0) @@ -164,11 +164,7 @@ func ReadArchive(in io.Reader) (*Archive, error) { case "https", "file": fs := arc.getFs(pfx) name = filepath.FromSlash(name) - if hdr.Typeflag == tar.TypeDir { - err = fs.Mkdir(name, os.FileMode(hdr.Mode)) - } else { - err = afero.WriteFile(fs, name, data, os.FileMode(hdr.Mode)) - } + err = afero.WriteFile(fs, name, data, os.FileMode(hdr.Mode)) if err != nil { return nil, err } diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go new file mode 100644 index 00000000000..a8b6a0d039f --- /dev/null +++ b/lib/old_archive_test.go @@ -0,0 +1,115 @@ +package lib + +import ( + "archive/tar" + "bytes" + "os" + "path" + "path/filepath" + "testing" + + "github.com/loadimpact/k6/lib/fsext" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func dumpMakeMapFsToBuf(fs afero.Fs) (*bytes.Buffer, error) { + var b = bytes.NewBuffer(nil) + var w = tar.NewWriter(b) + err := fsext.Walk(fs, afero.FilePathSeparator, + filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error { + if filePath == afero.FilePathSeparator { + return nil // skip the root + } + if err != nil { + return err + } + if info.IsDir() { + return w.WriteHeader(&tar.Header{ + Name: path.Clean(filepath.ToSlash(filePath)[1:]), + Mode: 0555, + Typeflag: tar.TypeDir, + }) + } + var data []byte + data, err = afero.ReadFile(fs, filePath) + if err != nil { + return err + } + err = w.WriteHeader(&tar.Header{ + Name: path.Clean(filepath.ToSlash(filePath)[1:]), + Mode: 0644, + Size: int64(len(data)), + Typeflag: tar.TypeReg, + }) + if err != nil { + return err + } + _, err = w.Write(data) + if err != nil { + return err + } + return nil + })) + if err != nil { + return nil, err + } + return b, w.Close() +} + +func TestOldArchive(t *testing.T) { + fs := makeMemMapFs(t, map[string][]byte{ + // files + "/files/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), + "/files/cdnjs.com/packages/Faker": []byte(`faker file`), + "/files/example.com/path/to.js": []byte(`example.com file`), + "/files/_/C/something/path": []byte(`windows file`), + "/files/_/absolulte/path": []byte(`unix file`), + + // scripts + "/scripts/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), + "/scripts/cdnjs.com/packages/Faker2": []byte(`faker script`), + "/scripts/example.com/path/too.js": []byte(`example.com script`), + "/scripts/_/C/something/path2": []byte(`windows script`), + "/scripts/_/absolulte/path2": []byte(`unix script`), + }) + buf, err := dumpMakeMapFsToBuf(fs) + require.NoError(t, err) + + var ( + expectedFilesystems = map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + "/C:/something/path": []byte(`windows file`), + "/absolulte/path": []byte(`unix file`), + "/C:/something/path2": []byte(`windows script`), + "/absolulte/path2": []byte(`unix script`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/example.com/path/to.js": []byte(`example.com file`), + "/example.com/path/too.js": []byte(`example.com script`), + "/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), + "/cdnjs.com/packages/Faker": []byte(`faker file`), + "/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), + "/cdnjs.com/packages/Faker2": []byte(`faker script`), + }), + } + ) + + arc, err := ReadArchive(buf) + require.NoError(t, err) + + diffMapFilesystems(t, expectedFilesystems, arc.Filesystems) +} + +func TestUnknownPrefix(t *testing.T) { + fs := makeMemMapFs(t, map[string][]byte{ + "/strange/something": []byte(`github file`), + }) + buf, err := dumpMakeMapFsToBuf(fs) + require.NoError(t, err) + + _, err = ReadArchive(buf) + require.Error(t, err) + require.Equal(t, err.Error(), + "unknown file prefix `strange` for file `strange/something`") +} From ef49dfdcd73bbee8d4fa84bf92dd283fdccffe75 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 08:34:15 +0300 Subject: [PATCH 08/45] Fix saving and loading archive filename and pwd --- api/v1/setup_teardown_routes_test.go | 3 +- cmd/collectors.go | 3 +- cmd/run.go | 14 ++++---- cmd/runtime_options_test.go | 5 +-- converter/har/converter_test.go | 3 +- core/engine_test.go | 11 +++--- core/local/local_test.go | 3 +- js/bundle.go | 21 ++++-------- js/bundle_test.go | 9 ++--- js/console_test.go | 5 +-- js/modules/k6/marshalling_test.go | 3 +- js/runner.go | 3 +- lib/archive.go | 49 +++++++++++++++++++++++---- lib/archive_test.go | 29 ++++++++++------ lib/models.go | 7 ---- loader/loader.go | 17 ++++++---- loader/loader_test.go | 50 +++++++++++++++------------- stats/cloud/collector.go | 3 +- stats/cloud/collector_test.go | 5 +-- 19 files changed, 147 insertions(+), 96 deletions(-) diff --git a/api/v1/setup_teardown_routes_test.go b/api/v1/setup_teardown_routes_test.go index 81a6db30d2c..14476da7122 100644 --- a/api/v1/setup_teardown_routes_test.go +++ b/api/v1/setup_teardown_routes_test.go @@ -35,6 +35,7 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/manyminds/api2go/jsonapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -133,7 +134,7 @@ func TestSetupData(t *testing.T) { testCase := testCase t.Run(testCase.name, func(t *testing.T) { runner, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script}, + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script}, nil, lib.RuntimeOptions{}, ) diff --git a/cmd/collectors.go b/cmd/collectors.go index 93979b9bb64..f92dc7dcb6f 100644 --- a/cmd/collectors.go +++ b/cmd/collectors.go @@ -29,6 +29,7 @@ import ( "github.com/kelseyhightower/envconfig" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/consts" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats/cloud" "github.com/loadimpact/k6/stats/datadog" "github.com/loadimpact/k6/stats/influxdb" @@ -61,7 +62,7 @@ func parseCollector(s string) (t, arg string) { } } -func newCollector(collectorName, arg string, src *lib.SourceData, conf Config) (lib.Collector, error) { +func newCollector(collectorName, arg string, src *loader.SourceData, conf Config) (lib.Collector, error) { getCollector := func() (lib.Collector, error) { switch collectorName { case collectorJSON: diff --git a/cmd/run.go b/cmd/run.go index 1b077eefd00..25126bf821e 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -510,22 +510,24 @@ func createFilesystems() map[string]afero.Fs { // written every time something is read from the real filesystem. This cache is then used for // successive spawns to read from (they have no access to the real disk). // Also initialize the same for `https` but the caching is handled manually in the loader package + osfs := afero.NewOsFs() + if runtime.GOOS == "windows" { + osfs = fsext.NewUnprependPathFs(osfs, afero.FilePathSeparator) + } return map[string]afero.Fs{ - "file": fsext.NewCacheOnReadFs( - fsext.NewUnprependPathFs(afero.NewOsFs(), afero.FilePathSeparator), - afero.NewMemMapFs(), 0), + "file": fsext.NewCacheOnReadFs(osfs, afero.NewMemMapFs(), 0), "https": afero.NewMemMapFs(), } } // Reads a source file from any supported destination. -func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*lib.SourceData, error) { +func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*loader.SourceData, error) { if src == "-" { data, err := ioutil.ReadAll(stdin) if err != nil { return nil, err } - return &lib.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil + return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil } pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} srcURL, err := loader.Resolve(pwdURL, filepath.ToSlash(src)) @@ -537,7 +539,7 @@ func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reade // Creates a new runner. func newRunner( - src *lib.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions, + src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions, ) (lib.Runner, error) { switch typ { case "": diff --git a/cmd/runtime_options_test.go b/cmd/runtime_options_test.go index 25dcc7ed9bf..183031b9c8c 100644 --- a/cmd/runtime_options_test.go +++ b/cmd/runtime_options_test.go @@ -30,6 +30,7 @@ import ( "testing" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/loader" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -219,7 +220,7 @@ func TestEnvVars(t *testing.T) { } runner, err := newRunner( - &lib.SourceData{ + &loader.SourceData{ Data: []byte(jsCode), URL: &url.URL{Path: "/script.js"}, }, @@ -235,7 +236,7 @@ func TestEnvVars(t *testing.T) { getRunnerErr := func(rtOpts lib.RuntimeOptions) (lib.Runner, error) { return newRunner( - &lib.SourceData{ + &loader.SourceData{ Data: archiveBuf.Bytes(), URL: &url.URL{Path: "/script.js"}, }, diff --git a/converter/har/converter_test.go b/converter/har/converter_test.go index d427dde6b0d..fdf92cce8cc 100644 --- a/converter/har/converter_test.go +++ b/converter/har/converter_test.go @@ -27,6 +27,7 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/loader" "github.com/stretchr/testify/assert" ) @@ -55,7 +56,7 @@ func TestBuildK6RequestObject(t *testing.T) { } v, err := buildK6RequestObject(req) assert.NoError(t, err) - _, err = js.New(&lib.SourceData{ + _, err = js.New(&loader.SourceData{ URL: &url.URL{Path: "/script.js"}, Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)), }, nil, lib.RuntimeOptions{}) diff --git a/core/engine_test.go b/core/engine_test.go index f4e084d004c..784915594ad 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -33,6 +33,7 @@ import ( "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" "github.com/loadimpact/k6/stats/dummy" log "github.com/sirupsen/logrus" @@ -556,7 +557,7 @@ func TestSentReceivedMetrics(t *testing.T) { runTest := func(t *testing.T, ts testScript, tc testCase, noConnReuse bool) (float64, float64) { r, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)}, + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(ts.Code)}, nil, lib.RuntimeOptions{}, ) @@ -697,7 +698,7 @@ func TestRunTags(t *testing.T) { `)) r, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, nil, lib.RuntimeOptions{}, ) @@ -797,7 +798,7 @@ func TestSetupTeardownThresholds(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, nil, lib.RuntimeOptions{}, ) @@ -860,7 +861,7 @@ func TestEmittedMetricsWhenScalingDown(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, nil, lib.RuntimeOptions{}, ) @@ -920,7 +921,7 @@ func TestMinIterationDuration(t *testing.T) { t.Parallel() runner, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(` + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: []byte(` import { Counter } from "k6/metrics"; let testCounter = new Counter("testcounter"); diff --git a/core/local/local_test.go b/core/local/local_test.go index 787bf05eef5..4af9be82b0c 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -30,6 +30,7 @@ import ( "time" "github.com/loadimpact/k6/lib/netext" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" @@ -481,7 +482,7 @@ func TestRealTimeAndSetupTeardownMetrics(t *testing.T) { }`) runner, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, nil, lib.RuntimeOptions{}, ) diff --git a/js/bundle.go b/js/bundle.go index f8f9dc97e84..be76417c64a 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -58,7 +58,7 @@ type BundleInstance struct { } // NewBundle creates a new bundle from a source file and a filesystem. -func NewBundle(src *lib.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { +func NewBundle(src *loader.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Bundle, error) { compiler, err := compiler.New() if err != nil { return nil, err @@ -137,21 +137,12 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, return nil, errors.Errorf("expected bundle type 'js', got '%s'", arc.Type) } - pgm, _, err := compiler.Compile(string(arc.Data), arc.Filename, "", "", true) - if err != nil { - return nil, err - } - pwdURL, err := loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd) - if err != nil { - return nil, err - } - - filenameURL, err := loader.Resolve(pwdURL, arc.Filename) + pgm, _, err := compiler.Compile(string(arc.Data), arc.FilenameURL.String(), "", "", true) if err != nil { return nil, err } - initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.Filesystems, pwdURL) + initctx := NewInitContext(goja.New(), compiler, new(context.Context), arc.Filesystems, arc.PwdURL) env := arc.Env if env == nil { @@ -163,7 +154,7 @@ func NewBundleFromArchive(arc *lib.Archive, rtOpts lib.RuntimeOptions) (*Bundle, } bundle := &Bundle{ - Filename: filenameURL, + Filename: arc.FilenameURL, Source: string(arc.Data), Program: pgm, Options: arc.Options, @@ -181,9 +172,9 @@ func (b *Bundle) makeArchive() *lib.Archive { Type: "js", Filesystems: b.BaseInitContext.filesystems, Options: b.Options, - Filename: b.Filename.String(), + FilenameURL: b.Filename, Data: []byte(b.Source), - Pwd: b.BaseInitContext.pwd.String(), + PwdURL: b.BaseInitContext.pwd, Env: make(map[string]string, len(b.Env)), K6Version: consts.Version, } diff --git a/js/bundle_test.go b/js/bundle_test.go index e15928f13b3..2635ea35da3 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -37,6 +37,7 @@ import ( "github.com/loadimpact/k6/lib/consts" "github.com/loadimpact/k6/lib/fsext" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -51,7 +52,7 @@ func getSimpleBundle(filename, data string) (*Bundle, error) { func getSimpleBundleWithOptions(filename, data string, options lib.RuntimeOptions) (*Bundle, error) { return NewBundle( - &lib.SourceData{ + &loader.SourceData{ URL: &url.URL{Path: filename, Scheme: "file"}, Data: []byte(data), }, @@ -62,7 +63,7 @@ func getSimpleBundleWithOptions(filename, data string, options lib.RuntimeOption func getSimpleBundleWithFs(filename, data string, fs afero.Fs) (*Bundle, error) { return NewBundle( - &lib.SourceData{ + &loader.SourceData{ URL: &url.URL{Path: filename, Scheme: "file"}, Data: []byte(data), }, @@ -402,9 +403,9 @@ func TestNewBundleFromArchive(t *testing.T) { arc := b.makeArchive() assert.Equal(t, "js", arc.Type) assert.Equal(t, lib.Options{VUs: null.IntFrom(12345)}, arc.Options) - assert.Equal(t, "file:///path/to/script.js", arc.Filename) + assert.Equal(t, "file:///path/to/script.js", arc.FilenameURL.String()) assert.Equal(t, data, string(arc.Data)) - assert.Equal(t, "file:///path/to/", arc.Pwd) + assert.Equal(t, "file:///path/to/", arc.PwdURL.String()) exclaimData, err := afero.ReadFile(arc.Filesystems["file"], "/path/to/exclaim.js") assert.NoError(t, err) diff --git a/js/console_test.go b/js/console_test.go index fc8750e77d4..8b0448d5fd1 100644 --- a/js/console_test.go +++ b/js/console_test.go @@ -33,6 +33,7 @@ import ( "github.com/dop251/goja" "github.com/loadimpact/k6/js/common" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" log "github.com/sirupsen/logrus" logtest "github.com/sirupsen/logrus/hooks/test" @@ -74,7 +75,7 @@ func getSimpleRunner(path, data string) (*Runner, error) { } func getSimpleRunnerWithOptions(path, data string, options lib.RuntimeOptions) (*Runner, error) { - return New(&lib.SourceData{ + return New(&loader.SourceData{ URL: &url.URL{Path: path, Scheme: "file"}, Data: []byte(data), }, map[string]afero.Fs{ @@ -84,7 +85,7 @@ func getSimpleRunnerWithOptions(path, data string, options lib.RuntimeOptions) ( } func getSimpleRunnerWithFileFs(path, data string, fileFs afero.Fs) (*Runner, error) { - return New(&lib.SourceData{ + return New(&loader.SourceData{ URL: &url.URL{Path: path, Scheme: "file"}, Data: []byte(data), }, map[string]afero.Fs{ diff --git a/js/modules/k6/marshalling_test.go b/js/modules/k6/marshalling_test.go index d24baf07661..8c4ca3ae3a8 100644 --- a/js/modules/k6/marshalling_test.go +++ b/js/modules/k6/marshalling_test.go @@ -30,6 +30,7 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -114,7 +115,7 @@ func TestSetupDataMarshalling(t *testing.T) { `)) runner, err := js.New( - &lib.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, + &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script}, nil, lib.RuntimeOptions{}, ) diff --git a/js/runner.go b/js/runner.go index c43ff6a96db..4d3f97635bc 100644 --- a/js/runner.go +++ b/js/runner.go @@ -34,6 +34,7 @@ import ( "github.com/loadimpact/k6/js/common" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/netext" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" "github.com/oxtoacart/bpool" "github.com/pkg/errors" @@ -63,7 +64,7 @@ type Runner struct { } // New returns a new Runner for the provide source -func New(src *lib.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) { +func New(src *loader.SourceData, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions) (*Runner, error) { bundle, err := NewBundle(src, filesystems, rtOpts) if err != nil { return nil, err diff --git a/lib/archive.go b/lib/archive.go index faaa8438c17..7d89a82c522 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "io/ioutil" + "net/url" "os" "path" "path/filepath" @@ -36,6 +37,7 @@ import ( "time" "github.com/loadimpact/k6/lib/fsext" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" ) @@ -80,12 +82,16 @@ type Archive struct { // Options to use. Options Options `json:"options"` + // TODO: rewrite the encoding, decoding of json to use another type with only the fields it + // needs in order to remove Filename and Pwd from this // Filename and contents of the main file being executed. - Filename string `json:"filename"` - Data []byte `json:"-"` + Filename string `json:"filename"` // only for json + FilenameURL *url.URL `json:"-"` + Data []byte `json:"-"` // Working directory for resolving relative paths. - Pwd string `json:"pwd"` + Pwd string `json:"pwd"` // only for json + PwdURL *url.URL `json:"-"` Filesystems map[string]afero.Fs `json:"-"` @@ -135,8 +141,29 @@ func ReadArchive(in io.Reader) (*Archive, error) { return nil, err } // Path separator normalization for older archives (<=0.20.0) - arc.Filename = NormalizeAndAnonymizePath(arc.Filename) - arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd) + if arc.K6Version == "" { + arc.Filename = NormalizeAndAnonymizePath(arc.Filename) + arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd) + arc.PwdURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd) + if err != nil { + return nil, err + } + arc.FilenameURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Filename) + if err != nil { + return nil, err + } + } else { + arc.FilenameURL, err = url.Parse(arc.Filename) + if err != nil { + return nil, err + } + + arc.PwdURL, err = url.Parse(arc.Pwd) + if err != nil { + return nil, err + } + } + continue case "data": arc.Data = data @@ -181,6 +208,12 @@ func ReadArchive(in io.Reader) (*Archive, error) { return arc, nil } +func normalizeAndAnonymizeURL(u *url.URL) { + if u.Scheme == "file" { + u.Path = NormalizeAndAnonymizePath(u.Path) + } +} + // Write serialises the archive to a writer. // // The format should be treated as opaque; currently it is simply a TAR rollup, but this may @@ -190,8 +223,10 @@ func (arc *Archive) Write(out io.Writer) error { w := tar.NewWriter(out) metaArc := *arc - metaArc.Filename = NormalizeAndAnonymizePath(metaArc.Filename) - metaArc.Pwd = NormalizeAndAnonymizePath(metaArc.Pwd) + normalizeAndAnonymizeURL(metaArc.FilenameURL) + normalizeAndAnonymizeURL(metaArc.PwdURL) + metaArc.Filename = metaArc.FilenameURL.String() + metaArc.Pwd = metaArc.PwdURL.String() metadata, err := metaArc.json() if err != nil { return err diff --git a/lib/archive_test.go b/lib/archive_test.go index 09c89147067..1d4deb1bf45 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -23,11 +23,13 @@ package lib import ( "bytes" "fmt" + "net/url" "os" "path/filepath" "runtime" "testing" + "github.com/loadimpact/k6/lib/consts" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -130,14 +132,15 @@ func diffFilesystemsDir(t *testing.T, first, second afero.Fs, dirname string) { func TestArchiveReadWrite(t *testing.T) { t.Run("Roundtrip", func(t *testing.T) { arc1 := &Archive{ - Type: "js", + Type: "js", + K6Version: consts.Version, Options: Options{ VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - Filename: "/path/to/script.js", - Data: []byte(`// contents...`), - Pwd: "/path/to", + FilenameURL: &url.URL{Scheme: "file", Path: "/path/to/script.js"}, + Data: []byte(`// contents...`), + PwdURL: &url.URL{Scheme: "file", Path: "/path/to"}, Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ "/path/to/a.js": []byte(`// a contents`), @@ -163,6 +166,8 @@ func TestArchiveReadWrite(t *testing.T) { arc2Filesystems := arc2.Filesystems arc2.Filesystems = nil + arc2.Filename = "" + arc2.Pwd = "" assert.Equal(t, arc1, arc2) @@ -183,9 +188,10 @@ func TestArchiveReadWrite(t *testing.T) { VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - Filename: fmt.Sprintf("%s/script.js", entry.Pwd), - Data: []byte(`// contents...`), - Pwd: entry.Pwd, + FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/script.js", entry.Pwd)}, + K6Version: consts.Version, + Data: []byte(`// contents...`), + PwdURL: &url.URL{Scheme: "file", Path: entry.Pwd}, Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ fmt.Sprintf("%s/a.js", entry.Pwd): []byte(`// a contents`), @@ -205,9 +211,10 @@ func TestArchiveReadWrite(t *testing.T) { VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - Filename: fmt.Sprintf("%s/script.js", entry.PwdNormAnon), - Data: []byte(`// contents...`), - Pwd: entry.PwdNormAnon, + FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/script.js", entry.PwdNormAnon)}, + K6Version: consts.Version, + Data: []byte(`// contents...`), + PwdURL: &url.URL{Scheme: "file", Path: entry.PwdNormAnon}, Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ @@ -231,6 +238,8 @@ func TestArchiveReadWrite(t *testing.T) { arc2, err := ReadArchive(buf) assert.NoError(t, err) + arc2.Filename = "" + arc2.Pwd = "" arc2Filesystems := arc2.Filesystems arc2.Filesystems = nil diff --git a/lib/models.go b/lib/models.go index 567a24a08fc..3944eeecba6 100644 --- a/lib/models.go +++ b/lib/models.go @@ -24,7 +24,6 @@ import ( "crypto/md5" "encoding/hex" "encoding/json" - "net/url" "strconv" "strings" "sync" @@ -41,12 +40,6 @@ const GroupSeparator = "::" // Error emitted if you attempt to instantiate a Group or Check that contains the separator. var ErrNameContainsGroupSeparator = errors.New("group and check names may not contain '::'") -// SourceData wraps a source file; data and filename. -type SourceData struct { - Data []byte - URL *url.URL -} - // StageFields defines the fields used for a Stage; this is a dumb hack to make the JSON code // cleaner. pls fix. type StageFields struct { diff --git a/loader/loader.go b/loader/loader.go index 2cd673e64ff..cc19d5bd419 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -31,12 +31,17 @@ import ( "strings" "time" - "github.com/loadimpact/k6/lib" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/afero" ) +// SourceData wraps a source file; data and filename. +type SourceData struct { + Data []byte + URL *url.URL +} + type loaderFunc func(path string, parts []string) (string, error) //nolint: gochecknoglobals @@ -112,7 +117,7 @@ func Dir(old *url.URL) *url.URL { // be made if the files is not found in the map and written to the map. func Load( filesystems map[string]afero.Fs, moduleSpecifier *url.URL, originalModuleSpecifier string, -) (*lib.SourceData, error) { +) (*SourceData, error) { log.WithFields( log.Fields{ "moduleSpecifier": moduleSpecifier, @@ -125,7 +130,7 @@ func Load( if err != nil { if os.IsNotExist(err) { if moduleSpecifier.Scheme == "https" { - var result *lib.SourceData + var result *SourceData result, err = loadRemoteURL(moduleSpecifier) if err != nil { return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, moduleSpecifier, err) @@ -140,7 +145,7 @@ func Load( return nil, err } - return &lib.SourceData{URL: moduleSpecifier, Data: data}, nil + return &SourceData{URL: moduleSpecifier, Data: data}, nil } func resolveUsingLoaders(name string) (string, error) { @@ -152,7 +157,7 @@ func resolveUsingLoaders(name string) (string, error) { return "", errNoLoaderMatched } -func loadRemoteURL(u *url.URL) (*lib.SourceData, error) { +func loadRemoteURL(u *url.URL) (*SourceData, error) { var oldQuery = u.RawQuery if u.RawQuery != "" { u.RawQuery += "&" @@ -174,7 +179,7 @@ func loadRemoteURL(u *url.URL) (*lib.SourceData, error) { // // - return &lib.SourceData{URL: u, Data: data}, nil + return &SourceData{URL: u, Data: data}, nil } func pickLoader(path string) (string, loaderFunc, []string) { diff --git a/loader/loader_test.go b/loader/loader_test.go index 2147907981a..94aaeb76674 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -18,7 +18,7 @@ * */ -package loader +package loader_test import ( "fmt" @@ -28,6 +28,7 @@ import ( "testing" "github.com/loadimpact/k6/lib/testutils" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -42,7 +43,7 @@ func TestDir(t *testing.T) { nameURL := &url.URL{Scheme: "file", Path: name} dirURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(dir)} t.Run("path="+name, func(t *testing.T) { - assert.Equal(t, dirURL, Dir(nameURL)) + assert.Equal(t, dirURL, loader.Dir(nameURL)) }) } } @@ -50,7 +51,7 @@ func TestDir(t *testing.T) { func TestResolve(t *testing.T) { t.Run("Blank", func(t *testing.T) { - _, err := Resolve(nil, "") + _, err := loader.Resolve(nil, "") assert.EqualError(t, err, "local or remote path required") }) @@ -59,21 +60,21 @@ func TestResolve(t *testing.T) { require.NoError(t, err) t.Run("Missing", func(t *testing.T) { - u, err := Resolve(root, "example.com/html") + u, err := loader.Resolve(root, "example.com/html") require.NoError(t, err) assert.Equal(t, u.String(), "https://example.com/html") // TODO: check that warning was emitted }) t.Run("WS", func(t *testing.T) { moduleSpecifier := "ws://example.com/html" - _, err := Resolve(root, moduleSpecifier) + _, err := loader.Resolve(root, moduleSpecifier) assert.EqualError(t, err, "only supported schemes for imports are file and https, "+moduleSpecifier+" has `ws`") }) t.Run("HTTP", func(t *testing.T) { moduleSpecifier := "http://example.com/html" - _, err := Resolve(root, moduleSpecifier) + _, err := loader.Resolve(root, moduleSpecifier) assert.EqualError(t, err, "only supported schemes for imports are file and https, "+moduleSpecifier+" has `http`") }) @@ -83,7 +84,7 @@ func TestResolve(t *testing.T) { pwdURL, err := url.Parse("https://example.com") require.NoError(t, err) - _, err = Resolve(pwdURL, "file:///etc/shadow") + _, err = loader.Resolve(pwdURL, "file:///etc/shadow") assert.EqualError(t, err, "origin (https://example.com) not allowed to load local file: file:///etc/shadow") }) @@ -117,10 +118,10 @@ func TestLoad(t *testing.T) { pwdURL, err := url.Parse("file://" + data.pwd) require.NoError(t, err) - moduleURL, err := Resolve(pwdURL, data.path) + moduleURL, err := loader.Resolve(pwdURL, data.path) require.NoError(t, err) - src, err := Load(filesystems, moduleURL, data.path) + src, err := loader.Load(filesystems, moduleURL, data.path) require.NoError(t, err) assert.Equal(t, "file:///path/to/file.txt", src.URL.String()) @@ -133,11 +134,14 @@ func TestLoad(t *testing.T) { require.NoError(t, err) path := filepath.FromSlash("/nonexistent") - pathURL, err := Resolve(root, "/nonexistent") + pathURL, err := loader.Resolve(root, "/nonexistent") require.NoError(t, err) - _, err = Load(filesystems, pathURL, path) - assert.EqualError(t, err, fmt.Sprintf(fileSchemeCouldntBeLoadedMsg, "file://"+filepath.ToSlash(path))) + _, err = loader.Load(filesystems, pathURL, path) + require.Error(t, err) + assert.Contains(t, err.Error(), + fmt.Sprintf(`The moduleSpecifier "file://%s" couldn't be found on local disk. `, + filepath.ToSlash(path))) }) }) @@ -149,10 +153,10 @@ func TestLoad(t *testing.T) { require.NoError(t, err) moduleSpecifier := sr("HTTPSBIN_URL/html") - moduleSpecifierURL, err := Resolve(root, moduleSpecifier) + moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier) require.NoError(t, err) - src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, src.URL, moduleSpecifierURL) assert.Contains(t, string(src.Data), "Herman Melville - Moby-Dick") @@ -163,10 +167,10 @@ func TestLoad(t *testing.T) { require.NoError(t, err) moduleSpecifier := sr("HTTPSBIN_URL/robots.txt") - moduleSpecifierURL, err := Resolve(pwdURL, moduleSpecifier) + moduleSpecifierURL, err := loader.Resolve(pwdURL, moduleSpecifier) require.NoError(t, err) - src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/robots.txt")) assert.Equal(t, string(src.Data), "User-agent: *\nDisallow: /deny\n") @@ -177,10 +181,10 @@ func TestLoad(t *testing.T) { require.NoError(t, err) moduleSpecifier := ("./robots.txt") - moduleSpecifierURL, err := Resolve(pwdURL, moduleSpecifier) + moduleSpecifierURL, err := loader.Resolve(pwdURL, moduleSpecifier) require.NoError(t, err) - src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, sr("HTTPSBIN_URL/robots.txt"), src.URL.String()) assert.Equal(t, "User-agent: *\nDisallow: /deny\n", string(src.Data)) @@ -202,11 +206,11 @@ func TestLoad(t *testing.T) { require.NoError(t, err) moduleSpecifier := sr("HTTPSBIN_URL/raw/something") - moduleSpecifierURL, err := Resolve(root, moduleSpecifier) + moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier) require.NoError(t, err) filesystems := map[string]afero.Fs{"https": afero.NewMemMapFs()} - src, err := Load(filesystems, moduleSpecifierURL, moduleSpecifier) + src, err := loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.NoError(t, err) assert.Equal(t, src.URL.String(), sr("HTTPSBIN_URL/raw/something")) @@ -222,7 +226,7 @@ func TestLoad(t *testing.T) { require.NoError(t, err) t.Run("IP URL", func(t *testing.T) { - _, err := Resolve(root, "192.168.0.%31") + _, err := loader.Resolve(root, "192.168.0.%31") require.Error(t, err) require.Contains(t, err.Error(), `invalid URL escape "%31"`) }) @@ -238,10 +242,10 @@ func TestLoad(t *testing.T) { for _, data := range testData { moduleSpecifier := data.moduleSpecifier t.Run(data.name, func(t *testing.T) { - moduleSpecifierURL, err := Resolve(root, moduleSpecifier) + moduleSpecifierURL, err := loader.Resolve(root, moduleSpecifier) require.NoError(t, err) - _, err = Load(filesystems, moduleSpecifierURL, moduleSpecifier) + _, err = loader.Load(filesystems, moduleSpecifierURL, moduleSpecifier) require.Error(t, err) }) } diff --git a/stats/cloud/collector.go b/stats/cloud/collector.go index 5df5c37f97d..8fa8ec07453 100644 --- a/stats/cloud/collector.go +++ b/stats/cloud/collector.go @@ -30,6 +30,7 @@ import ( "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext" "github.com/loadimpact/k6/lib/netext/httpext" + "github.com/loadimpact/k6/loader" "github.com/pkg/errors" "gopkg.in/guregu/null.v3" @@ -97,7 +98,7 @@ func MergeFromExternal(external map[string]json.RawMessage, conf *Config) error } // New creates a new cloud collector -func New(conf Config, src *lib.SourceData, opts lib.Options, version string) (*Collector, error) { +func New(conf Config, src *loader.SourceData, opts lib.Options, version string) (*Collector, error) { if err := MergeFromExternal(opts.External, &conf); err != nil { return nil, err } diff --git a/stats/cloud/collector_test.go b/stats/cloud/collector_test.go index 084b2ed5acd..a0f926271c0 100644 --- a/stats/cloud/collector_test.go +++ b/stats/cloud/collector_test.go @@ -43,6 +43,7 @@ import ( "github.com/loadimpact/k6/lib/netext/httpext" "github.com/loadimpact/k6/lib/testutils" "github.com/loadimpact/k6/lib/types" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats" ) @@ -135,7 +136,7 @@ func TestCloudCollector(t *testing.T) { })) defer tb.Cleanup() - script := &lib.SourceData{ + script := &loader.SourceData{ Data: []byte(""), URL: &url.URL{Path: "/script.js"}, } @@ -281,7 +282,7 @@ func TestCloudCollectorMaxPerPacket(t *testing.T) { })) defer tb.Cleanup() - script := &lib.SourceData{ + script := &loader.SourceData{ Data: []byte(""), URL: &url.URL{Path: "/script.js"}, } From 3047dece9aeff5e9dc4cd805a66b75257a449938 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 09:54:37 +0300 Subject: [PATCH 09/45] Fix pwd sometimes missing it's slash at the end --- loader/loader.go | 6 ++++++ loader/loader_test.go | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index cc19d5bd419..998999a4e7c 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -72,6 +72,12 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { return nil, errors.New("local or remote path required") } + // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we readd + // it if it's missing + if !strings.HasSuffix(pwd.Path, "/") { + pwd.Path += "/" + } + if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' { return pwd.Parse(moduleSpecifier) } diff --git a/loader/loader_test.go b/loader/loader_test.go index 94aaeb76674..00e0dca6cce 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -81,11 +81,21 @@ func TestResolve(t *testing.T) { }) t.Run("Remote Lifting Denied", func(t *testing.T) { - pwdURL, err := url.Parse("https://example.com") + pwdURL, err := url.Parse("https://example.com/") require.NoError(t, err) _, err = loader.Resolve(pwdURL, "file:///etc/shadow") - assert.EqualError(t, err, "origin (https://example.com) not allowed to load local file: file:///etc/shadow") + assert.EqualError(t, err, "origin (https://example.com/) not allowed to load local file: file:///etc/shadow") + }) + + t.Run("Fixes missing slash in pwd", func(t *testing.T) { + pwdURL, err := url.Parse("https://example.com/path/to") + require.NoError(t, err) + + moduleURL, err := loader.Resolve(pwdURL, "./something") + require.NoError(t, err) + require.Equal(t, "https://example.com/path/to/something", moduleURL.String()) + require.Equal(t, "https://example.com/path/to/", pwdURL.String()) }) } From 8add590f98a1377745e3c3588627c75bc1695597 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 10:05:34 +0300 Subject: [PATCH 10/45] Fix not recognizing local files which are not starting with . or / --- cmd/run.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/run.go b/cmd/run.go index 25126bf821e..c4bc4807dc7 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -530,6 +530,11 @@ func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reade return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil } pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} + srcLocalPath := filepath.Join(pwd, src) + if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok { + // there is file on the local disk ... lets use it :) + return loader.Load(filesystems, &url.URL{Scheme: "file", Path: srcLocalPath}, src) + } srcURL, err := loader.Resolve(pwdURL, filepath.ToSlash(src)) if err != nil { return nil, err From a484a3139f14f1d7bb59179a675d68f873704f97 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 12:09:23 +0300 Subject: [PATCH 11/45] Fix lifting config files from the filesystems in the memmapfs and than archives --- cmd/archive.go | 3 ++- cmd/cloud.go | 3 ++- cmd/run.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/archive.go b/cmd/archive.go index ed01a7f0170..30b36f0e4ba 100644 --- a/cmd/archive.go +++ b/cmd/archive.go @@ -23,6 +23,7 @@ package cmd import ( "os" + "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -70,7 +71,7 @@ An archive is a fully self-contained test run, and can be executed identically e if err != nil { return err } - conf, err := getConsolidatedConfig(filesystems["file"], Config{Options: cliOpts}, r) + conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r) if err != nil { return err } diff --git a/cmd/cloud.go b/cmd/cloud.go index 350a76e4dd5..9cd098124b9 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -35,6 +35,7 @@ import ( "github.com/loadimpact/k6/stats/cloud" "github.com/loadimpact/k6/ui" "github.com/pkg/errors" + "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -90,7 +91,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud if err != nil { return err } - conf, err := getConsolidatedConfig(filesystems["file"], Config{Options: cliOpts}, r) + conf, err := getConsolidatedConfig(afero.NewOsFs(), Config{Options: cliOpts}, r) if err != nil { return err } diff --git a/cmd/run.go b/cmd/run.go index c4bc4807dc7..1f44c1d8c50 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -140,7 +140,7 @@ a commandline interface for interacting with it.`, if err != nil { return err } - conf, err := getConsolidatedConfig(filesystems["file"], cliConf, r) + conf, err := getConsolidatedConfig(afero.NewOsFs(), cliConf, r) if err != nil { return err } From 3e65e7ec0b05254d04197fefc320ecdc90f4bcef Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 12:14:16 +0300 Subject: [PATCH 12/45] Add comment about UnprependPathFs --- cmd/run.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/run.go b/cmd/run.go index 1f44c1d8c50..7698acdedc5 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -512,6 +512,9 @@ func createFilesystems() map[string]afero.Fs { // Also initialize the same for `https` but the caching is handled manually in the loader package osfs := afero.NewOsFs() if runtime.GOOS == "windows" { + // This is done so that we can continue to use paths with /|"\" through the code but also to + // be easier to travers the cachedFs later as it doesn't work very well if you have windows + // volumes osfs = fsext.NewUnprependPathFs(osfs, afero.FilePathSeparator) } return map[string]afero.Fs{ From 083227396e494f26dfa9fbd9a63983523ff882bf Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 13:06:27 +0300 Subject: [PATCH 13/45] Better names and abstration for UnprependPathFs --- cmd/run.go | 2 +- js/bundle_test.go | 2 +- lib/fsext/unprependfs.go | 108 ++++++++++++++++++++------------------- 3 files changed, 58 insertions(+), 54 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 7698acdedc5..bd7f7265ff9 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -515,7 +515,7 @@ func createFilesystems() map[string]afero.Fs { // This is done so that we can continue to use paths with /|"\" through the code but also to // be easier to travers the cachedFs later as it doesn't work very well if you have windows // volumes - osfs = fsext.NewUnprependPathFs(osfs, afero.FilePathSeparator) + osfs = fsext.NewTrimFilePathSeparatorFs(osfs) } return map[string]afero.Fs{ "file": fsext.NewCacheOnReadFs(osfs, afero.NewMemMapFs(), 0), diff --git a/js/bundle_test.go b/js/bundle_test.go index 2635ea35da3..971922f9b28 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -520,7 +520,7 @@ func TestOpen(t *testing.T) { require.NoError(t, fs.MkdirAll(filepath.Join(prefix, "/path/to"), 0755)) require.NoError(t, afero.WriteFile(fs, filePath, []byte(`hi`), 0644)) if isWindows { - fs = fsext.NewUnprependPathFs(fs, afero.FilePathSeparator) + fs = fsext.NewTrimFilePathSeparatorFs(fs) } return fs, prefix, func() { require.NoError(t, os.RemoveAll(prefix)) } }, diff --git a/lib/fsext/unprependfs.go b/lib/fsext/unprependfs.go index bfdab3e53c8..5dc0925b21c 100644 --- a/lib/fsext/unprependfs.go +++ b/lib/fsext/unprependfs.go @@ -9,77 +9,81 @@ import ( "github.com/spf13/afero" ) -var _ afero.Lstater = (*UnprependPathFs)(nil) +var _ afero.Lstater = (*ChangePathFs)(nil) -// UnprependPathFs is a filesystem that wraps another afero.Fs and unprepend a given path from all -// file and directory names before calling the same method on the wrapped afero.Fs. +// ChangePathFs is a filesystem that wraps another afero.Fs and changes all given paths from all +// file and directory names, with a function, before calling the same method on the wrapped afero.Fs. // Heavily based on afero.BasePathFs -type UnprependPathFs struct { +type ChangePathFs struct { source afero.Fs - path string + fn ChangePathFunc } -// UnprependPathFile is a file from UnprependPathFs -type UnprependPathFile struct { +// ChangePathFile is a file from ChangePathFs +type ChangePathFile struct { afero.File - path string + fn ChangePathFunc } -// Name Returns the name of the file -func (f *UnprependPathFile) Name() string { - sourcename := f.File.Name() - return strings.TrimPrefix(sourcename, filepath.Clean(f.path)) -} +// ChangePathFunc is the function that will be called by ChangePathFs to change the path +type ChangePathFunc func(name string) (path string, err error) -// NewUnprependPathFs returns a new UnprependPathFs that will unprepend -func NewUnprependPathFs(source afero.Fs, path string) *UnprependPathFs { - return &UnprependPathFs{source: source, path: path} -} +// NewTrimFilePathSeparatorFs is ChangePathFs that trims a Afero.FilePathSeparator from all paths +// Heavily based on afero.BasePathFs +func NewTrimFilePathSeparatorFs(source afero.Fs) *ChangePathFs { + return &ChangePathFs{source: source, fn: ChangePathFunc(func(name string) (path string, err error) { + if !strings.HasPrefix(name, afero.FilePathSeparator) { + return name, os.ErrNotExist + } -func (b *UnprependPathFs) realPath(name string) (path string, err error) { - if !strings.HasPrefix(name, b.path) { - return name, os.ErrNotExist - } + return filepath.Clean(strings.TrimPrefix(name, afero.FilePathSeparator)), nil + + })} +} - return filepath.Clean(strings.TrimPrefix(name, b.path)), nil +// Name Returns the name of the file +func (f *ChangePathFile) Name() string { + // error shouldn't be possible + name, _ := f.fn(f.File.Name()) + return name } //Chtimes changes the access and modification times of the named file -func (b *UnprependPathFs) Chtimes(name string, atime, mtime time.Time) (err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) Chtimes(name string, atime, mtime time.Time) (err error) { + if name, err = b.fn(name); err != nil { return &os.PathError{Op: "chtimes", Path: name, Err: err} } return b.source.Chtimes(name, atime, mtime) } // Chmod changes the mode of the named file to mode. -func (b *UnprependPathFs) Chmod(name string, mode os.FileMode) (err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) Chmod(name string, mode os.FileMode) (err error) { + if name, err = b.fn(name); err != nil { return &os.PathError{Op: "chmod", Path: name, Err: err} } return b.source.Chmod(name, mode) } // Name return the name of this FileSystem -func (b *UnprependPathFs) Name() string { - return "UnprependPathFs" +func (b *ChangePathFs) Name() string { + return "ChangePathFs" } // Stat returns a FileInfo describing the named file, or an error, if any // happens. -func (b *UnprependPathFs) Stat(name string) (fi os.FileInfo, err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) Stat(name string) (fi os.FileInfo, err error) { + if name, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "stat", Path: name, Err: err} } return b.source.Stat(name) } // Rename renames a file. -func (b *UnprependPathFs) Rename(oldname, newname string) (err error) { - if oldname, err = b.realPath(oldname); err != nil { +func (b *ChangePathFs) Rename(oldname, newname string) (err error) { + if oldname, err = b.fn(oldname); err != nil { return &os.PathError{Op: "rename", Path: oldname, Err: err} } - if newname, err = b.realPath(newname); err != nil { + if newname, err = b.fn(newname); err != nil { return &os.PathError{Op: "rename", Path: newname, Err: err} } return b.source.Rename(oldname, newname) @@ -87,8 +91,8 @@ func (b *UnprependPathFs) Rename(oldname, newname string) (err error) { // RemoveAll removes a directory path and any children it contains. It // does not fail if the path does not exist (return nil). -func (b *UnprependPathFs) RemoveAll(name string) (err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) RemoveAll(name string) (err error) { + if name, err = b.fn(name); err != nil { return &os.PathError{Op: "remove_all", Path: name, Err: err} } return b.source.RemoveAll(name) @@ -96,41 +100,41 @@ func (b *UnprependPathFs) RemoveAll(name string) (err error) { // Remove removes a file identified by name, returning an error, if any // happens. -func (b *UnprependPathFs) Remove(name string) (err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) Remove(name string) (err error) { + if name, err = b.fn(name); err != nil { return &os.PathError{Op: "remove", Path: name, Err: err} } return b.source.Remove(name) } // OpenFile opens a file using the given flags and the given mode. -func (b *UnprependPathFs) OpenFile(name string, flag int, mode os.FileMode) (f afero.File, err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) OpenFile(name string, flag int, mode os.FileMode) (f afero.File, err error) { + if name, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "openfile", Path: name, Err: err} } sourcef, err := b.source.OpenFile(name, flag, mode) if err != nil { return nil, err } - return &UnprependPathFile{sourcef, b.path}, nil + return &ChangePathFile{File: sourcef, fn: b.fn}, nil } // Open opens a file, returning it or an error, if any happens. -func (b *UnprependPathFs) Open(name string) (f afero.File, err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) Open(name string) (f afero.File, err error) { + if name, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "open", Path: name, Err: err} } sourcef, err := b.source.Open(name) if err != nil { return nil, err } - return &UnprependPathFile{File: sourcef, path: b.path}, nil + return &ChangePathFile{File: sourcef, fn: b.fn}, nil } // Mkdir creates a directory in the filesystem, return an error if any // happens. -func (b *UnprependPathFs) Mkdir(name string, mode os.FileMode) (err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) Mkdir(name string, mode os.FileMode) (err error) { + if name, err = b.fn(name); err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} } return b.source.Mkdir(name, mode) @@ -138,8 +142,8 @@ func (b *UnprependPathFs) Mkdir(name string, mode os.FileMode) (err error) { // MkdirAll creates a directory path and all parents that does not exist // yet. -func (b *UnprependPathFs) MkdirAll(name string, mode os.FileMode) (err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) MkdirAll(name string, mode os.FileMode) (err error) { + if name, err = b.fn(name); err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} } return b.source.MkdirAll(name, mode) @@ -147,20 +151,20 @@ func (b *UnprependPathFs) MkdirAll(name string, mode os.FileMode) (err error) { // Create creates a file in the filesystem, returning the file and an // error, if any happens -func (b *UnprependPathFs) Create(name string) (f afero.File, err error) { - if name, err = b.realPath(name); err != nil { +func (b *ChangePathFs) Create(name string) (f afero.File, err error) { + if name, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "create", Path: name, Err: err} } sourcef, err := b.source.Create(name) if err != nil { return nil, err } - return &UnprependPathFile{File: sourcef, path: b.path}, nil + return &ChangePathFile{File: sourcef, fn: b.fn}, nil } // LstatIfPossible implements the afero.Lstater interface -func (b *UnprependPathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - name, err := b.realPath(name) +func (b *ChangePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + name, err := b.fn(name) if err != nil { return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} } From 56a4c3798abbc7cf62ff766dce7886227b4735c9 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 13:12:39 +0300 Subject: [PATCH 14/45] Refactor normalizedFs to be a ChangePathFs --- lib/archive.go | 20 +++++--------------- lib/fsext/unprependfs.go | 5 +++++ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/archive.go b/lib/archive.go index 7d89a82c522..b30584ddd8f 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -58,20 +58,10 @@ func NormalizeAndAnonymizePath(path string) string { return homeDirRE.ReplaceAllString(p, `$1/$2/nobody`) } -type normalizedFS struct { - afero.Fs -} - -func (m *normalizedFS) Open(name string) (afero.File, error) { - return m.Fs.Open(NormalizeAndAnonymizePath(name)) -} - -func (m *normalizedFS) OpenFile(name string, flag int, mode os.FileMode) (afero.File, error) { - return m.Fs.OpenFile(NormalizeAndAnonymizePath(name), flag, mode) -} - -func (m *normalizedFS) Stat(name string) (os.FileInfo, error) { - return m.Fs.Stat(NormalizeAndAnonymizePath(name)) +func newNormalizedFs(fs afero.Fs) afero.Fs { + return fsext.NewChangePathFs(fs, fsext.ChangePathFunc(func(name string) (string, error) { + return NormalizeAndAnonymizePath(name), nil + })) } // An Archive is a rollup of all resources and options needed to reproduce a test identically elsewhere. @@ -106,7 +96,7 @@ func (arc *Archive) getFs(name string) afero.Fs { if !ok { fs = afero.NewMemMapFs() if name == "file" { - fs = &normalizedFS{fs} + fs = newNormalizedFs(fs) } arc.Filesystems[name] = fs } diff --git a/lib/fsext/unprependfs.go b/lib/fsext/unprependfs.go index 5dc0925b21c..b979cf962ca 100644 --- a/lib/fsext/unprependfs.go +++ b/lib/fsext/unprependfs.go @@ -25,6 +25,11 @@ type ChangePathFile struct { fn ChangePathFunc } +// NewChangePathFs return a ChangePathFs where all paths will be change with the provided funcs +func NewChangePathFs(source afero.Fs, fn ChangePathFunc) *ChangePathFs { + return &ChangePathFs{source: source, fn: fn} +} + // ChangePathFunc is the function that will be called by ChangePathFs to change the path type ChangePathFunc func(name string) (path string, err error) From 2fe2d6324ac0ebdb3cb840541210830078c2b4ba Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 5 Jul 2019 13:19:23 +0300 Subject: [PATCH 15/45] rename changepathfs file --- lib/fsext/{unprependfs.go => changepathfs.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/fsext/{unprependfs.go => changepathfs.go} (100%) diff --git a/lib/fsext/unprependfs.go b/lib/fsext/changepathfs.go similarity index 100% rename from lib/fsext/unprependfs.go rename to lib/fsext/changepathfs.go From 622409c76b75ba3d5129353abd0e0af87da5a369 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 8 Jul 2019 16:34:15 +0300 Subject: [PATCH 16/45] Implement caching and reading of cache for github/cdnjs urls --- lib/archive.go | 26 ++++++------------ lib/old_archive_test.go | 50 ++++++++++++++++++++++++++++++++++ loader/cdnjs_test.go | 10 ++++--- loader/github_test.go | 33 ++++++++++++++++++----- loader/loader.go | 60 +++++++++++++++++++++++++++++++++-------- 5 files changed, 141 insertions(+), 38 deletions(-) diff --git a/lib/archive.go b/lib/archive.go index b30584ddd8f..716ac885c68 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -134,24 +134,14 @@ func ReadArchive(in io.Reader) (*Archive, error) { if arc.K6Version == "" { arc.Filename = NormalizeAndAnonymizePath(arc.Filename) arc.Pwd = NormalizeAndAnonymizePath(arc.Pwd) - arc.PwdURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd) - if err != nil { - return nil, err - } - arc.FilenameURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Filename) - if err != nil { - return nil, err - } - } else { - arc.FilenameURL, err = url.Parse(arc.Filename) - if err != nil { - return nil, err - } - - arc.PwdURL, err = url.Parse(arc.Pwd) - if err != nil { - return nil, err - } + } + arc.PwdURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Pwd) + if err != nil { + return nil, err + } + arc.FilenameURL, err = loader.Resolve(&url.URL{Scheme: "file", Path: "/"}, arc.Filename) + if err != nil { + return nil, err } continue diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go index a8b6a0d039f..671cfe8a897 100644 --- a/lib/old_archive_test.go +++ b/lib/old_archive_test.go @@ -3,6 +3,7 @@ package lib import ( "archive/tar" "bytes" + "net/url" "os" "path" "path/filepath" @@ -113,3 +114,52 @@ func TestUnknownPrefix(t *testing.T) { require.Equal(t, err.Error(), "unknown file prefix `strange` for file `strange/something`") } + +func TestFilenamePwdResolve(t *testing.T) { + var tests = []struct { + Filename, Pwd string + expectedFilenameURL, expectedPwdURL *url.URL + }{ + { + Filename: "/home/nobody/something.js", + Pwd: "/home/nobody", + expectedFilenameURL: &url.URL{Scheme: "file", Path: "/home/nobody/something.js"}, + expectedPwdURL: &url.URL{Scheme: "file", Path: "/home/nobody"}, + }, + { + Filename: "github.com/loadimpact/k6/samples/http2.js", + Pwd: "github.com/loadimpact/k6/samples", + expectedFilenameURL: &url.URL{Opaque: "github.com/loadimpact/k6/samples/http2.js"}, + expectedPwdURL: &url.URL{Opaque: "github.com/loadimpact/k6/samples"}, + }, + { + Filename: "cdnjs.com/libraries/Faker", + Pwd: "/home/nobody", + expectedFilenameURL: &url.URL{Opaque: "cdnjs.com/libraries/Faker"}, + expectedPwdURL: &url.URL{Scheme: "file", Path: "/home/nobody"}, + }, + { + Filename: "example.com/something/dot.js", + Pwd: "example.com/something/", + expectedFilenameURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something/dot.js"}, + expectedPwdURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something"}, + }, + } + + for _, test := range tests { + metadata := `{ + "Filename": "` + test.Filename + `", + "Pwd": "` + test.Pwd + `" + }` + + buf, err := dumpMakeMapFsToBuf(makeMemMapFs(t, map[string][]byte{ + "/metadata.json": []byte(metadata), + })) + require.NoError(t, err) + + arc, err := ReadArchive(buf) + require.NoError(t, err) + require.Equal(t, test.expectedFilenameURL, arc.FilenameURL) + require.Equal(t, test.expectedPwdURL, arc.PwdURL) + } +} diff --git a/loader/cdnjs_test.go b/loader/cdnjs_test.go index 80bdb87afad..55de95eadba 100644 --- a/loader/cdnjs_test.go +++ b/loader/cdnjs_test.go @@ -63,6 +63,8 @@ func TestCDNJS(t *testing.T) { `^https://cdnjs.cloudflare.com/ajax/libs/Faker/0.7.2/MinFaker.js$`, }, } + + var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"} for path, expected := range paths { path, expected := path, expected t.Run(path, func(t *testing.T) { @@ -74,12 +76,14 @@ func TestCDNJS(t *testing.T) { require.NoError(t, err) assert.Regexp(t, expected.src, src) - pathURL, err := url.Parse(src) + resolvedURL, err := Resolve(root, path) require.NoError(t, err) + require.Empty(t, resolvedURL.Scheme) + require.Equal(t, path, resolvedURL.Opaque) - data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path) + data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path) require.NoError(t, err) - assert.Equal(t, pathURL, data.URL) + assert.Equal(t, resolvedURL, data.URL) assert.NotEmpty(t, data.Data) }) } diff --git a/loader/github_test.go b/loader/github_test.go index 5297d25ca5f..bd110185b44 100644 --- a/loader/github_test.go +++ b/loader/github_test.go @@ -39,12 +39,16 @@ func TestGithub(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expectedEndSrc, src) - pathURL, err := url.Parse(src) + var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"} + resolvedURL, err := Resolve(root, path) require.NoError(t, err) + require.Empty(t, resolvedURL.Scheme) + require.Equal(t, path, resolvedURL.Opaque) t.Run("not cached", func(t *testing.T) { - data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path) + data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path) require.NoError(t, err) - assert.Equal(t, expectedEndSrc, data.URL.String()) + assert.Equal(t, data.URL, resolvedURL) + assert.Equal(t, path, data.URL.String()) assert.NotEmpty(t, data.Data) }) @@ -52,12 +56,29 @@ func TestGithub(t *testing.T) { fs := afero.NewMemMapFs() testData := []byte("test data") - err := afero.WriteFile(fs, "/raw.githubusercontent.com/github/gitignore/master/Go.gitignore", testData, 0644) + err := afero.WriteFile(fs, "/github.com/github/gitignore/Go.gitignore", testData, 0644) require.NoError(t, err) - data, err := Load(map[string]afero.Fs{"https": fs}, pathURL, path) + data, err := Load(map[string]afero.Fs{"https": fs}, resolvedURL, path) require.NoError(t, err) - assert.Equal(t, expectedEndSrc, data.URL.String()) + assert.Equal(t, path, data.URL.String()) assert.Equal(t, data.Data, testData) }) + + t.Run("relative", func(t *testing.T) { + var tests = map[string]string{ + "./something.else": "github.com/github/gitignore/something.else", + "../something.else": "github.com/github/something.else", + "/something.else": "github.com/something.else", + } + for relative, expected := range tests { + relativeURL, err := Resolve(Dir(resolvedURL), relative) + require.NoError(t, err) + assert.Equal(t, expected, relativeURL.String()) + } + }) + + t.Run("dir", func(t *testing.T) { + require.Equal(t, &url.URL{Opaque: "github.com/github/gitignore"}, Dir(resolvedURL)) + }) } diff --git a/loader/loader.go b/loader/loader.go index 998999a4e7c..414b1f0a156 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -74,11 +74,23 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we readd // it if it's missing - if !strings.HasSuffix(pwd.Path, "/") { + if pwd.Opaque != "" { + if !strings.HasSuffix(pwd.Opaque, "/") { + pwd.Opaque += "/" + } + } else if !strings.HasSuffix(pwd.Path, "/") { pwd.Path += "/" } if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' { + if pwd.Opaque != "" { // this is a loader reference + parts := strings.SplitN(pwd.Opaque, "/", 2) + if moduleSpecifier[0] == '/' { + return &url.URL{Opaque: path.Join(parts[0], moduleSpecifier)}, nil + } + return &url.URL{Opaque: path.Join(parts[0], path.Join(path.Dir(parts[1]), moduleSpecifier))}, nil + } + return pwd.Parse(moduleSpecifier) } @@ -97,9 +109,10 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { } return u, err } - - stringURL, err := resolveUsingLoaders(moduleSpecifier) - if err == errNoLoaderMatched { + // here we only care if a loader is pickable, if it is and later there is an error in the loading + // from it we don't want to try another resolve + _, loader, _ := pickLoader(moduleSpecifier) + if loader == nil { log.WithField("url", moduleSpecifier).Warning( "A url was resolved but it didn't have scheme. " + "This will be deprecated in the future and all remote modules will " + @@ -107,14 +120,14 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { return url.Parse("https://" + moduleSpecifier) } - if err != nil { - return nil, err - } - return url.Parse(stringURL) + return &url.URL{Opaque: moduleSpecifier}, nil } // Dir returns the directory for the path. func Dir(old *url.URL) *url.URL { + if old.Opaque != "" { // loader + return &url.URL{Opaque: path.Join(old.Opaque, "../")} + } return old.ResolveReference(&url.URL{Path: "./"}) } @@ -131,6 +144,27 @@ func Load( }).Debug("Loading...") pathOnFs := filepath.FromSlash(path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):])) + if moduleSpecifier.Opaque != "" { // This is loader + pathOnFs = filepath.Join(afero.FilePathSeparator, moduleSpecifier.Opaque) + data, err := afero.ReadFile(filesystems["https"], pathOnFs) + if err != nil { + var finalModuleSpecifierURL *url.URL + finalModuleSpecifierURL, err = resolveUsingLoaders(moduleSpecifier.Opaque) + if err != nil { + return nil, err + } + + var result *SourceData + result, err = loadRemoteURL(finalModuleSpecifierURL) + if err != nil { // TODO: have a separateError + return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, finalModuleSpecifierURL, err) + } + _ = afero.WriteFile(filesystems["https"], pathOnFs, result.Data, 0644) + result.URL = moduleSpecifier + return result, nil + } + return &SourceData{URL: moduleSpecifier, Data: data}, nil + } data, err := afero.ReadFile(filesystems[moduleSpecifier.Scheme], pathOnFs) if err != nil { @@ -154,13 +188,17 @@ func Load( return &SourceData{URL: moduleSpecifier, Data: data}, nil } -func resolveUsingLoaders(name string) (string, error) { +func resolveUsingLoaders(name string) (*url.URL, error) { _, loader, loaderArgs := pickLoader(name) if loader != nil { - return loader(name, loaderArgs) + urlString, err := loader(name, loaderArgs) + if err != nil { + return nil, err + } + return url.Parse(urlString) } - return "", errNoLoaderMatched + return nil, errNoLoaderMatched } func loadRemoteURL(u *url.URL) (*SourceData, error) { From ac764f379c0a7d75927b526195ec0a0e1ef665b1 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 8 Jul 2019 18:07:09 +0300 Subject: [PATCH 17/45] Refactor the walk function in lib.archive a bit --- lib/archive.go | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/lib/archive.go b/lib/archive.go index 716ac885c68..02402b30ba8 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -253,28 +253,24 @@ func (arc *Archive) Write(out io.Writer) error { infos := make(map[string]os.FileInfo) // ... fix this ? files := make(map[string][]byte) - err = fsext.Walk(filesystem, afero.FilePathSeparator, - filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - normalizedPath := NormalizeAndAnonymizePath(filePath) - - infos[normalizedPath] = info - if info.IsDir() { - foundDirs[normalizedPath] = true - return nil - } - - files[normalizedPath], err = afero.ReadFile(filesystem, filePath) - if err != nil { - return err - } - paths = append(paths, normalizedPath) + walkFunc := filepath.WalkFunc(func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + normalizedPath := NormalizeAndAnonymizePath(filePath) + + infos[normalizedPath] = info + if info.IsDir() { + foundDirs[normalizedPath] = true return nil - })) + } - if err != nil { + paths = append(paths, normalizedPath) + files[normalizedPath], err = afero.ReadFile(filesystem, filePath) + return err + }) + + if err = fsext.Walk(filesystem, afero.FilePathSeparator, walkFunc); err != nil { return err } if len(files) == 0 { From 2e6bce1edb4c3a9b106d8cf15ce049a1694ba81a Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 9 Jul 2019 11:53:28 +0300 Subject: [PATCH 18/45] Fix name of dumpMemMapFsToBuf --- lib/old_archive_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go index 671cfe8a897..d5082c5a498 100644 --- a/lib/old_archive_test.go +++ b/lib/old_archive_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func dumpMakeMapFsToBuf(fs afero.Fs) (*bytes.Buffer, error) { +func dumpMemMapFsToBuf(fs afero.Fs) (*bytes.Buffer, error) { var b = bytes.NewBuffer(nil) var w = tar.NewWriter(b) err := fsext.Walk(fs, afero.FilePathSeparator, @@ -74,7 +74,7 @@ func TestOldArchive(t *testing.T) { "/scripts/_/C/something/path2": []byte(`windows script`), "/scripts/_/absolulte/path2": []byte(`unix script`), }) - buf, err := dumpMakeMapFsToBuf(fs) + buf, err := dumpMemMapFsToBuf(fs) require.NoError(t, err) var ( @@ -106,7 +106,7 @@ func TestUnknownPrefix(t *testing.T) { fs := makeMemMapFs(t, map[string][]byte{ "/strange/something": []byte(`github file`), }) - buf, err := dumpMakeMapFsToBuf(fs) + buf, err := dumpMemMapFsToBuf(fs) require.NoError(t, err) _, err = ReadArchive(buf) @@ -152,7 +152,7 @@ func TestFilenamePwdResolve(t *testing.T) { "Pwd": "` + test.Pwd + `" }` - buf, err := dumpMakeMapFsToBuf(makeMemMapFs(t, map[string][]byte{ + buf, err := dumpMemMapFsToBuf(makeMemMapFs(t, map[string][]byte{ "/metadata.json": []byte(metadata), })) require.NoError(t, err) From edf5d790d4cd0553d086ad8351458b91949e82de Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 9 Jul 2019 17:52:35 +0300 Subject: [PATCH 19/45] Add test for ChangePathFs and some fixes --- lib/fsext/changepathfs.go | 83 ++++++++++-------- lib/fsext/changepathfs_test.go | 154 +++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 37 deletions(-) create mode 100644 lib/fsext/changepathfs_test.go diff --git a/lib/fsext/changepathfs.go b/lib/fsext/changepathfs.go index b979cf962ca..7f7a81d6f2f 100644 --- a/lib/fsext/changepathfs.go +++ b/lib/fsext/changepathfs.go @@ -22,7 +22,7 @@ type ChangePathFs struct { // ChangePathFile is a file from ChangePathFs type ChangePathFile struct { afero.File - fn ChangePathFunc + originalName string } // NewChangePathFs return a ChangePathFs where all paths will be change with the provided funcs @@ -42,31 +42,30 @@ func NewTrimFilePathSeparatorFs(source afero.Fs) *ChangePathFs { } return filepath.Clean(strings.TrimPrefix(name, afero.FilePathSeparator)), nil - })} } // Name Returns the name of the file func (f *ChangePathFile) Name() string { - // error shouldn't be possible - name, _ := f.fn(f.File.Name()) - return name + return f.originalName } //Chtimes changes the access and modification times of the named file func (b *ChangePathFs) Chtimes(name string, atime, mtime time.Time) (err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return &os.PathError{Op: "chtimes", Path: name, Err: err} } - return b.source.Chtimes(name, atime, mtime) + return b.source.Chtimes(newName, atime, mtime) } // Chmod changes the mode of the named file to mode. func (b *ChangePathFs) Chmod(name string, mode os.FileMode) (err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return &os.PathError{Op: "chmod", Path: name, Err: err} } - return b.source.Chmod(name, mode) + return b.source.Chmod(newName, mode) } // Name return the name of this FileSystem @@ -77,105 +76,115 @@ func (b *ChangePathFs) Name() string { // Stat returns a FileInfo describing the named file, or an error, if any // happens. func (b *ChangePathFs) Stat(name string) (fi os.FileInfo, err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "stat", Path: name, Err: err} } - return b.source.Stat(name) + return b.source.Stat(newName) } // Rename renames a file. -func (b *ChangePathFs) Rename(oldname, newname string) (err error) { - if oldname, err = b.fn(oldname); err != nil { - return &os.PathError{Op: "rename", Path: oldname, Err: err} +func (b *ChangePathFs) Rename(oldName, newName string) (err error) { + var newOldName, newNewName string + if newOldName, err = b.fn(oldName); err != nil { + return &os.PathError{Op: "rename", Path: oldName, Err: err} } - if newname, err = b.fn(newname); err != nil { - return &os.PathError{Op: "rename", Path: newname, Err: err} + if newNewName, err = b.fn(newName); err != nil { + return &os.PathError{Op: "rename", Path: newName, Err: err} } - return b.source.Rename(oldname, newname) + return b.source.Rename(newOldName, newNewName) } // RemoveAll removes a directory path and any children it contains. It // does not fail if the path does not exist (return nil). func (b *ChangePathFs) RemoveAll(name string) (err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return &os.PathError{Op: "remove_all", Path: name, Err: err} } - return b.source.RemoveAll(name) + return b.source.RemoveAll(newName) } // Remove removes a file identified by name, returning an error, if any // happens. func (b *ChangePathFs) Remove(name string) (err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return &os.PathError{Op: "remove", Path: name, Err: err} } - return b.source.Remove(name) + return b.source.Remove(newName) } // OpenFile opens a file using the given flags and the given mode. func (b *ChangePathFs) OpenFile(name string, flag int, mode os.FileMode) (f afero.File, err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "openfile", Path: name, Err: err} } - sourcef, err := b.source.OpenFile(name, flag, mode) + sourcef, err := b.source.OpenFile(newName, flag, mode) if err != nil { return nil, err } - return &ChangePathFile{File: sourcef, fn: b.fn}, nil + return &ChangePathFile{File: sourcef, originalName: name}, nil } // Open opens a file, returning it or an error, if any happens. func (b *ChangePathFs) Open(name string) (f afero.File, err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "open", Path: name, Err: err} } - sourcef, err := b.source.Open(name) + sourcef, err := b.source.Open(newName) if err != nil { return nil, err } - return &ChangePathFile{File: sourcef, fn: b.fn}, nil + return &ChangePathFile{File: sourcef, originalName: name}, nil } // Mkdir creates a directory in the filesystem, return an error if any // happens. func (b *ChangePathFs) Mkdir(name string, mode os.FileMode) (err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} } - return b.source.Mkdir(name, mode) + return b.source.Mkdir(newName, mode) } // MkdirAll creates a directory path and all parents that does not exist // yet. func (b *ChangePathFs) MkdirAll(name string, mode os.FileMode) (err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} } - return b.source.MkdirAll(name, mode) + return b.source.MkdirAll(newName, mode) } // Create creates a file in the filesystem, returning the file and an // error, if any happens func (b *ChangePathFs) Create(name string) (f afero.File, err error) { - if name, err = b.fn(name); err != nil { + var newName string + if newName, err = b.fn(name); err != nil { return nil, &os.PathError{Op: "create", Path: name, Err: err} } - sourcef, err := b.source.Create(name) + sourcef, err := b.source.Create(newName) if err != nil { return nil, err } - return &ChangePathFile{File: sourcef, fn: b.fn}, nil + return &ChangePathFile{File: sourcef, originalName: name}, nil } // LstatIfPossible implements the afero.Lstater interface func (b *ChangePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - name, err := b.fn(name) + var newName string + newName, err := b.fn(name) if err != nil { return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} } if lstater, ok := b.source.(afero.Lstater); ok { - return lstater.LstatIfPossible(name) + return lstater.LstatIfPossible(newName) } - fi, err := b.source.Stat(name) + fi, err := b.source.Stat(newName) return fi, false, err } diff --git a/lib/fsext/changepathfs_test.go b/lib/fsext/changepathfs_test.go new file mode 100644 index 00000000000..4e51bb777e5 --- /dev/null +++ b/lib/fsext/changepathfs_test.go @@ -0,0 +1,154 @@ +package fsext + +import ( + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestChangePathFs(t *testing.T) { + var m = afero.NewMemMapFs() + var prefix = "/another/" + var c = NewChangePathFs(m, ChangePathFunc(func(name string) (string, error) { + if !strings.HasPrefix(name, prefix) { + return "", fmt.Errorf("path %s doesn't start with `%s`", name, prefix) + } + return name[len(prefix):], nil + })) + + var filePath = "/another/path/to/file.txt" + + require.Equal(t, c.Name(), "ChangePathFs") + t.Run("Create", func(t *testing.T) { + f, err := c.Create(filePath) + require.NoError(t, err) + require.Equal(t, filePath, f.Name()) + + /** TODO figure out if this is error in MemMapFs + _, err = c.Create(filePath) + require.Error(t, err) + require.True(t, os.IsExist(err)) + */ + + _, err = c.Create("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + }) + + t.Run("Mkdir", func(t *testing.T) { + require.NoError(t, c.Mkdir("/another/path/too", 0644)) + checkErrorPath(t, c.Mkdir("/notanother/path/too", 0644), "/notanother/path/too") + }) + + t.Run("MkdirAll", func(t *testing.T) { + require.NoError(t, c.MkdirAll("/another/pattth/too", 0644)) + checkErrorPath(t, c.Mkdir("/notanother/pattth/too", 0644), "/notanother/pattth/too") + }) + + t.Run("Open", func(t *testing.T) { + f, err := c.Open(filePath) + require.NoError(t, err) + require.Equal(t, filePath, f.Name()) + + _, err = c.Open("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + }) + + t.Run("OpenFile", func(t *testing.T) { + f, err := c.OpenFile(filePath, os.O_RDWR, 0644) + require.NoError(t, err) + require.Equal(t, filePath, f.Name()) + + _, err = c.OpenFile("/notanother/path/to/file.txt", os.O_RDWR, 0644) + checkErrorPath(t, err, "/notanother/path/to/file.txt") + }) + + t.Run("Stat Chmod Chtimes", func(t *testing.T) { + info, err := c.Stat(filePath) + require.NoError(t, err) + require.Equal(t, "file.txt", info.Name()) + + sometime := time.Unix(10000, 13) + require.NotEqual(t, sometime, info.ModTime()) + require.NoError(t, c.Chtimes(filePath, time.Now(), sometime)) + require.Equal(t, sometime, info.ModTime()) + + mode := os.FileMode(0007) + require.NotEqual(t, mode, info.Mode()) + require.NoError(t, c.Chmod(filePath, mode)) + require.Equal(t, mode, info.Mode()) + + _, err = c.Stat("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + + checkErrorPath(t, c.Chtimes("/notanother/path/to/file.txt", time.Now(), time.Now()), "/notanother/path/to/file.txt") + + checkErrorPath(t, c.Chmod("/notanother/path/to/file.txt", mode), "/notanother/path/to/file.txt") + }) + + t.Run("Rename", func(t *testing.T) { + info, err := c.Stat(filePath) + require.NoError(t, err) + require.False(t, info.IsDir()) + + require.NoError(t, c.Rename(filePath, "/another/path/to/file.doc")) + + _, err = c.Stat(filePath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + info, err = c.Stat("/another/path/to/file.doc") + require.NoError(t, err) + require.False(t, info.IsDir()) + + checkErrorPath(t, + c.Rename("/notanother/path/to/file.txt", "/another/path/to/file.doc"), + "/notanother/path/to/file.txt") + + checkErrorPath(t, + c.Rename(filePath, "/notanother/path/to/file.doc"), + "/notanother/path/to/file.doc") + }) + + t.Run("Remove", func(t *testing.T) { + var removeFilePath = "/another/file/to/remove.txt" + _, err := c.Create(removeFilePath) + require.NoError(t, err) + + require.NoError(t, c.Remove(removeFilePath)) + + _, err = c.Stat(removeFilePath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = c.Create(removeFilePath) + require.NoError(t, err) + + require.NoError(t, c.RemoveAll(path.Dir(removeFilePath))) + + _, err = c.Stat(removeFilePath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + checkErrorPath(t, + c.Remove("/notanother/path/to/file.txt"), + "/notanother/path/to/file.txt") + + checkErrorPath(t, + c.RemoveAll("/notanother/path/to"), + "/notanother/path/to") + }) +} + +func checkErrorPath(t *testing.T, err error, path string) { + require.Error(t, err) + p, ok := err.(*os.PathError) + require.True(t, ok) + require.Equal(t, p.Path, path) + +} From 4bdd575d4b3ffd2aad50988a27e3565e455b296e Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 10 Jul 2019 14:01:21 +0300 Subject: [PATCH 20/45] Check usage of CacheOnReadFs in archive --- lib/archive_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/archive_test.go b/lib/archive_test.go index 1d4deb1bf45..a2b92067fea 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -30,6 +30,7 @@ import ( "testing" "github.com/loadimpact/k6/lib/consts" + "github.com/loadimpact/k6/lib/fsext" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -259,3 +260,36 @@ func TestArchiveJSONEscape(t *testing.T) { assert.NoError(t, err) assert.Contains(t, string(b), "test<.js") } + +func TestUsingCacheFromCacheOnReadFs(t *testing.T) { + var base = afero.NewMemMapFs() + var cached = afero.NewMemMapFs() + // we specifically have different contents in both places + require.NoError(t, afero.WriteFile(base, "/wrong", []byte(`ooops`), 0644)) + require.NoError(t, afero.WriteFile(cached, "/correct", []byte(`test`), 0644)) + + arc := &Archive{ + Type: "js", + FilenameURL: &url.URL{Scheme: "file", Path: "/somewhere"}, + K6Version: consts.Version, + Data: []byte(`// contents...`), + PwdURL: &url.URL{Scheme: "file", Path: "/"}, + Filesystems: map[string]afero.Fs{ + "file": fsext.NewCacheOnReadFs(base, cached, 0), + }, + } + + buf := bytes.NewBuffer(nil) + require.NoError(t, arc.Write(buf)) + + newArc, err := ReadArchive(buf) + assert.NoError(t, err) + + data, err := afero.ReadFile(newArc.Filesystems["file"], "/correct") + require.NoError(t, err) + require.Equal(t, string(data), "test") + + data, err = afero.ReadFile(newArc.Filesystems["file"], "/wrong") + require.Error(t, err) + require.Nil(t, data) +} From 5d40270b167e2caf4e1b552b018de533a4372340 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 10 Jul 2019 14:13:17 +0300 Subject: [PATCH 21/45] Add test for LstatIfPossible to ChangePathFs --- lib/fsext/changepathfs_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/fsext/changepathfs_test.go b/lib/fsext/changepathfs_test.go index 4e51bb777e5..5f92b90ee26 100644 --- a/lib/fsext/changepathfs_test.go +++ b/lib/fsext/changepathfs_test.go @@ -91,6 +91,16 @@ func TestChangePathFs(t *testing.T) { checkErrorPath(t, c.Chmod("/notanother/path/to/file.txt", mode), "/notanother/path/to/file.txt") }) + t.Run("LstatIfPossible", func(t *testing.T) { + info, ok, err := c.LstatIfPossible(filePath) + require.NoError(t, err) + require.False(t, ok) + require.Equal(t, "file.txt", info.Name()) + + _, _, err = c.LstatIfPossible("/notanother/path/to/file.txt") + checkErrorPath(t, err, "/notanother/path/to/file.txt") + }) + t.Run("Rename", func(t *testing.T) { info, err := c.Stat(filePath) require.NoError(t, err) From 518e58f9b5d02c8a29c0db65cf74c202b30d743f Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 10 Jul 2019 14:18:13 +0300 Subject: [PATCH 22/45] Fix wronly calling Mkdir instead of MkdirAll in test --- lib/fsext/changepathfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fsext/changepathfs_test.go b/lib/fsext/changepathfs_test.go index 5f92b90ee26..81251322fec 100644 --- a/lib/fsext/changepathfs_test.go +++ b/lib/fsext/changepathfs_test.go @@ -47,7 +47,7 @@ func TestChangePathFs(t *testing.T) { t.Run("MkdirAll", func(t *testing.T) { require.NoError(t, c.MkdirAll("/another/pattth/too", 0644)) - checkErrorPath(t, c.Mkdir("/notanother/pattth/too", 0644), "/notanother/pattth/too") + checkErrorPath(t, c.MkdirAll("/notanother/pattth/too", 0644), "/notanother/pattth/too") }) t.Run("Open", func(t *testing.T) { From 34809f113e844febe7cec3b66db1014331c7ff51 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 10 Jul 2019 14:18:33 +0300 Subject: [PATCH 23/45] test a little bit more of the ChangePathFs.OpenFile code --- lib/fsext/changepathfs_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/fsext/changepathfs_test.go b/lib/fsext/changepathfs_test.go index 81251322fec..5e91ea77f97 100644 --- a/lib/fsext/changepathfs_test.go +++ b/lib/fsext/changepathfs_test.go @@ -66,6 +66,9 @@ func TestChangePathFs(t *testing.T) { _, err = c.OpenFile("/notanother/path/to/file.txt", os.O_RDWR, 0644) checkErrorPath(t, err, "/notanother/path/to/file.txt") + + _, err = c.OpenFile("/another/nonexistant", os.O_RDWR, 0644) + require.True(t, os.IsNotExist(err)) }) t.Run("Stat Chmod Chtimes", func(t *testing.T) { From 701df42c21a385933ad937312a5f30b63f792b5c Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 10 Jul 2019 14:29:05 +0300 Subject: [PATCH 24/45] Add TrimPathSeparatorFs test --- lib/fsext/trimpathseparator_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 lib/fsext/trimpathseparator_test.go diff --git a/lib/fsext/trimpathseparator_test.go b/lib/fsext/trimpathseparator_test.go new file mode 100644 index 00000000000..3e55d0eff43 --- /dev/null +++ b/lib/fsext/trimpathseparator_test.go @@ -0,0 +1,25 @@ +package fsext + +import ( + "os" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestTrimAferoPathSeparatorFs(t *testing.T) { + m := afero.NewMemMapFs() + fs := NewTrimFilePathSeparatorFs(m) + expecteData := []byte("something") + err := afero.WriteFile(fs, "/path/to/somewhere", expecteData, 0644) + require.NoError(t, err) + data, err := afero.ReadFile(m, "/path/to/somewhere") + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + require.Nil(t, data) + + data, err = afero.ReadFile(m, "path/to/somewhere") + require.NoError(t, err) + require.Equal(t, expecteData, data) +} From fbda1ae9305bb8b95b7c259d33fed01194d03484 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 10 Jul 2019 14:44:27 +0300 Subject: [PATCH 25/45] Fix trimpathseparator on windows --- lib/fsext/trimpathseparator_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/fsext/trimpathseparator_test.go b/lib/fsext/trimpathseparator_test.go index 3e55d0eff43..a3ca40c34f5 100644 --- a/lib/fsext/trimpathseparator_test.go +++ b/lib/fsext/trimpathseparator_test.go @@ -2,6 +2,7 @@ package fsext import ( "os" + "path/filepath" "testing" "github.com/spf13/afero" @@ -12,7 +13,7 @@ func TestTrimAferoPathSeparatorFs(t *testing.T) { m := afero.NewMemMapFs() fs := NewTrimFilePathSeparatorFs(m) expecteData := []byte("something") - err := afero.WriteFile(fs, "/path/to/somewhere", expecteData, 0644) + err := afero.WriteFile(fs, filepath.FromSlash("/path/to/somewhere"), expecteData, 0644) require.NoError(t, err) data, err := afero.ReadFile(m, "/path/to/somewhere") require.Error(t, err) From d7be44611dd67e3abb2398c7f1a45d529f2cb7cd Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 10 Jul 2019 14:47:28 +0300 Subject: [PATCH 26/45] Add to TestTrimAferoPathSeparatorFs --- lib/fsext/trimpathseparator_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/fsext/trimpathseparator_test.go b/lib/fsext/trimpathseparator_test.go index a3ca40c34f5..678b0c1b758 100644 --- a/lib/fsext/trimpathseparator_test.go +++ b/lib/fsext/trimpathseparator_test.go @@ -23,4 +23,8 @@ func TestTrimAferoPathSeparatorFs(t *testing.T) { data, err = afero.ReadFile(m, "path/to/somewhere") require.NoError(t, err) require.Equal(t, expecteData, data) + + err = afero.WriteFile(fs, filepath.FromSlash("path/without/separtor"), expecteData, 0644) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) } From 8f34ca761d79d37e2c165c543227acfba6cddf26 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 11 Jul 2019 11:26:54 +0300 Subject: [PATCH 27/45] Fix file and directory permissions and times --- lib/archive.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/archive.go b/lib/archive.go index 02402b30ba8..c033635de98 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -202,6 +202,7 @@ func normalizeAndAnonymizeURL(u *url.URL) { func (arc *Archive) Write(out io.Writer) error { w := tar.NewWriter(out) + now := time.Now() metaArc := *arc normalizeAndAnonymizeURL(metaArc.FilenameURL) normalizeAndAnonymizeURL(metaArc.PwdURL) @@ -215,7 +216,7 @@ func (arc *Archive) Write(out io.Writer) error { Name: "metadata.json", Mode: 0644, Size: int64(len(metadata)), - ModTime: time.Now(), + ModTime: now, Typeflag: tar.TypeReg, }) if _, err = w.Write(metadata); err != nil { @@ -226,7 +227,7 @@ func (arc *Archive) Write(out io.Writer) error { Name: "data", Mode: 0644, Size: int64(len(arc.Data)), - ModTime: time.Now(), + ModTime: now, Typeflag: tar.TypeReg, }) if _, err = w.Write(arc.Data); err != nil { @@ -286,10 +287,10 @@ func (arc *Archive) Write(out io.Writer) error { for _, dirPath := range dirs { _ = w.WriteHeader(&tar.Header{ Name: path.Clean(path.Join(name, dirPath)), - Mode: int64(infos[dirPath].Mode()), - AccessTime: infos[dirPath].ModTime(), - ChangeTime: infos[dirPath].ModTime(), - ModTime: infos[dirPath].ModTime(), + Mode: 0755, // MemMapFs is buggy + AccessTime: now, // MemMapFs is buggy + ChangeTime: now, // MemMapFs is buggy + ModTime: now, // MemMapFs is buggy Typeflag: tar.TypeDir, }) } @@ -297,7 +298,7 @@ func (arc *Archive) Write(out io.Writer) error { for _, filePath := range paths { _ = w.WriteHeader(&tar.Header{ Name: path.Clean(path.Join(name, filePath)), - Mode: int64(infos[filePath].Mode()), + Mode: 0644, // MemMapFs is buggy Size: int64(len(files[filePath])), AccessTime: infos[filePath].ModTime(), ChangeTime: infos[filePath].ModTime(), From df56303b2f2ff4731216cf5d3c890c409c6e406b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 11 Jul 2019 18:00:40 +0300 Subject: [PATCH 28/45] Hardlink the to the 'data' file in the archive This not only saves space but in the future paves the way for dropping the keeping of the data in the archive as a separate field and just keep in the filesystem abstraction layer --- cmd/runtime_options_test.go | 7 ++- js/runner_test.go | 8 ++-- lib/archive.go | 54 +++++++++++++++++----- lib/archive_test.go | 18 ++++---- lib/old_archive_test.go | 90 ++++++++++++++++++++++--------------- 5 files changed, 115 insertions(+), 62 deletions(-) diff --git a/cmd/runtime_options_test.go b/cmd/runtime_options_test.go index 183031b9c8c..7be2e8c0245 100644 --- a/cmd/runtime_options_test.go +++ b/cmd/runtime_options_test.go @@ -31,6 +31,7 @@ import ( "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/loader" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -219,13 +220,15 @@ func TestEnvVars(t *testing.T) { } } + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(jsCode), 0644)) runner, err := newRunner( &loader.SourceData{ Data: []byte(jsCode), - URL: &url.URL{Path: "/script.js"}, + URL: &url.URL{Path: "/script.js", Scheme: "file"}, }, typeJS, - nil, + map[string]afero.Fs{"file": fs}, rtOpts, ) require.NoError(t, err) diff --git a/js/runner_test.go b/js/runner_test.go index 1053f93c6be..ddcb80fc2f9 100644 --- a/js/runner_test.go +++ b/js/runner_test.go @@ -1296,8 +1296,7 @@ func TestArchiveRunningIntegraty(t *testing.T) { defer tb.Cleanup() fs := afero.NewMemMapFs() - require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm)) - r1, err := getSimpleRunnerWithFileFs("/script.js", tb.Replacer.Replace(` + data := tb.Replacer.Replace(` let fput = open("/home/somebody/test.json"); export let options = { setupTimeout: "10s", teardownTimeout: "10s" }; export function setup() { @@ -1308,7 +1307,10 @@ func TestArchiveRunningIntegraty(t *testing.T) { throw new Error("incorrect answer " + data); } } - `), fs) + `) + require.NoError(t, afero.WriteFile(fs, "/home/somebody/test.json", []byte(`42`), os.ModePerm)) + require.NoError(t, afero.WriteFile(fs, "/script.js", []byte(data), os.ModePerm)) + r1, err := getSimpleRunnerWithFileFs("/script.js", data, fs) require.NoError(t, err) buf := bytes.NewBuffer(nil) diff --git a/lib/archive.go b/lib/archive.go index c033635de98..a81c86f3a63 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -183,7 +183,11 @@ func ReadArchive(in io.Reader) (*Archive, error) { return nil, fmt.Errorf("unknown file prefix `%s` for file `%s`", pfx, normPath) } } - // TODO write the data to pwd in the appropriate archive + scheme, pathOnFs := getURLPathOnFs(arc.FilenameURL) + err := afero.WriteFile(arc.getFs(scheme), pathOnFs, arc.Data, 0644) // TODO fix the mode ? + if err != nil { + return nil, err + } return arc, nil } @@ -194,6 +198,13 @@ func normalizeAndAnonymizeURL(u *url.URL) { } } +func getURLPathOnFs(u *url.URL) (scheme string, pathOnFs string) { + if u.Opaque != "" { + return "https", "/" + u.Opaque + } + return u.Scheme, path.Clean(u.String()[len(u.Scheme)+len(":/"):]) +} + // Write serialises the archive to a writer. // // The format should be treated as opaque; currently it is simply a TAR rollup, but this may @@ -208,6 +219,8 @@ func (arc *Archive) Write(out io.Writer) error { normalizeAndAnonymizeURL(metaArc.PwdURL) metaArc.Filename = metaArc.FilenameURL.String() metaArc.Pwd = metaArc.PwdURL.String() + var actualDataPath = path.Join(getURLPathOnFs(metaArc.FilenameURL)) + var madeLinkToData bool metadata, err := metaArc.json() if err != nil { return err @@ -296,20 +309,39 @@ func (arc *Archive) Write(out io.Writer) error { } for _, filePath := range paths { - _ = w.WriteHeader(&tar.Header{ - Name: path.Clean(path.Join(name, filePath)), - Mode: 0644, // MemMapFs is buggy - Size: int64(len(files[filePath])), - AccessTime: infos[filePath].ModTime(), - ChangeTime: infos[filePath].ModTime(), - ModTime: infos[filePath].ModTime(), - Typeflag: tar.TypeReg, - }) - if _, err := w.Write(files[filePath]); err != nil { + var fullFilePath = path.Clean(path.Join(name, filePath)) + // we either have opaque + if fullFilePath == actualDataPath { + madeLinkToData = true + err = w.WriteHeader(&tar.Header{ + Name: fullFilePath, + Size: 0, + Typeflag: tar.TypeLink, + Linkname: "data", + }) + } else { + err = w.WriteHeader(&tar.Header{ + Name: fullFilePath, + Mode: 0644, // MemMapFs is buggy + Size: int64(len(files[filePath])), + AccessTime: infos[filePath].ModTime(), + ChangeTime: infos[filePath].ModTime(), + ModTime: infos[filePath].ModTime(), + Typeflag: tar.TypeReg, + }) + if err == nil { + _, err = w.Write(files[filePath]) + } + } + if err != nil { return err } } } + if !madeLinkToData { + // This should never happen we should always link to `data` from inside the file/https directories + return fmt.Errorf("archive creation failed because the main script wasn't present in the cached filesystem") + } return w.Close() } diff --git a/lib/archive_test.go b/lib/archive_test.go index a2b92067fea..aad5624f04e 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -139,8 +139,8 @@ func TestArchiveReadWrite(t *testing.T) { VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - FilenameURL: &url.URL{Scheme: "file", Path: "/path/to/script.js"}, - Data: []byte(`// contents...`), + FilenameURL: &url.URL{Scheme: "file", Path: "/path/to/a.js"}, + Data: []byte(`// a contents`), PwdURL: &url.URL{Scheme: "file", Path: "/path/to"}, Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ @@ -189,9 +189,9 @@ func TestArchiveReadWrite(t *testing.T) { VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/script.js", entry.Pwd)}, + FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/a.js", entry.Pwd)}, K6Version: consts.Version, - Data: []byte(`// contents...`), + Data: []byte(`// a contents`), PwdURL: &url.URL{Scheme: "file", Path: entry.Pwd}, Filesystems: map[string]afero.Fs{ "file": makeMemMapFs(t, map[string][]byte{ @@ -212,9 +212,9 @@ func TestArchiveReadWrite(t *testing.T) { VUs: null.IntFrom(12345), SystemTags: GetTagSet(DefaultSystemTagList...), }, - FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/script.js", entry.PwdNormAnon)}, + FilenameURL: &url.URL{Scheme: "file", Path: fmt.Sprintf("%s/a.js", entry.PwdNormAnon)}, K6Version: consts.Version, - Data: []byte(`// contents...`), + Data: []byte(`// a contents`), PwdURL: &url.URL{Scheme: "file", Path: entry.PwdNormAnon}, Filesystems: map[string]afero.Fs{ @@ -270,9 +270,9 @@ func TestUsingCacheFromCacheOnReadFs(t *testing.T) { arc := &Archive{ Type: "js", - FilenameURL: &url.URL{Scheme: "file", Path: "/somewhere"}, + FilenameURL: &url.URL{Scheme: "file", Path: "/correct"}, K6Version: consts.Version, - Data: []byte(`// contents...`), + Data: []byte(`test`), PwdURL: &url.URL{Scheme: "file", Path: "/"}, Filesystems: map[string]afero.Fs{ "file": fsext.NewCacheOnReadFs(base, cached, 0), @@ -283,7 +283,7 @@ func TestUsingCacheFromCacheOnReadFs(t *testing.T) { require.NoError(t, arc.Write(buf)) newArc, err := ReadArchive(buf) - assert.NoError(t, err) + require.NoError(t, err) data, err := afero.ReadFile(newArc.Filesystems["file"], "/correct") require.NoError(t, err) diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go index d5082c5a498..20fb973ed6a 100644 --- a/lib/old_archive_test.go +++ b/lib/old_archive_test.go @@ -59,47 +59,63 @@ func dumpMemMapFsToBuf(fs afero.Fs) (*bytes.Buffer, error) { } func TestOldArchive(t *testing.T) { - fs := makeMemMapFs(t, map[string][]byte{ - // files - "/files/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), - "/files/cdnjs.com/packages/Faker": []byte(`faker file`), - "/files/example.com/path/to.js": []byte(`example.com file`), - "/files/_/C/something/path": []byte(`windows file`), - "/files/_/absolulte/path": []byte(`unix file`), + var testCases = map[string]string{ + // map of filename to data for each main file tested + "github.com/loadimpact/k6/samples/example.js": `github file`, + "cdnjs.com/packages/Faker": `faker file`, + "C:/something/path2": `windows script`, + "/absolulte/path2": `unix script`, + } + for filename, data := range testCases { + filename, data := filename, data + t.Run(filename, func(t *testing.T) { + metadata := `{"filename": "` + filename + `"}` + fs := makeMemMapFs(t, map[string][]byte{ + // files + "/files/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), + "/files/cdnjs.com/packages/Faker": []byte(`faker file`), + "/files/example.com/path/to.js": []byte(`example.com file`), + "/files/_/C/something/path": []byte(`windows file`), + "/files/_/absolulte/path": []byte(`unix file`), - // scripts - "/scripts/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), - "/scripts/cdnjs.com/packages/Faker2": []byte(`faker script`), - "/scripts/example.com/path/too.js": []byte(`example.com script`), - "/scripts/_/C/something/path2": []byte(`windows script`), - "/scripts/_/absolulte/path2": []byte(`unix script`), - }) - buf, err := dumpMemMapFsToBuf(fs) - require.NoError(t, err) + // scripts + "/scripts/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), + "/scripts/cdnjs.com/packages/Faker2": []byte(`faker script`), + "/scripts/example.com/path/too.js": []byte(`example.com script`), + "/scripts/_/C/something/path2": []byte(`windows script`), + "/scripts/_/absolulte/path2": []byte(`unix script`), + "/data": []byte(data), + "/metadata.json": []byte(metadata), + }) - var ( - expectedFilesystems = map[string]afero.Fs{ - "file": makeMemMapFs(t, map[string][]byte{ - "/C:/something/path": []byte(`windows file`), - "/absolulte/path": []byte(`unix file`), - "/C:/something/path2": []byte(`windows script`), - "/absolulte/path2": []byte(`unix script`), - }), - "https": makeMemMapFs(t, map[string][]byte{ - "/example.com/path/to.js": []byte(`example.com file`), - "/example.com/path/too.js": []byte(`example.com script`), - "/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), - "/cdnjs.com/packages/Faker": []byte(`faker file`), - "/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), - "/cdnjs.com/packages/Faker2": []byte(`faker script`), - }), - } - ) + buf, err := dumpMemMapFsToBuf(fs) + require.NoError(t, err) - arc, err := ReadArchive(buf) - require.NoError(t, err) + var ( + expectedFilesystems = map[string]afero.Fs{ + "file": makeMemMapFs(t, map[string][]byte{ + "/C:/something/path": []byte(`windows file`), + "/absolulte/path": []byte(`unix file`), + "/C:/something/path2": []byte(`windows script`), + "/absolulte/path2": []byte(`unix script`), + }), + "https": makeMemMapFs(t, map[string][]byte{ + "/example.com/path/to.js": []byte(`example.com file`), + "/example.com/path/too.js": []byte(`example.com script`), + "/github.com/loadimpact/k6/samples/example.js": []byte(`github file`), + "/cdnjs.com/packages/Faker": []byte(`faker file`), + "/github.com/loadimpact/k6/samples/example.js2": []byte(`github script`), + "/cdnjs.com/packages/Faker2": []byte(`faker script`), + }), + } + ) - diffMapFilesystems(t, expectedFilesystems, arc.Filesystems) + arc, err := ReadArchive(buf) + require.NoError(t, err) + + diffMapFilesystems(t, expectedFilesystems, arc.Filesystems) + }) + } } func TestUnknownPrefix(t *testing.T) { From 6b08d619bc6b65352bf4238966be37729f962e81 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 11 Jul 2019 18:43:12 +0300 Subject: [PATCH 29/45] Don't print the warning for schemeless URLs multiple times --- lib/archive.go | 8 +++-- lib/old_archive_test.go | 14 ++++++-- loader/loader.go | 71 ++++++++++++++++++++++++----------------- loader/loader_test.go | 4 +-- 4 files changed, 60 insertions(+), 37 deletions(-) diff --git a/lib/archive.go b/lib/archive.go index a81c86f3a63..89d8b878591 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -199,10 +199,14 @@ func normalizeAndAnonymizeURL(u *url.URL) { } func getURLPathOnFs(u *url.URL) (scheme string, pathOnFs string) { + scheme = "https" + if u.Scheme != "" { + scheme = u.Scheme + } if u.Opaque != "" { - return "https", "/" + u.Opaque + return scheme, "/" + u.Opaque } - return u.Scheme, path.Clean(u.String()[len(u.Scheme)+len(":/"):]) + return scheme, path.Clean(u.String()[len(u.Scheme)+len(":/"):]) } // Write serialises the archive to a writer. diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go index 20fb973ed6a..61dd6100b24 100644 --- a/lib/old_archive_test.go +++ b/lib/old_archive_test.go @@ -133,7 +133,7 @@ func TestUnknownPrefix(t *testing.T) { func TestFilenamePwdResolve(t *testing.T) { var tests = []struct { - Filename, Pwd string + Filename, Pwd, version string expectedFilenameURL, expectedPwdURL *url.URL }{ { @@ -157,15 +157,23 @@ func TestFilenamePwdResolve(t *testing.T) { { Filename: "example.com/something/dot.js", Pwd: "example.com/something/", + expectedFilenameURL: &url.URL{Host: "example.com", Scheme: "", Path: "/something/dot.js"}, + expectedPwdURL: &url.URL{Host: "example.com", Scheme: "", Path: "/something"}, + }, + { + Filename: "https://example.com/something/dot.js", + Pwd: "https://example.com/something", expectedFilenameURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something/dot.js"}, expectedPwdURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something"}, + version: "0.25.0", }, } for _, test := range tests { metadata := `{ - "Filename": "` + test.Filename + `", - "Pwd": "` + test.Pwd + `" + "filename": "` + test.Filename + `", + "pwd": "` + test.Pwd + `", + "k6version": "` + test.version + `" }` buf, err := dumpMemMapFsToBuf(makeMemMapFs(t, map[string][]byte{ diff --git a/loader/loader.go b/loader/loader.go index 414b1f0a156..aa980a6d390 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -113,12 +113,12 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { // from it we don't want to try another resolve _, loader, _ := pickLoader(moduleSpecifier) if loader == nil { - log.WithField("url", moduleSpecifier).Warning( - "A url was resolved but it didn't have scheme. " + - "This will be deprecated in the future and all remote modules will " + - "need to explicitly use `https` as scheme") - - return url.Parse("https://" + moduleSpecifier) + u, err := url.Parse("https://" + moduleSpecifier) + if err != nil { + return nil, err + } + u.Scheme = "" + return u, nil } return &url.URL{Opaque: moduleSpecifier}, nil } @@ -143,41 +143,52 @@ func Load( "original moduleSpecifier": originalModuleSpecifier, }).Debug("Loading...") - pathOnFs := filepath.FromSlash(path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):])) + scheme := moduleSpecifier.Scheme + if scheme == "" { + scheme = "https" + } + + var pathOnFs string if moduleSpecifier.Opaque != "" { // This is loader pathOnFs = filepath.Join(afero.FilePathSeparator, moduleSpecifier.Opaque) - data, err := afero.ReadFile(filesystems["https"], pathOnFs) - if err != nil { - var finalModuleSpecifierURL *url.URL - finalModuleSpecifierURL, err = resolveUsingLoaders(moduleSpecifier.Opaque) - if err != nil { - return nil, err - } - - var result *SourceData - result, err = loadRemoteURL(finalModuleSpecifierURL) - if err != nil { // TODO: have a separateError - return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, finalModuleSpecifierURL, err) - } - _ = afero.WriteFile(filesystems["https"], pathOnFs, result.Data, 0644) - result.URL = moduleSpecifier - return result, nil - } - return &SourceData{URL: moduleSpecifier, Data: data}, nil + } else if scheme == "" { + pathOnFs = path.Clean(moduleSpecifier.String()) + } else { + pathOnFs = path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):]) } - data, err := afero.ReadFile(filesystems[moduleSpecifier.Scheme], pathOnFs) + + pathOnFs = filepath.FromSlash(pathOnFs) + + data, err := afero.ReadFile(filesystems[scheme], pathOnFs) if err != nil { if os.IsNotExist(err) { - if moduleSpecifier.Scheme == "https" { + if scheme == "https" { + var finalModuleSpecifierURL = &url.URL{} + if moduleSpecifier.Opaque != "" { // This is loader + finalModuleSpecifierURL, err = resolveUsingLoaders(moduleSpecifier.Opaque) + if err != nil { + return nil, err + } + } else if moduleSpecifier.Scheme == "" { + log.WithField("url", moduleSpecifier).Warning( + "A url was resolved but it didn't have scheme. " + + "This will be deprecated in the future and all remote modules will " + + "need to explicitly use `https` as scheme") + *finalModuleSpecifierURL = *moduleSpecifier + finalModuleSpecifierURL.Scheme = "https" + } else { + finalModuleSpecifierURL = moduleSpecifier + } var result *SourceData - result, err = loadRemoteURL(moduleSpecifier) + result, err = loadRemoteURL(finalModuleSpecifierURL) if err != nil { - return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, moduleSpecifier, err) + return nil, errors.Errorf(httpsSchemeCouldntBeLoadedMsg, originalModuleSpecifier, finalModuleSpecifierURL, err) } + result.URL = moduleSpecifier // TODO maybe make an afero.Fs which makes request directly and than use CacheOnReadFs // on top of as with the `file` scheme fs - _ = afero.WriteFile(filesystems[moduleSpecifier.Scheme], pathOnFs, result.Data, 0644) + _ = afero.WriteFile(filesystems[scheme], pathOnFs, result.Data, 0644) return result, nil } return nil, errors.Errorf(fileSchemeCouldntBeLoadedMsg, moduleSpecifier) diff --git a/loader/loader_test.go b/loader/loader_test.go index 00e0dca6cce..5fb1d53444e 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -62,8 +62,8 @@ func TestResolve(t *testing.T) { t.Run("Missing", func(t *testing.T) { u, err := loader.Resolve(root, "example.com/html") require.NoError(t, err) - assert.Equal(t, u.String(), "https://example.com/html") - // TODO: check that warning was emitted + assert.Equal(t, u.String(), "//example.com/html") + // TODO: check that warning will be emited if Loaded }) t.Run("WS", func(t *testing.T) { moduleSpecifier := "ws://example.com/html" From b2b681c76d9e6a8d8689dbc29eb1874ac5eac40e Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 08:11:41 +0300 Subject: [PATCH 30/45] Fixes for golangci and not correctly reading scheme urls from the cache --- .golangci.yml | 2 +- lib/archive.go | 21 +++++++++++++++------ loader/loader.go | 27 ++++++++++++++------------- loader/loader_test.go | 2 +- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index ed38e6f2a27..361c5f6977f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -38,7 +38,7 @@ linters-settings: dupl: threshold: 150 goconst: - min-len: 5 + min-len: 10 min-occurrences: 4 linters: diff --git a/lib/archive.go b/lib/archive.go index 89d8b878591..00440afbbd4 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -200,15 +200,24 @@ func normalizeAndAnonymizeURL(u *url.URL) { func getURLPathOnFs(u *url.URL) (scheme string, pathOnFs string) { scheme = "https" - if u.Scheme != "" { - scheme = u.Scheme - } - if u.Opaque != "" { + switch { + case u.Opaque != "": return scheme, "/" + u.Opaque + case u.Scheme == "": + return scheme, path.Clean(u.String()[len("//"):]) + default: + scheme = u.Scheme } return scheme, path.Clean(u.String()[len(u.Scheme)+len(":/"):]) } +func getURLtoString(u *url.URL) string { + if u.Opaque == "" && u.Scheme == "" { + return u.String()[len("//"):] // https url without a scheme + } + return u.String() +} + // Write serialises the archive to a writer. // // The format should be treated as opaque; currently it is simply a TAR rollup, but this may @@ -221,8 +230,8 @@ func (arc *Archive) Write(out io.Writer) error { metaArc := *arc normalizeAndAnonymizeURL(metaArc.FilenameURL) normalizeAndAnonymizeURL(metaArc.PwdURL) - metaArc.Filename = metaArc.FilenameURL.String() - metaArc.Pwd = metaArc.PwdURL.String() + metaArc.Filename = getURLtoString(metaArc.FilenameURL) + metaArc.Pwd = getURLtoString(metaArc.PwdURL) var actualDataPath = path.Join(getURLPathOnFs(metaArc.FilenameURL)) var madeLinkToData bool metadata, err := metaArc.json() diff --git a/loader/loader.go b/loader/loader.go index aa980a6d390..443111cb06f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -143,41 +143,42 @@ func Load( "original moduleSpecifier": originalModuleSpecifier, }).Debug("Loading...") - scheme := moduleSpecifier.Scheme - if scheme == "" { - scheme = "https" - } - var pathOnFs string - if moduleSpecifier.Opaque != "" { // This is loader + switch { + case moduleSpecifier.Opaque != "": // This is loader pathOnFs = filepath.Join(afero.FilePathSeparator, moduleSpecifier.Opaque) - } else if scheme == "" { + case moduleSpecifier.Scheme == "": pathOnFs = path.Clean(moduleSpecifier.String()) - } else { + default: pathOnFs = path.Clean(moduleSpecifier.String()[len(moduleSpecifier.Scheme)+len(":/"):]) } + scheme := moduleSpecifier.Scheme + if scheme == "" { + scheme = "https" + } pathOnFs = filepath.FromSlash(pathOnFs) - data, err := afero.ReadFile(filesystems[scheme], pathOnFs) if err != nil { if os.IsNotExist(err) { if scheme == "https" { var finalModuleSpecifierURL = &url.URL{} - if moduleSpecifier.Opaque != "" { // This is loader + + switch { + case moduleSpecifier.Opaque != "": // This is loader finalModuleSpecifierURL, err = resolveUsingLoaders(moduleSpecifier.Opaque) if err != nil { return nil, err } - } else if moduleSpecifier.Scheme == "" { + case moduleSpecifier.Scheme == "": log.WithField("url", moduleSpecifier).Warning( "A url was resolved but it didn't have scheme. " + "This will be deprecated in the future and all remote modules will " + "need to explicitly use `https` as scheme") *finalModuleSpecifierURL = *moduleSpecifier - finalModuleSpecifierURL.Scheme = "https" - } else { + finalModuleSpecifierURL.Scheme = scheme + default: finalModuleSpecifierURL = moduleSpecifier } var result *SourceData diff --git a/loader/loader_test.go b/loader/loader_test.go index 5fb1d53444e..52b3a4d8549 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -63,7 +63,7 @@ func TestResolve(t *testing.T) { u, err := loader.Resolve(root, "example.com/html") require.NoError(t, err) assert.Equal(t, u.String(), "//example.com/html") - // TODO: check that warning will be emited if Loaded + // TODO: check that warning will be emitted if Loaded }) t.Run("WS", func(t *testing.T) { moduleSpecifier := "ws://example.com/html" From dee4693df4aef6a0c496aafca55e7999fb139a50 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 10:57:03 +0300 Subject: [PATCH 31/45] Add test testing error on main script not in fs --- lib/archive_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/archive_test.go b/lib/archive_test.go index aad5624f04e..78dff6c8f77 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -293,3 +293,21 @@ func TestUsingCacheFromCacheOnReadFs(t *testing.T) { require.Error(t, err) require.Nil(t, data) } + +func TestArchiveWithDataNotInFS(t *testing.T) { + t.Parallel() + + arc := &Archive{ + Type: "js", + FilenameURL: &url.URL{Scheme: "file", Path: "/script"}, + K6Version: consts.Version, + Data: []byte(`test`), + PwdURL: &url.URL{Scheme: "file", Path: "/"}, + Filesystems: nil, + } + + buf := bytes.NewBuffer(nil) + err := arc.Write(buf) + require.Error(t, err) + require.Contains(t, err.Error(), "the main script wasn't present in the cached filesystem") +} From e6b6ee2ba4e9725314d6df383560e171a1ee406c Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 11:23:05 +0300 Subject: [PATCH 32/45] Add tests for bad filename/pwd in archives The error message is not perfect, but this will be a very strange case either way. Mostly for coverage ;) --- lib/old_archive_test.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/old_archive_test.go b/lib/old_archive_test.go index 61dd6100b24..a72b1d975ce 100644 --- a/lib/old_archive_test.go +++ b/lib/old_archive_test.go @@ -135,6 +135,7 @@ func TestFilenamePwdResolve(t *testing.T) { var tests = []struct { Filename, Pwd, version string expectedFilenameURL, expectedPwdURL *url.URL + expectedError string }{ { Filename: "/home/nobody/something.js", @@ -167,6 +168,19 @@ func TestFilenamePwdResolve(t *testing.T) { expectedPwdURL: &url.URL{Host: "example.com", Scheme: "https", Path: "/something"}, version: "0.25.0", }, + { + Filename: "ftps://example.com/something/dot.js", + Pwd: "https://example.com/something", + expectedError: "only supported schemes for imports are file and https", + version: "0.25.0", + }, + + { + Filename: "https://example.com/something/dot.js", + Pwd: "ftps://example.com/something", + expectedError: "only supported schemes for imports are file and https", + version: "0.25.0", + }, } for _, test := range tests { @@ -182,8 +196,13 @@ func TestFilenamePwdResolve(t *testing.T) { require.NoError(t, err) arc, err := ReadArchive(buf) - require.NoError(t, err) - require.Equal(t, test.expectedFilenameURL, arc.FilenameURL) - require.Equal(t, test.expectedPwdURL, arc.PwdURL) + if test.expectedError != "" { + require.Error(t, err) + require.Contains(t, err.Error(), test.expectedError) + } else { + require.NoError(t, err) + require.Equal(t, test.expectedFilenameURL, arc.FilenameURL) + require.Equal(t, test.expectedPwdURL, arc.PwdURL) + } } } From 1ee2b1697aaa278e5b9c6da0c9fe67f0b4c361c8 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 11:39:27 +0300 Subject: [PATCH 33/45] Add test for malformed metadata in archive --- lib/archive_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/archive_test.go b/lib/archive_test.go index 78dff6c8f77..637a10a9aa3 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -311,3 +311,13 @@ func TestArchiveWithDataNotInFS(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "the main script wasn't present in the cached filesystem") } + +func TestMalformedMetadata(t *testing.T) { + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/metadata.json", []byte("{,}"), 0644)) + var b, err = dumpMemMapFsToBuf(fs) + require.NoError(t, err) + _, err = ReadArchive(b) + require.Error(t, err) + require.Equal(t, err.Error(), `invalid character ',' looking for beginning of object key string`) +} From ab0bc8a5a194c9c3c8fd16a07aff0ab05d6a9bcc Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 14:35:22 +0300 Subject: [PATCH 34/45] s/travers/traverse --- cmd/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/run.go b/cmd/run.go index bd7f7265ff9..2361467664a 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -513,7 +513,7 @@ func createFilesystems() map[string]afero.Fs { osfs := afero.NewOsFs() if runtime.GOOS == "windows" { // This is done so that we can continue to use paths with /|"\" through the code but also to - // be easier to travers the cachedFs later as it doesn't work very well if you have windows + // be easier to traverse the cachedFs later as it doesn't work very well if you have windows // volumes osfs = fsext.NewTrimFilePathSeparatorFs(osfs) } From 55b90d09562eb7ab8ccc35fecd1f21dcbbf199ee Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 18:34:40 +0300 Subject: [PATCH 35/45] Fix relative and absolute paths on windows --- cmd/run.go | 7 ++++--- lib/archive.go | 5 ++++- loader/loader.go | 13 ++++++++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 2361467664a..a04a9b14dcb 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -532,12 +532,13 @@ func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reade } return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil } - pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} - srcLocalPath := filepath.Join(pwd, src) + srcLocalPath := filepath.Clean(afero.FilePathSeparator + filepath.Join(pwd, src)) if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok { // there is file on the local disk ... lets use it :) - return loader.Load(filesystems, &url.URL{Scheme: "file", Path: srcLocalPath}, src) + return loader.Load(filesystems, &url.URL{Scheme: "file", Path: filepath.ToSlash(srcLocalPath)}, src) } + + pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} srcURL, err := loader.Resolve(pwdURL, filepath.ToSlash(src)) if err != nil { return nil, err diff --git a/lib/archive.go b/lib/archive.go index 00440afbbd4..b37e40237fe 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -232,7 +232,10 @@ func (arc *Archive) Write(out io.Writer) error { normalizeAndAnonymizeURL(metaArc.PwdURL) metaArc.Filename = getURLtoString(metaArc.FilenameURL) metaArc.Pwd = getURLtoString(metaArc.PwdURL) - var actualDataPath = path.Join(getURLPathOnFs(metaArc.FilenameURL)) + var actualDataPath, err = url.PathUnescape(path.Join(getURLPathOnFs(metaArc.FilenameURL))) + if err != nil { + return err + } var madeLinkToData bool metadata, err := metaArc.json() if err != nil { diff --git a/loader/loader.go b/loader/loader.go index 443111cb06f..461cc25db1c 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -81,8 +81,7 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { } else if !strings.HasSuffix(pwd.Path, "/") { pwd.Path += "/" } - - if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' { + if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' || filepath.IsAbs(moduleSpecifier) { if pwd.Opaque != "" { // this is a loader reference parts := strings.SplitN(pwd.Opaque, "/", 2) if moduleSpecifier[0] == '/' { @@ -91,6 +90,10 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { return &url.URL{Opaque: path.Join(parts[0], path.Join(path.Dir(parts[1]), moduleSpecifier))}, nil } + if filepath.VolumeName(moduleSpecifier) != "" { + return pwd.Parse("/" + filepath.ToSlash(moduleSpecifier)) + } + return pwd.Parse(moduleSpecifier) } @@ -157,7 +160,11 @@ func Load( scheme = "https" } - pathOnFs = filepath.FromSlash(pathOnFs) + pathOnFs, err := url.PathUnescape(filepath.FromSlash(pathOnFs)) + if err != nil { + return nil, err + } + data, err := afero.ReadFile(filesystems[scheme], pathOnFs) if err != nil { From 97a658e39b13bd5fe9acf448d9e3ecddff1e21d9 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 19:28:30 +0300 Subject: [PATCH 36/45] loader.Resolve: Don't change pwd when it's missing slash on the end --- loader/loader.go | 29 +++++++++++++++++------------ loader/loader_test.go | 2 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 461cc25db1c..508266e60b2 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -72,29 +72,34 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { return nil, errors.New("local or remote path required") } - // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we readd - // it if it's missing - if pwd.Opaque != "" { - if !strings.HasSuffix(pwd.Opaque, "/") { - pwd.Opaque += "/" - } - } else if !strings.HasSuffix(pwd.Path, "/") { - pwd.Path += "/" - } if moduleSpecifier[0] == '.' || moduleSpecifier[0] == '/' || filepath.IsAbs(moduleSpecifier) { if pwd.Opaque != "" { // this is a loader reference parts := strings.SplitN(pwd.Opaque, "/", 2) if moduleSpecifier[0] == '/' { return &url.URL{Opaque: path.Join(parts[0], moduleSpecifier)}, nil } - return &url.URL{Opaque: path.Join(parts[0], path.Join(path.Dir(parts[1]), moduleSpecifier))}, nil + return &url.URL{Opaque: path.Join(parts[0], path.Join(path.Dir(parts[1]+"/"), moduleSpecifier))}, nil } + // The file is in format like C:/something/path.js. But this will be decoded as scheme `C` + // ... which is not what we want we want it to be decode as file:///C:/something/path.js if filepath.VolumeName(moduleSpecifier) != "" { - return pwd.Parse("/" + filepath.ToSlash(moduleSpecifier)) + moduleSpecifier = "/" + moduleSpecifier } - return pwd.Parse(moduleSpecifier) + // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we readd + // it if it's missing + var finalPwd = pwd + if pwd.Opaque != "" { + if !strings.HasSuffix(pwd.Opaque, "/") { + finalPwd = &url.URL{Opaque: pwd.Opaque + "/"} + } + } else if !strings.HasSuffix(pwd.Path, "/") { + finalPwd = &url.URL{} + *finalPwd = *pwd + finalPwd.Path += "/" + } + return finalPwd.Parse(moduleSpecifier) } if strings.Contains(moduleSpecifier, "://") { diff --git a/loader/loader_test.go b/loader/loader_test.go index 52b3a4d8549..745d6007509 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -95,7 +95,7 @@ func TestResolve(t *testing.T) { moduleURL, err := loader.Resolve(pwdURL, "./something") require.NoError(t, err) require.Equal(t, "https://example.com/path/to/something", moduleURL.String()) - require.Equal(t, "https://example.com/path/to/", pwdURL.String()) + require.Equal(t, "https://example.com/path/to", pwdURL.String()) }) } From 723c49eeae50805e6794b2488043cf025fa70f4b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Mon, 15 Jul 2019 19:35:48 +0300 Subject: [PATCH 37/45] If a command gets an absolute path don't try it as relative --- cmd/run.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/run.go b/cmd/run.go index a04a9b14dcb..0d4cbc94bdb 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -532,7 +532,13 @@ func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reade } return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil } - srcLocalPath := filepath.Clean(afero.FilePathSeparator + filepath.Join(pwd, src)) + var srcLocalPath string + if filepath.IsAbs(src) { + srcLocalPath = src + } else { + srcLocalPath = filepath.Join(pwd, src) + } + srcLocalPath = filepath.Clean(afero.FilePathSeparator + srcLocalPath) if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok { // there is file on the local disk ... lets use it :) return loader.Load(filesystems, &url.URL{Scheme: "file", Path: filepath.ToSlash(srcLocalPath)}, src) From 3b3b90528a30beeab622d30cc6e8e8ba9224ed90 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 16 Jul 2019 09:03:50 +0300 Subject: [PATCH 38/45] Add test with funky paths to the archive --- lib/archive.go | 7 ++++++- lib/archive_test.go | 48 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/lib/archive.go b/lib/archive.go index b37e40237fe..4b0a857c049 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -184,7 +184,12 @@ func ReadArchive(in io.Reader) (*Archive, error) { } } scheme, pathOnFs := getURLPathOnFs(arc.FilenameURL) - err := afero.WriteFile(arc.getFs(scheme), pathOnFs, arc.Data, 0644) // TODO fix the mode ? + var err error + pathOnFs, err = url.PathUnescape(pathOnFs) + if err != nil { + return nil, err + } + err = afero.WriteFile(arc.getFs(scheme), pathOnFs, arc.Data, 0644) // TODO fix the mode ? if err != nil { return nil, err } diff --git a/lib/archive_test.go b/lib/archive_test.go index 637a10a9aa3..4527d7fb90b 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -25,6 +25,7 @@ import ( "fmt" "net/url" "os" + "path" "path/filepath" "runtime" "testing" @@ -321,3 +322,50 @@ func TestMalformedMetadata(t *testing.T) { require.Error(t, err) require.Equal(t, err.Error(), `invalid character ',' looking for beginning of object key string`) } + +func TestStrangePaths(t *testing.T) { + var pathsToChange = []string{ + `/path/with spaces/a.js`, + `/path/with spaces/a.js`, + `/path/with日本語/b.js`, + `/path/with spaces and 日本語/file1.txt`, + } + for _, pathToChange := range pathsToChange { + otherMap := make(map[string][]byte, len(pathsToChange)) + for _, other := range pathsToChange { + otherMap[other] = []byte(`// ` + other + ` contents`) + } + arc1 := &Archive{ + Type: "js", + K6Version: consts.Version, + Options: Options{ + VUs: null.IntFrom(12345), + SystemTags: GetTagSet(DefaultSystemTagList...), + }, + FilenameURL: &url.URL{Scheme: "file", Path: pathToChange}, + Data: []byte(`// ` + pathToChange + ` contents`), + PwdURL: &url.URL{Scheme: "file", Path: path.Dir(pathToChange)}, + Filesystems: map[string]afero.Fs{ + "file": makeMemMapFs(t, otherMap), + }, + } + + buf := bytes.NewBuffer(nil) + require.NoError(t, arc1.Write(buf), pathToChange) + + arc1Filesystems := arc1.Filesystems + arc1.Filesystems = nil + + arc2, err := ReadArchive(buf) + require.NoError(t, err, pathToChange) + + arc2Filesystems := arc2.Filesystems + arc2.Filesystems = nil + arc2.Filename = "" + arc2.Pwd = "" + + assert.Equal(t, arc1, arc2, pathToChange) + + diffMapFilesystems(t, arc1Filesystems, arc2Filesystems) + } +} From 67c6b135b9780dfd969412a7f2c4efa65c56176b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 16 Jul 2019 16:04:44 +0300 Subject: [PATCH 39/45] 100% test coverage for cmd#readSource --- cmd/read_source_test.go | 87 +++++++++++++++++++++++++++++++++++++++++ cmd/run.go | 3 +- 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 cmd/read_source_test.go diff --git a/cmd/read_source_test.go b/cmd/read_source_test.go new file mode 100644 index 00000000000..57b8d62bb7b --- /dev/null +++ b/cmd/read_source_test.go @@ -0,0 +1,87 @@ +package cmd + +import ( + "bytes" + "io" + "net/url" + "testing" + + "github.com/loadimpact/k6/loader" + "github.com/pkg/errors" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +type errorReader string + +func (e errorReader) Read(_ []byte) (int, error) { + return 0, errors.New((string)(e)) +} + +var _ io.Reader = errorReader("") + +func TestReadSourceSTDINError(t *testing.T) { + _, err := readSource("-", "", nil, errorReader("1234")) + require.Error(t, err) + require.Equal(t, "1234", err.Error()) +} + +func TestReadSourceSTDINCache(t *testing.T) { + var data = []byte(`test contents`) + var r = bytes.NewReader(data) + var fs = afero.NewMemMapFs() + sourceData, err := readSource("-", "", map[string]afero.Fs{"file": fs}, r) + require.NoError(t, err) + require.Equal(t, &loader.SourceData{ + URL: &url.URL{Scheme: "file", Path: "-"}, + Data: data}, sourceData) + fileData, err := afero.ReadFile(fs, "-") + require.NoError(t, err) + require.Equal(t, data, fileData) +} + +func TestReadSourceRelative(t *testing.T) { + var data = []byte(`test contents`) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/path/to/somewhere/script.js", data, 0644)) + sourceData, err := readSource("../somewhere/script.js", "/path/to/pwd", map[string]afero.Fs{"file": fs}, nil) + require.NoError(t, err) + require.Equal(t, &loader.SourceData{ + URL: &url.URL{Scheme: "file", Path: "/path/to/somewhere/script.js"}, + Data: data}, sourceData) +} + +func TestReadSourceAbsolute(t *testing.T) { + var data = []byte(`test contents`) + var r = bytes.NewReader(data) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/a/b", data, 0644)) + require.NoError(t, afero.WriteFile(fs, "/c/a/b", []byte("wrong"), 0644)) + sourceData, err := readSource("/a/b", "/c", map[string]afero.Fs{"file": fs}, r) + require.NoError(t, err) + require.Equal(t, &loader.SourceData{ + URL: &url.URL{Scheme: "file", Path: "/a/b"}, + Data: data}, sourceData) +} + +func TestReadSourceHttps(t *testing.T) { + var data = []byte(`test contents`) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644)) + sourceData, err := readSource("https://github.com/something", "/c", + map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil) + require.NoError(t, err) + require.Equal(t, &loader.SourceData{ + URL: &url.URL{Scheme: "https", Host: "github.com", Path: "/something"}, + Data: data}, sourceData) +} + +func TestReadSourceHttpError(t *testing.T) { + var data = []byte(`test contents`) + var fs = afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644)) + _, err := readSource("http://github.com/something", "/c", + map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil) + require.Error(t, err) + require.Contains(t, err.Error(), `only supported schemes for imports are file and https`) +} diff --git a/cmd/run.go b/cmd/run.go index 0d4cbc94bdb..74559138ec4 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -530,7 +530,8 @@ func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reade if err != nil { return nil, err } - return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, nil + err = afero.WriteFile(filesystems["file"], "-", data, 0644) + return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, err } var srcLocalPath string if filepath.IsAbs(src) { From fd5d380d50c3579b5918897ad80c28576c2c9fbf Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 16 Jul 2019 17:21:39 +0300 Subject: [PATCH 40/45] Support running and archiving when giving scripts with stdin --- cmd/read_source_test.go | 8 +++++--- cmd/run.go | 7 +++++-- lib/archive_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/cmd/read_source_test.go b/cmd/read_source_test.go index 57b8d62bb7b..07f1b7a7b3b 100644 --- a/cmd/read_source_test.go +++ b/cmd/read_source_test.go @@ -6,6 +6,7 @@ import ( "net/url" "testing" + "github.com/loadimpact/k6/lib/fsext" "github.com/loadimpact/k6/loader" "github.com/pkg/errors" "github.com/spf13/afero" @@ -30,12 +31,13 @@ func TestReadSourceSTDINCache(t *testing.T) { var data = []byte(`test contents`) var r = bytes.NewReader(data) var fs = afero.NewMemMapFs() - sourceData, err := readSource("-", "", map[string]afero.Fs{"file": fs}, r) + sourceData, err := readSource("-", "/path/to/pwd", + map[string]afero.Fs{"file": fsext.NewCacheOnReadFs(nil, fs, 0)}, r) require.NoError(t, err) require.Equal(t, &loader.SourceData{ - URL: &url.URL{Scheme: "file", Path: "-"}, + URL: &url.URL{Scheme: "file", Path: "/-"}, Data: data}, sourceData) - fileData, err := afero.ReadFile(fs, "-") + fileData, err := afero.ReadFile(fs, "/-") require.NoError(t, err) require.Equal(t, data, fileData) } diff --git a/cmd/run.go b/cmd/run.go index 74559138ec4..8a592ef7af6 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -530,8 +530,11 @@ func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reade if err != nil { return nil, err } - err = afero.WriteFile(filesystems["file"], "-", data, 0644) - return &loader.SourceData{URL: &url.URL{Path: "-", Scheme: "file"}, Data: data}, err + err = afero.WriteFile(filesystems["file"].(fsext.CacheOnReadFs).GetCachingFs(), "/-", data, 0644) + if err != nil { + return nil, errors.Wrap(err, "caching data read from -") + } + return &loader.SourceData{URL: &url.URL{Path: "/-", Scheme: "file"}, Data: data}, err } var srcLocalPath string if filepath.IsAbs(src) { diff --git a/lib/archive_test.go b/lib/archive_test.go index 4527d7fb90b..19e283a07d7 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -369,3 +369,31 @@ func TestStrangePaths(t *testing.T) { diffMapFilesystems(t, arc1Filesystems, arc2Filesystems) } } + +func TestStdinArchive(t *testing.T) { + var fs = afero.NewMemMapFs() + // we specifically have different contents in both places + require.NoError(t, afero.WriteFile(fs, "/-", []byte(`test`), 0644)) + + arc := &Archive{ + Type: "js", + FilenameURL: &url.URL{Scheme: "file", Path: "/-"}, + K6Version: consts.Version, + Data: []byte(`test`), + PwdURL: &url.URL{Scheme: "file", Path: "/"}, + Filesystems: map[string]afero.Fs{ + "file": fs, + }, + } + + buf := bytes.NewBuffer(nil) + require.NoError(t, arc.Write(buf)) + + newArc, err := ReadArchive(buf) + require.NoError(t, err) + + data, err := afero.ReadFile(newArc.Filesystems["file"], "/-") + require.NoError(t, err) + require.Equal(t, string(data), "test") + +} From 60f229587cad404ae9e111110a4b73e85b3ce6dd Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 16 Jul 2019 17:35:55 +0300 Subject: [PATCH 41/45] Move cmd#readSource to loader#ReadSource --- cmd/archive.go | 3 +- cmd/cloud.go | 3 +- cmd/inspect.go | 3 +- cmd/run.go | 39 +-------------- loader/readsource.go | 48 +++++++++++++++++++ .../readsource_test.go | 23 +++++---- 6 files changed, 66 insertions(+), 53 deletions(-) create mode 100644 loader/readsource.go rename cmd/read_source_test.go => loader/readsource_test.go (81%) diff --git a/cmd/archive.go b/cmd/archive.go index 30b36f0e4ba..3d3eb5aed5d 100644 --- a/cmd/archive.go +++ b/cmd/archive.go @@ -23,6 +23,7 @@ package cmd import ( "os" + "github.com/loadimpact/k6/loader" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -52,7 +53,7 @@ An archive is a fully self-contained test run, and can be executed identically e } filename := args[0] filesystems := createFilesystems() - src, err := readSource(filename, pwd, filesystems, os.Stdin) + src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } diff --git a/cmd/cloud.go b/cmd/cloud.go index 9cd098124b9..dccb8bbdb77 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -32,6 +32,7 @@ import ( "github.com/kelseyhightower/envconfig" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/consts" + "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/stats/cloud" "github.com/loadimpact/k6/ui" "github.com/pkg/errors" @@ -72,7 +73,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud filename := args[0] filesystems := createFilesystems() - src, err := readSource(filename, pwd, filesystems, os.Stdin) + src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } diff --git a/cmd/inspect.go b/cmd/inspect.go index 9a26b49e0f7..2640fefdda1 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -28,6 +28,7 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/loader" "github.com/spf13/cobra" ) @@ -43,7 +44,7 @@ var inspectCmd = &cobra.Command{ return err } filesystems := createFilesystems() - src, err := readSource(args[0], pwd, filesystems, os.Stdin) + src, err := loader.ReadSource(args[0], pwd, filesystems, os.Stdin) if err != nil { return err } diff --git a/cmd/run.go b/cmd/run.go index 8a592ef7af6..5aeff46e103 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -26,13 +26,9 @@ import ( "context" "encoding/json" "fmt" - "io" - "io/ioutil" "net/http" - "net/url" "os" "os/signal" - "path/filepath" "runtime" "strings" "syscall" @@ -119,7 +115,7 @@ a commandline interface for interacting with it.`, } filename := args[0] filesystems := createFilesystems() - src, err := readSource(filename, pwd, filesystems, os.Stdin) + src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err } @@ -523,39 +519,6 @@ func createFilesystems() map[string]afero.Fs { } } -// Reads a source file from any supported destination. -func readSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*loader.SourceData, error) { - if src == "-" { - data, err := ioutil.ReadAll(stdin) - if err != nil { - return nil, err - } - err = afero.WriteFile(filesystems["file"].(fsext.CacheOnReadFs).GetCachingFs(), "/-", data, 0644) - if err != nil { - return nil, errors.Wrap(err, "caching data read from -") - } - return &loader.SourceData{URL: &url.URL{Path: "/-", Scheme: "file"}, Data: data}, err - } - var srcLocalPath string - if filepath.IsAbs(src) { - srcLocalPath = src - } else { - srcLocalPath = filepath.Join(pwd, src) - } - srcLocalPath = filepath.Clean(afero.FilePathSeparator + srcLocalPath) - if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok { - // there is file on the local disk ... lets use it :) - return loader.Load(filesystems, &url.URL{Scheme: "file", Path: filepath.ToSlash(srcLocalPath)}, src) - } - - pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} - srcURL, err := loader.Resolve(pwdURL, filepath.ToSlash(src)) - if err != nil { - return nil, err - } - return loader.Load(filesystems, srcURL, src) -} - // Creates a new runner. func newRunner( src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions, diff --git a/loader/readsource.go b/loader/readsource.go new file mode 100644 index 00000000000..3efd4ce85a5 --- /dev/null +++ b/loader/readsource.go @@ -0,0 +1,48 @@ +package loader + +import ( + "io" + "io/ioutil" + "net/url" + "path/filepath" + + "github.com/loadimpact/k6/lib/fsext" + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +// ReadSource Reads a source file from any supported destination. +func ReadSource(src, pwd string, filesystems map[string]afero.Fs, stdin io.Reader) (*SourceData, error) { + if src == "-" { + data, err := ioutil.ReadAll(stdin) + if err != nil { + return nil, err + } + // TODO: don't do it in this way ... + err = afero.WriteFile(filesystems["file"].(fsext.CacheOnReadFs).GetCachingFs(), "/-", data, 0644) + if err != nil { + return nil, errors.Wrap(err, "caching data read from -") + } + return &SourceData{URL: &url.URL{Path: "/-", Scheme: "file"}, Data: data}, err + } + var srcLocalPath string + if filepath.IsAbs(src) { + srcLocalPath = src + } else { + srcLocalPath = filepath.Join(pwd, src) + } + // All paths should start with a / in all fses. This is mostly for windows where it will start + // with a volume name : C:\something.js + srcLocalPath = filepath.Clean(afero.FilePathSeparator + srcLocalPath) + if ok, _ := afero.Exists(filesystems["file"], srcLocalPath); ok { + // there is file on the local disk ... lets use it :) + return Load(filesystems, &url.URL{Scheme: "file", Path: filepath.ToSlash(srcLocalPath)}, src) + } + + pwdURL := &url.URL{Scheme: "file", Path: filepath.ToSlash(filepath.Clean(pwd)) + "/"} + srcURL, err := Resolve(pwdURL, filepath.ToSlash(src)) + if err != nil { + return nil, err + } + return Load(filesystems, srcURL, src) +} diff --git a/cmd/read_source_test.go b/loader/readsource_test.go similarity index 81% rename from cmd/read_source_test.go rename to loader/readsource_test.go index 07f1b7a7b3b..e962f304f87 100644 --- a/cmd/read_source_test.go +++ b/loader/readsource_test.go @@ -1,4 +1,4 @@ -package cmd +package loader import ( "bytes" @@ -7,7 +7,6 @@ import ( "testing" "github.com/loadimpact/k6/lib/fsext" - "github.com/loadimpact/k6/loader" "github.com/pkg/errors" "github.com/spf13/afero" "github.com/stretchr/testify/require" @@ -22,7 +21,7 @@ func (e errorReader) Read(_ []byte) (int, error) { var _ io.Reader = errorReader("") func TestReadSourceSTDINError(t *testing.T) { - _, err := readSource("-", "", nil, errorReader("1234")) + _, err := ReadSource("-", "", nil, errorReader("1234")) require.Error(t, err) require.Equal(t, "1234", err.Error()) } @@ -31,10 +30,10 @@ func TestReadSourceSTDINCache(t *testing.T) { var data = []byte(`test contents`) var r = bytes.NewReader(data) var fs = afero.NewMemMapFs() - sourceData, err := readSource("-", "/path/to/pwd", + sourceData, err := ReadSource("-", "/path/to/pwd", map[string]afero.Fs{"file": fsext.NewCacheOnReadFs(nil, fs, 0)}, r) require.NoError(t, err) - require.Equal(t, &loader.SourceData{ + require.Equal(t, &SourceData{ URL: &url.URL{Scheme: "file", Path: "/-"}, Data: data}, sourceData) fileData, err := afero.ReadFile(fs, "/-") @@ -46,9 +45,9 @@ func TestReadSourceRelative(t *testing.T) { var data = []byte(`test contents`) var fs = afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/path/to/somewhere/script.js", data, 0644)) - sourceData, err := readSource("../somewhere/script.js", "/path/to/pwd", map[string]afero.Fs{"file": fs}, nil) + sourceData, err := ReadSource("../somewhere/script.js", "/path/to/pwd", map[string]afero.Fs{"file": fs}, nil) require.NoError(t, err) - require.Equal(t, &loader.SourceData{ + require.Equal(t, &SourceData{ URL: &url.URL{Scheme: "file", Path: "/path/to/somewhere/script.js"}, Data: data}, sourceData) } @@ -59,9 +58,9 @@ func TestReadSourceAbsolute(t *testing.T) { var fs = afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/a/b", data, 0644)) require.NoError(t, afero.WriteFile(fs, "/c/a/b", []byte("wrong"), 0644)) - sourceData, err := readSource("/a/b", "/c", map[string]afero.Fs{"file": fs}, r) + sourceData, err := ReadSource("/a/b", "/c", map[string]afero.Fs{"file": fs}, r) require.NoError(t, err) - require.Equal(t, &loader.SourceData{ + require.Equal(t, &SourceData{ URL: &url.URL{Scheme: "file", Path: "/a/b"}, Data: data}, sourceData) } @@ -70,10 +69,10 @@ func TestReadSourceHttps(t *testing.T) { var data = []byte(`test contents`) var fs = afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644)) - sourceData, err := readSource("https://github.com/something", "/c", + sourceData, err := ReadSource("https://github.com/something", "/c", map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil) require.NoError(t, err) - require.Equal(t, &loader.SourceData{ + require.Equal(t, &SourceData{ URL: &url.URL{Scheme: "https", Host: "github.com", Path: "/something"}, Data: data}, sourceData) } @@ -82,7 +81,7 @@ func TestReadSourceHttpError(t *testing.T) { var data = []byte(`test contents`) var fs = afero.NewMemMapFs() require.NoError(t, afero.WriteFile(fs, "/github.com/something", data, 0644)) - _, err := readSource("http://github.com/something", "/c", + _, err := ReadSource("http://github.com/something", "/c", map[string]afero.Fs{"file": afero.NewMemMapFs(), "https": fs}, nil) require.Error(t, err) require.Contains(t, err.Error(), `only supported schemes for imports are file and https`) From 305bf816a7d7b80ede33a6bd94a402c533dcf4aa Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 16 Jul 2019 17:54:43 +0300 Subject: [PATCH 42/45] Move cmd#createFilesystems to loader#CreateFilesystems --- cmd/archive.go | 2 +- cmd/cloud.go | 2 +- cmd/inspect.go | 2 +- cmd/run.go | 21 +-------------------- loader/filesystems.go | 27 +++++++++++++++++++++++++++ 5 files changed, 31 insertions(+), 23 deletions(-) create mode 100644 loader/filesystems.go diff --git a/cmd/archive.go b/cmd/archive.go index 3d3eb5aed5d..adb8ad6cb7e 100644 --- a/cmd/archive.go +++ b/cmd/archive.go @@ -52,7 +52,7 @@ An archive is a fully self-contained test run, and can be executed identically e return err } filename := args[0] - filesystems := createFilesystems() + filesystems := loader.CreateFilesystems() src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err diff --git a/cmd/cloud.go b/cmd/cloud.go index dccb8bbdb77..7fdcd48686c 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -72,7 +72,7 @@ This will execute the test on the Load Impact cloud service. Use "k6 login cloud } filename := args[0] - filesystems := createFilesystems() + filesystems := loader.CreateFilesystems() src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err diff --git a/cmd/inspect.go b/cmd/inspect.go index 2640fefdda1..c48221d5cf1 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -43,7 +43,7 @@ var inspectCmd = &cobra.Command{ if err != nil { return err } - filesystems := createFilesystems() + filesystems := loader.CreateFilesystems() src, err := loader.ReadSource(args[0], pwd, filesystems, os.Stdin) if err != nil { return err diff --git a/cmd/run.go b/cmd/run.go index 5aeff46e103..caf9df0e3dc 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -40,7 +40,6 @@ import ( "github.com/loadimpact/k6/js" "github.com/loadimpact/k6/lib" "github.com/loadimpact/k6/lib/consts" - "github.com/loadimpact/k6/lib/fsext" "github.com/loadimpact/k6/lib/types" "github.com/loadimpact/k6/loader" "github.com/loadimpact/k6/ui" @@ -114,7 +113,7 @@ a commandline interface for interacting with it.`, return err } filename := args[0] - filesystems := createFilesystems() + filesystems := loader.CreateFilesystems() src, err := loader.ReadSource(filename, pwd, filesystems, os.Stdin) if err != nil { return err @@ -501,24 +500,6 @@ func init() { runCmd.Flags().AddFlagSet(runCmdFlagSet()) } -func createFilesystems() map[string]afero.Fs { - // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's - // written every time something is read from the real filesystem. This cache is then used for - // successive spawns to read from (they have no access to the real disk). - // Also initialize the same for `https` but the caching is handled manually in the loader package - osfs := afero.NewOsFs() - if runtime.GOOS == "windows" { - // This is done so that we can continue to use paths with /|"\" through the code but also to - // be easier to traverse the cachedFs later as it doesn't work very well if you have windows - // volumes - osfs = fsext.NewTrimFilePathSeparatorFs(osfs) - } - return map[string]afero.Fs{ - "file": fsext.NewCacheOnReadFs(osfs, afero.NewMemMapFs(), 0), - "https": afero.NewMemMapFs(), - } -} - // Creates a new runner. func newRunner( src *loader.SourceData, typ string, filesystems map[string]afero.Fs, rtOpts lib.RuntimeOptions, diff --git a/loader/filesystems.go b/loader/filesystems.go new file mode 100644 index 00000000000..2bfde5c1205 --- /dev/null +++ b/loader/filesystems.go @@ -0,0 +1,27 @@ +package loader + +import ( + "runtime" + + "github.com/loadimpact/k6/lib/fsext" + "github.com/spf13/afero" +) + +// CreateFilesystems creates the correct filesystem map for the current OS +func CreateFilesystems() map[string]afero.Fs { + // We want to eliminate disk access at runtime, so we set up a memory mapped cache that's + // written every time something is read from the real filesystem. This cache is then used for + // successive spawns to read from (they have no access to the real disk). + // Also initialize the same for `https` but the caching is handled manually in the loader package + osfs := afero.NewOsFs() + if runtime.GOOS == "windows" { + // This is done so that we can continue to use paths with /|"\" through the code but also to + // be easier to traverse the cachedFs later as it doesn't work very well if you have windows + // volumes + osfs = fsext.NewTrimFilePathSeparatorFs(osfs) + } + return map[string]afero.Fs{ + "file": fsext.NewCacheOnReadFs(osfs, afero.NewMemMapFs(), 0), + "https": afero.NewMemMapFs(), + } +} From f57c99233a2d13352e443f8036f5e5319bd3de80 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Wed, 17 Jul 2019 15:59:23 +0300 Subject: [PATCH 43/45] typo --- loader/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/loader.go b/loader/loader.go index 508266e60b2..eb94568a24f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -87,7 +87,7 @@ func Resolve(pwd *url.URL, moduleSpecifier string) (*url.URL, error) { moduleSpecifier = "/" + moduleSpecifier } - // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we readd + // we always want for the pwd to end in a slash, but filepath/path.Clean strips it so we read // it if it's missing var finalPwd = pwd if pwd.Opaque != "" { From a9498ebb4175b660c83b1f6b4392086a00785b1b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 18 Jul 2019 15:54:02 +0300 Subject: [PATCH 44/45] case insensitive anonymizaiton for windows paths --- lib/archive.go | 2 +- lib/archive_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/archive.go b/lib/archive.go index 4b0a857c049..64ef57e921d 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -45,7 +45,7 @@ import ( var ( volumeRE = regexp.MustCompile(`^[/\\]?([a-zA-Z]):(.*)`) sharedRE = regexp.MustCompile(`^\\\\([^\\]+)`) // matches a shared folder in Windows before backslack replacement. i.e \\VMBOXSVR\k6\script.js - homeDirRE = regexp.MustCompile(`^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`) + homeDirRE = regexp.MustCompile(`(?i)^(/[a-zA-Z])?/(Users|home|Documents and Settings)/(?:[^/]+)`) ) // NormalizeAndAnonymizePath Normalizes (to use a / path separator) and anonymizes a file path, by scrubbing usernames from home directories. diff --git a/lib/archive_test.go b/lib/archive_test.go index 19e283a07d7..bc1ffbb043f 100644 --- a/lib/archive_test.go +++ b/lib/archive_test.go @@ -50,6 +50,8 @@ func TestNormalizeAndAnonymizePath(t *testing.T) { "\\NOTSHARED\\dir\\dir\\myfile.txt": "/NOTSHARED/dir/dir/myfile.txt", "C:\\Users\\myname\\dir\\myfile.txt": "/C/Users/nobody/dir/myfile.txt", "D:\\Documents and Settings\\myname\\dir\\myfile.txt": "/D/Documents and Settings/nobody/dir/myfile.txt", + "C:\\uSers\\myname\\dir\\myfile.txt": "/C/uSers/nobody/dir/myfile.txt", + "D:\\doCUMENts aND Settings\\myname\\dir\\myfile.txt": "/D/doCUMENts aND Settings/nobody/dir/myfile.txt", } // TODO: fix this - the issue is that filepath.Clean replaces `/` with whatever the path // separator is on the current OS and as such this gets confused for shared folder on From 9249ba474bea92fce887380beb44ae14a13232d6 Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 23 Jul 2019 11:02:53 +0300 Subject: [PATCH 45/45] Add the os under which the archive was made in the archive --- js/bundle.go | 2 ++ lib/archive.go | 1 + 2 files changed, 3 insertions(+) diff --git a/js/bundle.go b/js/bundle.go index be76417c64a..9214d6a1bec 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -24,6 +24,7 @@ import ( "context" "encoding/json" "net/url" + "runtime" "github.com/loadimpact/k6/lib/consts" @@ -177,6 +178,7 @@ func (b *Bundle) makeArchive() *lib.Archive { PwdURL: b.BaseInitContext.pwd, Env: make(map[string]string, len(b.Env)), K6Version: consts.Version, + Goos: runtime.GOOS, } // Copy env so changes in the archive are not reflected in the source Bundle for k, v := range b.Env { diff --git a/lib/archive.go b/lib/archive.go index 64ef57e921d..f18db2d2c72 100644 --- a/lib/archive.go +++ b/lib/archive.go @@ -89,6 +89,7 @@ type Archive struct { Env map[string]string `json:"env"` K6Version string `json:"k6version"` + Goos string `json:"goos"` } func (arc *Archive) getFs(name string) afero.Fs {