From f0b5d442f287f7d2340158afda8ba17f2c1721d5 Mon Sep 17 00:00:00 2001 From: Daniel Larsson Date: Mon, 2 Nov 2020 09:26:59 +0100 Subject: [PATCH] Add possibility to override script values (#138) Add possibility to override script values --- cmd/execute.go | 14 ++++--- cmd/helpers.go | 72 +++++++++++++++++++++++++++++++++ cmd/script.go | 2 +- docs/README.md | 63 +++++++++++++++++++++++++++++ helpers/datapath.go | 25 ++++++++++-- helpers/datapath_test.go | 86 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 251 insertions(+), 11 deletions(-) diff --git a/cmd/execute.go b/cmd/execute.go index a6671c69..9ce2f9fa 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -40,12 +40,14 @@ type ( ) var ( - metricsPort int - metricsAddress string - metricsLabel string - metricsGroupings []string - profTyp string - objDefFile string + metricsPort int + metricsAddress string + metricsLabel string + metricsGroupings []string + profTyp string + objDefFile string + scriptOverrides []string + scriptOverrideFile string ) // *** Custom errors *** diff --git a/cmd/helpers.go b/cmd/helpers.go index 558207b7..eeb1a7a8 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -3,6 +3,8 @@ package cmd import ( "fmt" "io/ioutil" + "os" + "strings" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" @@ -27,6 +29,7 @@ var ( // AddAllSharedParameters add shared parameters to command func AddAllSharedParameters(cmd *cobra.Command) { AddConfigParameter(cmd) + AddOverrideParameters(cmd) } // AddConfigParameter add config file parameter to command @@ -34,6 +37,13 @@ func AddConfigParameter(cmd *cobra.Command) { cmd.Flags().StringVarP(&cfgFile, "config", "c", "", `Scenario config file.`) } +// AddOverrideParameters to command +func AddOverrideParameters(cmd *cobra.Command) { + cmd.Flags().StringArrayVarP(&scriptOverrides, "set", "s", nil, "Override a value in script with 'path/to/key=value'.") + cmd.Flags().StringVar(&scriptOverrideFile, "setfromfile", "", "Override values from file where each row is path/to/key=value.") +} + +// AddLoggingParameters add logging parameters to command func AddLoggingParameters(cmd *cobra.Command) { cmd.Flags().BoolVarP(&traffic, "traffic", "t", false, "Log traffic. Logging traffic is heavy and should only be done for debugging purposes.") cmd.Flags().BoolVarP(&trafficMetrics, "trafficmetrics", "m", false, "Log traffic metrics.") @@ -52,14 +62,63 @@ func unmarshalConfigFile() (*config.Config, error) { return nil, errors.Wrapf(err, "Error reading config from file<%s>", cfgFile) } + var overrides []string + cfgJSON, overrides, err = overrideScriptValues(cfgJSON) + if err != nil { + return nil, errors.WithStack(err) + } + var cfg config.Config if err = jsonit.Unmarshal(cfgJSON, &cfg); err != nil { return nil, errors.Wrap(err, "Failed to unmarshal config from json") } + if cfg.Settings.LogSettings.Format != config.LogFormatNoLogs { + PrintOverrides(overrides) + } + return &cfg, nil } +func overrideScriptValues(cfgJSON []byte) ([]byte, []string, error) { + var overrides []string + if scriptOverrideFile != "" { + overrideFile, err := helpers.NewRowFile(scriptOverrideFile) + if err != nil { + return nil, nil, errors.Wrapf(err, "Error reading overrides from file<%s>", scriptOverrideFile) + } + if scriptOverrides == nil { + scriptOverrides = make([]string, 0, len(overrideFile.Rows())) + } + scriptOverrides = append(overrideFile.Rows(), scriptOverrides...) // let command line overrides override file overrides + } + + overrides = make([]string, 0, len(scriptOverrides)) + for _, kvp := range scriptOverrides { + kvSplit := strings.SplitN(kvp, "=", 2) + if len(kvSplit) != 2 { + return cfgJSON, overrides, errors.Errorf("malformed override: %s, should be in the form 'path/to/key=value'", kvp) + } + + path := helpers.DataPath(kvSplit[0]) + rawOrig, err := path.Lookup(cfgJSON) + if err != nil { + return cfgJSON, overrides, errors.Wrap(err, "invalid script override") + } + cfgJSON, err = path.Set(cfgJSON, []byte(kvSplit[1])) + if err != nil { + return cfgJSON, overrides, errors.WithStack(err) + } + rawModified, err := path.Lookup(cfgJSON) + if err != nil { + return cfgJSON, overrides, errors.WithStack(err) + } + overrides = append(overrides, fmt.Sprintf("%s: %s -> %s\n", path, rawOrig, rawModified)) + } + + return cfgJSON, overrides, nil +} + func getLogFormatHelpString() string { buf := helpers.NewBuffer() buf.WriteString("Set a log format to be used. One of:\n") @@ -82,6 +141,7 @@ func getSummaryTypeHelpString() string { return buf.String() } +// ConfigOverrideLogSettings override log settings with parameters func ConfigOverrideLogSettings(cfg *config.Config) error { if trafficMetrics { cfg.SetTrafficMetricsLogging() @@ -113,3 +173,15 @@ func ConfigOverrideLogSettings(cfg *config.Config) error { return nil } + +// PrintOverrides to script +func PrintOverrides(overrides []string) { + if len(overrides) < 1 { + return + } + os.Stderr.WriteString("=== Script overrides ===\n") + for _, override := range overrides { + os.Stderr.WriteString(override) + } + os.Stderr.WriteString("========================\n") +} diff --git a/cmd/script.go b/cmd/script.go index 93130d40..12ba3e3a 100644 --- a/cmd/script.go +++ b/cmd/script.go @@ -3,9 +3,9 @@ package cmd import ( "context" "fmt" - "github.com/qlik-oss/gopherciser/appstructure" "os" + "github.com/qlik-oss/gopherciser/appstructure" "github.com/qlik-oss/gopherciser/config" "github.com/spf13/cobra" ) diff --git a/docs/README.md b/docs/README.md index 464bb27f..049068c7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -51,6 +51,7 @@ Flags: * `-c`, `--config string`: Load the specified scenario setup file. * `--debug`: Log debug information. +* `-d`, `--definitions`: Custom object definitions and overrides. * `-h`, `--help`: Show the help for the `execute` command. * `--logformat string`: Set the specified log format. The log format specified in the scenario setup file is used by default. If no log format is specified, `tsvfile` is used. * `0` or `tsvfile`: TSV file @@ -74,6 +75,7 @@ Flags: * `6` or `mutex`: Mutex * `7` or `trace`: Trace * `8` or `mem`: Mem +* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation. * `--summary string`: Set the type of summary to display after the test run. Defaults to `simple` for minimal performance impact. * `0` or `undefined`: Simple, single-row summary * `1` or `none`: No summary @@ -137,6 +139,7 @@ Sub-commands: * `-c`, `--config string`: Connect using the specified scenario config file. * `-h`, `--help`: Show the help for the `connect` command. +* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation. `structure` command flags: @@ -162,11 +165,15 @@ Sub-commands: * `4` or `full`: Currently the same as the `extended` summary, includes a list of all objects in the structure. * `-t`, `--traffic`: Log traffic information. * `-m`, `--trafficmetrics`: Log metrics information. +* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation. +* `--setfromfile`: Override values from file where each row is path/to/key=value. `validate` command flags: * `-c`, `--config string`: Load the specified scenario setup file. * `-h`, `--help`: Show the help for the `validate` command. +* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation. +* `--setfromfile`: Override values from file where each row is path/to/key=value. `template` command flags: @@ -174,6 +181,62 @@ Sub-commands: * `-f`, `--force`: Overwrite existing scenario setup file. * `-h`, `--help`: Show the help for the `template` command. +#### Using script overrides + +Script overrides overrides a value pointed to by a path to its key. If the key doesn't exist in the script there will an error, even if it's a valid value according to config. + +The syntax is path/to/key=value. A common thing to override would be the settings of the simple scheduler. + +```json +"scheduler": { + "type": "simple", + "settings": { + "executiontime": -1, + "iterations": 1, + "rampupdelay": 1.0, + "concurrentusers": 1 + } +} +``` + +`scheduler` is in the root of the JSON, so the path to the key of `concurrentusers` would be `scheduler/settings/concurrentusers`. To override concurrent users from command line: + +```bash +./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'scheduler/settings/concurrentusers=2' +``` + +Overriding a string, such as the server the servername requires it to be wrapped with double quotes. E.g. to override the server: + +```bash +./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'connectionSettings/server="127.0.0.1"' +``` + +Do note that the path is case sensitive. It needs to be `connectionSettings/server` as `connectionsettings/server` would try, and fail, to add new key called `connectionsettings`. + +Overrides could also be used to with more advanced paths. If the position in `scenario` is known for `openapp` we could replace e.g. the app opened, assuming `openapp` is the first action in `scenario`: + +```bash +./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'scenario/[0]/settings/app="mynewapp"' +``` + +It could even replace an entire JSON object such as the `connectionSettings` with one replace call: + +```bash +./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'connectionSettings={"mode":"ws","server":"127.0.0.1","port":19076}' +``` + +Overrides could also be defined in a file. Each row in the file should be in the same format as when using overrides from command line, although should not be wrapped with single quotes as this is for command line interpretation purposes. Using the same overrides as above, the file could look like the following: + +``` +connectionSettings/server="1.2.3.4" +scenario/[0]/settings/app="mynewapp" +connectionSettings={"mode":"ws","server":"127.0.0.1","port":19076} +``` + +Overrides will be executed from top to button, as such the third line will override the `server` overriden by the first line and script will execute towards `127.0.0.1:19076`. + +Any command line overrides will be executed _after_ the overrides defined in file. + ## Analyzing the test results A log file is recorded during each test execution. The `logs.filename` setting in the `settings` section of the load test script specifies the name of and the path to the log file (see [Setting up load scenarios](./settingup.md)). If a file with the specified filename already exists, a number is appended to the filename (for example, if the file `xxxxx.yyy` already exists, the log file is stored as `xxxxx-001.yyy`). diff --git a/helpers/datapath.go b/helpers/datapath.go index 884d2018..18665c8f 100644 --- a/helpers/datapath.go +++ b/helpers/datapath.go @@ -38,17 +38,17 @@ func (path *DataPath) String() string { return string(*path) } -//steps path substrings splitted on / (slash) +// steps path substrings splitted on / (slash) func (path *DataPath) steps() []string { return strings.Split(strings.Trim(path.String(), "/"), "/") } -//Lookup object in path, if data found is of type string it will be quoted with "" +// Lookup object in path, if data found is of type string it will be quoted with "" func (path DataPath) Lookup(data json.RawMessage) (json.RawMessage, error) { return path.lookup(data, true) } -//Lookup object in path, data of type string will not be quoted +// LookupNoQuotes object in path, data of type string will not be quoted func (path DataPath) LookupNoQuotes(data json.RawMessage) (json.RawMessage, error) { return path.lookup(data, false) } @@ -78,7 +78,24 @@ func (path DataPath) lookup(data json.RawMessage, quoteString bool) (json.RawMes } } -//LookupMulti objects with subpaths under an array in a path +// Set look object in path and set to new object +func (path DataPath) Set(data []byte, newValue []byte) ([]byte, error) { + steps := path.steps() + + if steps == nil || len(steps) < 1 { + return data, errors.WithStack(NoStepsError(string(path))) + } + + var err error + data, err = jsonparser.Set([]byte(data), newValue, steps...) + if err != nil { + return nil, errors.Wrapf(err, "failed to set data<%s> at path<%s>", newValue, path) + } + + return data, nil +} + +// LookupMulti objects with subpaths under an array in a path func (path DataPath) LookupMulti(data json.RawMessage, separator string) ([]json.RawMessage, error) { // todo change to use jsonparser if separator == "" || !strings.Contains(string(path), separator) { diff --git a/helpers/datapath_test.go b/helpers/datapath_test.go index 3a63b916..f414a6d7 100644 --- a/helpers/datapath_test.go +++ b/helpers/datapath_test.go @@ -2,6 +2,7 @@ package helpers import ( "encoding/json" + "fmt" "testing" ) @@ -44,6 +45,91 @@ func TestMultiLookup(t *testing.T) { } } +type LookupAndSetResult struct { + Node struct { + Intval int + Floatval float64 + Stringval string + } +} + +func TestLookupAndSet(t *testing.T) { + data := []byte(`{ + "node" : { + "intval" : 2, + "floatval": 4.2, + "stringval": "string1" + } +}`) + t.Logf("data: %s", data) + + tests := []struct { + Path DataPath + NewData []byte + ExpectedValue interface{} + }{ + { + Path: DataPath("node/intval"), + NewData: []byte("4"), + ExpectedValue: 4, + }, + { + Path: DataPath("node/floatval"), + NewData: []byte("5.2"), + ExpectedValue: 5.2, + }, + { + Path: DataPath("node/stringval"), + NewData: []byte(`"yadda"`), + ExpectedValue: "yadda", + }, + { + Path: DataPath("new/path/novaluehere"), + NewData: []byte("404"), + ExpectedValue: 404, + }, + { + Path: DataPath("nodenode"), + NewData: []byte(`{"newkey":"newvalue"}`), + ExpectedValue: `{"newkey":"newvalue"}`, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("TestLookupAndSet%d", i), func(t *testing.T) { + // Lookup and modify + t.Logf("%s;%s;%v", test.Path, string(test.NewData), test.ExpectedValue) + modifiedData, err := test.Path.Set(data, test.NewData) + if len(modifiedData) > 0 { + t.Logf("modified data: %s", modifiedData) + } + if err != nil { + t.Fatal(err) + } + + // unmarshal results + var result LookupAndSetResult + err = jsonit.Unmarshal(modifiedData, &result) + if err != nil { + t.Fatal(err) + } + + // validate data + newVal, err := test.Path.LookupNoQuotes(modifiedData) + if err != nil { + t.Fatal(err) + } + + // compare values as strings + newValueAsString := string(newVal) + expectedValueAsString := fmt.Sprintf("%v", test.ExpectedValue) + if newValueAsString != expectedValueAsString { + t.Errorf("unexpected result, got<%s> expected<%s>", newValueAsString, expectedValueAsString) + } + }) + } +} + //TODO Both Lookup and MultiLookup needs performance optimizations func BenchmarkMultiLookup(b *testing.B) {