Add possibility to override script values (#138)

Add possibility to override script values

cmd/execute.go

@@ -40,12 +40,14 @@ type (
 )
 
 var (
-	metricsPort      int
-	metricsAddress   string
-	metricsLabel     string
-	metricsGroupings []string
-	profTyp          string
-	objDefFile       string
+	metricsPort        int
+	metricsAddress     string
+	metricsLabel       string
+	metricsGroupings   []string
+	profTyp            string
+	objDefFile         string
+	scriptOverrides    []string
+	scriptOverrideFile string
 )
 
 // *** Custom errors ***

cmd/helpers.go

@@ -3,6 +3,8 @@ package cmd
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
+	"strings"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
@@ -27,13 +29,21 @@ var (
 // AddAllSharedParameters add shared parameters to command
 func AddAllSharedParameters(cmd *cobra.Command) {
 	AddConfigParameter(cmd)
+	AddOverrideParameters(cmd)
 }
 
 // AddConfigParameter add config file parameter to command
 func AddConfigParameter(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&cfgFile, "config", "c", "", `Scenario config file.`)
 }
 
+// AddOverrideParameters to command
+func AddOverrideParameters(cmd *cobra.Command) {
+	cmd.Flags().StringArrayVarP(&scriptOverrides, "set", "s", nil, "Override a value in script with 'path/to/key=value'.")
+	cmd.Flags().StringVar(&scriptOverrideFile, "setfromfile", "", "Override values from file where each row is path/to/key=value.")
+}
+
+// AddLoggingParameters add logging parameters to command
 func AddLoggingParameters(cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(&traffic, "traffic", "t", false, "Log traffic. Logging traffic is heavy and should only be done for debugging purposes.")
 	cmd.Flags().BoolVarP(&trafficMetrics, "trafficmetrics", "m", false, "Log traffic metrics.")
@@ -52,14 +62,63 @@ func unmarshalConfigFile() (*config.Config, error) {
 		return nil, errors.Wrapf(err, "Error reading config from file<%s>", cfgFile)
 	}
 
+	var overrides []string
+	cfgJSON, overrides, err = overrideScriptValues(cfgJSON)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
 	var cfg config.Config
 	if err = jsonit.Unmarshal(cfgJSON, &cfg); err != nil {
 		return nil, errors.Wrap(err, "Failed to unmarshal config from json")
 	}
 
+	if cfg.Settings.LogSettings.Format != config.LogFormatNoLogs {
+		PrintOverrides(overrides)
+	}
+
 	return &cfg, nil
 }
 
+func overrideScriptValues(cfgJSON []byte) ([]byte, []string, error) {
+	var overrides []string
+	if scriptOverrideFile != "" {
+		overrideFile, err := helpers.NewRowFile(scriptOverrideFile)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "Error reading overrides from file<%s>", scriptOverrideFile)
+		}
+		if scriptOverrides == nil {
+			scriptOverrides = make([]string, 0, len(overrideFile.Rows()))
+		}
+		scriptOverrides = append(overrideFile.Rows(), scriptOverrides...) // let command line overrides override file overrides
+	}
+
+	overrides = make([]string, 0, len(scriptOverrides))
+	for _, kvp := range scriptOverrides {
+		kvSplit := strings.SplitN(kvp, "=", 2)
+		if len(kvSplit) != 2 {
+			return cfgJSON, overrides, errors.Errorf("malformed override: %s, should be in the form 'path/to/key=value'", kvp)
+		}
+
+		path := helpers.DataPath(kvSplit[0])
+		rawOrig, err := path.Lookup(cfgJSON)
+		if err != nil {
+			return cfgJSON, overrides, errors.Wrap(err, "invalid script override")
+		}
+		cfgJSON, err = path.Set(cfgJSON, []byte(kvSplit[1]))
+		if err != nil {
+			return cfgJSON, overrides, errors.WithStack(err)
+		}
+		rawModified, err := path.Lookup(cfgJSON)
+		if err != nil {
+			return cfgJSON, overrides, errors.WithStack(err)
+		}
+		overrides = append(overrides, fmt.Sprintf("%s: %s -> %s\n", path, rawOrig, rawModified))
+	}
+
+	return cfgJSON, overrides, nil
+}
+
 func getLogFormatHelpString() string {
 	buf := helpers.NewBuffer()
 	buf.WriteString("Set a log format to be used. One of:\n")
@@ -82,6 +141,7 @@ func getSummaryTypeHelpString() string {
 	return buf.String()
 }
 
+// ConfigOverrideLogSettings override log settings with parameters
 func ConfigOverrideLogSettings(cfg *config.Config) error {
 	if trafficMetrics {
 		cfg.SetTrafficMetricsLogging()
@@ -113,3 +173,15 @@ func ConfigOverrideLogSettings(cfg *config.Config) error {
 
 	return nil
 }
+
+// PrintOverrides to script
+func PrintOverrides(overrides []string) {
+	if len(overrides) < 1 {
+		return
+	}
+	os.Stderr.WriteString("=== Script overrides ===\n")
+	for _, override := range overrides {
+		os.Stderr.WriteString(override)
+	}
+	os.Stderr.WriteString("========================\n")
+}

cmd/script.go

@@ -3,9 +3,9 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"github.com/qlik-oss/gopherciser/appstructure"
 	"os"
 
+	"github.com/qlik-oss/gopherciser/appstructure"
 	"github.com/qlik-oss/gopherciser/config"
 	"github.com/spf13/cobra"
 )

docs/README.md

@@ -51,6 +51,7 @@ Flags:
 
 * `-c`, `--config string`: Load the specified scenario setup file.
 * `--debug`: Log debug information.
+* `-d`, `--definitions`: Custom object definitions and overrides.
 * `-h`, `--help`: Show the help for the `execute` command.
 * `--logformat string`: Set the specified log format. The log format specified in the scenario setup file is used by default. If no log format is specified, `tsvfile` is used.
   * `0` or `tsvfile`: TSV file
@@ -74,6 +75,7 @@ Flags:
   * `6` or `mutex`: Mutex
   * `7` or `trace`: Trace
   * `8` or `mem`: Mem
+* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation.
 * `--summary string`: Set the type of summary to display after the test run. Defaults to `simple` for minimal performance impact.
   * `0` or `undefined`: Simple, single-row summary
   * `1` or `none`: No summary
@@ -137,6 +139,7 @@ Sub-commands:
 
 * `-c`, `--config string`: Connect using the specified scenario config file.
 * `-h`, `--help`: Show the help for the `connect` command.
+* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation.
 
 `structure` command flags:
 
@@ -162,18 +165,78 @@ Sub-commands:
   * `4` or `full`: Currently the same as the `extended` summary, includes a list of all objects in the structure.
 * `-t`, `--traffic`: Log traffic information.
 * `-m`, `--trafficmetrics`: Log metrics information.
+* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation.
+* `--setfromfile`: Override values from file where each row is path/to/key=value.
 
 `validate` command flags:
 
 * `-c`, `--config string`: Load the specified scenario setup file.
 * `-h`, `--help`: Show the help for the `validate` command.
+* `-s`, `--set`: Override a value in script with key.path=value. See [Using script overrides](#using-script-overrides) for further explanation.
+* `--setfromfile`: Override values from file where each row is path/to/key=value.
 
 `template` command flags:
 
 * `-c`, `--config string`: (optional) Create the specified scenario setup file. Defaults to `template.json`.
 * `-f`, `--force`: Overwrite existing scenario setup file.
 * `-h`, `--help`: Show the help for the `template` command.
 
+#### Using script overrides
+
+Script overrides overrides a value pointed to by a path to its key. If the key doesn't exist in the script there will an error, even if it's a valid value according to config.
+
+The syntax is path/to/key=value. A common thing to override would be the settings of the simple scheduler.
+
+```json
+"scheduler": {
+  "type": "simple",
+  "settings": {
+    "executiontime": -1,
+    "iterations": 1,
+    "rampupdelay": 1.0,
+    "concurrentusers": 1
+  }
+}
+```
+
+`scheduler` is in the root of the JSON, so the path to the key of `concurrentusers` would be `scheduler/settings/concurrentusers`. To override concurrent users from command line:
+
+```bash
+./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'scheduler/settings/concurrentusers=2'
+```
+
+Overriding a string, such as the server the servername requires it to be wrapped with double quotes. E.g. to override the server:
+
+```bash
+./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'connectionSettings/server="127.0.0.1"'
+```
+
+Do note that the path is case sensitive. It needs to be `connectionSettings/server` as `connectionsettings/server` would try, and fail, to add new key called `connectionsettings`.
+
+Overrides could also be used to with more advanced paths. If the position in `scenario` is known for `openapp` we could replace e.g. the app opened, assuming `openapp` is the first action in `scenario`:
+
+```bash
+./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'scenario/[0]/settings/app="mynewapp"'
+```
+
+It could even replace an entire JSON object such as the `connectionSettings` with one replace call:
+
+```bash
+./gopherciser x -c ./docs/examples/sheetChangerQlikCore.json -s 'connectionSettings={"mode":"ws","server":"127.0.0.1","port":19076}'
+```
+
+Overrides could also be defined in a file. Each row in the file should be in the same format as when using overrides from command line, although should not be wrapped with single quotes as this is for command line interpretation purposes. Using the same overrides as above, the file could look like the following:
+
+```
+connectionSettings/server="1.2.3.4"
+scenario/[0]/settings/app="mynewapp"
+connectionSettings={"mode":"ws","server":"127.0.0.1","port":19076}
+```
+
+Overrides will be executed from top to button, as such the third line will override the `server` overriden by the first line and script will execute towards `127.0.0.1:19076`.
+
+Any command line overrides will be executed _after_ the overrides defined in file.
+
 ## Analyzing the test results
 
 A log file is recorded during each test execution. The `logs.filename` setting in the `settings` section of the load test script specifies the name of and the path to the log file (see [Setting up load scenarios](./settingup.md)). If a file with the specified filename already exists, a number is appended to the filename (for example, if the file `xxxxx.yyy` already exists, the log file is stored as `xxxxx-001.yyy`). 

helpers/datapath.go

@@ -38,17 +38,17 @@ func (path *DataPath) String() string {
 	return string(*path)
 }
 
-//steps path substrings splitted on / (slash)
+// steps path substrings splitted on / (slash)
 func (path *DataPath) steps() []string {
 	return strings.Split(strings.Trim(path.String(), "/"), "/")
 }
 
-//Lookup object in path, if data found is of type string it will be quoted with ""
+// Lookup object in path, if data found is of type string it will be quoted with ""
 func (path DataPath) Lookup(data json.RawMessage) (json.RawMessage, error) {
 	return path.lookup(data, true)
 }
 
-//Lookup object in path, data of type string will not be quoted
+// LookupNoQuotes object in path, data of type string will not be quoted
 func (path DataPath) LookupNoQuotes(data json.RawMessage) (json.RawMessage, error) {
 	return path.lookup(data, false)
 }
@@ -78,7 +78,24 @@ func (path DataPath) lookup(data json.RawMessage, quoteString bool) (json.RawMes
 	}
 }
 
-//LookupMulti objects with subpaths under an array in a path
+// Set look object in path and set to new object
+func (path DataPath) Set(data []byte, newValue []byte) ([]byte, error) {
+	steps := path.steps()
+
+	if steps == nil || len(steps) < 1 {
+		return data, errors.WithStack(NoStepsError(string(path)))
+	}
+
+	var err error
+	data, err = jsonparser.Set([]byte(data), newValue, steps...)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to set data<%s> at path<%s>", newValue, path)
+	}
+
+	return data, nil
+}
+
+// LookupMulti objects with subpaths under an array in a path
 func (path DataPath) LookupMulti(data json.RawMessage, separator string) ([]json.RawMessage, error) {
 	// todo change to use jsonparser
 	if separator == "" || !strings.Contains(string(path), separator) {