From c23909e275c5b0250239752344ab16e24d86b7ac Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Wed, 27 Nov 2019 15:22:24 +0100 Subject: [PATCH] feat(executors): add sql executor to query databases (#213) contribution from @gwleclerc --- README.md | 1 + cli/venom/run/cmd.go | 2 + executors/sql/README.md | 65 +++++++++++++++++++ executors/sql/executor.go | 131 ++++++++++++++++++++++++++++++++++++++ lib/lib.go | 4 +- 5 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 executors/sql/README.md create mode 100644 executors/sql/executor.go diff --git a/README.md b/README.md index 3c339fc5..676f8a50 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Flags: * **web**: https://github.com/ovh/venom/tree/master/executors/web * **grpc**: https://github.com/ovh/venom/tree/master/executors/grpc * **rabbitmq**: https://github.com/ovh/venom/tree/master/executors/rabbitmq +* **sql**: https://github.com/ovh/venom/tree/master/executors/sql ## TestSuite files diff --git a/cli/venom/run/cmd.go b/cli/venom/run/cmd.go index 79955ee4..ac36b4c8 100644 --- a/cli/venom/run/cmd.go +++ b/cli/venom/run/cmd.go @@ -31,6 +31,7 @@ import ( "github.com/ovh/venom/executors/readfile" "github.com/ovh/venom/executors/redis" "github.com/ovh/venom/executors/smtp" + "github.com/ovh/venom/executors/sql" "github.com/ovh/venom/executors/ssh" "github.com/ovh/venom/executors/web" ) @@ -92,6 +93,7 @@ var Cmd = &cobra.Command{ v.RegisterExecutor(kafka.Name, kafka.New()) v.RegisterExecutor(grpc.Name, grpc.New()) v.RegisterExecutor(rabbitmq.Name, rabbitmq.New()) + v.RegisterExecutor(sql.Name, sql.New()) // Register Context v.RegisterTestCaseContext(defaultctx.Name, defaultctx.New()) diff --git a/executors/sql/README.md b/executors/sql/README.md new file mode 100644 index 00000000..ee814559 --- /dev/null +++ b/executors/sql/README.md @@ -0,0 +1,65 @@ +# Venom - Executor SQL + +Step to execute SQL queries into **MySQL** and **PostgreSQL** databases. + +It use the package `sqlx` under the hood: https://github.com/jmoiron/sqlx to retreive rows as a list of map[string]interface{} + +## Input + +In your yaml file, you declare tour step like this + +```yaml + - driver mandatory [mysql/postgres] + - dsn mandatory + - commands optional + - file optional + ``` + +- `commands` is a list of SQL queries. +- `file` parameter is only used as a fallback if `commands` is not used. + +Example usage (_mysql_): + +```yaml + +name: Title of TestSuite +testcases: + + - name: Query database + steps: + - type: sql + driver: mysql + dsn: user:password@(localhost:3306)/venom + commands: + - "SELECT * FROM employee;" + - "SELECT * FROM person;" + assertions: + - result.queries.__len__ ShouldEqual 2 + - result.queries.queries0.rows.rows0.name ShouldEqual Jack + - result.queries.queries1.rows.rows0.age ShouldEqual 21 + +``` + +```yaml + +name: Title of TestSuite +testcases: + + - name: Query database + steps: + - type: sql + database: mysql + dsn: user:password@(localhost:3306)/venom + file: ./test.sql + assertions: + - result.queries.__len__ ShouldEqual 1 +``` + +*note: in the example above, the results of each command is stored in the results array + +## SQL drivers + +This executor uses the following SQL drivers: + +- _MySQL_: https://github.com/go-sql-driver/mysql +- _PostgreSQL_: https://github.com/lib/pq diff --git a/executors/sql/executor.go b/executors/sql/executor.go new file mode 100644 index 00000000..64abf4c6 --- /dev/null +++ b/executors/sql/executor.go @@ -0,0 +1,131 @@ +package sql + +import ( + "fmt" + "io/ioutil" + "path" + + "github.com/mitchellh/mapstructure" + + // MySQL drivers + _ "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + + // Postgres driver + _ "github.com/lib/pq" + + "github.com/ovh/venom" + "github.com/ovh/venom/executors" +) + +// Name of the executor. +const Name = "sql" + +// New returns a new executor that can execute SQL queries +func New() venom.Executor { + return &Executor{} +} + +// Executor is a venom executor can execute SQL queries +type Executor struct { + File string `json:"file,omitempty" yaml:"file,omitempty"` + Commands []string `json:"commands,omitempty" yaml:"commands,omitempty"` + Driver string `json:"driver" yaml:"driver"` + DSN string `json:"dsn" yaml:"dsn"` +} + +// Rows represents an array of Row +type Rows []Row + +// Row represents a row return by a SQL query. +type Row map[string]interface{} + +// QueryResult represents a rows return by a SQL query execution. +type QueryResult struct { + Rows Rows `json:"rows,omitempty" yaml:"rows,omitempty"` +} + +// Result represents a step result. +type Result struct { + Executor Executor `json:"executor,omitempty" yaml:"executor,omitempty"` + Queries []QueryResult `json:"queries,omitempty" yaml:"queries,omitempty"` +} + +// Run implements the venom.Executor interface for Executor. +func (e Executor) Run(testCaseContext venom.TestCaseContext, l venom.Logger, step venom.TestStep, workdir string) (venom.ExecutorResult, error) { + // Transform step to Executor instance. + if err := mapstructure.Decode(step, &e); err != nil { + return nil, err + } + // Connect to the database and ping it. + l.Debugf("connecting to database %s, %s\n", e.Driver, e.DSN) + db, err := sqlx.Connect(e.Driver, e.DSN) + if err != nil { + return nil, fmt.Errorf("failed to connect to database: %v", err) + } + defer db.Close() + + results := []QueryResult{} + // Execute commands on database + // if the argument is specified. + if len(e.Commands) != 0 { + for i, s := range e.Commands { + l.Debugf("Executing command number %d\n", i) + rows, err := db.Queryx(s) + if err != nil { + return nil, fmt.Errorf("failed to exec command number %d : %v", i, err) + } + r, err := handleRows(rows) + if err != nil { + return nil, fmt.Errorf("failed to parse SQL rows for command number %d : %v", i, err) + } + results = append(results, QueryResult{Rows: r}) + } + } else if e.File != "" { + l.Debugf("loading SQL file from folder %s\n", e.File) + file := path.Join(workdir, e.File) + sbytes, errs := ioutil.ReadFile(file) + if errs != nil { + return nil, errs + } + rows, err := db.Queryx(string(sbytes)) + if err != nil { + return nil, fmt.Errorf("failed to exec SQL file %s : %v", file, err) + } + r, err := handleRows(rows) + if err != nil { + return nil, fmt.Errorf("failed to parse SQL rows for SQL file %s : %v", file, err) + } + results = append(results, QueryResult{Rows: r}) + } + r := Result{Executor: e, Queries: results} + return executors.Dump(r) +} + +// ZeroValueResult return an empty implemtation of this executor result +func (Executor) ZeroValueResult() venom.ExecutorResult { + r, _ := executors.Dump(Result{}) + return r +} + +// GetDefaultAssertions return the default assertions of the executor. +func (e Executor) GetDefaultAssertions() venom.StepAssertions { + return venom.StepAssertions{Assertions: []string{}} +} + +// handleRows iter on each SQL rows result sets and serialize it into a []Row. +func handleRows(rows *sqlx.Rows) ([]Row, error) { + defer rows.Close() + res := []Row{} + for rows.Next() { + row := make(Row) + if err := rows.MapScan(row); err != nil { + return nil, err + } + res = append(res, row) + } + if err := rows.Err(); err != nil { + return res, err + } + return res, nil +} diff --git a/lib/lib.go b/lib/lib.go index 5e3573bd..4b1e6877 100644 --- a/lib/lib.go +++ b/lib/lib.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/ovh/venom" - "github.com/ovh/venom/context/default" + defaultctx "github.com/ovh/venom/context/default" redisctx "github.com/ovh/venom/context/redis" "github.com/ovh/venom/context/webctx" "github.com/ovh/venom/executors/grpc" @@ -17,6 +17,7 @@ import ( "github.com/ovh/venom/executors/readfile" "github.com/ovh/venom/executors/redis" "github.com/ovh/venom/executors/smtp" + "github.com/ovh/venom/executors/sql" "github.com/ovh/venom/executors/ssh" "github.com/ovh/venom/executors/web" ) @@ -81,6 +82,7 @@ func TestCase(t *testing.T, name string, variables map[string]string) *T { v.RegisterExecutor(web.Name, web.New()) v.RegisterExecutor(redis.Name, redis.New()) v.RegisterExecutor(grpc.Name, grpc.New()) + v.RegisterExecutor(sql.Name, sql.New()) v.RegisterTestCaseContext(redisctx.Name, redisctx.New()) v.RegisterTestCaseContext(defaultctx.Name, defaultctx.New())