From 298649b0634ed350a1392e1bbffc8ca0c0f4ff20 Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Fri, 6 Sep 2019 15:31:29 +0300 Subject: [PATCH 1/4] Added `script` option to be able set a sql file as a query source --- .../postgresql_extensible.go | 27 +++++++++++++++++++ .../postgresql_extensible_test.go | 27 ++++++++++++++++++- plugins/inputs/postgresql_extensible/test.sql | 1 + 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 plugins/inputs/postgresql_extensible/test.sql diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible.go b/plugins/inputs/postgresql_extensible/postgresql_extensible.go index c2bcb7b600efc..7c2f241757bdc 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible.go @@ -3,7 +3,9 @@ package postgresql_extensible import ( "bytes" "fmt" + "io/ioutil" "log" + "os" "strings" // register in driver. @@ -25,6 +27,7 @@ type Postgresql struct { type query []struct { Sqlquery string + Script string Version int Withdbname bool Tagvalue string @@ -108,6 +111,20 @@ func (p *Postgresql) IgnoredColumns() map[string]bool { return ignoredColumns } +func (p *Postgresql) ReadQueryFromFile(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + qyery, err := ioutil.ReadAll(file) + if err != nil { + return "", err + } + return string(qyery), err +} + func (p *Postgresql) Gather(acc telegraf.Accumulator) error { var ( err error @@ -129,8 +146,18 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { // We loop in order to process each query // Query is not run if Database version does not match the query version. for i := range p.Query { + if p.Query[i].Sqlquery != "" && p.Query[i].Script != "" { + return fmt.Errorf("both 'sqlquery' and 'script' specified, please choose one option") + } sql_query = p.Query[i].Sqlquery + if sql_query == "" { + sql_query, err = p.ReadQueryFromFile(p.Query[i].Script) + if err != nil { + return err + } + } tag_value = p.Query[i].Tagvalue + if p.Query[i].Measurement != "" { meas_name = p.Query[i].Measurement } else { diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go index 1ed62a1cd62c0..0259db1086b49 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go @@ -145,7 +145,7 @@ func TestPostgresqlFieldOutput(t *testing.T) { } acc := queryRunner(t, query{{ - Sqlquery: "select * from pg_stat_database", + Script: "test.sql", Version: 901, Withdbname: false, Tagvalue: "", @@ -201,6 +201,31 @@ func TestPostgresqlFieldOutput(t *testing.T) { } } +func TestPostgresqlSqlQueryAndScriptScepifiedError(t *testing.T) { + q := query{{ + Script: "test.sql", + Sqlquery: "select * from pg_stat_database", + Version: 901, + Withdbname: false, + Tagvalue: "", + }} + p := &Postgresql{ + Service: postgresql.Service{ + Address: fmt.Sprintf( + "host=%s user=postgres sslmode=disable", + testutil.GetLocalHost(), + ), + IsPgBouncer: false, + }, + Databases: []string{"postgres"}, + Query: q, + } + var acc testutil.Accumulator + p.Start(&acc) + + require.Error(t, acc.GatherError(p.Gather)) +} + func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) { if testing.Short() { t.Skip("Skipping integration test in short mode") diff --git a/plugins/inputs/postgresql_extensible/test.sql b/plugins/inputs/postgresql_extensible/test.sql new file mode 100644 index 0000000000000..49ec02b258fe9 --- /dev/null +++ b/plugins/inputs/postgresql_extensible/test.sql @@ -0,0 +1 @@ +select * from pg_stat_database \ No newline at end of file From 5d90d599b9a8f7ce1002997908072e2614723b10 Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Fri, 6 Sep 2019 15:39:39 +0300 Subject: [PATCH 2/4] Readme update --- plugins/inputs/postgresql_extensible/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/postgresql_extensible/README.md b/plugins/inputs/postgresql_extensible/README.md index 29c5e36d853e4..af22de3341c9a 100644 --- a/plugins/inputs/postgresql_extensible/README.md +++ b/plugins/inputs/postgresql_extensible/README.md @@ -43,6 +43,9 @@ The example below has two queries are specified, with the following parameters: # withdbname was true. # Be careful that if the withdbname is set to false you don't have to define # the where clause (aka with the dbname) + # Also script option can be used to speciafy the .sql file path, + # in this way please make sure that sqlquery and script is not used at same time, + # coz this will cause error. # # the tagvalue field is used to define custom tags (separated by comas). # the query is expected to return columns which match the names of the @@ -61,7 +64,7 @@ The example below has two queries are specified, with the following parameters: withdbname=false tagvalue="" [[inputs.postgresql_extensible.query]] - sqlquery="SELECT * FROM pg_stat_bgwriter" + script="your_sql-filepath.sql" version=901 withdbname=false tagvalue="" From c0cbda8f13c66f0cf1dfe0dd43cf2d6de02c02da Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Wed, 11 Sep 2019 17:18:26 +0300 Subject: [PATCH 3/4] Readme update + comments rework --- plugins/inputs/postgresql_extensible/README.md | 6 +++--- .../postgresql_extensible/postgresql_extensible.go | 12 ++++++------ .../postgresql_extensible_test.go | 9 ++++----- .../postgresql_extensible/{ => testdata}/test.sql | 0 4 files changed, 13 insertions(+), 14 deletions(-) rename plugins/inputs/postgresql_extensible/{ => testdata}/test.sql (100%) diff --git a/plugins/inputs/postgresql_extensible/README.md b/plugins/inputs/postgresql_extensible/README.md index af22de3341c9a..75b653a84739f 100644 --- a/plugins/inputs/postgresql_extensible/README.md +++ b/plugins/inputs/postgresql_extensible/README.md @@ -43,9 +43,9 @@ The example below has two queries are specified, with the following parameters: # withdbname was true. # Be careful that if the withdbname is set to false you don't have to define # the where clause (aka with the dbname) - # Also script option can be used to speciafy the .sql file path, - # in this way please make sure that sqlquery and script is not used at same time, - # coz this will cause error. + # + # The script option can be used to speciafy the .sql file path, + # in case if script and sqlquery options specified at same time, sqlquery will be used # # the tagvalue field is used to define custom tags (separated by comas). # the query is expected to return columns which match the names of the diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible.go b/plugins/inputs/postgresql_extensible/postgresql_extensible.go index 7c2f241757bdc..d3dfba0078a9c 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible.go @@ -78,7 +78,10 @@ var sampleConfig = ` ## field is used to define custom tags (separated by commas) ## The optional "measurement" value can be used to override the default ## output measurement name ("postgresql"). - # + ## + ## The script option can be used to speciafy the .sql file path, + ## in case if script and sqlquery options specified at same time, sqlquery will be used + ## ## Structure : ## [[inputs.postgresql_extensible.query]] ## sqlquery string @@ -111,7 +114,7 @@ func (p *Postgresql) IgnoredColumns() map[string]bool { return ignoredColumns } -func (p *Postgresql) ReadQueryFromFile(filePath string) (string, error) { +func ReadQueryFromFile(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { return "", err @@ -146,12 +149,9 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { // We loop in order to process each query // Query is not run if Database version does not match the query version. for i := range p.Query { - if p.Query[i].Sqlquery != "" && p.Query[i].Script != "" { - return fmt.Errorf("both 'sqlquery' and 'script' specified, please choose one option") - } sql_query = p.Query[i].Sqlquery if sql_query == "" { - sql_query, err = p.ReadQueryFromFile(p.Query[i].Script) + sql_query, err = ReadQueryFromFile(p.Query[i].Script) if err != nil { return err } diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go index 0259db1086b49..78576d6cdef89 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go @@ -145,7 +145,7 @@ func TestPostgresqlFieldOutput(t *testing.T) { } acc := queryRunner(t, query{{ - Script: "test.sql", + Script: "select * from pg_stat_database", Version: 901, Withdbname: false, Tagvalue: "", @@ -201,10 +201,9 @@ func TestPostgresqlFieldOutput(t *testing.T) { } } -func TestPostgresqlSqlQueryAndScriptScepifiedError(t *testing.T) { +func TestPostgresqlSqlScript(t *testing.T) { q := query{{ - Script: "test.sql", - Sqlquery: "select * from pg_stat_database", + Script: "testdata/test.sql", Version: 901, Withdbname: false, Tagvalue: "", @@ -223,7 +222,7 @@ func TestPostgresqlSqlQueryAndScriptScepifiedError(t *testing.T) { var acc testutil.Accumulator p.Start(&acc) - require.Error(t, acc.GatherError(p.Gather)) + require.NoError(t, acc.GatherError(p.Gather)) } func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) { diff --git a/plugins/inputs/postgresql_extensible/test.sql b/plugins/inputs/postgresql_extensible/testdata/test.sql similarity index 100% rename from plugins/inputs/postgresql_extensible/test.sql rename to plugins/inputs/postgresql_extensible/testdata/test.sql From f744e85082614e4e5b5a715b8c81d732484c0571 Mon Sep 17 00:00:00 2001 From: bpopovschi Date: Thu, 12 Sep 2019 14:46:41 +0300 Subject: [PATCH 4/4] added Init func --- .../inputs/postgresql_extensible/README.md | 4 +-- .../postgresql_extensible.go | 27 ++++++++++++------- .../postgresql_extensible_test.go | 5 ++-- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/plugins/inputs/postgresql_extensible/README.md b/plugins/inputs/postgresql_extensible/README.md index 75b653a84739f..4621e46b54078 100644 --- a/plugins/inputs/postgresql_extensible/README.md +++ b/plugins/inputs/postgresql_extensible/README.md @@ -44,8 +44,8 @@ The example below has two queries are specified, with the following parameters: # Be careful that if the withdbname is set to false you don't have to define # the where clause (aka with the dbname) # - # The script option can be used to speciafy the .sql file path, - # in case if script and sqlquery options specified at same time, sqlquery will be used + # The script option can be used to specify the .sql file path. + # If script and sqlquery options specified at same time, sqlquery will be used # # the tagvalue field is used to define custom tags (separated by comas). # the query is expected to return columns which match the names of the diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible.go b/plugins/inputs/postgresql_extensible/postgresql_extensible.go index d3dfba0078a9c..05e57583f83b3 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible.go @@ -79,8 +79,8 @@ var sampleConfig = ` ## The optional "measurement" value can be used to override the default ## output measurement name ("postgresql"). ## - ## The script option can be used to speciafy the .sql file path, - ## in case if script and sqlquery options specified at same time, sqlquery will be used + ## The script option can be used to specify the .sql file path. + ## If script and sqlquery options specified at same time, sqlquery will be used ## ## Structure : ## [[inputs.postgresql_extensible.query]] @@ -102,6 +102,19 @@ var sampleConfig = ` tagvalue="postgresql.stats" ` +func (p *Postgresql) Init() error { + var err error + for i := range p.Query { + if p.Query[i].Sqlquery == "" { + p.Query[i].Sqlquery, err = ReadQueryFromFile(p.Query[i].Script) + if err != nil { + return err + } + } + } + return nil +} + func (p *Postgresql) SampleConfig() string { return sampleConfig } @@ -121,11 +134,11 @@ func ReadQueryFromFile(filePath string) (string, error) { } defer file.Close() - qyery, err := ioutil.ReadAll(file) + query, err := ioutil.ReadAll(file) if err != nil { return "", err } - return string(qyery), err + return string(query), err } func (p *Postgresql) Gather(acc telegraf.Accumulator) error { @@ -150,12 +163,6 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { // Query is not run if Database version does not match the query version. for i := range p.Query { sql_query = p.Query[i].Sqlquery - if sql_query == "" { - sql_query, err = ReadQueryFromFile(p.Query[i].Script) - if err != nil { - return err - } - } tag_value = p.Query[i].Tagvalue if p.Query[i].Measurement != "" { diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go index 78576d6cdef89..7fbc343028015 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go @@ -25,7 +25,7 @@ func queryRunner(t *testing.T, q query) *testutil.Accumulator { } var acc testutil.Accumulator p.Start(&acc) - + p.Init() require.NoError(t, acc.GatherError(p.Gather)) return &acc } @@ -145,7 +145,7 @@ func TestPostgresqlFieldOutput(t *testing.T) { } acc := queryRunner(t, query{{ - Script: "select * from pg_stat_database", + Sqlquery: "select * from pg_stat_database", Version: 901, Withdbname: false, Tagvalue: "", @@ -221,6 +221,7 @@ func TestPostgresqlSqlScript(t *testing.T) { } var acc testutil.Accumulator p.Start(&acc) + p.Init() require.NoError(t, acc.GatherError(p.Gather)) }