From a776544c837e24f2a825ee5875fb7563d9af38a0 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Wed, 29 Mar 2023 17:54:53 +0800 Subject: [PATCH 1/6] enh: log db support cachemode opts --- api/report.go | 42 +++++++++++++++++++++++--------- config/keeper.toml | 3 +++ infrastructure/config/config.go | 11 +++++++-- infrastructure/config/metrics.go | 13 +++++++--- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/api/report.go b/api/report.go index 3c81999..8d8c7b1 100644 --- a/api/report.go +++ b/api/report.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "context" "fmt" "strconv" @@ -32,21 +33,23 @@ var createList = []string{ } type Reporter struct { - username string - password string - host string - port int - dbname string - totalRep atomic.Value + username string + password string + host string + port int + dbname string + databaseOptions config.DatabaseOptions + totalRep atomic.Value } func NewReporter(conf *config.Config) *Reporter { r := &Reporter{ - username: conf.TDengine.Username, - password: conf.TDengine.Password, - host: conf.TDengine.Host, - port: conf.TDengine.Port, - dbname: conf.Metrics.Database, + username: conf.TDengine.Username, + password: conf.TDengine.Password, + host: conf.TDengine.Host, + port: conf.TDengine.Port, + dbname: conf.Metrics.Database, + databaseOptions: conf.Metrics.DatabaseOptions, } r.totalRep.Store(0) return r @@ -138,12 +141,27 @@ func (r *Reporter) createDatabase() { conn := r.getConn() defer r.closeConn(conn) - if _, err := conn.Exec(ctx, fmt.Sprintf("create database if not exists %s", r.dbname)); err != nil { + createDBSql := r.generateCreateDBSql() + logger.Warningf("create database sql: %s", createDBSql) + + if _, err := conn.Exec(ctx, createDBSql); err != nil { logger.WithError(err).Errorf("create database %s error %v", r.dbname, err) panic(err) } } +func (r *Reporter) generateCreateDBSql() string { + var buf bytes.Buffer + buf.WriteString("create database if not exists ") + buf.WriteString(r.dbname) + + if r.databaseOptions.CacheModel != "" && r.databaseOptions.CacheModel != "none" && r.databaseOptions.CacheModel != "NONE" { + buf.WriteString(" CACHEMODEL ") + buf.WriteString(r.databaseOptions.CacheModel) + } + return buf.String() +} + func (r *Reporter) creatTables() { ctx := context.Background() conn, err := db.NewConnectorWithDb(r.username, r.password, r.host, r.port, r.dbname) diff --git a/config/keeper.toml b/config/keeper.toml index 5028da6..6fa2130 100644 --- a/config/keeper.toml +++ b/config/keeper.toml @@ -33,6 +33,9 @@ database = "log" # export some tables that are not super table tables = [] +[metrics.databaseoptions] +cachemodel = "none" + [environment] # Whether running in cgroup. incgroup = false diff --git a/infrastructure/config/config.go b/infrastructure/config/config.go index f2802aa..4d8bffe 100644 --- a/infrastructure/config/config.go +++ b/infrastructure/config/config.go @@ -96,8 +96,11 @@ func InitConfig() *Config { conf.Metrics = MetricsConfig{ Prefix: viper.GetString("metrics.prefix"), Database: viper.GetString("metrics.database"), - Tables: map[string]struct{}{}, - Normals: viper.GetStringSlice("metrics.tables"), + DatabaseOptions: DatabaseOptions{ + CacheModel: viper.GetString("metrics.databaseoptions.cachemodel"), + }, + Tables: map[string]struct{}{}, + Normals: viper.GetStringSlice("metrics.tables"), } conf.Env = Environment{ InCGroup: viper.GetBool("environment.incgroup"), @@ -154,6 +157,10 @@ func init() { _ = viper.BindEnv("metrics.database", "TAOS_KEEPER_METRICS_DATABASE") pflag.String("metrics.database", "log", `database for storing metrics data. Env "TAOS_KEEPER_METRICS_DATABASE"`) + viper.SetDefault("metrics.databaseoptions.cacheModel", "none") + _ = viper.BindEnv("metrics.databaseoptions.cacheModel", "TAOS_KEEPER_METRICS_DATABASEOPTIONS_CACHEMODEL") + pflag.String("metrics.databaseoptions.cacheModel", "none", `cache model for database. Env "TAOS_KEEPER_METRICS_DATABASEOPTIONS_CACHEMODEL"`) + viper.SetDefault("metrics.tables", "") _ = viper.BindEnv("metrics.tables", "TAOS_KEEPER_METRICS_TABLES") pflag.String("metrics.tables", "", `export some tables that are not super table, multiple values split with white space. Env "TAOS_KEEPER_METRICS_TABLES"`) diff --git a/infrastructure/config/metrics.go b/infrastructure/config/metrics.go index 3b88b3d..2c3b31f 100644 --- a/infrastructure/config/metrics.go +++ b/infrastructure/config/metrics.go @@ -1,10 +1,11 @@ package config type MetricsConfig struct { - Prefix string `toml:"prefix"` - Database string `toml:"database"` - Tables map[string]struct{} - Normals []string `toml:"tables"` + Prefix string `toml:"prefix"` + Database string `toml:"database"` + DatabaseOptions DatabaseOptions `toml:"databaseoptions"` + Tables map[string]struct{} + Normals []string `toml:"tables"` } type TaosAdapter struct { @@ -22,3 +23,7 @@ type Metric struct { type Environment struct { InCGroup bool `toml:"whether running in cgroup"` } + +type DatabaseOptions struct { + CacheModel string `toml:"cachemodel"` +} From 2fed7bb104fb31dbcd0d6033bbfb14772ad56ae0 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Wed, 29 Mar 2023 19:50:29 +0800 Subject: [PATCH 2/6] fix for config --- api/report.go | 15 +++++++++---- config/keeper.toml | 1 + infrastructure/config/config.go | 37 +++++--------------------------- infrastructure/config/metrics.go | 10 +++------ 4 files changed, 20 insertions(+), 43 deletions(-) diff --git a/api/report.go b/api/report.go index 8d8c7b1..716f77d 100644 --- a/api/report.go +++ b/api/report.go @@ -38,7 +38,7 @@ type Reporter struct { host string port int dbname string - databaseOptions config.DatabaseOptions + databaseOptions map[string]interface{} totalRep atomic.Value } @@ -155,9 +155,16 @@ func (r *Reporter) generateCreateDBSql() string { buf.WriteString("create database if not exists ") buf.WriteString(r.dbname) - if r.databaseOptions.CacheModel != "" && r.databaseOptions.CacheModel != "none" && r.databaseOptions.CacheModel != "NONE" { - buf.WriteString(" CACHEMODEL ") - buf.WriteString(r.databaseOptions.CacheModel) + for k, v := range r.databaseOptions { + buf.WriteString(" ") + buf.WriteString(k) + switch v := v.(type) { + case string: + buf.WriteString(fmt.Sprintf(" '%s'", v)) + default: + buf.WriteString(fmt.Sprintf(" %v", v)) + } + buf.WriteString(" ") } return buf.String() } diff --git a/config/keeper.toml b/config/keeper.toml index 6fa2130..587f60b 100644 --- a/config/keeper.toml +++ b/config/keeper.toml @@ -33,6 +33,7 @@ database = "log" # export some tables that are not super table tables = [] +# database options for db storing metrics data [metrics.databaseoptions] cachemodel = "none" diff --git a/infrastructure/config/config.go b/infrastructure/config/config.go index 4d8bffe..0bf5110 100644 --- a/infrastructure/config/config.go +++ b/infrastructure/config/config.go @@ -73,39 +73,16 @@ func InitConfig() *Config { } } - conf := &Config{ - Debug: viper.GetBool("debug"), - Port: viper.GetInt("port"), - LogLevel: viper.GetString("logLevel"), - GoPoolSize: viper.GetInt("gopoolsize"), - RotationInterval: viper.GetString("RotationInterval"), + var conf Config + if err = viper.Unmarshal(&conf); err != nil { + panic(err) } + conf.Cors.Init() pool.Init(conf.GoPoolSize) log.Init(conf.LogLevel) - conf.TDengine = TDengineRestful{ - Host: viper.GetString("tdengine.host"), - Port: viper.GetInt("tdengine.port"), - Username: viper.GetString("tdengine.username"), - Password: viper.GetString("tdengine.password"), - } - conf.TaosAdapter = TaosAdapter{ - Addrs: viper.GetStringSlice("taosAdapter.address"), - } - conf.Metrics = MetricsConfig{ - Prefix: viper.GetString("metrics.prefix"), - Database: viper.GetString("metrics.database"), - DatabaseOptions: DatabaseOptions{ - CacheModel: viper.GetString("metrics.databaseoptions.cachemodel"), - }, - Tables: map[string]struct{}{}, - Normals: viper.GetStringSlice("metrics.tables"), - } - conf.Env = Environment{ - InCGroup: viper.GetBool("environment.incgroup"), - } - return conf + return &conf } func init() { @@ -157,10 +134,6 @@ func init() { _ = viper.BindEnv("metrics.database", "TAOS_KEEPER_METRICS_DATABASE") pflag.String("metrics.database", "log", `database for storing metrics data. Env "TAOS_KEEPER_METRICS_DATABASE"`) - viper.SetDefault("metrics.databaseoptions.cacheModel", "none") - _ = viper.BindEnv("metrics.databaseoptions.cacheModel", "TAOS_KEEPER_METRICS_DATABASEOPTIONS_CACHEMODEL") - pflag.String("metrics.databaseoptions.cacheModel", "none", `cache model for database. Env "TAOS_KEEPER_METRICS_DATABASEOPTIONS_CACHEMODEL"`) - viper.SetDefault("metrics.tables", "") _ = viper.BindEnv("metrics.tables", "TAOS_KEEPER_METRICS_TABLES") pflag.String("metrics.tables", "", `export some tables that are not super table, multiple values split with white space. Env "TAOS_KEEPER_METRICS_TABLES"`) diff --git a/infrastructure/config/metrics.go b/infrastructure/config/metrics.go index 2c3b31f..24ac0bf 100644 --- a/infrastructure/config/metrics.go +++ b/infrastructure/config/metrics.go @@ -1,9 +1,9 @@ package config type MetricsConfig struct { - Prefix string `toml:"prefix"` - Database string `toml:"database"` - DatabaseOptions DatabaseOptions `toml:"databaseoptions"` + Prefix string `toml:"prefix"` + Database string `toml:"database"` + DatabaseOptions map[string]interface{} `toml:"databaseoptions"` Tables map[string]struct{} Normals []string `toml:"tables"` } @@ -23,7 +23,3 @@ type Metric struct { type Environment struct { InCGroup bool `toml:"whether running in cgroup"` } - -type DatabaseOptions struct { - CacheModel string `toml:"cachemodel"` -} From 6b5ddaaba9c1a1058d64d0aca4a8bf8c9d80876c Mon Sep 17 00:00:00 2001 From: sunpeng Date: Wed, 29 Mar 2023 20:07:09 +0800 Subject: [PATCH 3/6] fix for config --- infrastructure/config/config.go | 4 ++-- infrastructure/config/metrics.go | 4 ++-- process/builder.go | 10 +++++----- process/handle.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/infrastructure/config/config.go b/infrastructure/config/config.go index 0bf5110..20c1ab0 100644 --- a/infrastructure/config/config.go +++ b/infrastructure/config/config.go @@ -134,9 +134,9 @@ func init() { _ = viper.BindEnv("metrics.database", "TAOS_KEEPER_METRICS_DATABASE") pflag.String("metrics.database", "log", `database for storing metrics data. Env "TAOS_KEEPER_METRICS_DATABASE"`) - viper.SetDefault("metrics.tables", "") + viper.SetDefault("metrics.tables", []string{}) _ = viper.BindEnv("metrics.tables", "TAOS_KEEPER_METRICS_TABLES") - pflag.String("metrics.tables", "", `export some tables that are not super table, multiple values split with white space. Env "TAOS_KEEPER_METRICS_TABLES"`) + pflag.StringArray("metrics.tables", []string{}, `export some tables that are not super table, multiple values split with white space. Env "TAOS_KEEPER_METRICS_TABLES"`) viper.SetDefault("environment.incgroup", false) _ = viper.BindEnv("environment.incgroup", "TAOS_KEEPER_ENVIRONMENT_INCGROUP") diff --git a/infrastructure/config/metrics.go b/infrastructure/config/metrics.go index 24ac0bf..83fd052 100644 --- a/infrastructure/config/metrics.go +++ b/infrastructure/config/metrics.go @@ -4,8 +4,8 @@ type MetricsConfig struct { Prefix string `toml:"prefix"` Database string `toml:"database"` DatabaseOptions map[string]interface{} `toml:"databaseoptions"` - Tables map[string]struct{} - Normals []string `toml:"tables"` + Tables []string `toml:"tables"` + TableMap map[string]struct{} } type TaosAdapter struct { diff --git a/process/builder.go b/process/builder.go index b58282c..f99bf12 100644 --- a/process/builder.go +++ b/process/builder.go @@ -13,15 +13,15 @@ var builderLogger = log.GetLogger("builder") func ExpandMetricsFromConfig(ctx context.Context, conn *db.Connector, cfg *config.MetricsConfig) error { - for _, name := range cfg.Normals { + for _, name := range cfg.Tables { builderLogger.Debug("normal table: ", name) - _, exist := cfg.Tables[name] + _, exist := cfg.TableMap[name] if exist { builderLogger.Debug(name, "is exist in config") continue } - cfg.Tables[name] = struct{}{} + cfg.TableMap[name] = struct{}{} } data, err := conn.Query(ctx, fmt.Sprintf("show %s.stables", cfg.Database)) @@ -34,12 +34,12 @@ func ExpandMetricsFromConfig(ctx context.Context, conn *db.Connector, cfg *confi name := info[0].(string) builderLogger.Debug("stable: ", info) - _, exist := cfg.Tables[name] + _, exist := cfg.TableMap[name] if exist { builderLogger.Debug(name, "is exist in config") continue } - cfg.Tables[name] = struct{}{} + cfg.TableMap[name] = struct{}{} } return err } diff --git a/process/handle.go b/process/handle.go index ddc4306..23b2ec8 100644 --- a/process/handle.go +++ b/process/handle.go @@ -176,7 +176,7 @@ func NewProcessor(conf *config.Config) *Processor { exitChan: make(chan struct{}), dbConn: conn, summaryTable: map[string]*Table{"taosadapter_restful_http_request_summary_milliseconds": nil}, - tables: conf.Metrics.Tables, + tables: conf.Metrics.TableMap, } p.Prepare() p.process() From 1cf7f4f4df8bc4cc051f77685f77fd51fa96adf7 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Thu, 30 Mar 2023 09:32:13 +0800 Subject: [PATCH 4/6] fix for table config --- infrastructure/config/metrics.go | 1 - process/builder.go | 16 ++++++++-------- process/handle.go | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/infrastructure/config/metrics.go b/infrastructure/config/metrics.go index 83fd052..435d53a 100644 --- a/infrastructure/config/metrics.go +++ b/infrastructure/config/metrics.go @@ -5,7 +5,6 @@ type MetricsConfig struct { Database string `toml:"database"` DatabaseOptions map[string]interface{} `toml:"databaseoptions"` Tables []string `toml:"tables"` - TableMap map[string]struct{} } type TaosAdapter struct { diff --git a/process/builder.go b/process/builder.go index f99bf12..d0920b4 100644 --- a/process/builder.go +++ b/process/builder.go @@ -11,22 +11,22 @@ import ( var builderLogger = log.GetLogger("builder") -func ExpandMetricsFromConfig(ctx context.Context, conn *db.Connector, cfg *config.MetricsConfig) error { - +func ExpandMetricsFromConfig(ctx context.Context, conn *db.Connector, cfg *config.MetricsConfig) (tables map[string]struct{}, err error) { + tables = make(map[string]struct{}) for _, name := range cfg.Tables { builderLogger.Debug("normal table: ", name) - _, exist := cfg.TableMap[name] + _, exist := tables[name] if exist { builderLogger.Debug(name, "is exist in config") continue } - cfg.TableMap[name] = struct{}{} + tables[name] = struct{}{} } data, err := conn.Query(ctx, fmt.Sprintf("show %s.stables", cfg.Database)) if err != nil { - return err + return nil, err } builderLogger.Debug("show stables") @@ -34,12 +34,12 @@ func ExpandMetricsFromConfig(ctx context.Context, conn *db.Connector, cfg *confi name := info[0].(string) builderLogger.Debug("stable: ", info) - _, exist := cfg.TableMap[name] + _, exist := tables[name] if exist { builderLogger.Debug(name, "is exist in config") continue } - cfg.TableMap[name] = struct{}{} + tables[name] = struct{}{} } - return err + return } diff --git a/process/handle.go b/process/handle.go index 23b2ec8..a0dffb4 100644 --- a/process/handle.go +++ b/process/handle.go @@ -163,7 +163,7 @@ func NewProcessor(conf *config.Config) *Processor { panic(err) } ctx := context.Background() - err = ExpandMetricsFromConfig(ctx, conn, &conf.Metrics) + tables, err := ExpandMetricsFromConfig(ctx, conn, &conf.Metrics) if err != nil { panic(err) } @@ -176,7 +176,7 @@ func NewProcessor(conf *config.Config) *Processor { exitChan: make(chan struct{}), dbConn: conn, summaryTable: map[string]*Table{"taosadapter_restful_http_request_summary_milliseconds": nil}, - tables: conf.Metrics.TableMap, + tables: tables, } p.Prepare() p.process() From b55177fbb0767fb91189d52911f8ed7407cbb5be Mon Sep 17 00:00:00 2001 From: sunpeng Date: Thu, 30 Mar 2023 09:48:24 +0800 Subject: [PATCH 5/6] fix for config --- api/adapter.go | 2 +- infrastructure/config/config.go | 28 ++++++++++++++-------------- infrastructure/config/metrics.go | 6 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/api/adapter.go b/api/adapter.go index a6e240b..e2603e4 100644 --- a/api/adapter.go +++ b/api/adapter.go @@ -56,7 +56,7 @@ func NewAdapterImporter(conf *config.Config) { host: conf.TDengine.Host, port: conf.TDengine.Port, database: conf.Metrics.Database, - adapters: conf.TaosAdapter.Addrs, + adapters: conf.TaosAdapter.Address, } imp.setNextTime(time.Now()) go imp.work() diff --git a/infrastructure/config/config.go b/infrastructure/config/config.go index 20c1ab0..3c7aedf 100644 --- a/infrastructure/config/config.go +++ b/infrastructure/config/config.go @@ -14,23 +14,23 @@ import ( ) type Config struct { - Cors web.CorsConfig - Debug bool - Port int - LogLevel string - GoPoolSize int - RotationInterval string - TDengine TDengineRestful - TaosAdapter TaosAdapter - Metrics MetricsConfig - Env Environment + Cors web.CorsConfig `toml:"cors"` + Debug bool `toml:"debug"` + Port int `toml:"port"` + LogLevel string `toml:"loglevel"` + GoPoolSize int `toml:"gopoolsize"` + RotationInterval string `toml:"RotationInterval"` + TDengine TDengineRestful `toml:"tdengine"` + TaosAdapter TaosAdapter `toml:"taosAdapter"` + Metrics MetricsConfig `toml:"metrics"` + Env Environment `toml:"environment"` } type TDengineRestful struct { - Host string - Port int - Username string - Password string + Host string `toml:"host"` + Port int `toml:"port"` + Username string `toml:"username"` + Password string `toml:"password"` } func InitConfig() *Config { diff --git a/infrastructure/config/metrics.go b/infrastructure/config/metrics.go index 435d53a..7335d8f 100644 --- a/infrastructure/config/metrics.go +++ b/infrastructure/config/metrics.go @@ -3,12 +3,12 @@ package config type MetricsConfig struct { Prefix string `toml:"prefix"` Database string `toml:"database"` - DatabaseOptions map[string]interface{} `toml:"databaseoptions"` Tables []string `toml:"tables"` + DatabaseOptions map[string]interface{} `toml:"databaseoptions"` } type TaosAdapter struct { - Addrs []string `toml:"address"` + Address []string `toml:"address"` } type Metric struct { @@ -20,5 +20,5 @@ type Metric struct { } type Environment struct { - InCGroup bool `toml:"whether running in cgroup"` + InCGroup bool `toml:"incgroup"` } From 9dce7fa91123886a14e27d39e28ac8a67c67a858 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Thu, 30 Mar 2023 09:59:43 +0800 Subject: [PATCH 6/6] add debug log --- api/adapter.go | 2 ++ infrastructure/config/config.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/api/adapter.go b/api/adapter.go index e2603e4..0a8085c 100644 --- a/api/adapter.go +++ b/api/adapter.go @@ -124,6 +124,8 @@ func (imp *AdapterImporter) queryMetrics() { logger.Errorf("error reading body: %s", err) continue } + + logger.Debug("## adapter metrics: ", string(body)) imp.lineWriteBody(body, addr) _ = resp.Body.Close() } diff --git a/infrastructure/config/config.go b/infrastructure/config/config.go index 3c7aedf..b9ff936 100644 --- a/infrastructure/config/config.go +++ b/infrastructure/config/config.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "fmt" "os" @@ -77,6 +78,10 @@ func InitConfig() *Config { if err = viper.Unmarshal(&conf); err != nil { panic(err) } + if conf.Debug { + j, _ := json.Marshal(conf) + fmt.Println("config file:", string(j)) + } conf.Cors.Init() pool.Init(conf.GoPoolSize)