From b3940bbbba10a2995de75661a02d603c044ed38c Mon Sep 17 00:00:00 2001 From: MingHao Li <1203558364@qq.com> Date: Fri, 28 May 2021 23:49:40 +0800 Subject: [PATCH] Refractor config_load.go (#158) * Refractor config_load.go * Refractor pixiu_start.go * Refractor pixiu_start.go * bug fix about read yml file. some key with _ can't read in object. * Change DiscoverService to DiscoveryService. Co-authored-by: zhangxun <18721825717@163.com> Former-commit-id: 51bb439739edaed119434c13b213ab7e8f3d4a99 --- pkg/common/constant/pixiu.go | 20 +++++ pkg/config/conf_test.yaml | 6 +- pkg/config/config_load.go | 153 +++++++++++++++++++-------------- pkg/config/config_load_test.go | 80 ++++++++++++++--- pkg/model/bootstrap.go | 2 +- pkg/model/cluster.go | 2 +- pkg/model/match.go | 6 +- pkg/model/router.go | 44 +++++----- 8 files changed, 205 insertions(+), 108 deletions(-) diff --git a/pkg/common/constant/pixiu.go b/pkg/common/constant/pixiu.go index 43b3ad23b..c9c6b101e 100644 --- a/pkg/common/constant/pixiu.go +++ b/pkg/common/constant/pixiu.go @@ -35,3 +35,23 @@ const ( ResponseStrategyNormal = "normal" ResponseStrategyHump = "hump" ) + +const ( + // DefaultDiscoveryType Set up default discovery type. + DefaultDiscoveryType = "EDS" + // DefaultLoadBalanceType Set up default load balance type. + DefaultLoadBalanceType = "RoundRobin" + // DefaultFilterType Set up default filter type. + DefaultFilterType = "dgp.filters.http_connect_manager" + // DefaultHTTPType Set up default HTTP Type. + DefaultHTTPType = "net/http" + // DefaultProtocolType Set up default protocol type. + DefaultProtocolType = "HTTP" +) + +const ( + // YAML .yaml + YAML = ".yaml" + //YML .yml + YML = ".yml" +) diff --git a/pkg/config/conf_test.yaml b/pkg/config/conf_test.yaml index 12515f5fb..c79be624f 100644 --- a/pkg/config/conf_test.yaml +++ b/pkg/config/conf_test.yaml @@ -78,7 +78,9 @@ static_resources: timeout: "60s" step_timeout: "10s" reject_policy: "immediacy" - pprofConf: enable: true - addr: "0.0.0.0:6060" \ No newline at end of file + address: + socket_address: + address: "0.0.0.0" + port: 6060 \ No newline at end of file diff --git a/pkg/config/config_load.go b/pkg/config/config_load.go index 476ac6878..4dffb46cf 100644 --- a/pkg/config/config_load.go +++ b/pkg/config/config_load.go @@ -18,7 +18,7 @@ package config import ( - "encoding/json" + "github.com/apache/dubbo-go-pixiu/pkg/common/constant" "io/ioutil" "log" "path/filepath" @@ -37,9 +37,12 @@ import ( var ( configPath string config *model.Bootstrap - configLoadFunc ConfigLoadFunc = DefaultConfigLoad + configLoadFunc LoadFunc = LoadYAMLConfig ) +// LoadFunc ConfigLoadFunc parse a input(usually file path) into a pixiu config +type LoadFunc func(path string) *model.Bootstrap + // GetBootstrap get config global, need a better name func GetBootstrap() *model.Bootstrap { return config @@ -48,142 +51,162 @@ func GetBootstrap() *model.Bootstrap { // Load config file and parse func Load(path string) *model.Bootstrap { logger.Infof("[dubbopixiu go] load path:%s", path) - configPath, _ = filepath.Abs(path) - if yamlFormat(path) { - RegisterConfigLoadFunc(YAMLConfigLoad) + if configPath != "" && CheckYamlFormat(configPath) { + RegisterConfigLoadFunc(LoadYAMLConfig) } - if cfg := configLoadFunc(path); cfg != nil { + if cfg := configLoadFunc(configPath); cfg != nil { config = cfg } - return config } -// ConfigLoadFunc parse a input(usually file path) into a pixiu config -type ConfigLoadFunc func(path string) *model.Bootstrap - // RegisterConfigLoadFunc can replace a new config load function instead of default -func RegisterConfigLoadFunc(f ConfigLoadFunc) { +func RegisterConfigLoadFunc(f LoadFunc) { configLoadFunc = f } -func yamlFormat(path string) bool { +func CheckYamlFormat(path string) bool { ext := filepath.Ext(path) - if ext == ".yaml" || ext == ".yml" { + if ext == constant.YAML || ext == constant.YML { return true } return false } -// YAMLConfigLoad config load yaml -func YAMLConfigLoad(path string) *model.Bootstrap { +// LoadYAMLConfig YAMLConfigLoad config load yaml +func LoadYAMLConfig(path string) *model.Bootstrap { log.Println("load config in YAML format from : ", path) content, err := ioutil.ReadFile(path) if err != nil { log.Fatalln("[config] [yaml load] load config failed, ", err) } cfg := &model.Bootstrap{} - - bytes, err := yaml.YAMLToJSON(content) + err = yaml.Unmarshal(content, cfg) if err != nil { log.Fatalln("[config] [yaml load] convert YAML to JSON failed, ", err) } - - err = json.Unmarshal(bytes, cfg) + err = Adapter(cfg) if err != nil { log.Fatalln("[config] [yaml load] yaml unmarshal config failed, ", err) } + return cfg +} - // other adapter +func Adapter(cfg *model.Bootstrap) (err error) { + if GetFilterChain(cfg) != nil || GetHttpConfig(cfg) != nil || GetProtocol(cfg) != nil || + GetLoadBalance(cfg) != nil || GetDiscoveryType(cfg) != nil { + return err + } + return nil +} - for i, l := range cfg.StaticResources.Listeners { +func GetProtocol(cfg *model.Bootstrap) (err error) { + if cfg == nil { + logger.Error("Bootstrap configuration is null") + return err + } + for _, l := range cfg.StaticResources.Listeners { if l.Address.SocketAddress.ProtocolStr == "" { - l.Address.SocketAddress.ProtocolStr = "HTTP" + l.Address.SocketAddress.ProtocolStr = constant.DefaultProtocolType } l.Address.SocketAddress.Protocol = model.ProtocolType(model.ProtocolTypeValue[l.Address.SocketAddress.ProtocolStr]) + } + return nil +} +func GetHttpConfig(cfg *model.Bootstrap) (err error) { + if cfg == nil { + logger.Error("Bootstrap configuration is null") + return err + } + for i, l := range cfg.StaticResources.Listeners { hc := &model.HttpConfig{} if l.Config != nil { if v, ok := l.Config.(map[string]interface{}); ok { switch l.Name { - case "net/http": + case constant.DefaultHTTPType: if err := mapstructure.Decode(v, hc); err != nil { logger.Error(err) } - - cfg.StaticResources.Listeners[i].Config = hc + cfg.StaticResources.Listeners[i].Config = *hc } } } + } + return nil +} +func GetFilterChain(cfg *model.Bootstrap) (err error) { + if cfg == nil { + logger.Error("Bootstrap configuration is null") + return err + } + for _, l := range cfg.StaticResources.Listeners { for _, fc := range l.FilterChains { if fc.Filters != nil { for i, fcf := range fc.Filters { hcm := &model.HttpConnectionManager{} if fcf.Config != nil { switch fcf.Name { - case "dgp.filters.http_connect_manager": + case constant.DefaultFilterType: if v, ok := fcf.Config.(map[string]interface{}); ok { if err := mapstructure.Decode(v, hcm); err != nil { logger.Error(err) } - - fc.Filters[i].Config = hcm + fc.Filters[i].Config = *hcm } } } } } } - } + return nil +} +func GetLoadBalance(cfg *model.Bootstrap) (err error) { + if cfg == nil { + logger.Error("Bootstrap configuration is null") + return err + } + var lbPolicy int32 for _, c := range cfg.StaticResources.Clusters { - var discoverType int32 + flag := true if c.TypeStr != "" { - if t, ok := model.DiscoveryTypeValue[c.TypeStr]; ok { - discoverType = t - } else { - c.TypeStr = "EDS" - discoverType = model.DiscoveryTypeValue[c.TypeStr] + if t, ok := model.LbPolicyValue[c.LbStr]; ok { + lbPolicy = t + flag = false } - } else { - c.TypeStr = "EDS" - discoverType = model.DiscoveryTypeValue[c.TypeStr] } - c.Type = model.DiscoveryType(discoverType) - - var lbPolicy int32 - if c.LbStr != "" { - if lb, ok := model.LbPolicyValue[c.LbStr]; ok { - lbPolicy = lb - } else { - c.LbStr = "RoundRobin" - lbPolicy = model.LbPolicyValue[c.LbStr] - } - } else { - c.LbStr = "RoundRobin" + if flag { + c.LbStr = constant.DefaultLoadBalanceType lbPolicy = model.LbPolicyValue[c.LbStr] } - c.Lb = model.LbPolicy(lbPolicy) + c.Type = model.DiscoveryType(lbPolicy) } - - return cfg + return nil } -// DefaultConfigLoad if not config, will load this -func DefaultConfigLoad(path string) *model.Bootstrap { - log.Println("load config from : ", path) - content, err := ioutil.ReadFile(path) - if err != nil { - log.Fatalln("[config] [default load] load config failed, ", err) +func GetDiscoveryType(cfg *model.Bootstrap) (err error) { + if cfg == nil { + logger.Error("Bootstrap configuration is null") + return err } - cfg := &model.Bootstrap{} - // translate to lower case - err = json.Unmarshal(content, cfg) - if err != nil { - log.Fatalln("[config] [default load] json unmarshal config failed, ", err) + var discoveryType int32 + for _, c := range cfg.StaticResources.Clusters { + flag := true + if c.TypeStr != "" { + if t, ok := model.DiscoveryTypeValue[c.TypeStr]; ok { + discoveryType = t + flag = false + } + } + if flag { + c.TypeStr = constant.DefaultDiscoveryType + discoveryType = model.DiscoveryTypeValue[c.TypeStr] + } + c.Type = model.DiscoveryType(discoveryType) } - return cfg + return nil } diff --git a/pkg/config/config_load_test.go b/pkg/config/config_load_test.go index 35b4da496..375ef92be 100644 --- a/pkg/config/config_load_test.go +++ b/pkg/config/config_load_test.go @@ -19,6 +19,8 @@ package config import ( "encoding/json" + "log" + "os" "testing" ) @@ -30,15 +32,11 @@ import ( "github.com/apache/dubbo-go-pixiu/pkg/model" ) -func TestLoad(t *testing.T) { - conf := Load("conf_test.yaml") +var b model.Bootstrap - assert.Equal(t, 1, len(conf.StaticResources.Listeners)) - assert.Equal(t, 1, len(conf.StaticResources.Clusters)) -} - -func TestStruct2JSON(t *testing.T) { - b := model.Bootstrap{ +func TestMain(m *testing.M) { + log.Println("Prepare Bootstrap") + b = model.Bootstrap{ StaticResources: model.StaticResources{ Listeners: []model.Listener{ { @@ -46,8 +44,8 @@ func TestStruct2JSON(t *testing.T) { Address: model.Address{ SocketAddress: model.SocketAddress{ ProtocolStr: "HTTP", - Address: "127.0.0.0", - Port: 8899, + Address: "0.0.0.0", + Port: 8888, }, }, Config: model.HttpConfig{ @@ -72,13 +70,20 @@ func TestStruct2JSON(t *testing.T) { { Match: model.RouterMatch{ Prefix: "/api/v1", + Headers: []model.HeaderMatcher{ + {Name: "X-DGP-WAY", + Value: "dubbo", + }, + }, }, Route: model.RouteAction{ - Cluster: "test_dubbo", + Cluster: "test_dubbo", + ClusterNotFoundResponseCode: 505, Cors: model.CorsPolicy{ AllowOrigin: []string{ "*", }, + Enabled: true, }, }, }, @@ -86,12 +91,20 @@ func TestStruct2JSON(t *testing.T) { }, HTTPFilters: []model.HTTPFilter{ { - Name: "dgp.filters.http.cors", + Name: "dgp.filters.http.api", + Config: interface{}(nil), + }, + { + Name: "dgp.filters.http.router", + Config: interface{}(nil), }, { - Name: "dgp.filters.http.router", + Name: "dgp.filters.http_transfer_dubbo", + Config: interface{}(nil), }, }, + ServerName: "test_http_dubbo", + GenerateRequestID: false, }, }, }, @@ -99,17 +112,56 @@ func TestStruct2JSON(t *testing.T) { }, }, }, - Clusters: []model.Cluster{ + Clusters: []*model.Cluster{ { Name: "test_dubbo", TypeStr: "EDS", + Type: model.EDS, LbStr: "RoundRobin", ConnectTimeoutStr: "5s", + RequestTimeoutStr: "10s", + Registries: map[string]model.Registry{ + "zookeeper": { + Timeout: "3s", + Address: "127.0.0.1:2182", + Username: "", + Password: "", + }, + "consul": { + Timeout: "3s", + Address: "127.0.0.1:8500", + }, + }, + }, + }, + ShutdownConfig: &model.ShutdownConfig{ + Timeout: "60s", + StepTimeout: "10s", + RejectPolicy: "immediacy", + }, + PprofConf: model.PprofConf{ + Enable: true, + Address: model.Address{ + SocketAddress: model.SocketAddress{ + Address: "0.0.0.0", + Port: 6060, + }, }, }, }, } + retCode := m.Run() + os.Exit(retCode) +} +func TestLoad(t *testing.T) { + conf := Load("conf_test.yaml") + assert.Equal(t, 1, len(conf.StaticResources.Listeners)) + assert.Equal(t, 1, len(conf.StaticResources.Clusters)) + assert.Equal(t, *conf, b) +} + +func TestStruct2JSON(t *testing.T) { if bytes, err := json.Marshal(b); err != nil { t.Fatal(err) } else { diff --git a/pkg/model/bootstrap.go b/pkg/model/bootstrap.go index 8fa4d48a0..1c99c3d07 100644 --- a/pkg/model/bootstrap.go +++ b/pkg/model/bootstrap.go @@ -55,7 +55,7 @@ func (bs *Bootstrap) ExistCluster(name string) bool { // StaticResources type StaticResources struct { Listeners []Listener `yaml:"listeners" json:"listeners" mapstructure:"listeners"` - Clusters []Cluster `yaml:"clusters" json:"clusters" mapstructure:"clusters"` + Clusters []*Cluster `yaml:"clusters" json:"clusters" mapstructure:"clusters"` ShutdownConfig *ShutdownConfig `yaml:"shutdown_config" json:"shutdown_config" mapstructure:"shutdown_config"` PprofConf PprofConf `yaml:"pprofConf" json:"pprofConf" mapstructure:"pprofConf"` AccessLogConfig AccessLogConfig `yaml:"accessLog" json:"accessLog" mapstructure:"accessLog"` diff --git a/pkg/model/cluster.go b/pkg/model/cluster.go index f4acc97f5..ee53b6f5b 100644 --- a/pkg/model/cluster.go +++ b/pkg/model/cluster.go @@ -22,7 +22,7 @@ type Cluster struct { Name string `yaml:"name" json:"name"` // Name the cluster unique name TypeStr string `yaml:"type" json:"type"` // Type the cluster discovery type string value Type DiscoveryType `yaml:",omitempty" json:",omitempty"` // Type the cluster discovery type - EdsClusterConfig EdsClusterConfig `yaml:"eds_cluster_config" json:"eds_cluster_config"` + EdsClusterConfig EdsClusterConfig `yaml:"eds_cluster_config" json:"eds_cluster_config" mapstructure:"eds_cluster_config"` LbStr string `yaml:"lb_policy" json:"lb_policy"` // Lb the cluster select node used loadBalance policy Lb LbPolicy `yaml:",omitempty" json:",omitempty"` // Lb the cluster select node used loadBalance policy ConnectTimeoutStr string `yaml:"connect_timeout" json:"connect_timeout"` // ConnectTimeout timeout for connect to cluster node diff --git a/pkg/model/match.go b/pkg/model/match.go index 2f046ea5d..72f8c0ac5 100644 --- a/pkg/model/match.go +++ b/pkg/model/match.go @@ -54,7 +54,7 @@ var MatcherTypeValue = map[string]int32{ // HeaderMatcher header matcher struct // Name header key, Value header value, Regex header value is regex type HeaderMatcher struct { - Name string `yaml:"name" json:"name"` - Value string `yaml:"value" json:"value"` - Regex bool `yaml:"regex" json:"regex"` + Name string `yaml:"name" json:"name" mapstructure:"name"` + Value string `yaml:"value" json:"value" mapstructure:"value"` + Regex bool `yaml:"regex" json:"regex" mapstructure:"regex"` } diff --git a/pkg/model/router.go b/pkg/model/router.go index 5fa8906f7..ab3d7e210 100644 --- a/pkg/model/router.go +++ b/pkg/model/router.go @@ -19,41 +19,41 @@ package model // Router struct type Router struct { - Match RouterMatch `yaml:"match" json:"match"` - Route RouteAction `yaml:"route" json:"route"` - Redirect RouteAction `yaml:"redirect" json:"redirect"` + Match RouterMatch `yaml:"match" json:"match" mapstructure:"match"` + Route RouteAction `yaml:"route" json:"route" mapstructure:"route"` + Redirect RouteAction `yaml:"redirect" json:"redirect" mapstructure:"redirect"` //"metadata": "{...}", //"decorator": "{...}" } // RouterMatch type RouterMatch struct { - Prefix string `yaml:"prefix" json:"prefix"` - Path string `yaml:"path" json:"path"` - Regex string `yaml:"regex" json:"regex"` + Prefix string `yaml:"prefix" json:"prefix" mapstructure:"prefix"` + Path string `yaml:"path" json:"path" mapstructure:"path"` + Regex string `yaml:"regex" json:"regex" mapstructure:"regex"` CaseSensitive bool // CaseSensitive default true - Headers []HeaderMatcher `yaml:"headers" json:"headers"` + Headers []HeaderMatcher `yaml:"headers" json:"headers" mapstructure:"headers"` } // RouteAction match route should do type RouteAction struct { - Cluster string `yaml:"cluster" json:"cluster"` // Cluster cluster name - ClusterNotFoundResponseCode int `yaml:"cluster_not_found_response_code" json:"cluster_not_found_response_code"` - PrefixRewrite string `yaml:"prefix_rewrite" json:"prefix_rewrite"` - HostRewrite string `yaml:"host_rewrite" json:"host_rewrite"` - Timeout string `yaml:"timeout" json:"timeout"` - Priority int8 `yaml:"priority" json:"priority"` - ResponseHeadersToAdd HeaderValueOption `yaml:"response_headers_to_add" json:"response_headers_to_add"` // ResponseHeadersToAdd add response head - ResponseHeadersToRemove []string `yaml:"response_headers_to_remove" json:"response_headers_to_remove"` // ResponseHeadersToRemove remove response head - RequestHeadersToAdd HeaderValueOption `yaml:"request_headers_to_add" json:"request_headers_to_add"` // RequestHeadersToAdd add request head - Cors CorsPolicy `yaml:"cors" json:"cors"` + Cluster string `yaml:"cluster" json:"cluster" mapstructure:"cluster"` + ClusterNotFoundResponseCode int `yaml:"cluster_not_found_response_code" json:"cluster_not_found_response_code" mapstructure:"cluster_not_found_response_code"` + PrefixRewrite string `yaml:"prefix_rewrite" json:"prefix_rewrite" mapstructure:"prefix_rewrite"` + HostRewrite string `yaml:"host_rewrite" json:"host_rewrite" mapstructure:"host_rewrite"` + Timeout string `yaml:"timeout" json:"timeout" mapstructure:"timeout"` + Priority int8 `yaml:"priority" json:"priority" mapstructure:"priority"` + ResponseHeadersToAdd HeaderValueOption `yaml:"response_headers_to_add" json:"response_headers_to_add" mapstructure:"response_headers_to_add"` // ResponseHeadersToAdd add response head + ResponseHeadersToRemove []string `yaml:"response_headers_to_remove" json:"response_headers_to_remove" mapstructure:"response_headers_to_remove"` // ResponseHeadersToRemove remove response head + RequestHeadersToAdd HeaderValueOption `yaml:"request_headers_to_add" json:"request_headers_to_add" mapstructure:"request_headers_to_add"` // RequestHeadersToAdd add request head + Cors CorsPolicy `yaml:"cors" json:"cors" mapstructure:"cors"` } // RouteConfiguration type RouteConfiguration struct { - InternalOnlyHeaders []string `yaml:"internal_only_headers" json:"internal_only_headers"` // InternalOnlyHeaders used internal, clear http request head - ResponseHeadersToAdd HeaderValueOption `yaml:"response_headers_to_add" json:"response_headers_to_add"` // ResponseHeadersToAdd add response head - ResponseHeadersToRemove []string `yaml:"response_headers_to_remove" json:"response_headers_to_remove"` // ResponseHeadersToRemove remove response head - RequestHeadersToAdd HeaderValueOption `yaml:"request_headers_to_add" json:"request_headers_to_add"` // RequestHeadersToAdd add request head - Routes []Router `yaml:"routes" json:"routes"` + InternalOnlyHeaders []string `yaml:"internal_only_headers" json:"internal_only_headers" mapstructure:"internal_only_headers"` // InternalOnlyHeaders used internal, clear http request head + ResponseHeadersToAdd HeaderValueOption `yaml:"response_headers_to_add" json:"response_headers_to_add" mapstructure:"response_headers_to_add"` // ResponseHeadersToAdd add response head + ResponseHeadersToRemove []string `yaml:"response_headers_to_remove" json:"response_headers_to_remove" mapstructure:"response_headers_to_remove"` // ResponseHeadersToRemove remove response head + RequestHeadersToAdd HeaderValueOption `yaml:"request_headers_to_add" json:"request_headers_to_add" mapstructure:"request_headers_to_add"` // RequestHeadersToAdd add request head + Routes []Router `yaml:"routes" json:"routes" mapstructure:"routes"` }