From 9646d3ce447b92036f85917b327e43bb9317a807 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Wed, 20 Oct 2021 10:26:39 +0800 Subject: [PATCH 01/75] update gopsutil to v3 --- agent.go | 2 +- go.mod | 9 ++------- manageHandler.go | 12 ++++++------ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/agent.go b/agent.go index 9efb298d..b62f72f7 100644 --- a/agent.go +++ b/agent.go @@ -16,7 +16,7 @@ import ( cfg "github.com/weibocom/motan-go/config" "gopkg.in/yaml.v2" - "github.com/shirou/gopsutil/process" + "github.com/shirou/gopsutil/v3/process" "github.com/valyala/fasthttp" "github.com/weibocom/motan-go/cluster" motan "github.com/weibocom/motan-go/core" diff --git a/go.mod b/go.mod index 51c916af..5bc1abb5 100644 --- a/go.mod +++ b/go.mod @@ -3,30 +3,25 @@ module github.com/weibocom/motan-go go 1.11 require ( - github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 github.com/beberlei/fastcgi-serve v0.0.0-20151230120321-4676005f65b7 github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/protobuf v1.2.0 github.com/juju/ratelimit v1.0.1 github.com/kr/pretty v0.1.0 // indirect github.com/mitchellh/mapstructure v1.1.2 github.com/opentracing/opentracing-go v1.0.2 - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec - github.com/shirou/gopsutil v2.18.12+incompatible - github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect + github.com/shirou/gopsutil/v3 v3.21.9 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect - github.com/stretchr/testify v1.2.2 + github.com/stretchr/testify v1.7.0 github.com/valyala/fasthttp v1.2.0 github.com/weibreeze/breeze-go v0.1.1 go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 golang.org/x/net v0.0.0-20181005035420-146acd28ed58 - golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9 // indirect google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect google.golang.org/grpc v1.15.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect diff --git a/manageHandler.go b/manageHandler.go index d2311f1b..3d3a9ca3 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -19,12 +19,12 @@ import ( "strings" "time" - "github.com/shirou/gopsutil/cpu" - "github.com/shirou/gopsutil/host" - "github.com/shirou/gopsutil/load" - "github.com/shirou/gopsutil/mem" - "github.com/shirou/gopsutil/net" - "github.com/shirou/gopsutil/process" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" + "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v3/process" "github.com/weibocom/motan-go/cluster" motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/filter" From 85f1a68c3f6acf5ae2e1ba92bc2af9df7abecc8e Mon Sep 17 00:00:00 2001 From: arraykeys Date: Wed, 20 Oct 2021 14:09:47 +0800 Subject: [PATCH 02/75] update gopsutil to v3 --- go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5bc1abb5..a0d8b232 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/mitchellh/mapstructure v1.1.2 github.com/opentracing/opentracing-go v1.0.2 + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec github.com/shirou/gopsutil/v3 v3.21.9 @@ -30,6 +31,7 @@ require ( replace ( cloud.google.com/go => github.com/GoogleCloudPlatform/google-cloud-go v0.30.0 + github.com/stretchr/testify => github.com/stretchr/testify v1.2.2 go.uber.org/atomic => github.com/uber-go/atomic v1.4.0 go.uber.org/multierr => github.com/uber-go/multierr v1.1.1-0.20180122172545-ddea229ff1df go.uber.org/zap => github.com/uber-go/zap v1.9.1 @@ -38,7 +40,6 @@ replace ( golang.org/x/net => github.com/golang/net v0.0.0-20181017193950-04a2e542c03f golang.org/x/oauth2 => github.com/golang/oauth2 v0.0.0-20181017192945-9dcd33a902f4 golang.org/x/sync => github.com/golang/sync v0.0.0-20180314180146-1d60e4601c6f - golang.org/x/sys => github.com/golang/sys v0.0.0-20181011152604-fa43e7bc11ba golang.org/x/text => github.com/golang/text v0.3.0 golang.org/x/time => github.com/golang/time v0.0.0-20181108054448-85acf8d2951c golang.org/x/tools => github.com/golang/tools v0.0.0-20181017214349-06f26fdaaa28 From 5e8ed1bf631cb973cf06f0b728b094b2113add97 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Wed, 20 Oct 2021 14:17:17 +0800 Subject: [PATCH 03/75] add testing on go1.17 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 73523b74..f4f55301 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: testing: strategy: matrix: - go-version: [1.12.x,1.13.x,1.14.x,1.15.x,1.16.x] + go-version: [1.12.x,1.13.x,1.14.x,1.15.x,1.16.x,1.17.x] platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} steps: From d2412935fa6c6b6070cfef06ad218547bfe66940 Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 19 Nov 2021 15:12:47 +0800 Subject: [PATCH 04/75] Feature/mix groups (#250) * merge dev (#249) * update gopsutil to v3 * add testing on go1.17 * support mix groups * add nil check in AccessLogFilter Co-authored-by: snail007 Co-authored-by: zhanglei28 --- cluster/command.go | 142 +++++++++++++++++++++++++++------------- cluster/command_test.go | 130 ++++++++++++++++++++++++++++++------ core/constants.go | 1 + filter/accessLog.go | 5 +- lb/lb.go | 88 +++++++++++++++---------- lb/lb_test.go | 27 ++++++-- 6 files changed, 289 insertions(+), 104 deletions(-) diff --git a/cluster/command.go b/cluster/command.go index 75761c8a..b63b15da 100644 --- a/cluster/command.go +++ b/cluster/command.go @@ -41,6 +41,7 @@ type CommandRegistryWrapper struct { mux sync.Mutex ownGroupURLs []*motan.URL otherGroupListener map[string]*serviceListener + staticTcCommand *ClientCommand // static traffic control command. tcCommand *ClientCommand //effective traffic control command degradeCommand *ClientCommand //effective degrade command switcherCommand *ClientCommand @@ -97,7 +98,7 @@ func (s *serviceListener) unSubscribe(registry motan.Registry) { } func (s *serviceListener) subscribe(registry motan.Registry) { - // this listener should not reuse, so let it crash when this listener is resubscribe after unsubscribe. + // this listener should not reuse, so let it crash when this listener is resubscribed after unsubscribe. registry.Subscribe(s.referURL, s) s.urls = registry.Discover(s.referURL) } @@ -145,6 +146,22 @@ func GetCommandRegistryWrapper(cluster *MotanCluster, registry motan.Registry) m cmdRegistry.ownGroupURLs = make([]*motan.URL, 0) cmdRegistry.otherGroupListener = make(map[string]*serviceListener) cmdRegistry.cluster = cluster + mixGroups := cluster.GetURL().GetParam(motan.MixGroups, "") + if mixGroups != "" { + groups := strings.Split(mixGroups, ",") + command := &ClientCommand{CommandType: CMDTrafficControl, Index: 0, Version: "1.0", MergeGroups: make([]string, 0, len(groups)+1)} + ownGroup := cluster.GetURL().Group + command.MergeGroups = append(command.MergeGroups, ownGroup) + for _, group := range groups { + group = strings.TrimSpace(group) + if group != "" && group != ownGroup { + command.MergeGroups = append(command.MergeGroups, group) + } + } + if len(command.MergeGroups) > 1 { // has other group + cmdRegistry.staticTcCommand = command + } + } return cmdRegistry } @@ -219,29 +236,46 @@ func (c *CommandRegistryWrapper) clear() { c.otherGroupListener = make(map[string]*serviceListener) } +func (c *CommandRegistryWrapper) getCurrentTcCommand() *ClientCommand { + // dynamic tc command > static tc command + currentCommand := c.tcCommand + if currentCommand == nil && c.staticTcCommand != nil { + currentCommand = c.staticTcCommand + vlog.Infof("%s use static command", c.cluster.GetIdentity()) + } + return currentCommand +} + func (c *CommandRegistryWrapper) getResultWithCommand(needNotify bool) []*motan.URL { c.mux.Lock() defer c.mux.Unlock() result := make([]*motan.URL, 0) - if c.tcCommand != nil { - vlog.Infof("%s get result with tc command.%+v", c.cluster.GetIdentity(), c.tcCommand) + currentCommand := c.getCurrentTcCommand() + if currentCommand != nil { + vlog.Infof("%s get result with tc command.%+v", c.cluster.GetIdentity(), currentCommand) var buffer bytes.Buffer - for _, group := range c.tcCommand.MergeGroups { + for _, group := range currentCommand.MergeGroups { + var urls []*motan.URL g := strings.Split(group, ":") //group name should not include ':' if c.cluster.GetURL().Group == g[0] { // own group vlog.Infof("%s get result from own group: %s, group result size:%d", c.cluster.GetIdentity(), g[0], len(c.ownGroupURLs)) - for _, u := range c.ownGroupURLs { - result = append(result, u) + urls = c.ownGroupURLs + } else { // other group + if l, ok := c.otherGroupListener[g[0]]; ok { + urls = l.urls + } else { + l := newSubscribe(c, g[0]) + c.otherGroupListener[g[0]] = l + urls = l.urls } - } else if l, ok := c.otherGroupListener[g[0]]; ok { - vlog.Infof("%s get result merge group: %s, group result size:%d", c.cluster.GetIdentity(), g[0], len(l.urls)) - for _, u := range l.urls { + vlog.Infof("%s get result merge group: %s, group result size:%d", c.cluster.GetIdentity(), g[0], len(urls)) + } + if urls != nil { + for _, u := range urls { result = append(result, u) } - } else { - vlog.Warningf("TC command merge group not found. refer:%s, group not found: %s, TC command %v", c.cluster.GetURL().GetIdentity(), g[0], c.tcCommand) - continue } + // build weight string if buffer.Len() > 0 { buffer.WriteString(",") } @@ -251,10 +285,10 @@ func (c *CommandRegistryWrapper) getResultWithCommand(needNotify bool) []*motan. url := buildRuleURL(buffer.String()) result = append(result, url) // add command rule url to the end of result. } - result = processRoute(result, c.tcCommand.RouteRules) + result = processRoute(result, currentCommand.RouteRules) if len(result) == 0 { result = c.ownGroupURLs - vlog.Warningf("TC command process failed, use default group. refer:%s, MergeGroups: %v, RouteRules %v", c.cluster.GetURL().GetIdentity(), c.tcCommand.MergeGroups, c.tcCommand.RouteRules) + vlog.Warningf("TC command process failed, use default group. refer:%s, MergeGroups: %v, RouteRules %v", c.cluster.GetURL().GetIdentity(), currentCommand.MergeGroups, currentCommand.RouteRules) } } else { result = c.ownGroupURLs @@ -262,7 +296,7 @@ func (c *CommandRegistryWrapper) getResultWithCommand(needNotify bool) []*motan. if needNotify { c.notifyListener.Notify(c.registry.GetURL(), result) } - vlog.Infof("%s get result with command. tcCommand: %t, degradeCommand:%t, result size %d, will notify:%t", c.cluster.GetURL().GetIdentity(), c.tcCommand != nil, c.degradeCommand != nil, len(result), needNotify) + vlog.Infof("%s get result with command. tcCommand: %t, degradeCommand:%t, result size %d, will notify:%t", c.cluster.GetURL().GetIdentity(), currentCommand != nil, c.degradeCommand != nil, len(result), needNotify) return result } @@ -365,44 +399,64 @@ func (c *CommandRegistryWrapper) processCommand(commandType int, commandInfo str if newTcCommand != nil || (c.tcCommand != nil && newTcCommand == nil) { needNotify = true } - //process all kinds commands - c.tcCommand = newTcCommand - if c.tcCommand == nil { + c.processTcCommand(newTcCommand) + c.processDegradeCommand(newDegradeCommand) + c.processSwitcherCommand(newSwitcherCommand) + return needNotify +} + +func (c *CommandRegistryWrapper) processTcCommand(newTcCommand *ClientCommand) { + removeListeners := make(map[string]*serviceListener) + for g, listener := range c.otherGroupListener { + removeListeners[g] = listener + } + newListeners := make(map[string]*serviceListener) + + if newTcCommand == nil && c.staticTcCommand == nil { vlog.Infof("%s process command result : no tc command. ", c.cluster.GetURL().GetIdentity()) - for _, v := range c.otherGroupListener { - v.unSubscribe(c.registry) - } - c.otherGroupListener = make(map[string]*serviceListener) } else { - vlog.Infof("%s process command result : has tc command. tc command will enable.command : %+v", c.cluster.GetURL().GetIdentity(), newTcCommand) - newOtherGroupListener := make(map[string]*serviceListener) - for _, group := range c.tcCommand.MergeGroups { + var groups []string + if newTcCommand != nil { + vlog.Infof("%s process command result : has dynamic tc command. tc command will enable.command : %+v", c.cluster.GetURL().GetIdentity(), newTcCommand) + groups = newTcCommand.MergeGroups + } else { + groups = c.staticTcCommand.MergeGroups + } + for _, group := range groups { g := strings.Split(group, ":") if c.cluster.GetURL().Group == g[0] { // own group already subscribe continue } if listener, ok := c.otherGroupListener[g[0]]; ok { // already exist vlog.Infof("commandWrapper %s process tc command. reuse group %s", c.cluster.GetURL().GetIdentity(), g[0]) - newOtherGroupListener[g[0]] = listener - delete(c.otherGroupListener, g[0]) + newListeners[g[0]] = listener + delete(removeListeners, g[0]) } else { - vlog.Infof("commandWrapper %s process tc command. subscribe new group %s", c.cluster.GetURL().GetIdentity(), g[0]) - newGroupURL := c.cluster.GetURL().Copy() - newGroupURL.Group = g[0] - l := &serviceListener{crw: c, referURL: newGroupURL} - l.subscribe(c.registry) - newOtherGroupListener[g[0]] = l + newListeners[g[0]] = newSubscribe(c, g[0]) } } + } - oldOtherGroupListener := c.otherGroupListener - c.otherGroupListener = newOtherGroupListener - // sync destroy - for _, v := range oldOtherGroupListener { - v.unSubscribe(c.registry) - } + c.otherGroupListener = newListeners + c.tcCommand = newTcCommand + + // destroy unused listeners + for _, v := range removeListeners { + v.unSubscribe(c.registry) } +} + +func newSubscribe(c *CommandRegistryWrapper, group string) *serviceListener { + vlog.Infof("commandWrapper %s subscribe new group %s", c.cluster.GetURL().GetIdentity(), group) + newGroupURL := c.cluster.GetURL().Copy() + newGroupURL.Group = group + l := &serviceListener{crw: c, referURL: newGroupURL} + l.subscribe(c.registry) + return l +} + +func (c *CommandRegistryWrapper) processDegradeCommand(newDegradeCommand *ClientCommand) { c.degradeCommand = newDegradeCommand if c.degradeCommand == nil { vlog.Infof("%s no degrade command. this cluster is available.", c.cluster.GetURL().GetIdentity()) @@ -411,8 +465,9 @@ func (c *CommandRegistryWrapper) processCommand(commandType int, commandInfo str vlog.Infof("%s has degrade command. this cluster will degrade.", c.cluster.GetURL().GetIdentity()) c.cluster.available = false } +} - //process switcher command +func (c *CommandRegistryWrapper) processSwitcherCommand(newSwitcherCommand *ClientCommand) { switcherManger := motan.GetSwitcherManager() newSwitcherMap := make(map[string]bool) if newSwitcherCommand != nil { @@ -441,8 +496,6 @@ func (c *CommandRegistryWrapper) processCommand(commandType int, commandInfo str } oldSwitcherMap = newSwitcherMap c.switcherCommand = newSwitcherCommand - - return needNotify } func mergeCommand(commandInfo string, url *motan.URL) (tcCommand *ClientCommand, degradeCommand *ClientCommand, switcherCommand *ClientCommand) { @@ -488,8 +541,9 @@ func (c *CommandRegistryWrapper) Notify(registryURL *motan.URL, urls []*motan.UR vlog.Infof("CommandRegistryWrapper notify urls size is %d. refer: %v, registry: %v", len(urls), c.cluster.GetURL(), registryURL) c.ownGroupURLs = urls needNotify := false - if c.tcCommand != nil { - for _, group := range c.tcCommand.MergeGroups { + currentCommand := c.getCurrentTcCommand() + if currentCommand != nil { + for _, group := range currentCommand.MergeGroups { g := strings.Split(group, ":") if g[0] == c.cluster.GetURL().Group { needNotify = true diff --git a/cluster/command_test.go b/cluster/command_test.go index 7ac9e2dc..4ed10ba4 100644 --- a/cluster/command_test.go +++ b/cluster/command_test.go @@ -28,12 +28,12 @@ func TestProcessRouter(t *testing.T) { //not match *motan.LocalIP = "10.75.0.8" result := processRoute(urls, router) - checksize(len(result), len(urls), t) + checkSize(len(result), len(urls), t) // prefix match *motan.LocalIP = "10.73.1.8" result = processRoute(urls, router) - checksize(len(result), 2, t) + checkSize(len(result), 2, t) checkHost(result, func(host string) bool { return !strings.HasPrefix(host, "10.75.1") }, t) @@ -42,7 +42,7 @@ func TestProcessRouter(t *testing.T) { router = newRouter("10.75.0.8 to 10.73.1.*") *motan.LocalIP = "10.75.0.8" result = processRoute(urls, router) - checksize(len(result), 2, t) + checkSize(len(result), 2, t) checkHost(result, func(host string) bool { return !strings.HasPrefix(host, "10.73.1") }, t) @@ -51,7 +51,7 @@ func TestProcessRouter(t *testing.T) { router = newRouter(" * to 10.75.*") *motan.LocalIP = "10.108.0.8" result = processRoute(urls, router) - checksize(len(result), 4, t) + checkSize(len(result), 4, t) checkHost(result, func(host string) bool { return !strings.HasPrefix(host, "10.75") }, t) @@ -60,18 +60,18 @@ func TestProcessRouter(t *testing.T) { router = newRouter(" * to 10.75.*", "10.108.* to 10.77.1.* ") *motan.LocalIP = "10.108.0.8" result = processRoute(urls, router) - checksize(len(result), 0, t) + checkSize(len(result), 0, t) router = newRouter(" * to 10.75.*", "10.108.* to 10.75.1.* ") result = processRoute(urls, router) - checksize(len(result), 2, t) + checkSize(len(result), 2, t) checkHost(result, func(host string) bool { return !(strings.HasPrefix(host, "10.77.1") || strings.HasPrefix(host, "10.75")) }, t) router = newRouter(" * to 10.*", "10.108.* to !10.73.1.* ") result = processRoute(urls, router) - checksize(len(result), 6, t) + checkSize(len(result), 6, t) checkHost(result, func(host string) bool { return !(strings.HasPrefix(host, "10.77.1") || strings.HasPrefix(host, "10.75")) }, t) @@ -79,21 +79,21 @@ func TestProcessRouter(t *testing.T) { router = newRouter(" 10.79.* to !10.75.1.*", "10.108.* to 10.73.1.* ", "10.108.* to !10.73.1.5") *motan.LocalIP = "10.79.0.8" result = processRoute(urls, router) - checksize(len(result), 6, t) + checkSize(len(result), 6, t) checkHost(result, func(host string) bool { return strings.HasPrefix(host, "10.75.1") }, t) *motan.LocalIP = "10.108.0.8" result = processRoute(urls, router) - checksize(len(result), 1, t) + checkSize(len(result), 1, t) checkHost(result, func(host string) bool { return host != "10.73.1.3" }, t) } func TestGetResultWithCommand(t *testing.T) { - crw := getDefalultCommandWarper() + crw := getDefaultCommandWrapper(false) cmds := make([]string, 0, 10) cmds = append(cmds, buildCmd(1, CMDTrafficControl, "*", "\"group0:3\",\"group1:5\"", "\" 10.79.* to !10.75.1.*\", \"10.108.* to 10.73.1.* \"")) cmds = append(cmds, buildCmd(1, CMDDegrade, "com.weibo.test.TestService", "", "")) @@ -124,24 +124,53 @@ func TestGetResultWithCommand(t *testing.T) { } // check urls - hasrule := false + hasRule := false for _, u := range urls { if u.Protocol == RuleProtocol { - hasrule = true + hasRule = true continue } if !strings.HasPrefix(u.Host, "10.73.1") { t.Errorf("notify urls host correct! url:%+v\n", u) } } - if !hasrule { + if !hasRule { t.Errorf("notify urls not has rule url! urls:%+v\n", urls) } + // has static command + crw = getDefaultCommandWrapper(true) + urls = crw.getResultWithCommand(false) + crw.notifyListener = listener + listener.urls = nil + validateResult(crw, []string{"test-group1", "test-group2", "test-group3"}, t) + + cmds = make([]string, 0, 10) + cmds = append(cmds, buildCmd(1, CMDTrafficControl, "*", "\"group0:3\",\"group1:5\"", "")) + crw.processCommand(ServiceCmd, buildCmdList(cmds)) + validateResult(crw, []string{"group0", "group1"}, t) + + crw.processCommand(ServiceCmd, "") + validateResult(crw, []string{"test-group1", "test-group2", "test-group3"}, t) +} + +func validateResult(crw *CommandRegistryWrapper, notifyGroups []string, t *testing.T) { + if len(crw.otherGroupListener) != len(notifyGroups) { + t.Errorf("other group listener with wrong size. otherGroupListener: %d, comand merge groups:%d, crw:%+v\n", len(crw.otherGroupListener), len(notifyGroups), crw) + } + size := 0 + for _, group := range notifyGroups { + crw.otherGroupListener[group].Notify(crw.registry.GetURL(), buildURLs(group)) + size += len(crw.otherGroupListener[group].urls) + } + urls := crw.getResultWithCommand(false) + if len(urls) != size+1 { // with weight rule + t.Errorf("result size not correct. expect size:%d, actual size:%d", size+1, len(urls)) + } } func TestProcessCommand(t *testing.T) { - crw := getDefalultCommandWarper() + crw := getDefaultCommandWrapper(false) cmds := make([]string, 0, 10) cmds = append(cmds, buildCmd(1, CMDTrafficControl, "*", "\"group0:3\",\"group1:5\"", "")) cmds = append(cmds, buildCmd(1, CMDDegrade, "com.weibo.test.TestService", "", "")) @@ -184,6 +213,26 @@ func TestProcessCommand(t *testing.T) { t.Errorf("abnormal command process not correct! notify: %t, crw:%+v\n", notify, crw) } fmt.Printf("notify:%t, crw:%+v\n", notify, crw) + + // has static command + crw = getDefaultCommandWrapper(true) + processServiceCmd(crw, cl, t) + notify = crw.processCommand(ServiceCmd, "") + if !notify { + t.Errorf("test empty service command fail! notify: %t, crw:%+v\n", notify, crw) + } + if len(crw.otherGroupListener) != len(crw.staticTcCommand.MergeGroups)-1 { + t.Errorf("other group listener with wrong size. otherGroupListener: %d, static comand mergeGroups:%d, crw:%+v\n", len(crw.otherGroupListener), len(crw.staticTcCommand.MergeGroups), crw) + } + for _, group := range crw.staticTcCommand.MergeGroups { + if group == crw.cluster.GetURL().Group { + continue + } + if _, ok := crw.otherGroupListener[group]; !ok { + t.Errorf("not have static group listener :%s, crw:%+v\n", group, crw) + } + } + } func processServiceCmd(crw *CommandRegistryWrapper, cl string, t *testing.T) { @@ -239,6 +288,46 @@ func TestMatchPattern(t *testing.T) { } } +func TestParseStaticCommand(t *testing.T) { + ownGroup := "ownGroup" + for _, unit := range []struct { + mixGroup string + shouldHasCommand bool + groups []string + }{ + {"test-group1", true, []string{ownGroup, "test-group1"}}, + {"test-group1 , test-group2", true, []string{ownGroup, "test-group1", "test-group2"}}, + {"test-group1 , test-group2,test-group3", true, []string{ownGroup, "test-group1", "test-group2", "test-group3"}}, + {"test-group1 , " + ownGroup + ",test-group2,test-group3", true, []string{ownGroup, "test-group1", "test-group2", "test-group3"}}, + {ownGroup, false, nil}, + {ownGroup + ", ", false, nil}, + {" ", false, nil}, + {" , ", false, nil}, + } { + cluster := initCluster() + cluster.GetURL().Group = ownGroup + cluster.url.Parameters[motan.MixGroups] = unit.mixGroup + registry := cluster.extFactory.GetRegistry(RegistryURL) + commandRegistry := GetCommandRegistryWrapper(cluster, registry).(*CommandRegistryWrapper) + command := commandRegistry.staticTcCommand + if unit.shouldHasCommand { + if command == nil { + t.Errorf("static command is nil. mix groups: %s", unit.mixGroup) + } + if len(unit.groups) != len(command.MergeGroups) { + t.Errorf("merge group size not correct. mix groups: %s, expect size: %d, actual size:%d", unit.mixGroup, len(unit.groups), len(command.MergeGroups)) + } + for i, group := range unit.groups { + if group != command.MergeGroups[i] { + t.Errorf("merge groups not same. mix groups: %s, index:%d, expect:%s, actual:%s", unit.mixGroup, i, group, command.MergeGroups[i]) + } + } + } else if command != nil { + t.Errorf("command should nil. mix groups: %s", unit.mixGroup) + } + } +} + func newRouter(rules ...string) []string { router := make([]string, 0, 20) for _, r := range rules { @@ -261,9 +350,9 @@ func buildURLs(group string) []*motan.URL { return urls } -func checksize(realsize int, expectsize int, t *testing.T) { - if realsize != expectsize { - t.Errorf("test router check size fail. real:%d, exp:%d\n", realsize, expectsize) +func checkSize(realSize int, expectSize int, t *testing.T) { + if realSize != expectSize { + t.Errorf("test router check size fail. real:%d, exp:%d\n", realSize, expectSize) } } @@ -275,8 +364,11 @@ func checkHost(urls []*motan.URL, f func(host string) bool, t *testing.T) { } } -func getDefalultCommandWarper() *CommandRegistryWrapper { +func getDefaultCommandWrapper(withStaticCommand bool) *CommandRegistryWrapper { cluster := initCluster() + if withStaticCommand { + cluster.url.Parameters[motan.MixGroups] = "test-group1,test-group2,test-group3" + } registry := cluster.extFactory.GetRegistry(RegistryURL) return GetCommandRegistryWrapper(cluster, registry).(*CommandRegistryWrapper) } @@ -317,5 +409,5 @@ func (m *MockListener) Notify(registryURL *motan.URL, urls []*motan.URL) { } func (m *MockListener) GetIdentity() string { - return "mocklistener" + return "mockListener" } diff --git a/core/constants.go b/core/constants.go index a91bfbbb..19ddac91 100644 --- a/core/constants.go +++ b/core/constants.go @@ -57,6 +57,7 @@ const ( ManagementUnixSockKey = "managementUnixSock" ManagementPortRangeKey = "managementPortRange" HTTPProxyUnixSockKey = "httpProxyUnixSock" + MixGroups = "mixGroups" ) // nodeType diff --git a/filter/accessLog.go b/filter/accessLog.go index 1596b375..9e73a79c 100644 --- a/filter/accessLog.go +++ b/filter/accessLog.go @@ -79,7 +79,10 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 reqCtx := request.GetRPCContext(true) resCtx := response.GetRPCContext(true) // response code should be same as upstream - responseCode := resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) + responseCode := "" + if resCtx.Meta != nil { + responseCode = resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) + } var exceptionData []byte if exception != nil { exceptionData, _ = json.Marshal(exception) diff --git a/lb/lb.go b/lb/lb.go index 469044a8..36fb4bbb 100644 --- a/lb/lb.go +++ b/lb/lb.go @@ -19,6 +19,7 @@ const ( const ( MaxSelectArraySize = 3 defaultWeight = 1 + maxWeight = 100 ) var ( @@ -35,45 +36,28 @@ func RegistDefaultLb(extFactory motan.ExtensionFactory) { })) } -// WeightedLbWraper support multi group weighted LB -type WeightedLbWraper struct { +// WeightedLbWrapper support multi group weighted LB +type WeightedLbWrapper struct { url *motan.URL - weightstring string + weightString string refers innerRefers newLb motan.NewLbFunc } func NewWeightLbFunc(newLb motan.NewLbFunc) motan.NewLbFunc { return func(url *motan.URL) motan.LoadBalance { - return &WeightedLbWraper{url: url, newLb: newLb, refers: &singleGroupRefers{lb: newLb(url)}} + return &WeightedLbWrapper{url: url, newLb: newLb, refers: &singleGroupRefers{lb: newLb(url)}} } } -func (w *WeightedLbWraper) OnRefresh(endpoints []motan.EndPoint) { - if w.weightstring == "" { //not weighted lb - if sgr, ok := w.refers.(*singleGroupRefers); ok { - sgr.lb.OnRefresh(endpoints) - } else { - lb := w.newLb(w.url) - lb.OnRefresh(endpoints) - w.refers = &singleGroupRefers{lb: lb} - } +func (w *WeightedLbWrapper) OnRefresh(endpoints []motan.EndPoint) { + if w.weightString == "" { //not weighted lb + w.onRefreshSingleGroup(endpoints) return } - // weighted lb - lbmutex.Lock() - defer lbmutex.Unlock() - groupEp := make(map[string][]motan.EndPoint) - for _, ep := range endpoints { - ges := groupEp[ep.GetURL().Group] - if ges == nil { - ges = make([]motan.EndPoint, 0, 32) - } - groupEp[ep.GetURL().Group] = append(ges, ep) - } - - weights := strings.Split(w.weightstring, ",") + weights := strings.Split(w.weightString, ",") + mixMode := true gws := make(map[string]int) for _, w := range weights { if w != "" { @@ -81,8 +65,15 @@ func (w *WeightedLbWraper) OnRefresh(endpoints []motan.EndPoint) { if len(groupWeight) == 1 { gws[groupWeight[0]] = defaultWeight } else { + mixMode = false // not mix groups if weight is set w, err := strconv.Atoi(groupWeight[1]) if err == nil { + //weight normalization + if w < 1 { + w = defaultWeight + } else if w > maxWeight { + w = maxWeight + } gws[groupWeight[0]] = w } else { gws[groupWeight[0]] = defaultWeight @@ -90,6 +81,24 @@ func (w *WeightedLbWraper) OnRefresh(endpoints []motan.EndPoint) { } } } + + if mixMode { + w.onRefreshSingleGroup(endpoints) + return + } + + // weighted lb + lbmutex.Lock() + defer lbmutex.Unlock() + groupEp := make(map[string][]motan.EndPoint) + for _, ep := range endpoints { + ges := groupEp[ep.GetURL().Group] + if ges == nil { + ges = make([]motan.EndPoint, 0, 32) + } + groupEp[ep.GetURL().Group] = append(ges, ep) + } + weightsArray := make([]int, 0, 16) wr := newWeightRefers() for g, e := range groupEp { @@ -99,9 +108,6 @@ func (w *WeightedLbWraper) OnRefresh(endpoints []motan.EndPoint) { wr.groupLb[g] = lb //build real weight wi := gws[g] - if wi < 1 || wi > 100 { //weight normalization - wi = defaultWeight - } wr.groupWeight[g] = wi weightsArray = append(weightsArray, wi) } @@ -109,9 +115,9 @@ func (w *WeightedLbWraper) OnRefresh(endpoints []motan.EndPoint) { gcd := findGcd(weightsArray) ring := make([]string, 0, 128) for k, v := range wr.groupWeight { - ringweight := v / gcd - wr.groupWeight[k] = ringweight - for i := 0; i < ringweight; i++ { + ringWeight := v / gcd + wr.groupWeight[k] = ringWeight + for i := 0; i < ringWeight; i++ { ring = append(ring, k) } } @@ -120,16 +126,26 @@ func (w *WeightedLbWraper) OnRefresh(endpoints []motan.EndPoint) { w.refers = wr } -func (w *WeightedLbWraper) Select(request motan.Request) motan.EndPoint { +func (w *WeightedLbWrapper) onRefreshSingleGroup(endpoints []motan.EndPoint) { + if sgr, ok := w.refers.(*singleGroupRefers); ok { + sgr.lb.OnRefresh(endpoints) + } else { + lb := w.newLb(w.url) + lb.OnRefresh(endpoints) + w.refers = &singleGroupRefers{lb: lb} + } +} + +func (w *WeightedLbWrapper) Select(request motan.Request) motan.EndPoint { return w.refers.selectNext(request) } -func (w *WeightedLbWraper) SelectArray(request motan.Request) []motan.EndPoint { +func (w *WeightedLbWrapper) SelectArray(request motan.Request) []motan.EndPoint { return w.refers.selectNextArray(request) } -func (w *WeightedLbWraper) SetWeight(weight string) { - w.weightstring = weight +func (w *WeightedLbWrapper) SetWeight(weight string) { + w.weightString = weight } type innerRefers interface { diff --git a/lb/lb_test.go b/lb/lb_test.go index f286940f..738f1bdd 100644 --- a/lb/lb_test.go +++ b/lb/lb_test.go @@ -8,16 +8,16 @@ import ( "github.com/weibocom/motan-go/endpoint" ) -func TestWeightedLbWraper(t *testing.T) { +func TestWeightedLbWrapper(t *testing.T) { url := &motan.URL{Parameters: make(map[string]string)} url.Parameters[motan.Lbkey] = Roundrobin defaultExtFactory := &motan.DefaultExtensionFactory{} defaultExtFactory.Initialize() RegistDefaultLb(defaultExtFactory) lb := defaultExtFactory.GetLB(url) - wlbw, ok := lb.(*WeightedLbWraper) + wlbw, ok := lb.(*WeightedLbWrapper) if !ok { - t.Errorf("lb type not WeightedLbWraper, lb: %v\n", lb) + t.Errorf("lb type not WeightedLbWrapper, lb: %v\n", lb) } //test weight ratio @@ -64,7 +64,7 @@ func TestWeightedLbWraper(t *testing.T) { request := &motan.MotanRequest{} // test lb - for i := 1; i < 21; i++ { // the first index of weightring used for select is 1 + for i := 1; i < 21; i++ { // the first index of weigh string used for select is 1 ep := wlbw.Select(request) rg := refers.weightRing[i%len(refers.weightRing)] if ep.GetURL().Group != rg { @@ -82,6 +82,25 @@ func TestWeightedLbWraper(t *testing.T) { //repeat wlbw.OnRefresh(endpoints) + // test mixGroup + wlbw.SetWeight("group0,group1,group2") + wlbw.OnRefresh(endpoints) + if refers, ok := wlbw.refers.(*singleGroupRefers); ok { + epSize := len((refers.lb.(*RoundrobinLB)).endpoints) + if len(endpoints) != epSize { + t.Errorf("endpoint size type not correct. expect size:%d, actual size:%d", len(endpoints), epSize) + } + } else { + t.Errorf("inner refers type not correct. refer:%+v", wlbw.refers) + } + + // test weight normalization + wlbw.SetWeight("group0:10,group1:0,group2:120") + wlbw.OnRefresh(endpoints) + refers, ok = wlbw.refers.(*weightedRefers) + if refers.groupWeight["group0"] != 10 || refers.groupWeight["group1"] != defaultWeight || refers.groupWeight["group2"] != maxWeight { + t.Errorf("weight not correct. %+v\n", refers.groupWeight) + } } func TestLBSelect(t *testing.T) { From dec9833071a42a3adbcc8bc7b8b913a7f058935f Mon Sep 17 00:00:00 2001 From: arraykeys Date: Fri, 19 Nov 2021 15:50:26 +0800 Subject: [PATCH 05/75] fix httpproxy testing case --- agent_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent_test.go b/agent_test.go index 238391b5..f0cc4546 100644 --- a/agent_test.go +++ b/agent_test.go @@ -22,7 +22,7 @@ import ( const ( goNum = 5 - requestNum = 10000 + requestNum = 100 ) var proxyClient *http.Client From 995cea73e34537d2ffc9a998213a786f9d2096fc Mon Sep 17 00:00:00 2001 From: zhanglei28 Date: Fri, 26 Nov 2021 16:17:37 +0800 Subject: [PATCH 06/75] add log for mix group --- cluster/command.go | 1 + lb/lb.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/cluster/command.go b/cluster/command.go index b63b15da..5cd7da20 100644 --- a/cluster/command.go +++ b/cluster/command.go @@ -160,6 +160,7 @@ func GetCommandRegistryWrapper(cluster *MotanCluster, registry motan.Registry) m } if len(command.MergeGroups) > 1 { // has other group cmdRegistry.staticTcCommand = command + vlog.Infof("set static command for cluster: %s, mixGroups: %s", cluster.GetIdentity(), mixGroups) } } return cmdRegistry diff --git a/lb/lb.go b/lb/lb.go index 36fb4bbb..d275ee70 100644 --- a/lb/lb.go +++ b/lb/lb.go @@ -1,6 +1,7 @@ package lb import ( + vlog "github.com/weibocom/motan-go/log" "math/rand" "strconv" "strings" @@ -52,6 +53,7 @@ func NewWeightLbFunc(newLb motan.NewLbFunc) motan.NewLbFunc { func (w *WeightedLbWrapper) OnRefresh(endpoints []motan.EndPoint) { if w.weightString == "" { //not weighted lb + vlog.Infof("WeightedLbWrapper: %s - OnRefresh:not have weight", w.url.GetIdentity()) w.onRefreshSingleGroup(endpoints) return } @@ -83,6 +85,7 @@ func (w *WeightedLbWrapper) OnRefresh(endpoints []motan.EndPoint) { } if mixMode { + vlog.Infof("WeightedLbWrapper: %s - OnRefresh:use mix mode. weight:%s", w.url.GetIdentity(), w.weightString) w.onRefreshSingleGroup(endpoints) return } @@ -124,6 +127,7 @@ func (w *WeightedLbWrapper) OnRefresh(endpoints []motan.EndPoint) { wr.weightRing = motan.SliceShuffle(ring) wr.ringSize = len(wr.weightRing) w.refers = wr + vlog.Infof("WeightedLbWrapper: %s - OnRefresh: weight:%s", w.url.GetIdentity(), w.weightString) } func (w *WeightedLbWrapper) onRefreshSingleGroup(endpoints []motan.EndPoint) { From 8b6bfd819386dd790b2d14915543e3297c101147 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Mon, 10 Jan 2022 15:26:46 +0800 Subject: [PATCH 07/75] =?UTF-8?q?=E5=A2=9E=E5=8A=A0backupRequestInitDelayT?= =?UTF-8?q?ime=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ha/backupRequestHA.go | 24 ++++++++++----- ha/backupRequestHA_test.go | 63 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/ha/backupRequestHA.go b/ha/backupRequestHA.go index c3f3d96c..d50c6e5b 100644 --- a/ha/backupRequestHA.go +++ b/ha/backupRequestHA.go @@ -54,11 +54,7 @@ func (br *BackupRequestHA) Call(request motan.Request, loadBalance motan.LoadBal if retries <= 0 { return br.doCall(request, ep) } - // TODO: we should use metrics of the cluster, with traffic control the group may changed - item := metrics.GetStatItem(metrics.Escape(request.GetAttachment(protocol.MGroup)), metrics.Escape(request.GetAttachment(protocol.MPath))) - if item == nil || item.LastSnapshot() == nil { - return br.doCall(request, ep) - } + var resp motan.Response backupRequestDelayRatio := br.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), "backupRequestDelayRatio", defaultBackupRequestDelayRatio) backupRequestMaxRetryRatio := br.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), "backupRequestMaxRetryRatio", defaultBackupRequestMaxRetryRatio) @@ -69,9 +65,21 @@ func (br *BackupRequestHA) Call(request motan.Request, loadBalance motan.LoadBal defer deadline.Stop() successCh := make(chan motan.Response, retries+1) - if delay <= 0 { - // if no delay time configuration, use pXX time as delay time - delay = (int)(item.LastSnapshot().Percentile(getKey(request), float64(backupRequestDelayRatio)/100.0)) + if delay <= 0 { //no delay time configuration + // TODO: we should use metrics of the cluster, with traffic control the group may changed + item := metrics.GetStatItem(metrics.Escape(request.GetAttachment(protocol.MGroup)), metrics.Escape(request.GetAttachment(protocol.MPath))) + if item == nil || item.LastSnapshot() == nil { + initDelay := int(br.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), "backupRequestInitDelayTime", 0)) + if initDelay == 0 { + // request LastSnapshot is nil and no init delay time configuration, we just do common call. + return br.doCall(request, ep) + } + // request LastSnapshot is nil and has init delay time configuration, we use init delay as delay time + delay = initDelay + } else { + // use pXX as delay time + delay = (int)(item.LastSnapshot().Percentile(getKey(request), float64(backupRequestDelayRatio)/100.0)) + } if delay < 10 { delay = 10 // min 10ms } diff --git a/ha/backupRequestHA_test.go b/ha/backupRequestHA_test.go index 27f93ef4..c9cb5fa8 100644 --- a/ha/backupRequestHA_test.go +++ b/ha/backupRequestHA_test.go @@ -57,6 +57,69 @@ metrics: } } +func TestBackupRequestHA_Call2(t *testing.T) { + params := make(map[string]string) + params[motan.RetriesKey] = "1" + params["backupRequestInitDelayTime"] = "1" + url := &motan.URL{Protocol: "motan2", Path: TestService, Parameters: params} + ha := &BackupRequestHA{url: url} + ha.Initialize() + haName := ha.GetName() + if haName != BackupRequest { + t.Error("Test Case failed.") + } + request := &motan.MotanRequest{ServiceName: TestService, Method: TestMethod} + request.SetAttachment(protocol.MGroup, TestGroup) + request.SetAttachment(protocol.MPath, TestService) + nlb := &lb.RoundrobinLB{} + + ctx := &motan.Context{} + config, _ := config.NewConfigFromReader(bytes.NewReader([]byte(``))) + ctx.Config = config + //init histogram + nlb.OnRefresh([]motan.EndPoint{getEP(1)}) + res := ha.Call(request, nlb) + time.Sleep(10*time.Millisecond + 1*time.Second) + ep1 := getEP(1) // third round + testEndpoints := []motan.EndPoint{ep1} + nlb.OnRefresh(testEndpoints) + res = ha.Call(request, nlb) + if res.GetProcessTime() != 1 { + t.Errorf("backupRequest call fail. res:%+v", res) + } +} +func TestBackupRequestHA_Call3(t *testing.T) { + params := make(map[string]string) + params[motan.RetriesKey] = "1" + //params["backupRequestInitDelayTime"] = "1" + url := &motan.URL{Protocol: "motan2", Path: TestService, Parameters: params} + ha := &BackupRequestHA{url: url} + ha.Initialize() + haName := ha.GetName() + if haName != BackupRequest { + t.Error("Test Case failed.") + } + request := &motan.MotanRequest{ServiceName: TestService, Method: TestMethod} + request.SetAttachment(protocol.MGroup, TestGroup) + request.SetAttachment(protocol.MPath, TestService) + nlb := &lb.RoundrobinLB{} + + ctx := &motan.Context{} + config, _ := config.NewConfigFromReader(bytes.NewReader([]byte(``))) + ctx.Config = config + //init histogram + nlb.OnRefresh([]motan.EndPoint{getEP(1)}) + res := ha.Call(request, nlb) + time.Sleep(10*time.Millisecond + 1*time.Second) + ep1 := getEP(1) // third round + testEndpoints := []motan.EndPoint{ep1} + nlb.OnRefresh(testEndpoints) + res = ha.Call(request, nlb) + if res.GetProcessTime() != 1 { + t.Errorf("backupRequest call fail. res:%+v", res) + } +} + func getEP(processTime int64) motan.EndPoint { fep := &motan.FilterEndPoint{Caller: &motan.TestEndPoint{ProcessTime: processTime}} mf := &filter.MetricsFilter{} From a4d423ecb661850d8be659101c04443806a58c93 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Wed, 12 Jan 2022 15:13:07 +0800 Subject: [PATCH 08/75] modify rateLimit and circuitBreaker filter (#253) modify rateLimit and circuitBreaker filter --- filter/circuitBreaker.go | 71 +++--- filter/circuitBreaker_test.go | 157 ++++++++++++-- filter/clusterCircuitBreaker_test.go | 146 +++++++++++++ filter/rateLimit.go | 133 +++++++++--- filter/rateLimit_test.go | 308 +++++++++++++++++++++++---- provider/httpProvider_test.go | 12 +- 6 files changed, 700 insertions(+), 127 deletions(-) create mode 100644 filter/clusterCircuitBreaker_test.go diff --git a/filter/circuitBreaker.go b/filter/circuitBreaker.go index a0b24724..0c986de1 100644 --- a/filter/circuitBreaker.go +++ b/filter/circuitBreaker.go @@ -2,18 +2,19 @@ package filter import ( "errors" - "strconv" - "github.com/afex/hystrix-go/hystrix" motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/log" + "strconv" ) const ( RequestVolumeThresholdField = "circuitBreaker.requestThreshold" SleepWindowField = "circuitBreaker.sleepWindow" //ms ErrorPercentThreshold = "circuitBreaker.errorPercent" //% + MaxConcurrentField = "circuitBreaker.maxConcurrent" IncludeBizException = "circuitBreaker.bizException" + defaultMaxConcurrent = 5000 ) type CircuitBreakerFilter struct { @@ -69,9 +70,9 @@ func newCircuitBreaker(filterName string, url *motan.URL) bool { bizException, err := strconv.ParseBool(bizExceptionStr) if err != nil { bizException = true - vlog.Warningf("[%s] parse config %s error, use default", filterName, IncludeBizException) + vlog.Warningf("[%s] parse config %s error, use default: true", filterName, IncludeBizException) } - commandConfig := buildCommandConfig(filterName, url) + commandConfig := buildCommandConfig(url) hystrix.ConfigureCommand(url.GetIdentity(), *commandConfig) if _, _, err = hystrix.GetCircuit(url.GetIdentity()); err != nil { vlog.Errorf("[%s] new circuit fail. err:%s, url:%v, config{%s}", err.Error(), filterName, url.GetIdentity(), getConfigStr(commandConfig)+"bizException:"+bizExceptionStr) @@ -81,31 +82,29 @@ func newCircuitBreaker(filterName string, url *motan.URL) bool { return bizException } -func buildCommandConfig(filterName string, url *motan.URL) *hystrix.CommandConfig { - hystrix.DefaultMaxConcurrent = 1000 - hystrix.DefaultTimeout = int(url.GetPositiveIntValue(motan.TimeOutKey, int64(hystrix.DefaultTimeout))) * 2 - commandConfig := &hystrix.CommandConfig{} - if v, ok := url.Parameters[RequestVolumeThresholdField]; ok { +func getConfigValue(url *motan.URL, key string, defaultValue int) int { + if v, ok := url.Parameters[key]; ok { if temp, _ := strconv.Atoi(v); temp > 0 { - commandConfig.RequestVolumeThreshold = temp - } else { - vlog.Warningf("[%s] parse config %s error, use default", filterName, RequestVolumeThresholdField) - } - } - if v, ok := url.Parameters[SleepWindowField]; ok { - if temp, _ := strconv.Atoi(v); temp > 0 { - commandConfig.SleepWindow = temp - } else { - vlog.Warningf("[%s] parse config %s error, use default", filterName, SleepWindowField) - } - } - if v, ok := url.Parameters[ErrorPercentThreshold]; ok { - if temp, _ := strconv.Atoi(v); temp > 0 && temp <= 100 { - commandConfig.ErrorPercentThreshold = temp - } else { - vlog.Warningf("[%s] parse config %s error, use default", filterName, ErrorPercentThreshold) + if key == ErrorPercentThreshold { + if temp <= 100 { + return temp + } + } else { + return temp + } } } + vlog.Warningf("[%s] parse config %s error, use default: %d", CircuitBreaker, key, defaultValue) + return defaultValue +} + +func buildCommandConfig(url *motan.URL) *hystrix.CommandConfig { + commandConfig := &hystrix.CommandConfig{} + commandConfig.RequestVolumeThreshold = getConfigValue(url, RequestVolumeThresholdField, hystrix.DefaultVolumeThreshold) + commandConfig.SleepWindow = getConfigValue(url, SleepWindowField, hystrix.DefaultSleepWindow) + commandConfig.ErrorPercentThreshold = getConfigValue(url, ErrorPercentThreshold, hystrix.DefaultErrorPercentThreshold) + commandConfig.MaxConcurrentRequests = getConfigValue(url, MaxConcurrentField, defaultMaxConcurrent) + commandConfig.Timeout = getConfigValue(url, motan.TimeOutKey, hystrix.DefaultTimeout) return commandConfig } @@ -124,21 +123,11 @@ func defaultErrMotanResponse(request motan.Request, errMsg string) motan.Respons func getConfigStr(config *hystrix.CommandConfig) string { var ret string - if config.RequestVolumeThreshold != 0 { - ret += "requestThreshold:" + strconv.Itoa(config.RequestVolumeThreshold) + " " - } else { - ret += "requestThreshold:" + strconv.Itoa(hystrix.DefaultVolumeThreshold) + " " - } - if config.SleepWindow != 0 { - ret += "sleepWindow:" + strconv.Itoa(config.SleepWindow) + " " - } else { - ret += "sleepWindow:" + strconv.Itoa(hystrix.DefaultSleepWindow) + " " - } - if config.ErrorPercentThreshold != 0 { - ret += "errorPercent:" + strconv.Itoa(config.ErrorPercentThreshold) + " " - } else { - ret += "errorPercent:" + strconv.Itoa(hystrix.DefaultErrorPercentThreshold) + " " - } + ret += "requestThreshold:" + strconv.Itoa(config.RequestVolumeThreshold) + " " + ret += "sleepWindow:" + strconv.Itoa(config.SleepWindow) + " " + ret += "errorPercent:" + strconv.Itoa(config.ErrorPercentThreshold) + " " + ret += "maxConcurrent:" + strconv.Itoa(config.MaxConcurrentRequests) + " " + ret += "timeout:" + strconv.Itoa(config.Timeout) + "ms " return ret } diff --git a/filter/circuitBreaker_test.go b/filter/circuitBreaker_test.go index 5ea70e56..37ed4cc1 100644 --- a/filter/circuitBreaker_test.go +++ b/filter/circuitBreaker_test.go @@ -1,14 +1,15 @@ package filter import ( + "github.com/afex/hystrix-go/hystrix" + assert2 "github.com/stretchr/testify/assert" + "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/endpoint" + "github.com/weibocom/motan-go/log" "sync" "sync/atomic" "testing" "time" - - "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/endpoint" - "github.com/weibocom/motan-go/log" ) var ( @@ -29,13 +30,14 @@ func TestCircuitBreakerFilter(t *testing.T) { request := &core.MotanRequest{Method: "testMethod"} //Test NewFilter - param := map[string]string{core.TimeOutKey: "2", SleepWindowField: "300"} + param := map[string]string{core.TimeOutKey: "2", SleepWindowField: "300", IncludeBizException: "jia"} filterURL := &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint", Parameters: param} f := defaultExtFactory.GetFilter(CircuitBreaker) if f == nil { t.Error("Can not find circuitBreaker filter!") + } else { + f = f.NewFilter(filterURL) } - f = f.NewFilter(filterURL) ef := f.(core.EndPointFilter) ef.SetNext(new(mockEndPointFilter)) @@ -67,31 +69,131 @@ func TestCircuitBreakerFilter(t *testing.T) { filterSleepTimeLock.Lock() filterSleepTime = 0 * time.Millisecond filterSleepTimeLock.Unlock() - for i := 0; i < 100; i++ { + for i := 0; i < 20; i++ { ef.Filter(caller, request) } - time.Sleep(10 * time.Millisecond) //wait until async call complete + time.Sleep(100 * time.Millisecond) //wait until async call complete filterSleepTimeLock.Lock() filterSleepTime = 7 * time.Millisecond filterSleepTimeLock.Unlock() - for i := 0; i < 50; i++ { + for i := 0; i < 30; i++ { ef.Filter(caller, request) } - time.Sleep(10 * time.Millisecond) //wait until async call complete + time.Sleep(10000 * time.Millisecond) //wait until async call complete countLock.RLock() - if count != 171 && count != 172 { + if count != 61 && count != 62 { t.Error("Test sleepWindow failed! count:", count) } countLock.RUnlock() } +func TestGetConfigStr(t *testing.T) { + assert := assert2.New(t) + conf := &hystrix.CommandConfig{ + RequestVolumeThreshold: 10, + SleepWindow: 300, + MaxConcurrentRequests: 500, + ErrorPercentThreshold: 50, + Timeout: 50, + } + res := getConfigStr(conf) + assert.Equal(res, "requestThreshold:10 sleepWindow:300 errorPercent:50 maxConcurrent:500 timeout:50ms ") +} + +func TestBuildConfig(t *testing.T) { + assert := assert2.New(t) + valid := core.URL{ + Parameters: map[string]string{ + RequestVolumeThresholdField: "10", + core.TimeOutKey: "10", + SleepWindowField: "10", + MaxConcurrentField: "10", + ErrorPercentThreshold: "10", + }, + } + conf := buildCommandConfig(&valid) + assert.Equal(conf.ErrorPercentThreshold, 10) + assert.Equal(conf.Timeout, 10) + assert.Equal(conf.MaxConcurrentRequests, 10) + assert.Equal(conf.SleepWindow, 10) + assert.Equal(conf.RequestVolumeThreshold, 10) + invalid := core.URL{ + Parameters: map[string]string{ + RequestVolumeThresholdField: "-1", + core.TimeOutKey: "-1", + SleepWindowField: "-1", + MaxConcurrentField: "-1", + ErrorPercentThreshold: "200", + }, + } + conf = buildCommandConfig(&invalid) + assert.Equal(conf.ErrorPercentThreshold, 50) + assert.Equal(conf.Timeout, 1000) + assert.Equal(conf.MaxConcurrentRequests, defaultMaxConcurrent) + assert.Equal(conf.SleepWindow, 5000) + assert.Equal(conf.RequestVolumeThreshold, 20) + empty := core.URL{ + Parameters: map[string]string{ + RequestVolumeThresholdField: "-1", + core.TimeOutKey: "-1", + SleepWindowField: "-1", + MaxConcurrentField: "-1", + ErrorPercentThreshold: "200", + }, + } + conf = buildCommandConfig(&empty) + assert.Equal(conf.ErrorPercentThreshold, 50) + assert.Equal(conf.Timeout, 1000) + assert.Equal(conf.MaxConcurrentRequests, defaultMaxConcurrent) + assert.Equal(conf.SleepWindow, 5000) + assert.Equal(conf.RequestVolumeThreshold, 20) +} + +func TestOtherCB(t *testing.T) { + assert := assert2.New(t) + defaultExtFactory := &core.DefaultExtensionFactory{} + defaultExtFactory.Initialize() + RegistDefaultFilters(defaultExtFactory) + endpoint.RegistDefaultEndpoint(defaultExtFactory) + f := defaultExtFactory.GetFilter(CircuitBreaker) + assert.Equal(f.GetIndex(), 20) + assert.Equal(f.HasNext(), false) + assert.Equal(f.GetName(), CircuitBreaker) + assert.Equal(int(f.GetType()), core.EndPointFilterType) +} + +func TestMockException(t *testing.T) { + assert := assert2.New(t) + defaultExtFactory := &core.DefaultExtensionFactory{} + defaultExtFactory.Initialize() + RegistDefaultFilters(defaultExtFactory) + endpoint.RegistDefaultEndpoint(defaultExtFactory) + url := &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint"} + caller := defaultExtFactory.GetEndPoint(url) + request := &core.MotanRequest{Method: "testMethod"} + + //Test NewFilter + param := map[string]string{core.TimeOutKey: "2", SleepWindowField: "300", IncludeBizException: "jia"} + filterURL := &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint", Parameters: param} + f := defaultExtFactory.GetFilter(CircuitBreaker) + if f == nil { + t.Error("Can not find circuitBreaker filter!") + } else { + f = f.NewFilter(filterURL) + } + ef := f.(core.EndPointFilter) + ef.SetNext(new(mockExceptionEPFilter)) + res := ef.Filter(caller, request) + assert.NotNil(res.GetException()) +} + type mockEndPointFilter struct{} func (m *mockEndPointFilter) GetName() string { return "mockEndPointFilter" } -func (m *mockEndPointFilter) NewFilter(url *core.URL) core.Filter { +func (m *mockEndPointFilter) NewFilter(*core.URL) core.Filter { return core.GetLastEndPointFilter() } @@ -121,3 +223,34 @@ func (m *mockEndPointFilter) GetIndex() int { func (m *mockEndPointFilter) GetType() int32 { return core.EndPointFilterType } + +type mockExceptionEPFilter struct{} + +func (m *mockExceptionEPFilter) GetName() string { + return "mockEndPointFilter" +} + +func (m *mockExceptionEPFilter) NewFilter(*core.URL) core.Filter { + return core.GetLastEndPointFilter() +} + +func (m *mockExceptionEPFilter) Filter(_ core.Caller, request core.Request) core.Response { + return defaultErrMotanResponse(request, "mock exception") +} + +func (m *mockExceptionEPFilter) HasNext() bool { + return false +} + +func (m *mockExceptionEPFilter) SetNext(nextFilter core.EndPointFilter) { + vlog.Errorf("should not set next in mockEndPointFilter! filer:%s", nextFilter.GetName()) +} +func (m *mockExceptionEPFilter) GetNext() core.EndPointFilter { + return nil +} +func (m *mockExceptionEPFilter) GetIndex() int { + return 100 +} +func (m *mockExceptionEPFilter) GetType() int32 { + return core.EndPointFilterType +} diff --git a/filter/clusterCircuitBreaker_test.go b/filter/clusterCircuitBreaker_test.go new file mode 100644 index 00000000..6f370018 --- /dev/null +++ b/filter/clusterCircuitBreaker_test.go @@ -0,0 +1,146 @@ +package filter + +import ( + assert2 "github.com/stretchr/testify/assert" + "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/endpoint" + lb2 "github.com/weibocom/motan-go/lb" + "sync" + "sync/atomic" + "testing" + "time" +) + +var ( + clusterCount = int64(0) + clusterCountLock sync.RWMutex + clusterFilterSleepTime = 7 * time.Millisecond + clusterFilterSleepTimeLock sync.RWMutex +) + +func TestClusterCircuitBreakerFilter(t *testing.T) { + //Init + defaultExtFactory := &core.DefaultExtensionFactory{} + defaultExtFactory.Initialize() + RegistDefaultFilters(defaultExtFactory) + lb2.RegistDefaultLb(defaultExtFactory) + endpoint.RegistDefaultEndpoint(defaultExtFactory) + url := &core.URL{Host: "127.0.0.1", Port: 8888, Protocol: "mockEndpoint"} + ha := new(mockHA) + lb := defaultExtFactory.GetLB(url) + lb.OnRefresh([]core.EndPoint{defaultExtFactory.GetEndPoint(url)}) + request := &core.MotanRequest{Method: "testMethod"} + + //Test NewFilter + param := map[string]string{core.TimeOutKey: "2", SleepWindowField: "300", IncludeBizException: "jia"} + filterURL := &core.URL{Host: "127.0.0.1", Port: 8888, Protocol: "mockEndpoint", Parameters: param} + f := defaultExtFactory.GetFilter(ClusterCircuitBreaker) + if f == nil { + t.Error("Can not find circuitBreaker filter!") + } else { + f = f.NewFilter(filterURL) + } + ef := f.(core.ClusterFilter) + ef.SetNext(new(mockClusterFilter)) + + //Test circuitBreakerTimeout & requestVolumeThreshold + for i := 0; i < 30; i++ { + ef.Filter(ha, lb, request) + } + time.Sleep(10 * time.Millisecond) //wait until async call complete + clusterCountLock.RLock() + if clusterCount != 20 && clusterCount != 21 { + t.Error("Test clusterCircuitBreakerTimeout failed! count:", clusterCount) + } + clusterCountLock.RUnlock() + + //Test sleepWindow + time.Sleep(350 * time.Millisecond) //wait until SleepWindowField + for i := 0; i < 5; i++ { + ef.Filter(ha, lb, request) + } + time.Sleep(10 * time.Millisecond) //wait until async call complete + clusterCountLock.RLock() + if clusterCount != 21 && clusterCount != 22 { + t.Error("Test sleepWindow failed! count:", clusterCount) + } + clusterCountLock.RUnlock() +} + +func TestClusterCircuitBreakerOther(t *testing.T) { + assert := assert2.New(t) + defaultExtFactory := &core.DefaultExtensionFactory{} + defaultExtFactory.Initialize() + RegistDefaultFilters(defaultExtFactory) + f := defaultExtFactory.GetFilter(ClusterCircuitBreaker) + assert.Equal(int(f.GetType()), core.ClusterFilterType) + assert.Equal(f.GetIndex(), 20) + assert.Equal(f.HasNext(), false) + assert.Equal(f.GetName(), ClusterCircuitBreaker) +} + +type mockClusterFilter struct{} + +func (c *mockClusterFilter) GetIndex() int { + return 101 +} + +func (c *mockClusterFilter) NewFilter(*core.URL) core.Filter { + return core.GetLastClusterFilter() +} + +func (c *mockClusterFilter) Filter(ha core.HaStrategy, lb core.LoadBalance, request core.Request) core.Response { + clusterCountLock.Lock() + atomic.AddInt64(&clusterCount, 1) + clusterCountLock.Unlock() + clusterFilterSleepTimeLock.RLock() + time.Sleep(clusterFilterSleepTime) + clusterFilterSleepTimeLock.RUnlock() + return ha.Call(request, lb) +} + +func (c *mockClusterFilter) GetName() string { + return ClusterCircuitBreaker +} + +func (c *mockClusterFilter) HasNext() bool { + return false +} + +func (c *mockClusterFilter) GetType() int32 { + return core.ClusterFilterType +} + +func (c *mockClusterFilter) SetNext(core.ClusterFilter) { + return +} + +func (c *mockClusterFilter) GetNext() core.ClusterFilter { + return nil +} + +type mockHA struct { + url *core.URL +} + +func (f *mockHA) GetName() string { + return "mockHA" +} +func (f *mockHA) GetURL() *core.URL { + return f.url +} +func (f *mockHA) SetURL(url *core.URL) { + f.url = url +} +func (f *mockHA) Call(request core.Request, loadBalance core.LoadBalance) core.Response { + ep := loadBalance.Select(request) + if ep == nil { + return defaultErrMotanResponse(request, "no ep") + } + response := ep.Call(request) + if response.GetException() == nil || response.GetException().ErrType == core.BizException { + return response + } + lastErr := response.GetException() + return defaultErrMotanResponse(request, lastErr.ErrMsg) +} diff --git a/filter/rateLimit.go b/filter/rateLimit.go index 229cc753..85d5ac22 100644 --- a/filter/rateLimit.go +++ b/filter/rateLimit.go @@ -1,8 +1,11 @@ package filter import ( + "fmt" + mpro "github.com/weibocom/motan-go/protocol" "strconv" "strings" + "time" "github.com/juju/ratelimit" "github.com/weibocom/motan-go/core" @@ -10,64 +13,101 @@ import ( ) const ( - defaultCapacity = 1000 + capacityKey = "capacity" methodConfigPrefix = "rateLimit." + defaultTimeout = 1000 + maxCapacity = 5000 + minCapacity = 1000 ) type RateLimitFilter struct { - switcher *core.Switcher - bucket *ratelimit.Bucket //limit service - methodBuckets map[string]*ratelimit.Bucket //limit method - next core.EndPointFilter + switcher *core.Switcher + serviceBucket *ratelimit.Bucket //limit service + serviceMaxDuration time.Duration + methodBuckets map[string]*ratelimit.Bucket //limit method + next core.EndPointFilter + url *core.URL } func (r *RateLimitFilter) NewFilter(url *core.URL) core.Filter { ret := &RateLimitFilter{} - //init bucket - if rate, err := strconv.ParseFloat(url.GetParam(RateLimit, ""), 64); err == nil { - ret.bucket = ratelimit.NewBucketWithRate(rate, defaultCapacity) + rlp := url.GetParam(RateLimit, "") + if rlp == "" { + vlog.Warningf("[rateLimit] limit rate config is empty, service %s initialize failed", RateLimit) } else { - vlog.Warningf("[rateLimit] parse %s config error:%v", RateLimit, err) + if rate, err := strconv.ParseFloat(rlp, 64); err == nil { + if rate <= 0 { + vlog.Warningf("[rateLimit] %s: service rateLimit config invalid, rate: %f", url.GetIdentity(), rate) + } else { + capacity := getCapacity(url, rate) + ret.serviceBucket = ratelimit.NewBucketWithRate(rate, capacity) + ret.serviceMaxDuration = url.GetTimeDuration(core.TimeOutKey, time.Millisecond, time.Duration(defaultTimeout)*time.Millisecond) + vlog.Infof("[rateLimit] %s %s service config success, rate:%f, capacity:%d, wait max duration:%s", url.GetIdentity(), RateLimit, rate, capacity, ret.serviceMaxDuration.String()) + } + } else { + vlog.Warningf("[rateLimit] parse %s config error:%v", RateLimit, err) + } } - //init methodBucket + //read config methodBuckets := make(map[string]*ratelimit.Bucket) + methodRate := make(map[string]float64) for key, value := range url.Parameters { - if temp := strings.Split(key, methodConfigPrefix); len(temp) == 2 { - if methodRate, err := strconv.ParseFloat(value, 64); err == nil && temp[1] != "" { - methodBuckets[temp[1]] = ratelimit.NewBucketWithRate(methodRate, defaultCapacity) - } else { - vlog.Warningf("[rateLimit] parse %s config error:%s", key, err.Error()) - } - } else { - vlog.Warningf("[rateLimit] parse %s config error", key) + if k, v, ok := getKeyValue(key, value, methodConfigPrefix); ok { + methodRate[k] = v } } - ret.methodBuckets = methodBuckets + //init methodBucket + for key, value := range methodRate { + capacity := getMethodCapacity(url, value, key) + methodBuckets[key] = ratelimit.NewBucketWithRate(value, capacity) + vlog.Infof("[rateLimit] %s %s method: %s config success, rate:%f, capacity:%d", url.GetIdentity(), RateLimit, key, value, capacity) + } + if len(methodBuckets) == 0 && ret.serviceBucket == nil { + vlog.Warningf("[rateLimit] %s: no service or method rateLimit takes effect", url.GetIdentity()) + } + ret.methodBuckets = methodBuckets //init switcher switcherName := GetRateLimitSwitcherName(url) core.GetSwitcherManager().Register(switcherName, true) ret.switcher = core.GetSwitcherManager().GetSwitcher(switcherName) - + ret.url = url return ret } func (r *RateLimitFilter) Filter(caller core.Caller, request core.Request) core.Response { if r.switcher.IsOpen() { - if r.bucket != nil { - r.bucket.Wait(1) + if r.serviceBucket != nil { + callable := r.serviceBucket.WaitMaxDuration(1, r.serviceMaxDuration) + if !callable { + return defaultErrMotanResponse(request, fmt.Sprintf("[rateLimit] wait time exceed timeout(%s)", r.serviceMaxDuration.String())) + } } if methodBucket, ok := r.methodBuckets[request.GetMethod()]; ok { - methodBucket.Wait(1) + var methodTimeout int64 + tRequest := request.GetAttachment(mpro.MTimeout) + if tRequest != "" { + if v, err := strconv.ParseInt(tRequest, 10, 64); err == nil { + methodTimeout = v + } + } else { + v := r.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), core.TimeOutKey, defaultTimeout) + methodTimeout = v + } + methodDuration := time.Duration(methodTimeout) * time.Millisecond + callable := methodBucket.WaitMaxDuration(1, methodDuration) + if !callable { + return defaultErrMotanResponse(request, fmt.Sprintf("[rateLimit] method %s wait time exceed timeout(%s)", request.GetMethod(), methodDuration.String())) + } } } return r.GetNext().Filter(caller, request) } func GetRateLimitSwitcherName(url *core.URL) string { - return url.GetParam("conf-id", "") + "_rateLimit" + return url.GetParam(core.URLConfKey, "") + "_rateLimit" } func (r *RateLimitFilter) SetNext(nextFilter core.EndPointFilter) { @@ -93,3 +133,48 @@ func (r *RateLimitFilter) GetIndex() int { func (r *RateLimitFilter) GetType() int32 { return core.EndPointFilterType } + +func getKeyValue(key, value, prefix string) (string, float64, bool) { + if strings.HasPrefix(key, prefix) { + if temp := strings.Split(key, prefix); len(temp) == 2 { + if r, err := strconv.ParseFloat(value, 64); err == nil && temp[1] != "" && r > 0 { + return temp[1], r, true + } else { + if err != nil { + vlog.Warningf("[rateLimit] parse %s config error:%s", key, err.Error()) + } else { + if r <= 0 { + vlog.Warningf("[rateLimit] parse %s config error: value is 0 or negative", key) + } + } + if temp[1] == "" { + vlog.Warningf("[rateLimit] parse %s config error: key is empty", key) + } + } + } + } + return "", 0, false +} + +func getCapacity(url *core.URL, rate float64) int64 { + return url.GetPositiveIntValue(capacityKey, getDefaultCapacity(rate)) +} + +func getMethodCapacity(url *core.URL, rate float64, methodName string) int64 { + return url.GetMethodPositiveIntValue(methodName, "", capacityKey, getDefaultCapacity(rate)) +} + +// dynamically get +func getDefaultCapacity(rate float64) (res int64) { + capicity := rate * 2 + if capicity >= minCapacity && capicity <= maxCapacity { + res = int64(capicity) + return + } + if capicity < minCapacity { + res = minCapacity + return + } + res = maxCapacity + return +} diff --git a/filter/rateLimit_test.go b/filter/rateLimit_test.go index 05fa6d95..33abf1ce 100644 --- a/filter/rateLimit_test.go +++ b/filter/rateLimit_test.go @@ -1,6 +1,8 @@ package filter import ( + assert2 "github.com/stretchr/testify/assert" + mpro "github.com/weibocom/motan-go/protocol" "strconv" "testing" "time" @@ -12,60 +14,280 @@ import ( var offset = 2 var rate = 10 // times per second +var defaultCap = 1000 + func TestRateLimitFilter(t *testing.T) { - //Init - defaultExtFactory := &core.DefaultExtensionFactory{} - defaultExtFactory.Initialize() - RegistDefaultFilters(defaultExtFactory) - endpoint.RegistDefaultEndpoint(defaultExtFactory) - url := &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint"} - caller := defaultExtFactory.GetEndPoint(url) request := &core.MotanRequest{Method: "testMethod"} - - //Test NewFilter - param := map[string]string{"rateLimit": strconv.Itoa(rate), "conf-id": "zha"} - filterURL := &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint", Parameters: param} - f := defaultExtFactory.GetFilter("rateLimit") - if f == nil { - t.Error("Can not find rateLimit filter!") + testItems := []testItem{ + { + title: "plain service rateLimit", + param: map[string]string{"rateLimit": strconv.Itoa(rate), "conf-id": "zha"}, + }, + { + title: "plain method rateLimit", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "zha"}, + }, + { + title: "rateLimit switcher off", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "zha"}, + preCall: func() { + core.GetSwitcherManager().GetSwitcher("zha_rateLimit").SetValue(false) + }, + afterCall: func() { + core.GetSwitcherManager().GetSwitcher("zha_rateLimit").SetValue(true) + }, + }, } - f = f.NewFilter(filterURL) - ef := f.(core.EndPointFilter) - ef.SetNext(core.GetLastEndPointFilter()) + for i, j := range testItems { + if j.preCall != nil { + j.preCall() + } + caller, ef := getEf(j.param) + if ef == nil { + t.Error("Can not find rateLimit filter!") + return + } + startTime := time.Now() + for i := 0; i < defaultCap+offset; i++ { + ef.Filter(caller, request) + } + elapsed := time.Since(startTime) + if i == 2 { + if elapsed > time.Duration(offset-1)*time.Second/time.Duration(rate) { + t.Error("Test ", j.title, " failed! elapsed:", elapsed) + } + } else { + if elapsed < time.Duration(offset-1)*time.Second/time.Duration(rate) { + t.Error("Test ", j.title, " failed! elapsed:", elapsed) + } + } + if j.afterCall != nil { + j.afterCall() + } + } +} - //Test serviceFilter - startTime := time.Now() - for i := 0; i < defaultCapacity+offset; i++ { - ef.Filter(caller, request) +func TestWrongParam(t *testing.T) { + assert := assert2.New(t) + testItems := []testItem{ + { + title: "wrong type of service rate value", + param: map[string]string{"rateLimit": "jia", "conf-id": "zha"}, + }, + { + title: "wrong type of method rate value", + param: map[string]string{"rateLimit.testMethod": "jia", "conf-id": "jia"}, + }, + { + title: "wrong method rate key", + param: map[string]string{"rateLimit.": "10", "conf-id": "jia"}, + }, + { + title: "wrong type of method timeout value", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().requestTimeout": "jia"}, + }, + { + title: "negative value of method timeout", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().requestTimeout": "-1"}, + }, + { + title: "wrong service rate value and wrong type of method timeout", + param: map[string]string{"rateLimit.testMethod": "-1", "conf-id": "Jia", "testMethod().requestTimeout": "jia"}, + }, + { + title: "negative service rate value", + param: map[string]string{"rateLimit": "-1", "conf-id": "jia"}, + }, } - if elapsed := time.Since(startTime); elapsed < time.Duration(offset-1)*time.Second/time.Duration(rate) { - t.Error("Test serviceFilter failed! elapsed:", elapsed) + request := &core.MotanRequest{Method: "testMethod"} + for _, j := range testItems { + caller, ef := getEf(j.param) + for i := 0; i < defaultCap+offset; i++ { + resp := ef.Filter(caller, request) + assert.Nil(resp.GetException()) + } } +} - //Test methodFilter - param = map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "zha"} - filterURL = &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint", Parameters: param} - ef = defaultExtFactory.GetFilter("rateLimit").NewFilter(filterURL).(core.EndPointFilter) - ef.SetNext(core.GetLastEndPointFilter()) - startTime = time.Now() - for i := 0; i < defaultCapacity+offset; i++ { - ef.Filter(caller, request) +func TestRateLimitTimeout(t *testing.T) { + assert := assert2.New(t) + request := &core.MotanRequest{Method: "testMethod"} + + testItems := []testItem{ + { + title: "plain service max duration(timeout)", + param: map[string]string{"rateLimit": strconv.Itoa(rate), "conf-id": "Jia", "requestTimeout": "50"}, + errMsg: "[rateLimit] wait time exceed timeout(50ms)", + }, + { + title: "plain method max duration(timeout)", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().requestTimeout": "50"}, + errMsg: "[rateLimit] method testMethod wait time exceed timeout(50ms)", + }, } - if elapsed := time.Since(startTime); elapsed < time.Duration(offset-1)*time.Second/time.Duration(rate) { - t.Error("Test methodFilter failed! elapsed:", elapsed) + for _, j := range testItems { + caller, ef := getEf(j.param) + for i := 0; i < defaultCap+offset; i++ { + resp := ef.Filter(caller, request) + if i >= defaultCap { + assert.NotNil(resp.GetException()) + assert.Equal(resp.GetException().ErrMsg, j.errMsg) + } + } } //Test switcher - param = map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "zha"} - filterURL = &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint", Parameters: param} - ef = defaultExtFactory.GetFilter("rateLimit").NewFilter(filterURL).(core.EndPointFilter) - ef.SetNext(core.GetLastEndPointFilter()) - core.GetSwitcherManager().GetSwitcher("zha_rateLimit").SetValue(false) - startTime = time.Now() - for i := 0; i < defaultCapacity+offset; i++ { - ef.Filter(caller, request) + param := map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().requestTimeout": "50"} + caller, ef := getEf(param) + core.GetSwitcherManager().GetSwitcher("Jia_rateLimit").SetValue(false) + for i := 0; i < defaultCap+offset; i++ { + resp := ef.Filter(caller, request) + assert.Nil(resp.GetException()) + } + core.GetSwitcherManager().GetSwitcher("Jia_rateLimit").SetValue(true) + + //Test set timeout in request + request.SetAttachment(mpro.MTimeout, "50") + param = map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().requestTimeout": "100"} + caller, ef = getEf(param) + for i := 0; i < defaultCap+offset; i++ { + resp := ef.Filter(caller, request) + if i >= defaultCap { + assert.NotNil(resp.GetException()) + // timeout value will use the one in request attachment(50) + assert.Equal(resp.GetException().ErrMsg, "[rateLimit] method testMethod wait time exceed timeout(50ms)") + } } - if elapsed := time.Since(startTime); elapsed > time.Duration(offset+1)*time.Second/time.Duration(rate) { - t.Error("Test switcher failed! elapsed:", elapsed) +} + +func TestCapacity(t *testing.T) { + capacity := 200 + request := &core.MotanRequest{Method: "testMethod"} + testItems := []testItem{ + { + title: "plain service capacity", + param: map[string]string{"rateLimit": strconv.Itoa(rate), "conf-id": "Jia", "capacity": strconv.Itoa(capacity)}, + }, + { + title: "plain method capacity", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().capacity": strconv.Itoa(capacity)}, + }, + { + title: "wrong type of service capacity value", + param: map[string]string{"rateLimit": strconv.Itoa(rate), "conf-id": "Jia", "capacity": "jia"}, + }, + { + title: "wrong type of method capacity value", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().capacity": "jia"}, + }, + { + title: "wrong value of method capacity", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "testMethod().capacity": "-1"}, + }, } + for k, j := range testItems { + caller, ef := getEf(j.param) + startTime := time.Now() + for i := 0; i < capacity+offset; i++ { + ef.Filter(caller, request) + } + if k >= 2 { + if elapsed := time.Since(startTime); elapsed > time.Duration(offset-1)*time.Second/time.Duration(rate) { + t.Error("Test ", j.title, " failed! elapsed:", elapsed.String(), " ", time.Duration(offset-1)*time.Second/time.Duration(rate)) + } + } else { + if elapsed := time.Since(startTime); elapsed < time.Duration(offset-1)*time.Second/time.Duration(rate) { + t.Error("Test ", j.title, " failed! elapsed:", elapsed.String(), " ", time.Duration(offset-1)*time.Second/time.Duration(rate)) + } + } + } + param := map[string]string{"rateLimit.testMethod": strconv.Itoa(rate), "conf-id": "Jia", "capacity.testMethod": "5"} + caller, ef := getEf(param) + // rateLimit will use default capacity + for i := 0; i < defaultCap+offset; i++ { + resp := ef.Filter(caller, request) + assert2.Nil(t, resp.GetException()) + } +} + +func TestCapacityEdge(t *testing.T) { + request := &core.MotanRequest{Method: "testMethod"} + bigRate := 2501 + properRate := 600 + properCapacity := properRate * 2 + testItems := []testItem{ + { + title: "max service capacity", + param: map[string]string{"rateLimit": strconv.Itoa(bigRate), "conf-id": "Jia"}, + }, + { + title: "max method capacity", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(bigRate), "conf-id": "Jia"}, + }, + { + title: "proper service capacity", + param: map[string]string{"rateLimit": strconv.Itoa(properRate), "conf-id": "Jia"}, + }, + { + title: "proper method capacity", + param: map[string]string{"rateLimit.testMethod": strconv.Itoa(properRate), "conf-id": "Jia"}, + }, + } + for k, j := range testItems { + caller, ef := getEf(j.param) + startTime := time.Now() + if k >= 2 { + for i := 0; i < properCapacity+offset; i++ { + ef.Filter(caller, request) + } + if elapsed := time.Since(startTime); elapsed < time.Duration(offset-1)*time.Second/time.Duration(properRate) { + t.Error("Test ", j.title, " failed! elapsed:", elapsed.String(), " ", time.Duration(offset-1)*time.Second/time.Duration(rate)) + } + } else { + for i := 0; i < maxCapacity+offset; i++ { + ef.Filter(caller, request) + } + if elapsed := time.Since(startTime); elapsed < time.Duration(offset-1)*time.Second/time.Duration(bigRate) { + t.Error("Test ", j.title, " failed! elapsed:", elapsed.String(), " ", time.Duration(offset-1)*time.Second/time.Duration(rate)) + } + } + + } + +} + +func TestOther(t *testing.T) { + assert := assert2.New(t) + defaultExtFactory := &core.DefaultExtensionFactory{} + defaultExtFactory.Initialize() + RegistDefaultFilters(defaultExtFactory) + endpoint.RegistDefaultEndpoint(defaultExtFactory) + f := defaultExtFactory.GetFilter("rateLimit") + assert.Equal(f.GetIndex(), 3) + assert.Equal(f.GetName(), RateLimit) + assert.Equal(f.HasNext(), false) + assert.Equal(int(f.GetType()), core.EndPointFilterType) +} + +func getEf(param map[string]string) (core.EndPoint, core.EndPointFilter) { + defaultExtFactory := &core.DefaultExtensionFactory{} + defaultExtFactory.Initialize() + RegistDefaultFilters(defaultExtFactory) + endpoint.RegistDefaultEndpoint(defaultExtFactory) + url := &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint"} + caller := defaultExtFactory.GetEndPoint(url) + filterURL := &core.URL{Host: "127.0.0.1", Port: 7888, Protocol: "mockEndpoint", Parameters: param} + f := defaultExtFactory.GetFilter("rateLimit") + f = f.NewFilter(filterURL) + ef := f.(core.EndPointFilter) + ef.SetNext(core.GetLastEndPointFilter()) + return caller, ef +} + +type testItem struct { + title string + param map[string]string + preCall func() + afterCall func() + errMsg string } diff --git a/provider/httpProvider_test.go b/provider/httpProvider_test.go index 25664af4..fe83e19f 100644 --- a/provider/httpProvider_test.go +++ b/provider/httpProvider_test.go @@ -2,16 +2,16 @@ package provider import ( "bytes" - "net/http" - "os" - "testing" - "github.com/stretchr/testify/assert" "github.com/valyala/fasthttp" "github.com/weibocom/motan-go/config" "github.com/weibocom/motan-go/core" mhttp "github.com/weibocom/motan-go/http" "github.com/weibocom/motan-go/serialize" + "net/http" + "os" + "testing" + "time" ) const httpProviderTestData = ` @@ -22,19 +22,16 @@ http-locations: upstream: test1 rewriteRules: - 'exact /Test2/1 /(.*) /test' - - match: ^/test2/.* type: regexp upstream: test2 rewriteRules: - '!iregexp ^/Test2/1/.* ^/test2/(.*) /test/$1' - - match: ^/test3/.* type: iregexp upstream: test3 rewriteRules: - 'start / ^/(.*) /test/$1' - - match: ^(/|/2/)(p1|p2).* type: regexp upstream: test4 @@ -82,5 +79,6 @@ func TestMain(m *testing.M) { }) http.ListenAndServe(addr, handler) }() + time.Sleep(time.Second * 2) os.Exit(m.Run()) } From 92d3ca06c86d6ede6f640e422bcffefd40603fa2 Mon Sep 17 00:00:00 2001 From: cocowh <596527851@qq.com> Date: Thu, 10 Feb 2022 14:55:46 +0800 Subject: [PATCH 09/75] Fix httpproxy contenttype (#256) * fix: set httrequest contentType by attachment * refactor: attachment convert to httpheader not case sensitive --- http/httpProxy.go | 15 ++++++++++----- http/httpRpc_test.go | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/http/httpProxy.go b/http/httpProxy.go index f6764619..8a99fc7c 100644 --- a/http/httpProxy.go +++ b/http/httpProxy.go @@ -20,6 +20,10 @@ const ( QueryString = "HTTP_QueryString" ) +const ( + HeaderContentType = "Content-Type" +) + const ( DomainKey = "domain" KeepaliveTimeoutKey = "keepaliveTimeout" @@ -426,18 +430,19 @@ func MotanRequestToFasthttpRequest(motanRequest core.Request, fasthttpRequest *f motanRequest.GetAttachments().Range(func(k, v string) bool { // ignore some specified key for _, attachmentKey := range httpProxySpecifiedAttachments { - if attachmentKey == k { + if strings.EqualFold(k, attachmentKey) { return true } } - if k == core.HostKey { - return true - } // fasthttp will use a special field to store this header - if k == "Host" { + if strings.EqualFold(k, core.HostKey) { fasthttpRequest.Header.SetHost(v) return true } + if strings.EqualFold(k, HeaderContentType) { + fasthttpRequest.Header.SetContentType(v) + return true + } k = strings.Replace(k, "M_", "MOTAN-", -1) fasthttpRequest.Header.Add(k, v) return true diff --git a/http/httpRpc_test.go b/http/httpRpc_test.go index 6f99da2e..1990fdd6 100644 --- a/http/httpRpc_test.go +++ b/http/httpRpc_test.go @@ -2,12 +2,14 @@ package http_test import ( "bytes" + "encoding/json" "fmt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/weibocom/motan-go" "github.com/weibocom/motan-go/config" motancore "github.com/weibocom/motan-go/core" + http2 "github.com/weibocom/motan-go/http" "github.com/weibocom/motan-go/protocol" "math/rand" "net/http" @@ -46,6 +48,10 @@ motan-service: requestTimeout: 2000 ` +var ( + httpRequestHeader http.Header +) + type APITestSuite struct { suite.Suite } @@ -97,9 +103,17 @@ func (s *APITestSuite) TestRequestResponse() { resp, err = callTimeOutWrongArgumentCount("127.0.0.1", "9989", "test", "test.domain", 30000, "", "/", []interface{}{argumentString}, attachments) assert.NotNil(s.T(), err) assert.Nil(s.T(), resp) + // json content-type request + bodyBytes,_ := json.Marshal(arguments) + attachments[http2.HeaderContentType] = "application/json" + resp, err = callTimeOutWithAttachment("127.0.0.1", "9989", "test", "test.domain", 30000, "", "/", []interface{}{bodyBytes}, attachments) + assert.Nil(s.T(), err) + assert.NotNil(s.T(), resp) + assert.Equal(s.T(), "application/json", httpRequestHeader.Get(http2.HeaderContentType)) } func indexHandler(w http.ResponseWriter, r *http.Request) { + httpRequestHeader = r.Header fmt.Fprintf(w, "hello world") } From a50d017728dddfe6f15ec8666803f86f538e4b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=9E=A8=E7=85=8A?= Date: Fri, 1 Apr 2022 09:44:18 +0800 Subject: [PATCH 10/75] support global filter fix #257 (#259) log dir permission set 0755 build support windows --- .gitignore | 3 + core/constants.go | 2 + core/globalContext.go | 45 ++++++++++++++- core/globalContext_test.go | 38 +++++++++++++ core/testdata/applications/globalFilter.yaml | 56 +++++++++++++++++++ .../motan.TestGlobalFilterService1.yaml | 21 +++++++ .../motan.TestGlobalFilterService2.yaml | 21 +++++++ .../motan.TestGlobalFilterService3.yaml | 21 +++++++ core/util.go | 12 ++++ log/rotate.go | 13 +---- log/rotate_notwin.go | 22 ++++++++ log/rotate_win.go | 15 +++++ 12 files changed, 256 insertions(+), 13 deletions(-) create mode 100644 core/testdata/applications/globalFilter.yaml create mode 100644 core/testdata/services/motan.TestGlobalFilterService1.yaml create mode 100644 core/testdata/services/motan.TestGlobalFilterService2.yaml create mode 100644 core/testdata/services/motan.TestGlobalFilterService3.yaml create mode 100644 log/rotate_notwin.go create mode 100644 log/rotate_win.go diff --git a/.gitignore b/.gitignore index fb9d914e..6160bf02 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ tmtags pretty/zj zj* +# Windows +*.exe + .idea/* # gdb *.gdb_history diff --git a/core/constants.go b/core/constants.go index 19ddac91..320075e8 100644 --- a/core/constants.go +++ b/core/constants.go @@ -33,6 +33,8 @@ const ( ApplicationKey = "application" VersionKey = "version" FilterKey = "filter" + GlobalFilter = "globalFilter" + DisableGlobalFilter = "disableGlobalFilter" RegistryKey = "registry" WeightKey = "weight" SerializationKey = "serialization" diff --git a/core/globalContext.go b/core/globalContext.go index a0841693..423275e7 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -232,11 +232,11 @@ func (c *Context) Initialize() { } c.parseRegistrys() + c.parseHostURL() c.parseBasicRefers() c.parseRefers() c.parserBasicServices() c.parseServices() - c.parseHostURL() c.parseHTTPClients() } @@ -396,11 +396,54 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { } else { newURL = url } + + // merge agent globalFilter, filter + c.mergeGlobalFilter(newURL) + newURLs[key] = newURL } return newURLs } +func (c *Context) mergeGlobalFilter(newURL *URL) { + if c.AgentURL != nil { + finalFilterSet := make(map[string]bool) + globalFilterStr := c.AgentURL.GetStringParamsWithDefault(GlobalFilter, "") + // disable globalFilter + if globalFilterStr != "" { + finalFilterSet = TrimSplitSet(globalFilterStr, ",") + disableGlobalFilterStr := newURL.GetStringParamsWithDefault(DisableGlobalFilter, "") + if disableGlobalFilterStr != "" { + vlog.Infoln("disable global filter: " + disableGlobalFilterStr) + disableGlobalFilterArr := TrimSplit(disableGlobalFilterStr, ",") + for _, disableFilter := range disableGlobalFilterArr { + if _, ok := finalFilterSet[disableFilter]; ok { + delete(finalFilterSet, disableFilter) + } + } + } + } + // append filter + filterStr := newURL.GetStringParamsWithDefault(FilterKey, "") + if filterStr != "" { + filterArr := TrimSplit(filterStr, ",") + for _, filter := range filterArr { + finalFilterSet[filter] = true + } + } + // make new filter string + var finalFilter string + for filter := range finalFilterSet { + if filter != "" { + finalFilter += filter + "," + } + } + finalFilter = strings.TrimRight(finalFilter, ",") + newURL.PutParam(FilterKey, finalFilter) + } + +} + func (c *Context) parseRefers() { c.RefersURLs = c.basicConfToURLs(refersSection) } diff --git a/core/globalContext_test.go b/core/globalContext_test.go index c4629ba8..7434a5e3 100644 --- a/core/globalContext_test.go +++ b/core/globalContext_test.go @@ -3,6 +3,7 @@ package core import ( "flag" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -19,6 +20,8 @@ func TestGetContext(t *testing.T) { assert.NotNil(t, rs.RefersURLs, "parse refers urls fail.") assert.NotNil(t, rs.RefersURLs["status-rpc-json"], "parse refer section fail.") assert.Equal(t, "test-group", rs.RefersURLs["status-rpc-json"].Group, "get refer key fail.") + assert.Contains(t, rs.RefersURLs["status-rpc-json"].Parameters["filter"], "accessLog", "get refer filter fail.") + assert.Contains(t, rs.RefersURLs["status-rpc-json"].Parameters["filter"], "metrics", "get refer filter fail.") assert.NotEqual(t, 0, len(rs.ServiceURLs), "parse service urls fail") assert.Equal(t, "motan-demo-rpc", rs.ServiceURLs["mytest-motan2"].Group, "parse serivce key fail") } @@ -62,4 +65,39 @@ func TestNewContext(t *testing.T) { assert.NotNil(t, context.Config) context = NewContext("testdata", "app", "") assert.NotNil(t, context.Config) + + globalFilterContext := NewContext("testdata", "globalFilter", "app-idc1") + assert.NotNil(t, globalFilterContext) + + filterArrs := make(map[string][]string) + for _, section := range []string{"test-motan-refer", "test-global-filter-service-1-refer", "test-global-filter-service-2-refer"} { + filterArrs[section] = strings.Split(globalFilterContext.RefersURLs[section].Parameters["filter"], ",") + } + for _, section := range []string{"test-global-filter-0-service", "test-global-filter-1-service", "test-global-filter-2-service"} { + filterArrs[section] = strings.Split(globalFilterContext.ServiceURLs[section].Parameters["filter"], ",") + } + + for key, filterArr := range filterArrs { + t.Run(key, func(t *testing.T) { + assert.Contains(t, filterArr, "testGlobalFilter1") + assert.Contains(t, filterArr, "testGlobalFilter2") + assert.Contains(t, filterArr, "testGlobalFilter3") + assert.Contains(t, filterArr, "accessLog") + assert.Contains(t, filterArr, "metrics") + }) + } + + filterArrs = map[string][]string{ + "test-global-filter-service-3-refer": strings.Split(globalFilterContext.RefersURLs["test-global-filter-service-3-refer"].Parameters["filter"], ","), + "test-global-filter-3-service": strings.Split(globalFilterContext.ServiceURLs["test-global-filter-3-service"].Parameters["filter"], ","), + } + for key, filterArr := range filterArrs { + t.Run(key, func(t *testing.T) { + assert.Contains(t, filterArr, "testGlobalFilter1") + assert.NotContains(t, filterArr, "testGlobalFilter2") + assert.NotContains(t, filterArr, "testGlobalFilter3") + assert.Contains(t, filterArr, "accessLog") + assert.Contains(t, filterArr, "metrics") + }) + } } diff --git a/core/testdata/applications/globalFilter.yaml b/core/testdata/applications/globalFilter.yaml new file mode 100644 index 00000000..ce2953b2 --- /dev/null +++ b/core/testdata/applications/globalFilter.yaml @@ -0,0 +1,56 @@ +motan-agent: + port: 9981 + mport: 8002 + log_dir: logs + registry: consul + application: testGlobal + globalFilter: "testGlobalFilter1,testGlobalFilter2,testGlobalFilter3" + +motan-basicService: + test-basic-service: + group: motan-demo-rpc + protocol: motan2 + registry: consul + +motan-service: + test-global-filter-0-service: + path: motan.GlobalFilter1.TestService + proxy: "motan2:8100" + export: "motan2:9982" + provider: motan2 + requestTimeout: 2000 + filter: "accessLog,metrics" + basicService: test-basic-service + test-global-filter-1-service: + path: motan.GlobalFilter1.TestService + proxy: "motan2:8100" + export: "motan2:9982" + provider: motan2 + requestTimeout: 2000 + filter: "accessLog,metrics,testGlobalFilter3" + basicService: test-basic-service + test-global-filter-2-service: + path: motan.GlobalFilter1.TestService + proxy: "motan2:8100" + export: "motan2:9982" + provider: motan2 + requestTimeout: 2000 + filter: "accessLog,metrics,testGlobalFilter3" + disableGlobalFilter: "testGlobalFilter3" + basicService: test-basic-service + test-global-filter-3-service: + path: motan.GlobalFilter1.TestService + proxy: "motan2:8100" + export: "motan2:9982" + provider: motan2 + requestTimeout: 2000 + filter: "accessLog,metrics" + disableGlobalFilter: "testGlobalFilter2,testGlobalFilter3" + basicService: test-basic-service + +import-refer: + - motan.TestService + - motan.TestGlobalFilterService1 + - motan.TestGlobalFilterService2 + - motan.TestGlobalFilterService3 + diff --git a/core/testdata/services/motan.TestGlobalFilterService1.yaml b/core/testdata/services/motan.TestGlobalFilterService1.yaml new file mode 100644 index 00000000..dd78abdc --- /dev/null +++ b/core/testdata/services/motan.TestGlobalFilterService1.yaml @@ -0,0 +1,21 @@ +motan-basicRefer: + test-basic-refer: + group: motan-demo-rpc + protocol: motan2 + registry: consul + requestTimeout: 1000 + haStrategy: failover + loadbalance: random + filter: accessLog,metrics,testGlobalFilter3 + maxClientConnection: 10 + minClientConnection: 1 + retries: 0 + application: test + +motan-refer: + test-global-filter-service-1-refer: + path: motan.TestGlobalFilterService1 + serialization: simple + basicRefer: test-basic-refer + requestTimeout: 1000 + diff --git a/core/testdata/services/motan.TestGlobalFilterService2.yaml b/core/testdata/services/motan.TestGlobalFilterService2.yaml new file mode 100644 index 00000000..e6ef9dca --- /dev/null +++ b/core/testdata/services/motan.TestGlobalFilterService2.yaml @@ -0,0 +1,21 @@ +motan-basicRefer: + test-basic-refer: + group: motan-demo-rpc + protocol: motan2 + registry: consul + requestTimeout: 1000 + haStrategy: failover + loadbalance: random + maxClientConnection: 10 + minClientConnection: 1 + retries: 0 + application: test + +motan-refer: + test-global-filter-service-2-refer: + path: motan.TestGlobalFilterService2 + serialization: simple + basicRefer: test-basic-refer + requestTimeout: 1000 + filter: accessLog,metrics,testGlobalFilter3 + disableGlobalFilter: "testGlobalFilter3" diff --git a/core/testdata/services/motan.TestGlobalFilterService3.yaml b/core/testdata/services/motan.TestGlobalFilterService3.yaml new file mode 100644 index 00000000..9cbc3a95 --- /dev/null +++ b/core/testdata/services/motan.TestGlobalFilterService3.yaml @@ -0,0 +1,21 @@ +motan-basicRefer: + test-basic-refer: + group: motan-demo-rpc + protocol: motan2 + registry: consul + requestTimeout: 1000 + haStrategy: failover + loadbalance: random + maxClientConnection: 10 + minClientConnection: 1 + retries: 0 + application: test + +motan-refer: + test-global-filter-service-3-refer: + path: motan.TestGlobalFilterService3 + serialization: simple + basicRefer: test-basic-refer + requestTimeout: 1000 + filter: accessLog,metrics + disableGlobalFilter: "testGlobalFilter2,testGlobalFilter3" diff --git a/core/util.go b/core/util.go index 6ba37657..88bcb9a9 100644 --- a/core/util.go +++ b/core/util.go @@ -158,6 +158,18 @@ func TrimSplit(s string, sep string) []string { return a[:i+1] } +// TrimSplitSet slices string and convert to map set +func TrimSplitSet(s string, sep string) map[string]bool { + slice := TrimSplit(s, sep) + set := make(map[string]bool, len(slice)) + + for _, item := range slice { + set[item] = true + } + + return set +} + // ListenUnixSock try to listen a unix socket address // this method using by create motan agent server, management server and http proxy server func ListenUnixSock(unixSockAddr string) (net.Listener, error) { diff --git a/log/rotate.go b/log/rotate.go index 50b0b6e9..18d7ca7b 100644 --- a/log/rotate.go +++ b/log/rotate.go @@ -34,7 +34,6 @@ import ( "sort" "strings" "sync" - "syscall" "time" ) @@ -214,7 +213,7 @@ func (l *RotateWriter) rotate() error { // openNew opens a new log file for writing, moving any old log file out of the // way. This methods assumes the file has already been closed. func (l *RotateWriter) openNew() error { - err := os.MkdirAll(l.dir(), 0744) + err := os.MkdirAll(l.dir(), 0755) if err != nil { return fmt.Errorf("can't make directories for new logfile: %s", err) } @@ -588,13 +587,3 @@ func (b byFormatTime) Len() int { // osChown is a var so we can mock it out during tests. var osChown = os.Chown - -func chown(name string, info os.FileInfo) error { - f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) - if err != nil { - return err - } - _ = f.Close() - stat := info.Sys().(*syscall.Stat_t) - return osChown(name, int(stat.Uid), int(stat.Gid)) -} diff --git a/log/rotate_notwin.go b/log/rotate_notwin.go new file mode 100644 index 00000000..a50c96c5 --- /dev/null +++ b/log/rotate_notwin.go @@ -0,0 +1,22 @@ +//go:build !windows +// +build !windows + +package vlog + +import ( + "os" + "syscall" +) + +// create new file with file info's mode,user,group +func chown(name string, info os.FileInfo) error { + f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) + if err != nil { + return err + } + _ = f.Close() + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + return osChown(name, int(stat.Uid), int(stat.Gid)) + } + return nil +} diff --git a/log/rotate_win.go b/log/rotate_win.go new file mode 100644 index 00000000..7fe87639 --- /dev/null +++ b/log/rotate_win.go @@ -0,0 +1,15 @@ +//go:build windows +// +build windows + +package vlog + +import "os" + +// create new file with file info's mode only, windows is not support user,group +func chown(name string, info os.FileInfo) error { + f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) + if err == nil { + _ = f.Close() + } + return err +} From ac0afecf709a070c0c4512eb352897b8f13b1e79 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Thu, 21 Apr 2022 11:15:50 +0800 Subject: [PATCH 11/75] modify time.After function in manageHandler.go, which might cause memory leek. (#267) --- filter/circuitBreaker_test.go | 2 +- manageHandler.go | 4 +++- manageHandler_test.go | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 manageHandler_test.go diff --git a/filter/circuitBreaker_test.go b/filter/circuitBreaker_test.go index 37ed4cc1..687d2a3f 100644 --- a/filter/circuitBreaker_test.go +++ b/filter/circuitBreaker_test.go @@ -81,7 +81,7 @@ func TestCircuitBreakerFilter(t *testing.T) { } time.Sleep(10000 * time.Millisecond) //wait until async call complete countLock.RLock() - if count != 61 && count != 62 { + if count < 61 { t.Error("Test sleepWindow failed! count:", count) } countLock.RUnlock() diff --git a/manageHandler.go b/manageHandler.go index 3d3a9ca3..604e95ff 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -696,8 +696,10 @@ func sleep(w http.ResponseWriter, d time.Duration) { if cn, ok := w.(http.CloseNotifier); ok { clientGone = cn.CloseNotify() } + timer := time.NewTimer(d) + defer timer.Stop() select { - case <-time.After(d): + case <-timer.C: case <-clientGone: } } diff --git a/manageHandler_test.go b/manageHandler_test.go new file mode 100644 index 00000000..583f62b2 --- /dev/null +++ b/manageHandler_test.go @@ -0,0 +1,15 @@ +package motan + +import ( + assert2 "github.com/stretchr/testify/assert" + "net/http/httptest" + "testing" + "time" +) + +func TestSleep(t *testing.T) { + start := time.Now() + sleep(&httptest.ResponseRecorder{}, time.Second*3) + end := time.Now() + assert2.True(t, end.Sub(start).Seconds() >= 3) +} From 8bef2b49cbf77d5c7ac4563368c507467bd78947 Mon Sep 17 00:00:00 2001 From: cocowh Date: Tue, 10 May 2022 15:08:14 +0800 Subject: [PATCH 12/75] log filter (#269) add vlog filter --- agent.go | 12 +++- agent_test.go | 29 ++++++++++ client.go | 7 ++- client_test.go | 36 ++++++++++++ go.mod | 1 + log/filter.go | 139 +++++++++++++++++++++++++++++++++++++++++++++ log/filter_test.go | 97 +++++++++++++++++++++++++++++++ log/log.go | 25 +++++--- server.go | 7 ++- server_test.go | 88 +++++++++++++++++++++++++++- 10 files changed, 429 insertions(+), 12 deletions(-) create mode 100644 log/filter.go create mode 100644 log/filter_test.go diff --git a/agent.go b/agent.go index b62f72f7..c2c5285a 100644 --- a/agent.go +++ b/agent.go @@ -295,7 +295,12 @@ func (a *Agent) initParam() { if section != nil && section["log_level"] != nil { logLevel = section["log_level"].(string) } - initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel) + logFilterCaller := "" + if section != nil && section["log_filter_caller"] != nil { + logFilterCaller = strconv.FormatBool(section["log_filter_caller"].(bool)) + } + + initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel, logFilterCaller) registerSwitchers(a.Context) port := *motan.Port @@ -861,7 +866,7 @@ func getClusterKey(group, version, protocol, path string) string { return group + "_" + version + "_" + protocol + "_" + path } -func initLog(logDir, logAsync, logStructured, rotatePerHour string, logLevel string) { +func initLog(logDir, logAsync, logStructured, rotatePerHour string, logLevel string, logFilterCaller string) { // TODO: remove after a better handle if logDir == "stdout" { return @@ -880,6 +885,9 @@ func initLog(logDir, logAsync, logStructured, rotatePerHour string, logLevel str if logLevel != "" { _ = flag.Set("log_level", logLevel) } + if logFilterCaller != "" { + _ = flag.Set("log_filter_caller", logFilterCaller) + } vlog.LogInit(nil) } diff --git a/agent_test.go b/agent_test.go index f0cc4546..830866db 100644 --- a/agent_test.go +++ b/agent_test.go @@ -2,6 +2,7 @@ package motan import ( "bytes" + "github.com/weibocom/motan-go/config" "io/ioutil" "math/rand" "net/http" @@ -66,6 +67,34 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +func Test_initParam(t *testing.T) { + assert := assert.New(t) + conf, err := config.NewConfigFromFile(filepath.Join("testdata", "agent.yaml")) + assert.Nil(err) + a := NewAgent(nil) + a.Context = &core.Context{Config: conf} + logFilterCallerFalseConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + log_filter_caller: false +`))) + conf.Merge(logFilterCallerFalseConfig) + section, err := conf.GetSection("motan-agent") + assert.Nil(err) + assert.Equal(false, section["log_filter_caller"].(bool)) + a.initParam() + + logFilterCallerTrueConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + log_filter_caller: true +`))) + assert.Nil(err) + conf.Merge(logFilterCallerTrueConfig) + section, err = conf.GetSection("motan-agent") + assert.Nil(err) + assert.Equal(true, section["log_filter_caller"].(bool)) + a.initParam() +} + func TestHTTPProxyBodySize(t *testing.T) { body := bytes.NewReader(make([]byte, 1000)) resp, _ := proxyClient.Post("http://test.domain/tst/test", "application/octet-stream", body) diff --git a/client.go b/client.go index 122832aa..cc880eec 100644 --- a/client.go +++ b/client.go @@ -130,7 +130,12 @@ func NewClientContextFromConfig(conf *config.Config) (mc *MCContext) { if section != nil && section["log_level"] != nil { logLevel = section["log_level"].(string) } - initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel) + logFilterCaller := "" + if section != nil && section["log_filter_caller"] != nil { + logFilterCaller = strconv.FormatBool(section["log_filter_caller"].(bool)) + } + + initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel, logFilterCaller) registerSwitchers(mc.context) return mc } diff --git a/client_test.go b/client_test.go index 8cd5ad39..ddc0d60e 100644 --- a/client_test.go +++ b/client_test.go @@ -41,6 +41,42 @@ motan-refer: err = mclient.BaseCall(req, &reply) // sync call assert.Nil(err) assert.Equal("Hello Ray", reply) + + confWithCallertrue, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-client: + log_filter_caller: true +`))) + assert.Nil(err) + conf.Merge(confWithCallertrue) + section, err := conf.GetSection("motan-client") + assert.Nil(err) + assert.Equal(true, section["log_filter_caller"].(bool)) + mccontext = NewClientContextFromConfig(conf) + mccontext.Start(ext) + time.Sleep(time.Second) + mclient = mccontext.GetClient("mytest") + req = mclient.BuildRequest("hello", []interface{}{"Ray"}) + err = mclient.BaseCall(req, &reply) // sync call + assert.Nil(err) + assert.Equal("Hello Ray", reply) + + confWithCallerFalse, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-client: + log_filter_caller: false +`))) + assert.Nil(err) + conf.Merge(confWithCallerFalse) + section, err = conf.GetSection("motan-client") + assert.Nil(err) + assert.Equal(false, section["log_filter_caller"].(bool)) + mccontext = NewClientContextFromConfig(conf) + mccontext.Start(ext) + time.Sleep(time.Second) + mclient = mccontext.GetClient("mytest") + req = mclient.BuildRequest("hello", []interface{}{"Ray"}) + err = mclient.BaseCall(req, &reply) // sync call + assert.Nil(err) + assert.Equal("Hello Ray", reply) } func StartServer() { diff --git a/go.mod b/go.mod index a0d8b232..e71ca457 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 golang.org/x/net v0.0.0-20181005035420-146acd28ed58 + golang.org/x/time v0.0.0-00010101000000-000000000000 // indirect google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect google.golang.org/grpc v1.15.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect diff --git a/log/filter.go b/log/filter.go new file mode 100644 index 00000000..20f0361f --- /dev/null +++ b/log/filter.go @@ -0,0 +1,139 @@ +package vlog + +import ( + "encoding/json" + "fmt" + "golang.org/x/time/rate" + "runtime" + "sync" + "time" +) + +type Entrypoint int + +const ( + EntrypointInfoln Entrypoint = iota + EntrypointInfof + EntrypointWarningln + EntrypointWarningf + EntrypointErrorln + EntrypointErrorf + EntrypointFatalln + EntrypointFatalf + EntrypointAccessLog + EntrypointMetricsLog +) + +var ( + asyncFilterItemChan = make(chan *FilterItem, 5000) + asyncFilterLock = sync.RWMutex{} + asyncFilters = make(map[string]AsyncFilter) + // limit log frequency when channel full + asyncLimit = rate.NewLimiter(rate.Every(time.Second*5), 1) +) + +type Caller struct { + Ok bool + File string + Line int +} + +type FilterItem struct { + Level LogLevel + Content string + Entrypoint Entrypoint + Caller *Caller +} + +type AsyncFilter interface { + GetChannel() chan *FilterItem +} + +func init() { + startAsyncFilterConsumer() +} + +func AddAsyncFilter(filterName string, filter AsyncFilter) { + asyncFilterLock.Lock() + defer asyncFilterLock.Unlock() + asyncFilters[filterName] = filter +} + +func startAsyncFilterConsumer() { + go func() { + for { + select { + case item := <-asyncFilterItemChan: + asyncFilterLock.RLock() + for filterName, filter := range asyncFilters { + callAsyncFilter(filterName, filter, item) + } + asyncFilterLock.RUnlock() + } + } + }() +} + +func callAsyncFilter(filterName string, filter AsyncFilter, item *FilterItem) { + defer func() { + //limit high frequency log + if err := recover(); err != nil && asyncLimit.Allow() { + Errorf("asyncFilter %s consumer error, err:%v", filterName, err) + } + }() + select { + case filter.GetChannel() <- item: + default: + //limit high frequency log + if asyncLimit.Allow() { + Errorf("asyncFilter %s channel is full, drop log", filterName) + } + } +} + +func doAsyncFilters(level LogLevel, entrypoint Entrypoint, format string, fields ...interface{}) { + if len(asyncFilters) == 0 { + return + } + item := &FilterItem{ + Level: level, + Content: getLogContent(entrypoint, format, fields...), + Entrypoint: entrypoint, + Caller: getCaller(), + } + + select { + case asyncFilterItemChan <- item: + default: + //limit high frequency log + if asyncLimit.Allow() { + Errorf("vlog asyncFilter chan is full, drop log") + } + } +} + +func getCaller() *Caller { + if !*filterCallerSwitch { + return nil + } + _, file, line, ok := runtime.Caller(3) + return &Caller{ + Ok: ok, + File: file, + Line: line, + } +} + +func getLogContent(entrypoint Entrypoint, format string, fields ...interface{}) string { + if entrypoint == EntrypointAccessLog { + accessLog, _ := json.Marshal(fields[0].(*AccessLogEntity)) + return string(accessLog) + } + msg := format + if msg == "" && len(fields) > 0 { + msg = fmt.Sprint(fields...) + } else if msg != "" && len(fields) > 0 { + msg = fmt.Sprintf(format, fields...) + } + return msg +} diff --git a/log/filter_test.go b/log/filter_test.go new file mode 100644 index 00000000..77e8efde --- /dev/null +++ b/log/filter_test.go @@ -0,0 +1,97 @@ +package vlog + +import ( + "encoding/json" + assert2 "github.com/stretchr/testify/assert" + "testing" +) + +var ( + asyncLogChanTest = make(chan *FilterItem, 100) +) + +func Test_AsyncFilter(t *testing.T) { + assert := assert2.New(t) + assert.Equal(0, len(asyncFilters)) + f1 := &testAsyncFilter{ + channel: asyncLogChanTest, + } + AddAsyncFilter("test1", f1) + assert.Equal(1, len(asyncFilters)) + + doAsyncFilters(InfoLevel, EntrypointInfof, "", "3123") + doAsyncFilters(InfoLevel, EntrypointInfof, "", "3123") + doAsyncFilters(InfoLevel, EntrypointInfof, "", "3123") + doAsyncFilters(InfoLevel, EntrypointInfof, "", "3123") + assert.True(true, len(asyncLogChanTest)+len(asyncFilterItemChan) >= 4) + + f2 := &testAsyncFilter{ + channel: asyncLogChanTest, + } + AddAsyncFilter("test2", f2) + assert.Equal(2, len(asyncFilters)) + doAsyncFilters(InfoLevel, EntrypointInfof, "", "3123") + assert.True(true, len(asyncLogChanTest)+len(asyncFilterItemChan) > 5) + + f3 := &testAsyncFilter{ + channel: asyncLogChanTest, + } + AddAsyncFilter("test3", f3) + assert.Equal(3, len(asyncFilters)) + doAsyncFilters(InfoLevel, EntrypointInfof, "", "3123") + assert.True(true, len(asyncLogChanTest)+len(asyncFilterItemChan) > 6) + + accessEntry := &AccessLogEntity{ + FilterName: "", + Role: "", + RequestID: 0, + Service: "", + Method: "", + Desc: "", + RemoteAddress: "", + ReqSize: 0, + ResSize: 0, + BizTime: 0, + TotalTime: 0, + Success: false, + ResponseCode: "", + Exception: "", + } + accessLogBytes, _ := json.Marshal(accessEntry) + getLogContentTestCases := []map[string]interface{}{ + { + "format": "", + "fields": []interface{}{}, + "entrypoint": EntrypointInfof, + "expect": "", + }, + { + "format": "s%s", + "fields": []interface{}{ + "1", + }, + "entrypoint": EntrypointInfof, + "expect": "s1", + }, + { + "format": "", + "fields": []interface{}{ + accessEntry, + }, + "entrypoint": EntrypointAccessLog, + "expect": string(accessLogBytes), + }, + } + for _, testCase := range getLogContentTestCases { + res := getLogContent(testCase["entrypoint"].(Entrypoint), testCase["format"].(string), testCase["fields"].([]interface{})...) + assert.Equal(testCase["expect"].(string), res) + } +} + +type testAsyncFilter struct { + channel chan *FilterItem +} + +func (f *testAsyncFilter) GetChannel() chan *FilterItem { + return f.channel +} diff --git a/log/log.go b/log/log.go index 90e3c1cd..5cc49bd0 100644 --- a/log/log.go +++ b/log/log.go @@ -16,13 +16,14 @@ import ( ) var ( - loggerInstance Logger - once sync.Once - logDir = flag.String("log_dir", ".", "If non-empty, write log files in this directory") - logAsync = flag.Bool("log_async", true, "If false, write log sync, default is true") - logLevel = flag.String("log_level", "info", "Init log level, default is info.") - logStructured = flag.Bool("log_structured", false, "If true, write accessLog structured, default is false") - rotatePerHour = flag.Bool("rotate_per_hour", true, "") + loggerInstance Logger + once sync.Once + logDir = flag.String("log_dir", ".", "If non-empty, write log files in this directory") + logAsync = flag.Bool("log_async", true, "If false, write log sync, default is true") + logLevel = flag.String("log_level", "info", "Init log level, default is info.") + logStructured = flag.Bool("log_structured", false, "If true, write accessLog structured, default is false") + rotatePerHour = flag.Bool("rotate_per_hour", true, "") + filterCallerSwitch = flag.Bool("log_filter_caller", false, "If true, set caller for filter item, default is false") ) const ( @@ -89,6 +90,7 @@ func LogInit(logger Logger) { } func Infoln(args ...interface{}) { + doAsyncFilters(InfoLevel, EntrypointInfoln, "", args...) if loggerInstance != nil { loggerInstance.Infoln(args...) } else { @@ -97,6 +99,7 @@ func Infoln(args ...interface{}) { } func Infof(format string, args ...interface{}) { + doAsyncFilters(InfoLevel, EntrypointInfof, format, args...) if loggerInstance != nil { loggerInstance.Infof(format, args...) } else { @@ -105,6 +108,7 @@ func Infof(format string, args ...interface{}) { } func Warningln(args ...interface{}) { + doAsyncFilters(WarnLevel, EntrypointWarningln, "", args...) if loggerInstance != nil { loggerInstance.Warningln(args...) } else { @@ -113,6 +117,7 @@ func Warningln(args ...interface{}) { } func Warningf(format string, args ...interface{}) { + doAsyncFilters(WarnLevel, EntrypointWarningf, format, args...) if loggerInstance != nil { loggerInstance.Warningf(format, args...) } else { @@ -121,6 +126,7 @@ func Warningf(format string, args ...interface{}) { } func Errorln(args ...interface{}) { + doAsyncFilters(ErrorLevel, EntrypointErrorln, "", args...) if loggerInstance != nil { loggerInstance.Errorln(args...) } else { @@ -129,6 +135,7 @@ func Errorln(args ...interface{}) { } func Errorf(format string, args ...interface{}) { + doAsyncFilters(ErrorLevel, EntrypointErrorf, format, args...) if loggerInstance != nil { loggerInstance.Errorf(format, args...) } else { @@ -137,6 +144,7 @@ func Errorf(format string, args ...interface{}) { } func Fatalln(args ...interface{}) { + doAsyncFilters(FatalLevel, EntrypointFatalln, "", args...) if loggerInstance != nil { loggerInstance.Fatalln(args...) } else { @@ -145,6 +153,7 @@ func Fatalln(args ...interface{}) { } func Fatalf(format string, args ...interface{}) { + doAsyncFilters(FatalLevel, EntrypointFatalf, format, args...) if loggerInstance != nil { loggerInstance.Fatalf(format, args...) } else { @@ -153,12 +162,14 @@ func Fatalf(format string, args ...interface{}) { } func AccessLog(logType *AccessLogEntity) { + doAsyncFilters(InfoLevel, EntrypointAccessLog, "", logType) if loggerInstance != nil { loggerInstance.AccessLog(logType) } } func MetricsLog(msg string) { + doAsyncFilters(InfoLevel, EntrypointMetricsLog, "", msg) if loggerInstance != nil { loggerInstance.MetricsLog(msg) } diff --git a/server.go b/server.go index c99adc63..e93013bf 100644 --- a/server.go +++ b/server.go @@ -70,7 +70,12 @@ func NewMotanServerContextFromConfig(conf *config.Config) (ms *MSContext) { if section != nil && section["log_level"] != nil { logLevel = section["log_level"].(string) } - initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel) + logFilterCaller := "" + if section != nil && section["log_filter_caller"] != nil { + logFilterCaller = strconv.FormatBool(section["log_filter_caller"].(bool)) + } + + initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel, logFilterCaller) registerSwitchers(ms.context) return ms diff --git a/server_test.go b/server_test.go index 661555c2..2ac08611 100644 --- a/server_test.go +++ b/server_test.go @@ -10,6 +10,88 @@ import ( "time" ) +func TestServerContextConfig(t *testing.T) { + assert := assert2.New(t) + cfgTpl := ` +motan-server: + log_dir: "stdout" + application: "app-golang" # server identify. + +motan-registry: + direct: + protocol: direct + +#conf of services +motan-service: + mytest-motan2: + path: %s + group: bj + protocol: motan2 + registry: direct + serialization: simple + ref : "serviceID" + export: "motan2:%d" +` + path := "helloService" + port := 64332 + cfgText := fmt.Sprintf(cfgTpl, path, port) + conf, err := config.NewConfigFromReader(bytes.NewReader([]byte(cfgText))) + assert.Nil(err) + + logFilterCallerTrueConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-server: + log_filter_caller: true +`))) + assert.Nil(err) + conf.Merge(logFilterCallerTrueConfig) + section, err := conf.GetSection("motan-server") + assert.Nil(err) + assert.Equal(true, section["log_filter_caller"].(bool)) + + ext := startServerFromConfig(assert, conf, path, port) + clientExt := GetDefaultExtFactory() + u := motan.FromExtInfo("motan2://127.0.0.1:64332/helloService?serialization=simple") + assert.NotNil(u) + ep := clientExt.GetEndPoint(u) + assert.NotNil(ep) + ep.SetSerialization(motan.GetSerialization(u, ext)) + motan.Initialize(ep) + // wait ha + time.Sleep(time.Second * 1) + request := newRequest("helloService", "hello", "Ray") + request.Attachment = motan.NewStringMap(motan.DefaultAttachmentSize) + + resp := ep.Call(request) + assert.Nil(resp.GetException()) + assert.Equal("Hello Ray from motan server", resp.GetValue()) + + port = 64222 + logFilterCallerFalseConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-server: + log_filter_caller: false +`))) + assert.Nil(err) + cfgText = fmt.Sprintf(cfgTpl, path, port) + conf, err = config.NewConfigFromReader(bytes.NewReader([]byte(cfgText))) + assert.Nil(err) + conf.Merge(logFilterCallerFalseConfig) + section, err = conf.GetSection("motan-server") + assert.Nil(err) + assert.Equal(false, section["log_filter_caller"].(bool)) + + ext = startServerFromConfig(assert, conf, path, port) + u = motan.FromExtInfo("motan2://127.0.0.1:64222/helloService?serialization=simple") + assert.NotNil(u) + ep = clientExt.GetEndPoint(u) + assert.NotNil(ep) + ep.SetSerialization(motan.GetSerialization(u, ext)) + motan.Initialize(ep) + // wait ha + time.Sleep(time.Second * 1) + request = newRequest("helloService", "hello", "Ray") + request.Attachment = motan.NewStringMap(motan.DefaultAttachmentSize) +} + func TestNewMotanServerContextFromConfig(t *testing.T) { assert := assert2.New(t) @@ -56,9 +138,13 @@ motan-service: assert := assert2.New(t) conf, err := config.NewConfigFromReader(bytes.NewReader([]byte(cfgText))) assert.Nil(err) + return startServerFromConfig(assert, conf, path, port) +} + +func startServerFromConfig(assert *assert2.Assertions, conf *config.Config, path string, port int) motan.ExtensionFactory { ext := GetDefaultExtFactory() mscontext := NewMotanServerContextFromConfig(conf) - err = mscontext.RegisterService(&HelloService{}, "serviceID") + err := mscontext.RegisterService(&HelloService{}, "serviceID") assert.Nil(err) mscontext.Start(ext) mscontext.ServicesAvailable() From fdbf27a4a3ba51246cf267e67a14328112af7b6d Mon Sep 17 00:00:00 2001 From: cocowh Date: Wed, 11 May 2022 15:27:59 +0800 Subject: [PATCH 13/75] fix mergeGlobalFilter (#270) --- core/globalContext.go | 5 +++-- core/globalContext_test.go | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/core/globalContext.go b/core/globalContext.go index 423275e7..efe14559 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -439,9 +439,10 @@ func (c *Context) mergeGlobalFilter(newURL *URL) { } } finalFilter = strings.TrimRight(finalFilter, ",") - newURL.PutParam(FilterKey, finalFilter) + if finalFilter != "" { + newURL.PutParam(FilterKey, finalFilter) + } } - } func (c *Context) parseRefers() { diff --git a/core/globalContext_test.go b/core/globalContext_test.go index 7434a5e3..80e15870 100644 --- a/core/globalContext_test.go +++ b/core/globalContext_test.go @@ -1,7 +1,9 @@ package core import ( + "bytes" "flag" + "github.com/weibocom/motan-go/config" "path/filepath" "strings" "testing" @@ -67,6 +69,7 @@ func TestNewContext(t *testing.T) { assert.NotNil(t, context.Config) globalFilterContext := NewContext("testdata", "globalFilter", "app-idc1") + assert.NotNil(t, globalFilterContext) filterArrs := make(map[string][]string) @@ -101,3 +104,44 @@ func TestNewContext(t *testing.T) { }) } } + +func Test_fixMergeGlobalFilter(t *testing.T) { + testCases := []map[string]interface{}{ + { + "config": ` +motan-server: + application: testFix + +motan-service: + mytest: + path: test +`, + "expect_value": "", + "expect_ok": false, + }, + { + "config": ` +motan-server: + application: testFix + +motan-service: + mytest: + path: test + filter: test +`, + "expect_value": "test", + "expect_ok": true, + }, + } + for _, testCase := range testCases { + conf, err := config.NewConfigFromReader(bytes.NewReader([]byte(testCase["config"].(string)))) + assert.Nil(t, err) + context := NewContextFromConfig(conf, "", "") + Initialize(context) + for _, url := range context.ServiceURLs { + v, ok := url.Parameters[FilterKey] + assert.Equal(t, testCase["expect_value"].(string), v) + assert.Equal(t, testCase["expect_ok"].(bool), ok) + } + } +} From 4a16caeab937471471e550f440d14a28876530a2 Mon Sep 17 00:00:00 2001 From: cocowh Date: Wed, 25 May 2022 11:25:20 +0800 Subject: [PATCH 14/75] reconnect graphite udp connection every write interval (#271) reconnect graphite udp connection every write interval --- metrics/graphite.go | 12 ++++++++---- metrics/graphite_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/metrics/graphite.go b/metrics/graphite.go index 58b944df..93f80cb5 100644 --- a/metrics/graphite.go +++ b/metrics/graphite.go @@ -59,12 +59,16 @@ func (g *graphite) Write(snapshots []Snapshot) error { g.lock.Lock() defer g.lock.Unlock() - if g.conn == nil { - if g.conn = getUDPConn(g.Host, g.Port); g.conn == nil { - return errors.New("open graphite conn failed") - } + if g.conn = getUDPConn(g.Host, g.Port); g.conn == nil { + return errors.New("open graphite conn failed") } + defer func() { + if g.conn != nil { + g.conn.Close() + } + }() + if g.localIP == "" { g.localIP = strings.Replace(strings.Split(g.conn.LocalAddr().String(), ":")[0], ".", "_", -1) } diff --git a/metrics/graphite_test.go b/metrics/graphite_test.go index 4401c924..fceb856d 100644 --- a/metrics/graphite_test.go +++ b/metrics/graphite_test.go @@ -35,6 +35,16 @@ func Test_graphite_Write(t *testing.T) { time.Sleep(100 * time.Millisecond) server.stop() assert.Equal(t, 589, server.data.Len(), "send data size") + + // illegal udp connection address + g = newGraphite("1.1.1.1", "test pool 2", 1234) + item.AddCounter(keyPrefix+"c1", 1) + item.AddHistograms(keyPrefix+" h1", 100) + server.data.Reset() + assert.Equal(t, 0, server.data.Len(), "illegal udp address data size") + err = g.Write([]Snapshot{item.SnapshotAndClear()}) + assert.Equal(t, nil, err, "illegal udp address") + assert.Equal(t, 0, server.data.Len(), "illegal udp address data size") } func TestGenGraphiteMessages(t *testing.T) { From d34512b33a1a6f44bd991282e5688bc68f408b87 Mon Sep 17 00:00:00 2001 From: snail007 Date: Tue, 19 Jul 2022 17:24:05 +0800 Subject: [PATCH 15/75] add motan-service group support of multiple comma split group name (#272) * add motan-service group support of multiple comma split group name --- agent.go | 3 ++ core/globalContext.go | 38 ++++++++++++++++----- core/globalContext_test.go | 32 +++++++++++++++++- core/util.go | 6 ++-- dynamicConfig.go | 67 +++++++++++++++++++++++--------------- dynamicConfig_test.go | 29 +++++++++++++++++ go.mod | 2 +- 7 files changed, 137 insertions(+), 40 deletions(-) create mode 100644 dynamicConfig_test.go diff --git a/agent.go b/agent.go index c2c5285a..4b543dbc 100644 --- a/agent.go +++ b/agent.go @@ -800,6 +800,9 @@ func (a *Agent) doExportService(url *motan.URL) { a.agentPortServer[url.Port] = server } else if canShareChannel(*url, *server.GetURL()) { server.GetMessageHandler().AddProvider(provider) + }else{ + vlog.Errorf("service can't find a share channel , url:%v", url) + return } err := exporter.Export(server, a.extFactory, globalContext) if err != nil { diff --git a/core/globalContext.go b/core/globalContext.go index efe14559..eab4fafb 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -35,14 +35,15 @@ const ( basicServiceKey = "basicService" // const for application pool - basicConfig = "basic.yaml" - servicePath = "services/" - applicationPath = "applications/" - poolPath = "pools/" - httpServicePath = "http/service/" - httpLocationPath = "http/location/" - httpPoolPath = "http/pools/" - poolNameSeparator = "-" + basicConfig = "basic.yaml" + servicePath = "services/" + applicationPath = "applications/" + poolPath = "pools/" + httpServicePath = "http/service/" + httpLocationPath = "http/location/" + httpPoolPath = "http/pools/" + poolNameSeparator = "-" + groupNameSeparator = "," ) // Context for agent, client, server. context is created depends on config file @@ -445,6 +446,23 @@ func (c *Context) mergeGlobalFilter(newURL *URL) { } } +// parseMultipleServiceGroup add motan-service group support of multiple comma split group name +func (c *Context) parseMultipleServiceGroup(motanServiceMap map[string]*URL) { + for k, serviceURL := range motanServiceMap { + if !strings.Contains(serviceURL.Group, groupNameSeparator) { + continue + } + groups := TrimSplit(serviceURL.Group, groupNameSeparator) + serviceURL.Group = groups[0] + for idx, g := range groups[1:] { + key := fmt.Sprintf("%v-%v", k, idx) + newService := serviceURL.Copy() + newService.Group = g + motanServiceMap[key] = newService + } + } +} + func (c *Context) parseRefers() { c.RefersURLs = c.basicConfToURLs(refersSection) } @@ -454,7 +472,9 @@ func (c *Context) parseBasicRefers() { } func (c *Context) parseServices() { - c.ServiceURLs = c.basicConfToURLs(servicesSection) + urlsMap := c.basicConfToURLs(servicesSection) + c.parseMultipleServiceGroup(urlsMap) + c.ServiceURLs = urlsMap } func (c *Context) parserBasicServices() { diff --git a/core/globalContext_test.go b/core/globalContext_test.go index 80e15870..13dabefa 100644 --- a/core/globalContext_test.go +++ b/core/globalContext_test.go @@ -3,11 +3,12 @@ package core import ( "bytes" "flag" - "github.com/weibocom/motan-go/config" "path/filepath" "strings" "testing" + "github.com/weibocom/motan-go/config" + "github.com/stretchr/testify/assert" ) @@ -145,3 +146,32 @@ motan-service: } } } + +func TestContext_parseMultipleServiceGroup(t *testing.T) { + data0 := map[string]*URL{ + "service1": { + Group: "", + }, + } + data1 := map[string]*URL{ + "service1": { + Group: "hello", + }, + } + data2 := map[string]*URL{ + "service1": { + Group: "hello,hello1,hello2", + }, + } + ctx := Context{} + ctx.parseMultipleServiceGroup(map[string]*URL{}) + ctx.parseMultipleServiceGroup(data0) + assert.Len(t, data0, 1) + ctx.parseMultipleServiceGroup(data1) + assert.Len(t, data1, 1) + ctx.parseMultipleServiceGroup(data2) + assert.Len(t, data2, 3) + assert.Equal(t, data2["service1"].Group, "hello") + assert.Equal(t, data2["service1-0"].Group, "hello1") + assert.Equal(t, data2["service1-1"].Group, "hello2") +} diff --git a/core/util.go b/core/util.go index 88bcb9a9..4d557168 100644 --- a/core/util.go +++ b/core/util.go @@ -138,12 +138,12 @@ func HandlePanic(f func()) { // returns a slice of the substrings between those separators, // specially trim all substrings. func TrimSplit(s string, sep string) []string { - n := strings.Count(s, sep) + 1 - a := make([]string, n) - i := 0 if sep == "" { return strings.Split(s, sep) } + n := strings.Count(s, sep) + 1 + a := make([]string, n) + i := 0 for { m := strings.Index(s, sep) if m < 0 { diff --git a/dynamicConfig.go b/dynamicConfig.go index 200082bb..8fc81372 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -201,18 +201,27 @@ func (h *DynamicConfigurerHandler) ServeHTTP(res http.ResponseWriter, req *http. res.WriteHeader(http.StatusNotFound) } } - -func (h *DynamicConfigurerHandler) getURL(req *http.Request) (*core.URL, error) { +func (h *DynamicConfigurerHandler) readURLs(req *http.Request) ([]*core.URL, error) { bytes, err := ioutil.ReadAll(req.Body) if err != nil { return nil, err } - url := new(core.URL) err = json.Unmarshal(bytes, url) if err != nil { return nil, err } + groups := strings.Split(url.Group, ",") + urls := []*core.URL{} + for _, group := range groups { + u := url.Copy() + u.Group = group + urls = append(urls, u) + } + return urls, nil +} + +func (h *DynamicConfigurerHandler) parseURL(url *core.URL) (*core.URL, error) { if url.Group == "" { url.Group = url.GetParam(core.GroupKey, "") delete(url.Parameters, core.GroupKey) @@ -278,51 +287,57 @@ func (h *DynamicConfigurerHandler) getURL(req *http.Request) (*core.URL, error) } func (h *DynamicConfigurerHandler) register(res http.ResponseWriter, req *http.Request) { - url, err := h.getURL(req) + urls, err := h.readURLs(req) if err != nil { writeHandlerResponse(res, http.StatusBadRequest, err.Error(), nil) return } - url.PutParam(core.ProxyKey, url.Protocol+":"+url.GetPortStr()) - url.PutParam(core.ExportKey, url.Protocol+":"+strconv.Itoa(h.agent.eport)) - h.agent.initProxyServiceURL(url) - err = h.agent.configurer.Register(url) - if err != nil { - writeHandlerResponse(res, http.StatusInternalServerError, err.Error(), nil) - return + for _, url := range urls { + url.PutParam(core.ProxyKey, url.Protocol+":"+url.GetPortStr()) + url.PutParam(core.ExportKey, url.Protocol+":"+strconv.Itoa(h.agent.eport)) + h.agent.initProxyServiceURL(url) + err = h.agent.configurer.Register(url) + if err != nil { + writeHandlerResponse(res, http.StatusInternalServerError, err.Error(), nil) + return + } } writeHandlerResponse(res, http.StatusOK, "ok", nil) } func (h *DynamicConfigurerHandler) unregister(res http.ResponseWriter, req *http.Request) { - url, err := h.getURL(req) + urls, err := h.readURLs(req) if err != nil { writeHandlerResponse(res, http.StatusBadRequest, err.Error(), nil) return } - url.PutParam(core.ProxyKey, url.Protocol+":"+url.GetPortStr()) - url.PutParam(core.ExportKey, url.Protocol+":"+strconv.Itoa(h.agent.eport)) - h.agent.initProxyServiceURL(url) - err = h.agent.configurer.Unregister(url) - if err != nil { - writeHandlerResponse(res, http.StatusInternalServerError, err.Error(), nil) - return + for _, url := range urls { + url.PutParam(core.ProxyKey, url.Protocol+":"+url.GetPortStr()) + url.PutParam(core.ExportKey, url.Protocol+":"+strconv.Itoa(h.agent.eport)) + h.agent.initProxyServiceURL(url) + err = h.agent.configurer.Unregister(url) + if err != nil { + writeHandlerResponse(res, http.StatusInternalServerError, err.Error(), nil) + return + } } writeHandlerResponse(res, http.StatusOK, "ok", nil) } func (h *DynamicConfigurerHandler) subscribe(res http.ResponseWriter, req *http.Request) { - url, err := h.getURL(req) + urls, err := h.readURLs(req) if err != nil { writeHandlerResponse(res, http.StatusBadRequest, err.Error(), nil) return } - url.Host = "" - url.Port = 0 - err = h.agent.configurer.Subscribe(url) - if err != nil { - writeHandlerResponse(res, http.StatusInternalServerError, err.Error(), nil) - return + for _, url := range urls { + url.Host = "" + url.Port = 0 + err = h.agent.configurer.Subscribe(url) + if err != nil { + writeHandlerResponse(res, http.StatusInternalServerError, err.Error(), nil) + return + } } writeHandlerResponse(res, http.StatusOK, "ok", nil) } diff --git a/dynamicConfig_test.go b/dynamicConfig_test.go new file mode 100644 index 00000000..26a43232 --- /dev/null +++ b/dynamicConfig_test.go @@ -0,0 +1,29 @@ +package motan + +import ( + "bytes" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDynamicConfigurerHandler_readURLs(t *testing.T) { + body1 := `{"protocol":"motan2","host":"10.10.64.11","port":1880,"path":"com.company.HelloService","group":"hello","parameters":{"conf-id":"com.company.HelloService","export":"motan2:1880","nodeType":"service","proxyRegistry":"direct://127.0.0.1:1880","ref":"com.company.HelloService","registry":"mesh-registry","requestTimeout":"600000","serialization":"breeze"}}` + body2 := `{"protocol":"motan2","host":"10.10.64.11","port":1880,"path":"com.company.HelloService","group":"hello,hello1,hello2","parameters":{"conf-id":"com.company.HelloService","export":"motan2:1880","nodeType":"service","proxyRegistry":"direct://127.0.0.1:1880","ref":"com.company.HelloService","registry":"mesh-registry","requestTimeout":"600000","serialization":"breeze"}}` + d := &DynamicConfigurerHandler{} + req1 := httptest.NewRequest("POST", "/register", bytes.NewBufferString(body1)) + req2 := httptest.NewRequest("POST", "/register", bytes.NewBufferString(body2)) + req3 := httptest.NewRequest("POST", "/register", bytes.NewBufferString("}")) + urls, err := d.readURLs(req1) + assert.Equal(t, len(urls), 1) + assert.Nil(t, err) + urls, err = d.readURLs(req2) + assert.Equal(t, len(urls), 3) + assert.Nil(t, err) + assert.Equal(t, urls[0].Group, "hello") + assert.Equal(t, urls[1].Group, "hello1") + assert.Equal(t, urls[2].Group, "hello2") + _, err = d.readURLs(req3) + assert.NotNil(t, err) +} diff --git a/go.mod b/go.mod index e71ca457..a93b245d 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 golang.org/x/net v0.0.0-20181005035420-146acd28ed58 - golang.org/x/time v0.0.0-00010101000000-000000000000 // indirect + golang.org/x/time v0.0.0-00010101000000-000000000000 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect google.golang.org/grpc v1.15.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect From 63012a07e2b547774cc6971c884229dfbde2545a Mon Sep 17 00:00:00 2001 From: snail007 Date: Thu, 21 Jul 2022 09:16:59 +0800 Subject: [PATCH 16/75] fix readURLs add h.parseURL(u) --- dynamicConfig.go | 1 + 1 file changed, 1 insertion(+) diff --git a/dynamicConfig.go b/dynamicConfig.go index 8fc81372..363778cf 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -216,6 +216,7 @@ func (h *DynamicConfigurerHandler) readURLs(req *http.Request) ([]*core.URL, err for _, group := range groups { u := url.Copy() u.Group = group + h.parseURL(u) urls = append(urls, u) } return urls, nil From 71adc808f898569071aa684cbd84656a7c7a7954 Mon Sep 17 00:00:00 2001 From: snail007 Date: Thu, 21 Jul 2022 09:30:32 +0800 Subject: [PATCH 17/75] fix readURLs add h.parseURLs() --- dynamicConfig.go | 25 +++++++++++++++++++++++-- dynamicConfig_test.go | 6 +++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dynamicConfig.go b/dynamicConfig.go index 363778cf..f6d2cccc 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -201,7 +201,7 @@ func (h *DynamicConfigurerHandler) ServeHTTP(res http.ResponseWriter, req *http. res.WriteHeader(http.StatusNotFound) } } -func (h *DynamicConfigurerHandler) readURLs(req *http.Request) ([]*core.URL, error) { +func (h *DynamicConfigurerHandler) readURLsFromRequest(req *http.Request) ([]*core.URL, error) { bytes, err := ioutil.ReadAll(req.Body) if err != nil { return nil, err @@ -216,12 +216,33 @@ func (h *DynamicConfigurerHandler) readURLs(req *http.Request) ([]*core.URL, err for _, group := range groups { u := url.Copy() u.Group = group - h.parseURL(u) urls = append(urls, u) } return urls, nil } +func (h *DynamicConfigurerHandler) readURLs(req *http.Request) ([]*core.URL, error) { + urls, err := h.readURLsFromRequest(req) + if err != nil { + return nil, err + } + err = h.parseURLs(urls) + if err != nil { + return nil, err + } + return urls, nil +} + +func (h *DynamicConfigurerHandler) parseURLs(urls []*core.URL) error { + for _, u := range urls { + _, e := h.parseURL(u) + if e != nil { + return e + } + } + return nil +} + func (h *DynamicConfigurerHandler) parseURL(url *core.URL) (*core.URL, error) { if url.Group == "" { url.Group = url.GetParam(core.GroupKey, "") diff --git a/dynamicConfig_test.go b/dynamicConfig_test.go index 26a43232..6ce1af9e 100644 --- a/dynamicConfig_test.go +++ b/dynamicConfig_test.go @@ -15,15 +15,15 @@ func TestDynamicConfigurerHandler_readURLs(t *testing.T) { req1 := httptest.NewRequest("POST", "/register", bytes.NewBufferString(body1)) req2 := httptest.NewRequest("POST", "/register", bytes.NewBufferString(body2)) req3 := httptest.NewRequest("POST", "/register", bytes.NewBufferString("}")) - urls, err := d.readURLs(req1) + urls, err := d.readURLsFromRequest(req1) assert.Equal(t, len(urls), 1) assert.Nil(t, err) - urls, err = d.readURLs(req2) + urls, err = d.readURLsFromRequest(req2) assert.Equal(t, len(urls), 3) assert.Nil(t, err) assert.Equal(t, urls[0].Group, "hello") assert.Equal(t, urls[1].Group, "hello1") assert.Equal(t, urls[2].Group, "hello2") - _, err = d.readURLs(req3) + _, err = d.readURLsFromRequest(req3) assert.NotNil(t, err) } From 70fb5a540836ba7ec7ebbef510eee5ed62e913cf Mon Sep 17 00:00:00 2001 From: snail007 Date: Wed, 27 Jul 2022 14:43:55 +0800 Subject: [PATCH 18/75] support add service group from environment variable (#274) --- core/globalContext.go | 37 +++++++++++++++++++++++++------------ core/globalContext_test.go | 26 ++++++++++++++++++++++++++ core/util.go | 13 +++++++++++++ core/util_test.go | 9 +++++++++ dynamicConfig.go | 11 ++++++++++- 5 files changed, 83 insertions(+), 13 deletions(-) diff --git a/core/globalContext.go b/core/globalContext.go index eab4fafb..25af1e1b 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -35,15 +35,16 @@ const ( basicServiceKey = "basicService" // const for application pool - basicConfig = "basic.yaml" - servicePath = "services/" - applicationPath = "applications/" - poolPath = "pools/" - httpServicePath = "http/service/" - httpLocationPath = "http/location/" - httpPoolPath = "http/pools/" - poolNameSeparator = "-" - groupNameSeparator = "," + basicConfig = "basic.yaml" + servicePath = "services/" + applicationPath = "applications/" + poolPath = "pools/" + httpServicePath = "http/service/" + httpLocationPath = "http/location/" + httpPoolPath = "http/pools/" + poolNameSeparator = "-" + GroupNameSeparator = "," + GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" ) // Context for agent, client, server. context is created depends on config file @@ -448,19 +449,31 @@ func (c *Context) mergeGlobalFilter(newURL *URL) { // parseMultipleServiceGroup add motan-service group support of multiple comma split group name func (c *Context) parseMultipleServiceGroup(motanServiceMap map[string]*URL) { + addMotanServiceMap := map[string]*URL{} for k, serviceURL := range motanServiceMap { - if !strings.Contains(serviceURL.Group, groupNameSeparator) { + //add additional service group from environment + if v := os.Getenv(GroupEnvironmentName); v != "" { + if serviceURL.Group == "" { + serviceURL.Group = v + } else { + serviceURL.Group += "," + v + } + } + if !strings.Contains(serviceURL.Group, GroupNameSeparator) { continue } - groups := TrimSplit(serviceURL.Group, groupNameSeparator) + groups := SlicesUnique(TrimSplit(serviceURL.Group, GroupNameSeparator)) serviceURL.Group = groups[0] for idx, g := range groups[1:] { key := fmt.Sprintf("%v-%v", k, idx) newService := serviceURL.Copy() newService.Group = g - motanServiceMap[key] = newService + addMotanServiceMap[key] = newService } } + for k, v := range addMotanServiceMap { + motanServiceMap[k] = v + } } func (c *Context) parseRefers() { diff --git a/core/globalContext_test.go b/core/globalContext_test.go index 13dabefa..86e86507 100644 --- a/core/globalContext_test.go +++ b/core/globalContext_test.go @@ -3,6 +3,7 @@ package core import ( "bytes" "flag" + "os" "path/filepath" "strings" "testing" @@ -163,15 +164,40 @@ func TestContext_parseMultipleServiceGroup(t *testing.T) { Group: "hello,hello1,hello2", }, } + data3 := map[string]*URL{ + "service1": { + Group: "hello,hello1", + }, + } + data4 := map[string]*URL{ + "service1": { + Group: "", + }, + } ctx := Context{} ctx.parseMultipleServiceGroup(map[string]*URL{}) + ctx.parseMultipleServiceGroup(data0) assert.Len(t, data0, 1) + ctx.parseMultipleServiceGroup(data1) assert.Len(t, data1, 1) + ctx.parseMultipleServiceGroup(data2) assert.Len(t, data2, 3) assert.Equal(t, data2["service1"].Group, "hello") assert.Equal(t, data2["service1-0"].Group, "hello1") assert.Equal(t, data2["service1-1"].Group, "hello2") + + os.Setenv(GroupEnvironmentName,"hello2") + ctx.parseMultipleServiceGroup(data3) + assert.Len(t, data3, 3) + assert.Equal(t, data3["service1"].Group, "hello") + assert.Equal(t, data3["service1-0"].Group, "hello1") + assert.Equal(t, data3["service1-1"].Group, "hello2") + + os.Setenv(GroupEnvironmentName,"hello") + ctx.parseMultipleServiceGroup(data4) + assert.Len(t, data4, 1) + assert.Equal(t, data3["service1"].Group, "hello") } diff --git a/core/util.go b/core/util.go index 4d557168..656a640d 100644 --- a/core/util.go +++ b/core/util.go @@ -185,3 +185,16 @@ func ListenUnixSock(unixSockAddr string) (net.Listener, error) { } return listener, nil } + +// c convert slices to set slices +func SlicesUnique(src []string) []string { + var dst []string + set := map[string]bool{} + for _, v := range src { + if !set[v] { + set[v] = true + dst = append(dst, v) + } + } + return dst +} diff --git a/core/util_test.go b/core/util_test.go index 406fce07..21600cfa 100644 --- a/core/util_test.go +++ b/core/util_test.go @@ -101,3 +101,12 @@ func TestSplitTrim(t *testing.T) { assert.Equal(t, tt.expect, ret) } } + +func TestSlicesUnique(t *testing.T) { + a := []string{"a", "a", "b"} + b := []string{"a", "b"} + var c []string + assert.Equal(t, SlicesUnique(a), b) + assert.Equal(t, SlicesUnique(b), b) + assert.Equal(t, SlicesUnique(c), c) +} diff --git a/dynamicConfig.go b/dynamicConfig.go index f6d2cccc..236c9393 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" URL "net/url" + "os" "path/filepath" "strconv" "strings" @@ -211,7 +212,15 @@ func (h *DynamicConfigurerHandler) readURLsFromRequest(req *http.Request) ([]*co if err != nil { return nil, err } - groups := strings.Split(url.Group, ",") + //add additional service group from environment + if v := os.Getenv(core.GroupEnvironmentName); v != "" { + if url.Group == "" { + url.Group = v + } else { + url.Group += "," + v + } + } + groups := core.SlicesUnique(core.TrimSplit(url.Group, core.GroupNameSeparator)) urls := []*core.URL{} for _, group := range groups { u := url.Copy() From 83c38995ead76095fee5dede1e76050042df1ede Mon Sep 17 00:00:00 2001 From: arraykeys Date: Thu, 28 Jul 2022 08:47:12 +0800 Subject: [PATCH 19/75] move GroupNameSeparator GroupEnvironmentName to constants.go --- core/constants.go | 2 ++ core/globalContext.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/constants.go b/core/constants.go index 320075e8..cbae1da8 100644 --- a/core/constants.go +++ b/core/constants.go @@ -83,6 +83,8 @@ const ( const ( DefaultWriteTimeout = 5 * time.Second + GroupNameSeparator = "," + GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" ) // meta keys diff --git a/core/globalContext.go b/core/globalContext.go index 25af1e1b..5a7d186f 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -43,8 +43,6 @@ const ( httpLocationPath = "http/location/" httpPoolPath = "http/pools/" poolNameSeparator = "-" - GroupNameSeparator = "," - GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" ) // Context for agent, client, server. context is created depends on config file From 609202d53fa8e80dff38ea7039327234a0975d61 Mon Sep 17 00:00:00 2001 From: snail007 Date: Thu, 28 Jul 2022 16:28:12 +0800 Subject: [PATCH 20/75] support add defaultFilter and disableDefaultFilter (#277) * support add defaultFilter and disableDefaultFilter --- core/constants.go | 2 + core/globalContext.go | 115 +++++++++++++++++++++++-------------- core/globalContext_test.go | 62 +++++++++++++++++++- 3 files changed, 134 insertions(+), 45 deletions(-) diff --git a/core/constants.go b/core/constants.go index cbae1da8..8a462523 100644 --- a/core/constants.go +++ b/core/constants.go @@ -35,6 +35,8 @@ const ( FilterKey = "filter" GlobalFilter = "globalFilter" DisableGlobalFilter = "disableGlobalFilter" + DefaultFilter = "defaultFilter" + DisableDefaultFilter = "disableDefaultFilter" RegistryKey = "registry" WeightKey = "weight" SerializationKey = "serialization" diff --git a/core/globalContext.go b/core/globalContext.go index 5a7d186f..eef6ebb5 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -35,14 +35,14 @@ const ( basicServiceKey = "basicService" // const for application pool - basicConfig = "basic.yaml" - servicePath = "services/" - applicationPath = "applications/" - poolPath = "pools/" - httpServicePath = "http/service/" - httpLocationPath = "http/location/" - httpPoolPath = "http/pools/" - poolNameSeparator = "-" + basicConfig = "basic.yaml" + servicePath = "services/" + applicationPath = "applications/" + poolPath = "pools/" + httpServicePath = "http/service/" + httpLocationPath = "http/location/" + httpPoolPath = "http/pools/" + poolNameSeparator = "-" ) // Context for agent, client, server. context is created depends on config file @@ -397,52 +397,81 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { newURL = url } - // merge agent globalFilter, filter - c.mergeGlobalFilter(newURL) + //final filters: defaultFilter + globalFilter + filters + finalFilters := c.mergeFilterSet( + c.getDefaultFilterSet(newURL), + c.getGlobalFilterSet(newURL), + c.getFilterSet(newURL.GetStringParamsWithDefault(FilterKey, ""), ""), + ) + if len(finalFilters) > 0 { + newURL.PutParam(FilterKey, c.filterSetToStr(finalFilters)) + } newURLs[key] = newURL } return newURLs } -func (c *Context) mergeGlobalFilter(newURL *URL) { - if c.AgentURL != nil { - finalFilterSet := make(map[string]bool) - globalFilterStr := c.AgentURL.GetStringParamsWithDefault(GlobalFilter, "") - // disable globalFilter - if globalFilterStr != "" { - finalFilterSet = TrimSplitSet(globalFilterStr, ",") - disableGlobalFilterStr := newURL.GetStringParamsWithDefault(DisableGlobalFilter, "") - if disableGlobalFilterStr != "" { - vlog.Infoln("disable global filter: " + disableGlobalFilterStr) - disableGlobalFilterArr := TrimSplit(disableGlobalFilterStr, ",") - for _, disableFilter := range disableGlobalFilterArr { - if _, ok := finalFilterSet[disableFilter]; ok { - delete(finalFilterSet, disableFilter) - } - } - } +func (c *Context) filterSetToStr(f map[string]bool) string { + var dst []string + for k := range f { + dst = append(dst, k) + } + return strings.Join(dst, ",") +} + +func (c *Context) getFilterSet(filterStr, disableFilterStr string) (dst map[string]bool) { + if filterStr == "" { + return + } + dst = map[string]bool{} + + for _, k := range strings.Split(filterStr, ",") { + k = strings.TrimSpace(k) + if k == "" { + continue } - // append filter - filterStr := newURL.GetStringParamsWithDefault(FilterKey, "") - if filterStr != "" { - filterArr := TrimSplit(filterStr, ",") - for _, filter := range filterArr { - finalFilterSet[filter] = true - } + dst[k] = true + } + + for _, k := range strings.Split(disableFilterStr, ",") { + k = strings.TrimSpace(k) + if k == "" { + continue } - // make new filter string - var finalFilter string - for filter := range finalFilterSet { - if filter != "" { - finalFilter += filter + "," + delete(dst, k) + } + return +} + +func (c *Context) mergeFilterSet(sets ...map[string]bool) (dst map[string]bool) { + dst = map[string]bool{} + for _, set := range sets { + for k := range set { + k = strings.TrimSpace(k) + if k == "" { + continue } + dst[k] = true } - finalFilter = strings.TrimRight(finalFilter, ",") - if finalFilter != "" { - newURL.PutParam(FilterKey, finalFilter) - } } + return +} + +func (c *Context) getDefaultFilterSet(newURL *URL) map[string]bool { + if c.AgentURL == nil { + return nil + } + return c.getFilterSet(c.AgentURL.GetStringParamsWithDefault(DefaultFilter, ""), + newURL.GetStringParamsWithDefault(DisableDefaultFilter, "")) +} + +func (c *Context) getGlobalFilterSet(newURL *URL) map[string]bool { + if c.AgentURL == nil { + return nil + } + return c.getFilterSet(c.AgentURL.GetStringParamsWithDefault(GlobalFilter, ""), + newURL.GetStringParamsWithDefault(DisableGlobalFilter, "")) } // parseMultipleServiceGroup add motan-service group support of multiple comma split group name diff --git a/core/globalContext_test.go b/core/globalContext_test.go index 86e86507..98680474 100644 --- a/core/globalContext_test.go +++ b/core/globalContext_test.go @@ -189,15 +189,73 @@ func TestContext_parseMultipleServiceGroup(t *testing.T) { assert.Equal(t, data2["service1-0"].Group, "hello1") assert.Equal(t, data2["service1-1"].Group, "hello2") - os.Setenv(GroupEnvironmentName,"hello2") + os.Setenv(GroupEnvironmentName, "hello2") ctx.parseMultipleServiceGroup(data3) assert.Len(t, data3, 3) assert.Equal(t, data3["service1"].Group, "hello") assert.Equal(t, data3["service1-0"].Group, "hello1") assert.Equal(t, data3["service1-1"].Group, "hello2") - os.Setenv(GroupEnvironmentName,"hello") + os.Setenv(GroupEnvironmentName, "hello") ctx.parseMultipleServiceGroup(data4) assert.Len(t, data4, 1) assert.Equal(t, data3["service1"].Group, "hello") } + +func TestContext_mergeDefaultFilter(t *testing.T) { + c := Context{AgentURL: &URL{ + Parameters: map[string]string{"defaultFilter": "a,b,d"}, + }} + u1 := &URL{ + Parameters: map[string]string{"filter": "a,c", "disableDefaultFilter": "b"}, + } + u2 := &URL{ + Parameters: map[string]string{"filter": "a,c", "disableDefaultFilter": "b"}, + } + + u1.Parameters[FilterKey] = c.filterSetToStr( + c.mergeFilterSet( + c.getDefaultFilterSet(u1), c.getFilterSet(u1.Parameters[FilterKey], ""), + ), + ) + + for _, v := range strings.Split("a,d,c", ",") { + assert.Contains(t, u1.Parameters["filter"], v) + } + + c = Context{} + u2.Parameters[FilterKey] = c.filterSetToStr( + c.mergeFilterSet( + c.getDefaultFilterSet(u1), c.getFilterSet(u2.Parameters[FilterKey], ""), + ), + ) + for _, v := range strings.Split("a,c", ",") { + assert.Contains(t, u1.Parameters["filter"], v) + } + + c = Context{AgentURL: &URL{}} + u2.Parameters[FilterKey] = c.filterSetToStr( + c.mergeFilterSet( + c.getDefaultFilterSet(u1), c.getFilterSet(u2.Parameters[FilterKey], ""), + ), + ) + for _, v := range strings.Split("a,c", ",") { + assert.Contains(t, u2.Parameters["filter"], v) + } +} + +func TestContext_getFilterSet(t *testing.T) { + c := Context{} + a := "a,b," + b := "b," + assert.Equal(t, c.getFilterSet("a", ""), c.getFilterSet(a, b)) +} + +func TestContext_mergeFilterSet(t *testing.T) { + c := Context{} + a := c.getFilterSet("a,b,c,", "") + b := c.getFilterSet("b,", "") + for v := range c.mergeFilterSet(a, b) { + assert.Contains(t, "a,b,c", v) + } +} From 1cc3f7af54164bd575a8810b5be9833ed8613aee Mon Sep 17 00:00:00 2001 From: arraykeys Date: Thu, 28 Jul 2022 16:59:44 +0800 Subject: [PATCH 21/75] remove unnecessary filters process --- dynamicConfig.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dynamicConfig.go b/dynamicConfig.go index 236c9393..7b47d289 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -308,9 +308,7 @@ func (h *DynamicConfigurerHandler) parseURL(url *core.URL) (*core.URL, error) { if len(agentFilter) > 0 { filters = strings.Join(agentFilter, ",") } - if filters == "" { - filters = h.agent.Context.AgentURL.GetParam(core.FilterKey, "") - } + if filters != "" { url.PutParam(core.FilterKey, filters) } From d9c07b182fb86724f026fccab82e0cd22695e090 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Fri, 29 Jul 2022 17:21:02 +0800 Subject: [PATCH 22/75] add get all service interface (#279) expose agent dynamic register service --- agent.go | 6 +++++- default.go | 1 + manageHandler.go | 14 ++++++++++++++ manageHandler_test.go | 22 ++++++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/agent.go b/agent.go index 4b543dbc..ae81290f 100644 --- a/agent.go +++ b/agent.go @@ -124,6 +124,10 @@ func (a *Agent) RegisterCommandHandler(f CommandHandler) { a.commandHandlers = append(a.commandHandlers, f) } +func (a *Agent) GetDynamicRegistryInfo() *registrySnapInfoStorage { + return a.configurer.getRegistryInfo() +} + func (a *Agent) callAfterStart() { time.AfterFunc(time.Second*5, func() { for _, f := range a.onAfterStart { @@ -800,7 +804,7 @@ func (a *Agent) doExportService(url *motan.URL) { a.agentPortServer[url.Port] = server } else if canShareChannel(*url, *server.GetURL()) { server.GetMessageHandler().AddProvider(provider) - }else{ + } else { vlog.Errorf("service can't find a share channel , url:%v", url) return } diff --git a/default.go b/default.go index 172fa106..e681a818 100644 --- a/default.go +++ b/default.go @@ -45,6 +45,7 @@ func GetDefaultManageHandlers() map[string]http.Handler { info := &InfoHandler{} defaultManageHandlers["/getConfig"] = info defaultManageHandlers["/getReferService"] = info + defaultManageHandlers["/getAllService"] = info debug := &DebugHandler{} defaultManageHandlers["/debug/pprof/"] = debug diff --git a/manageHandler.go b/manageHandler.go index 604e95ff..4a4a8365 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -151,9 +151,23 @@ func (i *InfoHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rw.Write(i.a.getConfigData()) case "/getReferService": rw.Write(i.getReferService()) + case "/getAllService": + rw.Write(i.getAllServices()) } } +func (i *InfoHandler) getAllServices() []byte { + sMap := make(map[string][]*motan.URL) + var serviceList []*motan.URL + for _, s := range i.a.Context.ServiceURLs { + serviceList = append(serviceList, s) + } + sMap["services"] = serviceList + sMap["dynamic_services"] = i.a.GetDynamicRegistryInfo().RegisterNodes + b, _ := json.Marshal(sMap) + return b +} + func (i *InfoHandler) getReferService() []byte { mbody := body{Service: []rpcService{}} i.a.clusterMap.Range(func(k, v interface{}) bool { diff --git a/manageHandler_test.go b/manageHandler_test.go index 583f62b2..55bafec9 100644 --- a/manageHandler_test.go +++ b/manageHandler_test.go @@ -1,7 +1,9 @@ package motan import ( + "encoding/json" assert2 "github.com/stretchr/testify/assert" + motan "github.com/weibocom/motan-go/core" "net/http/httptest" "testing" "time" @@ -13,3 +15,23 @@ func TestSleep(t *testing.T) { end := time.Now() assert2.True(t, end.Sub(start).Seconds() >= 3) } + +func TestGetAllService(t *testing.T) { + i := InfoHandler{} + ctx := &motan.Context{ + ServiceURLs: map[string]*motan.URL{ + "test": &motan.URL{ + Group: "testgroup", + Path: "testpath", + }, + }, + } + agent := &Agent{Context: ctx} + agent.configurer = NewDynamicConfigurer(agent) + i.a = agent + serviceInfo := i.getAllServices() + resMap := make(map[string][]*motan.URL) + err := json.Unmarshal(serviceInfo, &resMap) + assert2.Nil(t, err) + assert2.Equal(t, len(resMap["services"]), 1) +} From 0ecd9442ea4255d12abac1884024b23327148fc8 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Wed, 3 Aug 2022 12:22:34 +0800 Subject: [PATCH 23/75] support DynamicConfigurerHandler add filters from defaultFilter and globalFilter --- core/globalContext.go | 24 ++++++++++++------------ core/globalContext_test.go | 26 +++++++++++++------------- dynamicConfig.go | 10 ++++++++++ 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/core/globalContext.go b/core/globalContext.go index eef6ebb5..c910199c 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -398,13 +398,13 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { } //final filters: defaultFilter + globalFilter + filters - finalFilters := c.mergeFilterSet( - c.getDefaultFilterSet(newURL), - c.getGlobalFilterSet(newURL), - c.getFilterSet(newURL.GetStringParamsWithDefault(FilterKey, ""), ""), + finalFilters := c.MergeFilterSet( + c.GetDefaultFilterSet(newURL), + c.GetGlobalFilterSet(newURL), + c.GetFilterSet(newURL.GetStringParamsWithDefault(FilterKey, ""), ""), ) if len(finalFilters) > 0 { - newURL.PutParam(FilterKey, c.filterSetToStr(finalFilters)) + newURL.PutParam(FilterKey, c.FilterSetToStr(finalFilters)) } newURLs[key] = newURL @@ -412,7 +412,7 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { return newURLs } -func (c *Context) filterSetToStr(f map[string]bool) string { +func (c *Context) FilterSetToStr(f map[string]bool) string { var dst []string for k := range f { dst = append(dst, k) @@ -420,7 +420,7 @@ func (c *Context) filterSetToStr(f map[string]bool) string { return strings.Join(dst, ",") } -func (c *Context) getFilterSet(filterStr, disableFilterStr string) (dst map[string]bool) { +func (c *Context) GetFilterSet(filterStr, disableFilterStr string) (dst map[string]bool) { if filterStr == "" { return } @@ -444,7 +444,7 @@ func (c *Context) getFilterSet(filterStr, disableFilterStr string) (dst map[stri return } -func (c *Context) mergeFilterSet(sets ...map[string]bool) (dst map[string]bool) { +func (c *Context) MergeFilterSet(sets ...map[string]bool) (dst map[string]bool) { dst = map[string]bool{} for _, set := range sets { for k := range set { @@ -458,19 +458,19 @@ func (c *Context) mergeFilterSet(sets ...map[string]bool) (dst map[string]bool) return } -func (c *Context) getDefaultFilterSet(newURL *URL) map[string]bool { +func (c *Context) GetDefaultFilterSet(newURL *URL) map[string]bool { if c.AgentURL == nil { return nil } - return c.getFilterSet(c.AgentURL.GetStringParamsWithDefault(DefaultFilter, ""), + return c.GetFilterSet(c.AgentURL.GetStringParamsWithDefault(DefaultFilter, ""), newURL.GetStringParamsWithDefault(DisableDefaultFilter, "")) } -func (c *Context) getGlobalFilterSet(newURL *URL) map[string]bool { +func (c *Context) GetGlobalFilterSet(newURL *URL) map[string]bool { if c.AgentURL == nil { return nil } - return c.getFilterSet(c.AgentURL.GetStringParamsWithDefault(GlobalFilter, ""), + return c.GetFilterSet(c.AgentURL.GetStringParamsWithDefault(GlobalFilter, ""), newURL.GetStringParamsWithDefault(DisableGlobalFilter, "")) } diff --git a/core/globalContext_test.go b/core/globalContext_test.go index 98680474..ff6c5ad8 100644 --- a/core/globalContext_test.go +++ b/core/globalContext_test.go @@ -213,9 +213,9 @@ func TestContext_mergeDefaultFilter(t *testing.T) { Parameters: map[string]string{"filter": "a,c", "disableDefaultFilter": "b"}, } - u1.Parameters[FilterKey] = c.filterSetToStr( - c.mergeFilterSet( - c.getDefaultFilterSet(u1), c.getFilterSet(u1.Parameters[FilterKey], ""), + u1.Parameters[FilterKey] = c.FilterSetToStr( + c.MergeFilterSet( + c.GetDefaultFilterSet(u1), c.GetFilterSet(u1.Parameters[FilterKey], ""), ), ) @@ -224,9 +224,9 @@ func TestContext_mergeDefaultFilter(t *testing.T) { } c = Context{} - u2.Parameters[FilterKey] = c.filterSetToStr( - c.mergeFilterSet( - c.getDefaultFilterSet(u1), c.getFilterSet(u2.Parameters[FilterKey], ""), + u2.Parameters[FilterKey] = c.FilterSetToStr( + c.MergeFilterSet( + c.GetDefaultFilterSet(u1), c.GetFilterSet(u2.Parameters[FilterKey], ""), ), ) for _, v := range strings.Split("a,c", ",") { @@ -234,9 +234,9 @@ func TestContext_mergeDefaultFilter(t *testing.T) { } c = Context{AgentURL: &URL{}} - u2.Parameters[FilterKey] = c.filterSetToStr( - c.mergeFilterSet( - c.getDefaultFilterSet(u1), c.getFilterSet(u2.Parameters[FilterKey], ""), + u2.Parameters[FilterKey] = c.FilterSetToStr( + c.MergeFilterSet( + c.GetDefaultFilterSet(u1), c.GetFilterSet(u2.Parameters[FilterKey], ""), ), ) for _, v := range strings.Split("a,c", ",") { @@ -248,14 +248,14 @@ func TestContext_getFilterSet(t *testing.T) { c := Context{} a := "a,b," b := "b," - assert.Equal(t, c.getFilterSet("a", ""), c.getFilterSet(a, b)) + assert.Equal(t, c.GetFilterSet("a", ""), c.GetFilterSet(a, b)) } func TestContext_mergeFilterSet(t *testing.T) { c := Context{} - a := c.getFilterSet("a,b,c,", "") - b := c.getFilterSet("b,", "") - for v := range c.mergeFilterSet(a, b) { + a := c.GetFilterSet("a,b,c,", "") + b := c.GetFilterSet("b,", "") + for v := range c.MergeFilterSet(a, b) { assert.Contains(t, "a,b,c", v) } } diff --git a/dynamicConfig.go b/dynamicConfig.go index 7b47d289..86d9d2eb 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -312,6 +312,16 @@ func (h *DynamicConfigurerHandler) parseURL(url *core.URL) (*core.URL, error) { if filters != "" { url.PutParam(core.FilterKey, filters) } + + //final filters: defaultFilter + globalFilter + filters + finalFilters := h.agent.Context.MergeFilterSet( + h.agent.Context.GetDefaultFilterSet(url), + h.agent.Context.GetGlobalFilterSet(url), + h.agent.Context.GetFilterSet(url.GetStringParamsWithDefault(core.FilterKey, ""), ""), + ) + if len(finalFilters) > 0 { + url.PutParam(core.FilterKey, h.agent.Context.FilterSetToStr(finalFilters)) + } return url, nil } From 7ab5123d48a90ca8fa544731fd51196d201ec81d Mon Sep 17 00:00:00 2001 From: liangwei3 Date: Thu, 11 Aug 2022 14:08:56 +0800 Subject: [PATCH 24/75] fix empty err threshold bug --- endpoint/motanEndpoint.go | 15 +++++++----- endpoint/motanEndpoint_test.go | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 7c9a9d0a..c096619c 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -218,12 +218,15 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { } func (m *MotanEndpoint) recordErrAndKeepalive() { - errCount := atomic.AddUint32(&m.errorCount, 1) - // ensure trigger keepalive - if errCount >= uint32(m.errorCountThreshold) { - m.setAvailable(false) - vlog.Infoln("Referer disable:" + m.url.GetIdentity()) - go m.keepalive() + // errorCountThreshold <= 0 means not trigger keepalive + if m.errorCountThreshold > 0 { + errCount := atomic.AddUint32(&m.errorCount, 1) + // ensure trigger keepalive + if errCount >= uint32(m.errorCountThreshold) { + m.setAvailable(false) + vlog.Infoln("Referer disable:" + m.url.GetIdentity()) + go m.keepalive() + } } } diff --git a/endpoint/motanEndpoint_test.go b/endpoint/motanEndpoint_test.go index 30da9953..1b64b4e3 100644 --- a/endpoint/motanEndpoint_test.go +++ b/endpoint/motanEndpoint_test.go @@ -38,6 +38,49 @@ func TestGetName(t *testing.T) { fmt.Printf("res:%+v\n", res) } +func TestRecordErrEmptyThreshold(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan2"} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + for j := 0; j < 5; j++ { + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + ep.Call(request) + assert.True(t, ep.IsAvailable()) + } + ep.Destroy() +} + +func TestRecordErrWithErrThreshold(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan2"} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "5") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + for j := 0; j < 10; j++ { + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + ep.Call(request) + if j < 4 { + assert.True(t, ep.IsAvailable()) + } else { + assert.False(t, ep.IsAvailable()) + } + } + ep.Destroy() +} + func TestMotanEndpoint_Call(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") From 44c8176da4a63c19ca6eac419d16faf96c30a22d Mon Sep 17 00:00:00 2001 From: cocowh Date: Tue, 23 Aug 2022 18:41:33 +0800 Subject: [PATCH 25/75] http proxy header:host http proxy header:host --- http/httpProxy.go | 7 +++++-- http/httpProxy_test.go | 31 +++++++++++++++++++++++++++++++ http/httpRpc_test.go | 14 -------------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/http/httpProxy.go b/http/httpProxy.go index 8a99fc7c..779ace69 100644 --- a/http/httpProxy.go +++ b/http/httpProxy.go @@ -21,6 +21,7 @@ const ( ) const ( + HeaderPrefix = "http_" HeaderContentType = "Content-Type" ) @@ -58,7 +59,7 @@ const ( var ( WhitespaceSplitPattern = regexp.MustCompile(`\s+`) findRewriteVarPattern = regexp.MustCompile(`\{[0-9a-zA-Z_-]+\}`) - httpProxySpecifiedAttachments = []string{Proxy, Method, QueryString} + httpProxySpecifiedAttachments = []string{Proxy, Method, QueryString, core.HostKey} rewriteVarFunc = func(condType ProxyRewriteType, uri string, queryBytes []byte) string { if condType != proxyRewriteTypeRegexpVar || len(queryBytes) == 0 { return uri @@ -435,7 +436,7 @@ func MotanRequestToFasthttpRequest(motanRequest core.Request, fasthttpRequest *f } } // fasthttp will use a special field to store this header - if strings.EqualFold(k, core.HostKey) { + if strings.EqualFold(k, HeaderPrefix+core.HostKey) { fasthttpRequest.Header.SetHost(v) return true } @@ -444,6 +445,8 @@ func MotanRequestToFasthttpRequest(motanRequest core.Request, fasthttpRequest *f return true } k = strings.Replace(k, "M_", "MOTAN-", -1) + // http header private prefix + k = strings.Replace(k, HeaderPrefix, "", -1) fasthttpRequest.Header.Add(k, v) return true }) diff --git a/http/httpProxy_test.go b/http/httpProxy_test.go index 4f688fe9..16173283 100644 --- a/http/httpProxy_test.go +++ b/http/httpProxy_test.go @@ -4,6 +4,7 @@ import ( "bytes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/valyala/fasthttp" "github.com/weibocom/motan-go/config" "github.com/weibocom/motan-go/core" "testing" @@ -181,6 +182,36 @@ http-locations: } } +func (s *APITestSuite) TestMotanRequestToFasthttpRequest() { + method := "GET" + attachment := core.NewStringMap(0) + motanReq := &core.MotanRequest{ + RequestID: 0, + ServiceName: "", + Method: "", + MethodDesc: "", + Arguments: nil, + Attachment: attachment, + RPCContext: nil, + } + httpReq := fasthttp.AcquireRequest() + + attachment.Store(HeaderContentType, "application/json") + attachment.Store(core.HostKey, "127.0.0.1") + attachment.Store(HeaderPrefix+"testKey", "testValue") + + err := MotanRequestToFasthttpRequest(motanReq, httpReq, method) + assert.Nil(s.T(), err) + assert.Equal(s.T(), []byte("testValue"), httpReq.Header.Peek("testKey")) + assert.Equal(s.T(), []byte(nil), httpReq.Header.Peek("Host")) + assert.Equal(s.T(), []byte("application/json"), httpReq.Header.ContentType()) + + attachment.Store(HeaderPrefix+core.HostKey, "test.com") + err = MotanRequestToFasthttpRequest(motanReq, httpReq, method) + assert.Nil(s.T(), err) + assert.Equal(s.T(), []byte("test.com"), httpReq.Header.Peek("Host")) +} + func (s *APITestSuite) TestNewRewriteRuleError() { wrongRules := []string{ `exact /Test2/1 /(.*) /test /test`, diff --git a/http/httpRpc_test.go b/http/httpRpc_test.go index 1990fdd6..6f99da2e 100644 --- a/http/httpRpc_test.go +++ b/http/httpRpc_test.go @@ -2,14 +2,12 @@ package http_test import ( "bytes" - "encoding/json" "fmt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/weibocom/motan-go" "github.com/weibocom/motan-go/config" motancore "github.com/weibocom/motan-go/core" - http2 "github.com/weibocom/motan-go/http" "github.com/weibocom/motan-go/protocol" "math/rand" "net/http" @@ -48,10 +46,6 @@ motan-service: requestTimeout: 2000 ` -var ( - httpRequestHeader http.Header -) - type APITestSuite struct { suite.Suite } @@ -103,17 +97,9 @@ func (s *APITestSuite) TestRequestResponse() { resp, err = callTimeOutWrongArgumentCount("127.0.0.1", "9989", "test", "test.domain", 30000, "", "/", []interface{}{argumentString}, attachments) assert.NotNil(s.T(), err) assert.Nil(s.T(), resp) - // json content-type request - bodyBytes,_ := json.Marshal(arguments) - attachments[http2.HeaderContentType] = "application/json" - resp, err = callTimeOutWithAttachment("127.0.0.1", "9989", "test", "test.domain", 30000, "", "/", []interface{}{bodyBytes}, attachments) - assert.Nil(s.T(), err) - assert.NotNil(s.T(), resp) - assert.Equal(s.T(), "application/json", httpRequestHeader.Get(http2.HeaderContentType)) } func indexHandler(w http.ResponseWriter, r *http.Request) { - httpRequestHeader = r.Header fmt.Fprintf(w, "hello world") } From 89187d34c88db8fc560fc4a81866e04d6d654d66 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Mon, 19 Sep 2022 16:42:17 +0800 Subject: [PATCH 26/75] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E9=80=9A=E8=BF=87=E5=90=AF=E5=8A=A8=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E6=8C=87=E5=AE=9A=E5=8A=9F=E8=83=BD=20(#285)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: liangwei3 --- agent.go | 11 ++++++++++- agent_test.go | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/agent.go b/agent.go index ae81290f..1fc903f6 100644 --- a/agent.go +++ b/agent.go @@ -277,7 +277,16 @@ func (a *Agent) initParam() { fmt.Println("get config of \"motan-agent\" fail! err " + err.Error()) } logDir := "" - if section != nil && section["log_dir"] != nil { + isFound := false + for _, j := range os.Args { + if j == "-log_dir" { + isFound = true + break + } + } + if isFound { + logDir = flag.Lookup("log_dir").Value.String() + } else if section != nil && section["log_dir"] != nil { logDir = section["log_dir"].(string) } if logDir == "" { diff --git a/agent_test.go b/agent_test.go index 830866db..c96deae7 100644 --- a/agent_test.go +++ b/agent_test.go @@ -2,6 +2,7 @@ package motan import ( "bytes" + "flag" "github.com/weibocom/motan-go/config" "io/ioutil" "math/rand" @@ -93,6 +94,21 @@ motan-agent: assert.Nil(err) assert.Equal(true, section["log_filter_caller"].(bool)) a.initParam() + + logDirConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + log_dir: "./test/abcd" +`))) + assert.Nil(err) + conf.Merge(logDirConfig) + section, err = conf.GetSection("motan-agent") + assert.Nil(err) + a.initParam() + assert.Equal(a.logdir, "./test/abcd") + os.Args = append(os.Args, "-log_dir", "./test/cdef") + _ = flag.Set("log_dir", "./test/cdef") + a.initParam() + assert.Equal(a.logdir, "./test/cdef") } func TestHTTPProxyBodySize(t *testing.T) { From b34ea7e4cb21ec5cb2a161776b2f1229d2ef3a15 Mon Sep 17 00:00:00 2001 From: Ray Date: Tue, 20 Sep 2022 12:10:18 +0800 Subject: [PATCH 27/75] Feat/support motan1 (#284) * support motan v1 protocol * support v1 group in serverAgentMessageHandler --- agent.go | 7 +- core/constants.go | 14 +- core/util.go | 15 + core/util_test.go | 14 + dynamicConfig_test.go | 2 +- endpoint/endpoint.go | 13 +- endpoint/motanCommonEndpoint.go | 799 ++++++++++++++++++++++++++++++++ endpoint/motanEndpoint.go | 116 ++--- ha/failoverHA.go | 21 +- protocol/motan1Protocol.go | 588 +++++++++++++++++++++++ protocol/motan1Protocol_test.go | 264 +++++++++++ protocol/motanProtocol.go | 39 +- provider/motanProvider.go | 3 + server/motanserver.go | 127 +++-- 14 files changed, 1915 insertions(+), 107 deletions(-) create mode 100644 endpoint/motanCommonEndpoint.go create mode 100644 protocol/motan1Protocol.go create mode 100644 protocol/motan1Protocol_test.go diff --git a/agent.go b/agent.go index 1fc903f6..24b348c9 100644 --- a/agent.go +++ b/agent.go @@ -850,7 +850,12 @@ func (sa *serverAgentMessageHandler) Call(request motan.Request) (res motan.Resp res = motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "provider call panic", ErrType: motan.ServiceException}) vlog.Errorf("provider call panic. req:%s", motan.GetReqInfo(request)) }) - serviceKey := getServiceKey(request.GetAttachment(mpro.MGroup), request.GetServiceName()) + // todo: add GetGroup() method in Request + group := request.GetAttachment(mpro.MGroup) + if group == "" { // compatible with motan v1 + group = request.GetAttachment(motan.GroupKey) + } + serviceKey := getServiceKey(group, request.GetServiceName()) if p := sa.providers.LoadOrNil(serviceKey); p != nil { p := p.(motan.Provider) res = p.Call(request) diff --git a/core/constants.go b/core/constants.go index 8a462523..309b531f 100644 --- a/core/constants.go +++ b/core/constants.go @@ -35,8 +35,8 @@ const ( FilterKey = "filter" GlobalFilter = "globalFilter" DisableGlobalFilter = "disableGlobalFilter" - DefaultFilter = "defaultFilter" - DisableDefaultFilter = "disableDefaultFilter" + DefaultFilter = "defaultFilter" + DisableDefaultFilter = "disableDefaultFilter" RegistryKey = "registry" WeightKey = "weight" SerializationKey = "serialization" @@ -62,6 +62,7 @@ const ( ManagementPortRangeKey = "managementPortRange" HTTPProxyUnixSockKey = "httpProxyUnixSock" MixGroups = "mixGroups" + MaxContentLength = "maxContentLength" ) // nodeType @@ -84,9 +85,10 @@ const ( ) const ( - DefaultWriteTimeout = 5 * time.Second - GroupNameSeparator = "," - GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" + DefaultWriteTimeout = 5 * time.Second + DefaultMaxContentLength = 10 * 1024 * 1024 + GroupNameSeparator = "," + GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" ) // meta keys @@ -98,4 +100,6 @@ const ( const ( ENoEndpoints = 1001 ENoChannel = 1002 + EUnkonwnMsg = 1003 + EConvertMsg = 1004 ) diff --git a/core/util.go b/core/util.go index 656a640d..b20d3040 100644 --- a/core/util.go +++ b/core/util.go @@ -122,6 +122,21 @@ func GetReqInfo(request Request) string { return "" } +func GetResInfo(response Response) string { + if response != nil { + var buffer bytes.Buffer + buffer.WriteString("res{") + buffer.WriteString(strconv.FormatUint(response.GetRequestID(), 10)) + buffer.WriteString(",") + if response.GetException() != nil { + buffer.WriteString(response.GetException().ErrMsg) + } + buffer.WriteString("}") + return buffer.String() + } + return "" +} + func HandlePanic(f func()) { if err := recover(); err != nil { vlog.Errorf("recover panic. error:%v, stack: %s", err, debug.Stack()) diff --git a/core/util_test.go b/core/util_test.go index 21600cfa..18a76a01 100644 --- a/core/util_test.go +++ b/core/util_test.go @@ -110,3 +110,17 @@ func TestSlicesUnique(t *testing.T) { assert.Equal(t, SlicesUnique(b), b) assert.Equal(t, SlicesUnique(c), c) } + +func TestGetReqInfo(t *testing.T) { + req := &MotanRequest{RequestID: 34789798073, ServiceName: "testServiceName", Method: "testMethod"} + assert.Equal(t, "", GetReqInfo(nil)) + assert.Equal(t, "req{34789798073,testServiceName,testMethod}", GetReqInfo(req)) +} + +func TestGetResInfo(t *testing.T) { + res := &MotanResponse{RequestID: 374867809809} + resE := &MotanResponse{RequestID: 374867809809, Exception: &Exception{ErrType: ServiceException, ErrCode: 503, ErrMsg: "testErrMsg"}} + assert.Equal(t, "", GetResInfo(nil)) + assert.Equal(t, "res{374867809809,}", GetResInfo(res)) + assert.Equal(t, "res{374867809809,testErrMsg}", GetResInfo(resE)) +} diff --git a/dynamicConfig_test.go b/dynamicConfig_test.go index 6ce1af9e..477af59e 100644 --- a/dynamicConfig_test.go +++ b/dynamicConfig_test.go @@ -25,5 +25,5 @@ func TestDynamicConfigurerHandler_readURLs(t *testing.T) { assert.Equal(t, urls[1].Group, "hello1") assert.Equal(t, urls[2].Group, "hello2") _, err = d.readURLsFromRequest(req3) - assert.NotNil(t, err) + assert.NotNil(t, err) } diff --git a/endpoint/endpoint.go b/endpoint/endpoint.go index ee0183e2..4ea84c15 100644 --- a/endpoint/endpoint.go +++ b/endpoint/endpoint.go @@ -10,10 +10,11 @@ import ( // ext name const ( - Grpc = "grpc" - Motan2 = "motan2" - Local = "local" - Mock = "mockEndpoint" + Grpc = "grpc" + Motan2 = "motan2" + Local = "local" + Mock = "mockEndpoint" + MotanV1Compatible = "motanV1Compatible" ) const ( @@ -39,6 +40,10 @@ func RegistDefaultEndpoint(extFactory motan.ExtensionFactory) { extFactory.RegistExtEndpoint(Mock, func(url *motan.URL) motan.EndPoint { return &MockEndpoint{URL: url} }) + + extFactory.RegistExtEndpoint(MotanV1Compatible, func(url *motan.URL) motan.EndPoint { + return &MotanCommonEndpoint{url: url} + }) } func GetRequestGroup(r motan.Request) string { diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go new file mode 100644 index 00000000..daa14f16 --- /dev/null +++ b/endpoint/motanCommonEndpoint.go @@ -0,0 +1,799 @@ +package endpoint + +import ( + "bufio" + "errors" + motan "github.com/weibocom/motan-go/core" + vlog "github.com/weibocom/motan-go/log" + mpro "github.com/weibocom/motan-go/protocol" + "net" + "strconv" + "sync" + "sync/atomic" + "time" +) + +// MotanCommonEndpoint supports motan v1, v2 protocols +type MotanCommonEndpoint struct { + url *motan.URL + lock sync.Mutex + channels *ChannelPool + destroyed bool + destroyCh chan struct{} + available bool + errorCount uint32 + proxy bool + errorCountThreshold int64 + keepaliveInterval time.Duration + requestTimeoutMillisecond int64 + minRequestTimeoutMillisecond int64 + maxRequestTimeoutMillisecond int64 + clientConnection int + maxContentLength int + heartbeatVersion int + + keepaliveRunning bool + serialization motan.Serialization + + DefaultVersion int // default encode version +} + +func (m *MotanCommonEndpoint) setAvailable(available bool) { + m.available = available +} + +func (m *MotanCommonEndpoint) SetSerialization(s motan.Serialization) { + m.serialization = s +} + +func (m *MotanCommonEndpoint) SetProxy(proxy bool) { + m.proxy = proxy +} + +func (m *MotanCommonEndpoint) Initialize() { + m.destroyCh = make(chan struct{}, 1) + connectTimeout := m.url.GetTimeDuration(motan.ConnectTimeoutKey, time.Millisecond, defaultConnectTimeout) + connectRetryInterval := m.url.GetTimeDuration(motan.ConnectRetryIntervalKey, time.Millisecond, defaultConnectRetryInterval) + m.errorCountThreshold = m.url.GetIntValue(motan.ErrorCountThresholdKey, int64(defaultErrorCountThreshold)) + m.keepaliveInterval = m.url.GetTimeDuration(motan.KeepaliveIntervalKey, time.Millisecond, defaultKeepaliveInterval) + m.requestTimeoutMillisecond = m.url.GetPositiveIntValue(motan.TimeOutKey, int64(defaultRequestTimeout/time.Millisecond)) + m.minRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MinTimeOutKey) + m.maxRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MaxTimeOutKey) + m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) + m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) + m.heartbeatVersion = -1 + m.DefaultVersion = mpro.Version2 + factory := func() (net.Conn, error) { + return net.DialTimeout("tcp", m.url.GetAddressStr(), connectTimeout) + } + config := &ChannelConfig{MaxContentLength: m.maxContentLength, Serialization: m.serialization} + channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization) + if err != nil { + vlog.Errorf("Channel pool init failed. url: %v, err:%s", m.url, err.Error()) + // retry connect + go func() { + defer motan.HandlePanic(nil) + // TODO: retry after 2^n * timeUnit + ticker := time.NewTicker(connectRetryInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization) + if err == nil { + m.channels = channels + m.setAvailable(true) + vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + return + } + case <-m.destroyCh: + return + } + } + }() + } else { + m.channels = channels + m.setAvailable(true) + vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + } +} + +func (m *MotanCommonEndpoint) Destroy() { + m.lock.Lock() + defer m.lock.Unlock() + if m.destroyed { + return + } + m.setAvailable(false) + m.destroyCh <- struct{}{} + m.destroyed = true + if m.channels != nil { + vlog.Infof("motan2 endpoint %s will destroyed", m.url.GetAddressStr()) + m.channels.Close() + } +} + +func (m *MotanCommonEndpoint) GetRequestTimeout(request motan.Request) time.Duration { + timeout := m.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), motan.TimeOutKey, m.requestTimeoutMillisecond) + minTimeout := m.minRequestTimeoutMillisecond + maxTimeout := m.maxRequestTimeoutMillisecond + if minTimeout == 0 { + minTimeout = timeout / 2 + } + if maxTimeout == 0 { + maxTimeout = timeout * 2 + } + reqTimeout, _ := strconv.ParseInt(request.GetAttachment(mpro.MTimeout), 10, 64) + if reqTimeout >= minTimeout && reqTimeout <= maxTimeout { + timeout = reqTimeout + } + return time.Duration(timeout) * time.Millisecond +} + +func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { + rc := request.GetRPCContext(true) + rc.Proxy = m.proxy + rc.GzipSize = int(m.url.GetIntValue(motan.GzipSizeKey, 0)) + + if m.channels == nil { + vlog.Errorf("motanEndpoint %s error: channels is null", m.url.GetAddressStr()) + m.recordErrAndKeepalive() + return m.defaultErrMotanResponse(request, "motanEndpoint error: channels is null") + } + startTime := time.Now().UnixNano() + if rc.AsyncCall { + rc.Result.StartTime = startTime + } + // get a channel + channel, err := m.channels.Get() + if err != nil { + vlog.Errorf("motanEndpoint %s error: can not get a channel, msg: %s", m.url.GetAddressStr(), err.Error()) + m.recordErrAndKeepalive() + return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ + ErrCode: motan.ENoChannel, + ErrMsg: "can not get a channel", + ErrType: motan.ServiceException, + }) + } + deadline := m.GetRequestTimeout(request) + // do call + group := GetRequestGroup(request) + if group != m.url.Group && m.url.Group != "" { + request.SetAttachment(mpro.MGroup, m.url.Group) + } + response, err := channel.Call(request, deadline, rc) + if err != nil { + vlog.Errorf("motanEndpoint call fail. ep:%s, req:%s, error: %s", m.url.GetAddressStr(), motan.GetReqInfo(request), err.Error()) + m.recordErrAndKeepalive() + return m.defaultErrMotanResponse(request, "channel call error:"+err.Error()) + } + if rc.AsyncCall { + return defaultAsyncResponse + } + excep := response.GetException() + if excep != nil && excep.ErrCode == 503 { + m.recordErrAndKeepalive() + } else { + // reset errorCount + m.resetErr() + if m.heartbeatVersion == -1 { // init heartbeat version with first request version + if _, ok := rc.OriginalMessage.(*mpro.MotanV1Message); ok { + m.heartbeatVersion = mpro.Version1 + } else { + m.heartbeatVersion = mpro.Version2 + } + } + } + if !m.proxy { + if err = response.ProcessDeserializable(rc.Reply); err != nil { + return m.defaultErrMotanResponse(request, err.Error()) + } + } + return response +} + +func (m *MotanCommonEndpoint) recordErrAndKeepalive() { + errCount := atomic.AddUint32(&m.errorCount, 1) + // ensure trigger keepalive + if errCount >= uint32(m.errorCountThreshold) { + m.setAvailable(false) + vlog.Infoln("Referer disable:" + m.url.GetIdentity()) + go m.keepalive() + } +} + +func (m *MotanCommonEndpoint) resetErr() { + atomic.StoreUint32(&m.errorCount, 0) +} + +func (m *MotanCommonEndpoint) keepalive() { + m.lock.Lock() + // if the endpoint has been destroyed, we should not do keepalive + if m.destroyed { + m.lock.Unlock() + return + } + // ensure only one keepalive handler + if m.keepaliveRunning { + m.lock.Unlock() + return + } + m.keepaliveRunning = true + m.lock.Unlock() + + defer func() { + m.lock.Lock() + m.keepaliveRunning = false + m.lock.Unlock() + }() + + defer motan.HandlePanic(nil) + ticker := time.NewTicker(m.keepaliveInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if channel, err := m.channels.Get(); err != nil { + vlog.Infof("[keepalive] failed. url:%s, err:%s", m.url.GetIdentity(), err.Error()) + } else { + _, err = channel.HeartBeat(m.heartbeatVersion) + if err == nil { + m.setAvailable(true) + m.resetErr() + vlog.Infof("[keepalive] heartbeat success. url: %s", m.url.GetIdentity()) + return + } + vlog.Infof("[keepalive] heartbeat failed. url:%s, err:%s", m.url.GetIdentity(), err.Error()) + } + case <-m.destroyCh: + return + } + } +} + +func (m *MotanCommonEndpoint) defaultErrMotanResponse(request motan.Request, errMsg string) motan.Response { + response := &motan.MotanResponse{ + RequestID: request.GetRequestID(), + Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), + Exception: &motan.Exception{ + ErrCode: 400, + ErrMsg: errMsg, + ErrType: motan.ServiceException, + }, + } + return response +} + +func (m *MotanCommonEndpoint) GetName() string { + return "motanCommonsEndpoint" +} + +func (m *MotanCommonEndpoint) GetURL() *motan.URL { + return m.url +} + +func (m *MotanCommonEndpoint) SetURL(url *motan.URL) { + m.url = url +} + +func (m *MotanCommonEndpoint) IsAvailable() bool { + return m.available +} + +type Channel struct { + // config + config *ChannelConfig + serialization motan.Serialization + address string + + // connection + conn net.Conn + bufRead *bufio.Reader + + // send + sendCh chan sendReady + + // stream + streams map[uint64]*Stream + streamLock sync.Mutex + // heartbeat + heartbeats map[uint64]*Stream + heartbeatLock sync.Mutex + + // shutdown + shutdown bool + shutdownErr error + shutdownCh chan struct{} + shutdownLock sync.Mutex +} + +type Stream struct { + channel *Channel + streamId uint64 // RequestID is communication identifier, it is owned by channel + req motan.Request // for send + res motan.Response // for receive + recvNotifyCh chan struct{} + deadline time.Time // for timeout + rc *motan.RPCContext + isClose atomic.Value // bool + isHeartbeat bool // for heartbeat + heartbeatVersion int // for heartbeat +} + +func (s *Stream) Send() (err error) { + timer := time.NewTimer(s.deadline.Sub(time.Now())) + defer timer.Stop() + + var bytes []byte + var msg *mpro.Message + if s.isHeartbeat { + if s.heartbeatVersion == mpro.Version1 { + bytes = mpro.BuildV1HeartbeatReq(s.streamId) + } else { // motan2 as default heartbeat + msg = mpro.BuildHeartbeat(s.streamId, mpro.Req) + } + } else { // normal request + // s.rc should not nil while send request + if _, ok := s.rc.OriginalMessage.(*mpro.MotanV1Message); ok { // encode motan v1 + bytes, err = mpro.EncodeMotanV1Request(s.req, s.streamId) + if err != nil { + vlog.Errorf("encode v1 request fail, not contains MotanV1Message. req:%s", motan.GetReqInfo(s.req)) + return err + } + } else { // encode motan v2 + msg, err = mpro.ConvertToReqMessage(s.req, s.channel.config.Serialization) + if err != nil { + vlog.Errorf("convert motan request fail! ep: %s, req: %s, err:%s", s.channel.address, motan.GetReqInfo(s.req), err.Error()) + return err + } + msg.Header.RequestID = s.streamId + if s.rc.Tc != nil { + s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) + } + } + } + + if msg != nil { // encode v2 message + bytes = msg.Encode().Bytes() + } + if s.rc != nil && s.rc.Tc != nil { + s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Encode, Addr: s.channel.address, Time: time.Now()}) + } + ready := sendReady{data: bytes} + select { + case s.channel.sendCh <- ready: + if s.rc != nil { + sendTime := time.Now() + s.rc.RequestSendTime = sendTime + if s.rc.Tc != nil { + s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Send, Addr: s.channel.address, Time: sendTime}) + } + } + return nil + case <-timer.C: + return ErrSendRequestTimeout + case <-s.channel.shutdownCh: + return ErrChannelShutdown + } +} + +// Recv sync recv +func (s *Stream) Recv() (motan.Response, error) { + defer func() { + s.Close() + }() + timer := time.NewTimer(s.deadline.Sub(time.Now())) + defer timer.Stop() + select { + case <-s.recvNotifyCh: + msg := s.res + if msg == nil { + return nil, errors.New("recv err: recvMsg is nil") + } + return msg, nil + case <-timer.C: + return nil, ErrRecvRequestTimeout + case <-s.channel.shutdownCh: + return nil, ErrChannelShutdown + } +} + +func (s *Stream) notify(msg interface{}, t time.Time) { + defer func() { + s.Close() + }() + decodeTime := time.Now() + var res motan.Response + var v2Msg *mpro.Message + var err error + if mres, ok := msg.(*motan.MotanResponse); ok { // response (v1 decoded) + if s.req != nil { + mres.RequestID = s.req.GetRequestID() + } + res = mres + } else if v2Msg, ok = msg.(*mpro.Message); ok { // v2 message + if s.rc != nil { + v2Msg.Header.SetProxy(s.rc.Proxy) + } + if s.req != nil { + v2Msg.Header.RequestID = s.req.GetRequestID() + } + res, err = mpro.ConvertToResponse(v2Msg, s.channel.serialization) + if err != nil { + vlog.Errorf("convert to response fail. ep: %s, requestId:%d, err:%s", s.channel.address, v2Msg.Header.RequestID, err.Error()) + res = motan.BuildExceptionResponse(v2Msg.Header.RequestID, &motan.Exception{ + ErrCode: motan.EConvertMsg, + ErrMsg: "convert response fail", + ErrType: motan.ServiceException, + }) + } + } else { // unknown message + vlog.Errorf("stream notify: unsupported msg. msg:%v. ep: %s", msg, s.channel.address) + err = ErrUnsupportedMessage + rid := s.streamId // stream id as default rid(for heartbeat request) + if s.req != nil { + rid = s.req.GetRequestID() + } + res = motan.BuildExceptionResponse(rid, &motan.Exception{ + ErrCode: motan.EUnkonwnMsg, + ErrMsg: "receive unsupported msg", + ErrType: motan.ServiceException, + }) + } + if s.rc != nil { // not heartbeat + s.rc.ResponseReceiveTime = t + if s.rc.Tc != nil { + s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Receive, Addr: s.channel.address, Time: t}) + s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Decode, Addr: s.channel.address, Time: decodeTime}) + s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) + } + if s.rc.AsyncCall { + result := s.rc.Result + if err != nil { + result.Error = err + result.Done <- result + return + } + if err = res.ProcessDeserializable(result.Reply); err != nil { + result.Error = err + } + res.SetProcessTime((time.Now().UnixNano() - result.StartTime) / 1000000) + result.Done <- result + return + } + } + s.res = res + s.recvNotifyCh <- struct{}{} +} + +func (s *Stream) SetDeadline(deadline time.Duration) { + s.deadline = time.Now().Add(deadline) +} + +func (c *Channel) NewStream(req motan.Request, rc *motan.RPCContext) (*Stream, error) { + if c.IsClosed() { + return nil, ErrChannelShutdown + } + s := &Stream{ + streamId: GenerateRequestID(), + channel: c, + req: req, + recvNotifyCh: make(chan struct{}, 1), + deadline: time.Now().Add(defaultRequestTimeout), // default deadline + rc: rc, + } + s.isClose.Store(false) + c.streamLock.Lock() + c.streams[s.streamId] = s + c.streamLock.Unlock() + return s, nil +} + +func (c *Channel) NewHeartbeatStream(heartbeatVersion int) (*Stream, error) { + if c.IsClosed() { + return nil, ErrChannelShutdown + } + s := &Stream{ + streamId: GenerateRequestID(), + channel: c, + isHeartbeat: true, + heartbeatVersion: heartbeatVersion, + recvNotifyCh: make(chan struct{}, 1), + deadline: time.Now().Add(defaultRequestTimeout), + } + s.isClose.Store(false) + c.heartbeatLock.Lock() + c.heartbeats[s.streamId] = s + c.heartbeatLock.Unlock() + return s, nil +} + +func (s *Stream) Close() { + if !s.isClose.Load().(bool) { + if s.isHeartbeat { + s.channel.heartbeatLock.Lock() + delete(s.channel.heartbeats, s.streamId) + s.channel.heartbeatLock.Unlock() + } else { + s.channel.streamLock.Lock() + delete(s.channel.streams, s.streamId) + s.channel.streamLock.Unlock() + } + s.isClose.Store(true) + } +} + +// Call send request to the server. +// about return: exception in response will record error count, err will not. +func (c *Channel) Call(req motan.Request, deadline time.Duration, rc *motan.RPCContext) (motan.Response, error) { + stream, err := c.NewStream(req, rc) + if err != nil { + return nil, err + } + stream.SetDeadline(deadline) + err = stream.Send() + if err != nil { + return nil, err + } + if rc != nil && rc.AsyncCall { + return nil, nil + } + return stream.Recv() +} + +func (c *Channel) HeartBeat(heartbeatVersion int) (motan.Response, error) { + stream, err := c.NewHeartbeatStream(heartbeatVersion) + if err != nil { + return nil, err + } + err = stream.Send() + if err != nil { + return nil, err + } + return stream.Recv() +} + +func (c *Channel) IsClosed() bool { + return c.shutdown +} + +func (c *Channel) recv() { + defer motan.HandlePanic(func() { + c.closeOnErr(errPanic) + }) + if err := c.recvLoop(); err != nil { + c.closeOnErr(err) + } +} + +func (c *Channel) recvLoop() error { + for { + v, err := mpro.CheckMotanVersion(c.bufRead) + if err != nil { + return err + } + var msg interface{} + var t time.Time + if v == mpro.Version1 { + msg, t, err = mpro.ReadV1Message(c.bufRead, c.config.MaxContentLength) + } else if v == mpro.Version2 { + msg, t, err = mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) + } else { + vlog.Warningf("unsupported motan version! version:%d con:%s.", v, c.conn.RemoteAddr().String()) + err = mpro.ErrVersion + } + if err != nil { + return err + } + go c.handleMsg(msg, t) + } +} + +func (c *Channel) handleMsg(msg interface{}, t time.Time) { + var isHeartbeat bool + var rid uint64 + if v1msg, ok := msg.(*mpro.MotanV1Message); ok { + res, err := mpro.DecodeMotanV1Response(v1msg) + if err != nil { + vlog.Errorf("decode v1 response fail. v1msg:%v, err:%v", v1msg, err) + } + isHeartbeat = mpro.IsV1HeartbeatRes(res) + rid = v1msg.Rid + msg = res + } else if v2msg, ok := msg.(*mpro.Message); ok { + isHeartbeat = v2msg.Header.IsHeartbeat() + rid = v2msg.Header.RequestID + } else { + //should not here + vlog.Errorf("unsupported msg:%v", msg) + return + } + + var stream *Stream + if isHeartbeat { + c.heartbeatLock.Lock() + stream = c.heartbeats[rid] + c.heartbeatLock.Unlock() + } else { + c.streamLock.Lock() + stream = c.streams[rid] + c.streamLock.Unlock() + } + if stream == nil { + vlog.Warningf("handle message, missing stream: %d, ep:%s, isHeartbeat:%t", rid, c.address, isHeartbeat) + return + } + stream.notify(msg, t) +} + +func (c *Channel) send() { + defer motan.HandlePanic(func() { + c.closeOnErr(errPanic) + }) + for { + select { + case ready := <-c.sendCh: + if ready.data != nil { + c.conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) + sent := 0 + for sent < len(ready.data) { + n, err := c.conn.Write(ready.data[sent:]) + if err != nil { + vlog.Errorf("Failed to write channel. ep: %s, err: %s", c.address, err.Error()) + c.closeOnErr(err) + return + } + sent += n + } + } + case <-c.shutdownCh: + return + } + } +} + +func (c *Channel) closeOnErr(err error) { + c.shutdownLock.Lock() + if c.shutdownErr == nil { + c.shutdownErr = err + } + shutdown := c.shutdown + c.shutdownLock.Unlock() + if !shutdown { // not normal close + if err != nil && err.Error() != "EOF" { + vlog.Warningf("motan channel will close. ep:%s, err: %s\n", c.address, err.Error()) + } + c.Close() + } +} + +func (c *Channel) Close() error { + c.shutdownLock.Lock() + defer c.shutdownLock.Unlock() + if c.shutdown { + return nil + } + c.shutdown = true + close(c.shutdownCh) + c.conn.Close() + return nil +} + +type ChannelPool struct { + channels chan *Channel + channelsLock sync.Mutex + factory ConnFactory + config *ChannelConfig + serialization motan.Serialization +} + +func (c *ChannelPool) getChannels() chan *Channel { + channels := c.channels + return channels +} + +func (c *ChannelPool) Get() (*Channel, error) { + channels := c.getChannels() + if channels == nil { + return nil, errors.New("channels is nil") + } + channel, ok := <-channels + if ok && (channel == nil || channel.IsClosed()) { + conn, err := c.factory() + if err != nil { + vlog.Errorf("create channel failed. err:%s", err.Error()) + } + channel = buildChannel(conn, c.config, c.serialization) + } + if err := retChannelPool(channels, channel); err != nil && channel != nil { + channel.closeOnErr(err) + } + if channel == nil { + return nil, errors.New("channel is nil") + } + return channel, nil +} + +func retChannelPool(channels chan *Channel, channel *Channel) (error error) { + defer func() { + if err := recover(); err != nil { + error = errors.New("ChannelPool has been closed") + } + }() + if channels == nil { + return errors.New("channels is nil") + } + channels <- channel + return nil +} + +func (c *ChannelPool) Close() error { + c.channelsLock.Lock() // to prevent channels closed many times + channels := c.channels + c.channels = nil + c.factory = nil + c.config = nil + c.channelsLock.Unlock() + if channels == nil { + return nil + } + close(channels) + for channel := range channels { + if channel != nil { + channel.Close() + } + } + return nil +} + +func NewChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, serialization motan.Serialization) (*ChannelPool, error) { + if poolCap <= 0 { + return nil, errors.New("invalid capacity settings") + } + channelPool := &ChannelPool{ + channels: make(chan *Channel, poolCap), + factory: factory, + config: config, + serialization: serialization, + } + for i := 0; i < poolCap; i++ { + conn, err := factory() + if err != nil { + channelPool.Close() + return nil, err + } + _ = conn.(*net.TCPConn).SetNoDelay(true) + channelPool.channels <- buildChannel(conn, config, serialization) + } + return channelPool, nil +} + +func buildChannel(conn net.Conn, config *ChannelConfig, serialization motan.Serialization) *Channel { + if conn == nil { + return nil + } + if config == nil { + config = DefaultConfig() + } + if err := VerifyConfig(config); err != nil { + vlog.Errorf("can not build Channel, ChannelConfig check fail. err:%v", err) + return nil + } + channel := &Channel{ + conn: conn, + config: config, + bufRead: bufio.NewReader(conn), + sendCh: make(chan sendReady, 256), + streams: make(map[uint64]*Stream, 64), + heartbeats: make(map[uint64]*Stream), + shutdownCh: make(chan struct{}), + serialization: serialization, + address: conn.RemoteAddr().String(), + } + + go channel.recv() + + go channel.send() + + return channel +} diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index c096619c..e28dec22 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -22,9 +22,10 @@ var ( defaultKeepaliveInterval = 1000 * time.Millisecond defaultConnectRetryInterval = 60 * time.Second defaultErrorCountThreshold = 10 - ErrChannelShutdown = fmt.Errorf("The channel has been shutdown") - ErrSendRequestTimeout = fmt.Errorf("Timeout err: send request timeout") - ErrRecvRequestTimeout = fmt.Errorf("Timeout err: receive request timeout") + ErrChannelShutdown = fmt.Errorf("the channel has been shutdown") + ErrSendRequestTimeout = fmt.Errorf("timeout err: send request timeout") + ErrRecvRequestTimeout = fmt.Errorf("timeout err: receive request timeout") + ErrUnsupportedMessage = fmt.Errorf("stream notify : Unsupported message") defaultAsyncResponse = &motan.MotanResponse{Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), RPCContext: &motan.RPCContext{AsyncCall: true}} @@ -34,7 +35,7 @@ var ( type MotanEndpoint struct { url *motan.URL lock sync.Mutex - channels *ChannelPool + channels *V2ChannelPool destroyed bool destroyCh chan struct{} available bool @@ -46,6 +47,7 @@ type MotanEndpoint struct { minRequestTimeoutMillisecond int64 maxRequestTimeoutMillisecond int64 clientConnection int + maxContentLength int // for heartbeat requestID keepaliveID uint64 @@ -75,10 +77,12 @@ func (m *MotanEndpoint) Initialize() { m.minRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MinTimeOutKey) m.maxRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MaxTimeOutKey) m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) + m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) factory := func() (net.Conn, error) { return net.DialTimeout("tcp", m.url.GetAddressStr(), connectTimeout) } - channels, err := NewChannelPool(m.clientConnection, factory, nil, m.serialization) + config := &ChannelConfig{MaxContentLength: m.maxContentLength} + channels, err := NewV2ChannelPool(m.clientConnection, factory, config, m.serialization) if err != nil { vlog.Errorf("Channel pool init failed. url: %v, err:%s", m.url, err.Error()) // retry connect @@ -90,7 +94,7 @@ func (m *MotanEndpoint) Initialize() { for { select { case <-ticker.C: - channels, err := NewChannelPool(m.clientConnection, factory, nil, m.serialization) + channels, err := NewV2ChannelPool(m.clientConnection, factory, config, m.serialization) if err == nil { m.channels = channels m.setAvailable(true) @@ -309,27 +313,28 @@ func (m *MotanEndpoint) IsAvailable() bool { return m.available } -// Config : Config -type Config struct { - RequestTimeout time.Duration +// ChannelConfig : ChannelConfig +type ChannelConfig struct { + MaxContentLength int + Serialization motan.Serialization } -func DefaultConfig() *Config { - return &Config{ - RequestTimeout: defaultRequestTimeout, +func DefaultConfig() *ChannelConfig { + return &ChannelConfig{ + MaxContentLength: motan.DefaultMaxContentLength, } } -func VerifyConfig(config *Config) error { - if config.RequestTimeout <= 0 { - return fmt.Errorf("RequestTimeout interval must be positive") +func VerifyConfig(config *ChannelConfig) error { + if config.MaxContentLength <= 0 { + return fmt.Errorf("MaxContentLength of ChannelConfig must be positive") } return nil } -type Channel struct { +type V2Channel struct { // config - config *Config + config *ChannelConfig serialization motan.Serialization address string @@ -341,10 +346,10 @@ type Channel struct { sendCh chan sendReady // stream - streams map[uint64]*Stream + streams map[uint64]*V2Stream streamLock sync.Mutex // heartbeat - heartbeats map[uint64]*Stream + heartbeats map[uint64]*V2Stream heartbeatLock sync.Mutex // shutdown @@ -354,8 +359,8 @@ type Channel struct { shutdownLock sync.Mutex } -type Stream struct { - channel *Channel +type V2Stream struct { + channel *V2Channel sendMsg *mpro.Message // recv msg recvMsg *mpro.Message @@ -368,7 +373,7 @@ type Stream struct { isHeartBeat bool } -func (s *Stream) Send() error { +func (s *V2Stream) Send() error { timer := time.NewTimer(s.deadline.Sub(time.Now())) defer timer.Stop() @@ -395,7 +400,7 @@ func (s *Stream) Send() error { } // Recv sync recv -func (s *Stream) Recv() (*mpro.Message, error) { +func (s *V2Stream) Recv() (*mpro.Message, error) { defer func() { s.Close() }() @@ -415,7 +420,7 @@ func (s *Stream) Recv() (*mpro.Message, error) { } } -func (s *Stream) notify(msg *mpro.Message, t time.Time) { +func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { defer func() { s.Close() }() @@ -451,18 +456,18 @@ func (s *Stream) notify(msg *mpro.Message, t time.Time) { s.recvNotifyCh <- struct{}{} } -func (s *Stream) SetDeadline(deadline time.Duration) { +func (s *V2Stream) SetDeadline(deadline time.Duration) { s.deadline = time.Now().Add(deadline) } -func (c *Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*Stream, error) { +func (c *V2Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Stream, error) { if msg == nil || msg.Header == nil { return nil, errors.New("msg is invalid") } if c.IsClosed() { return nil, ErrChannelShutdown } - s := &Stream{ + s := &V2Stream{ channel: c, sendMsg: msg, recvNotifyCh: make(chan struct{}, 1), @@ -485,7 +490,7 @@ func (c *Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*Stream, e return s, nil } -func (s *Stream) Close() { +func (s *V2Stream) Close() { if !s.isClose.Load().(bool) { if s.isHeartBeat { s.channel.heartbeatLock.Lock() @@ -504,7 +509,7 @@ type sendReady struct { data []byte } -func (c *Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RPCContext) (*mpro.Message, error) { +func (c *V2Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RPCContext) (*mpro.Message, error) { stream, err := c.NewStream(msg, rc) if err != nil { return nil, err @@ -519,11 +524,11 @@ func (c *Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RPCC return stream.Recv() } -func (c *Channel) IsClosed() bool { +func (c *V2Channel) IsClosed() bool { return c.shutdown } -func (c *Channel) recv() { +func (c *V2Channel) recv() { defer motan.HandlePanic(func() { c.closeOnErr(errPanic) }) @@ -532,9 +537,9 @@ func (c *Channel) recv() { } } -func (c *Channel) recvLoop() error { +func (c *V2Channel) recvLoop() error { for { - res, t, err := mpro.DecodeWithTime(c.bufRead) + res, t, err := mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) if err != nil { return err } @@ -551,7 +556,7 @@ func (c *Channel) recvLoop() error { } } -func (c *Channel) send() { +func (c *V2Channel) send() { defer motan.HandlePanic(func() { c.closeOnErr(errPanic) }) @@ -578,7 +583,7 @@ func (c *Channel) send() { } } -func (c *Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { +func (c *V2Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { c.heartbeatLock.Lock() stream := c.heartbeats[msg.Header.RequestID] c.heartbeatLock.Unlock() @@ -590,7 +595,7 @@ func (c *Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { return nil } -func (c *Channel) handleMessage(msg *mpro.Message, t time.Time) error { +func (c *V2Channel) handleMessage(msg *mpro.Message, t time.Time) error { c.streamLock.Lock() stream := c.streams[msg.Header.RequestID] c.streamLock.Unlock() @@ -602,7 +607,7 @@ func (c *Channel) handleMessage(msg *mpro.Message, t time.Time) error { return nil } -func (c *Channel) closeOnErr(err error) { +func (c *V2Channel) closeOnErr(err error) { c.shutdownLock.Lock() if c.shutdownErr == nil { c.shutdownErr = err @@ -615,7 +620,7 @@ func (c *Channel) closeOnErr(err error) { } } -func (c *Channel) Close() error { +func (c *V2Channel) Close() error { c.shutdownLock.Lock() defer c.shutdownLock.Unlock() if c.shutdown { @@ -629,20 +634,20 @@ func (c *Channel) Close() error { type ConnFactory func() (net.Conn, error) -type ChannelPool struct { - channels chan *Channel +type V2ChannelPool struct { + channels chan *V2Channel channelsLock sync.Mutex factory ConnFactory - config *Config + config *ChannelConfig serialization motan.Serialization } -func (c *ChannelPool) getChannels() chan *Channel { +func (c *V2ChannelPool) getChannels() chan *V2Channel { channels := c.channels return channels } -func (c *ChannelPool) Get() (*Channel, error) { +func (c *V2ChannelPool) Get() (*V2Channel, error) { channels := c.getChannels() if channels == nil { return nil, errors.New("channels is nil") @@ -653,9 +658,9 @@ func (c *ChannelPool) Get() (*Channel, error) { if err != nil { vlog.Errorf("create channel failed. err:%s", err.Error()) } - channel = buildChannel(conn, c.config, c.serialization) + channel = buildV2Channel(conn, c.config, c.serialization) } - if err := retChannelPool(channels, channel); err != nil && channel != nil { + if err := retV2ChannelPool(channels, channel); err != nil && channel != nil { channel.closeOnErr(err) } if channel == nil { @@ -664,7 +669,7 @@ func (c *ChannelPool) Get() (*Channel, error) { return channel, nil } -func retChannelPool(channels chan *Channel, channel *Channel) (error error) { +func retV2ChannelPool(channels chan *V2Channel, channel *V2Channel) (error error) { defer func() { if err := recover(); err != nil { error = errors.New("ChannelPool has been closed") @@ -677,7 +682,7 @@ func retChannelPool(channels chan *Channel, channel *Channel) (error error) { return nil } -func (c *ChannelPool) Close() error { +func (c *V2ChannelPool) Close() error { c.channelsLock.Lock() // to prevent channels closed many times channels := c.channels c.channels = nil @@ -696,12 +701,12 @@ func (c *ChannelPool) Close() error { return nil } -func NewChannelPool(poolCap int, factory ConnFactory, config *Config, serialization motan.Serialization) (*ChannelPool, error) { +func NewV2ChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, serialization motan.Serialization) (*V2ChannelPool, error) { if poolCap <= 0 { return nil, errors.New("invalid capacity settings") } - channelPool := &ChannelPool{ - channels: make(chan *Channel, poolCap), + channelPool := &V2ChannelPool{ + channels: make(chan *V2Channel, poolCap), factory: factory, config: config, serialization: serialization, @@ -713,12 +718,12 @@ func NewChannelPool(poolCap int, factory ConnFactory, config *Config, serializat return nil, err } _ = conn.(*net.TCPConn).SetNoDelay(true) - channelPool.channels <- buildChannel(conn, config, serialization) + channelPool.channels <- buildV2Channel(conn, config, serialization) } return channelPool, nil } -func buildChannel(conn net.Conn, config *Config, serialization motan.Serialization) *Channel { +func buildV2Channel(conn net.Conn, config *ChannelConfig, serialization motan.Serialization) *V2Channel { if conn == nil { return nil } @@ -726,15 +731,16 @@ func buildChannel(conn net.Conn, config *Config, serialization motan.Serializati config = DefaultConfig() } if err := VerifyConfig(config); err != nil { + vlog.Errorf("can not build Channel, ChannelConfig check fail. err:%v", err) return nil } - channel := &Channel{ + channel := &V2Channel{ conn: conn, config: config, bufRead: bufio.NewReader(conn), sendCh: make(chan sendReady, 256), - streams: make(map[uint64]*Stream, 64), - heartbeats: make(map[uint64]*Stream), + streams: make(map[uint64]*V2Stream, 64), + heartbeats: make(map[uint64]*V2Stream), shutdownCh: make(chan struct{}), serialization: serialization, address: conn.RemoteAddr().String(), diff --git a/ha/failoverHA.go b/ha/failoverHA.go index 2d742c66..1701cf50 100644 --- a/ha/failoverHA.go +++ b/ha/failoverHA.go @@ -27,6 +27,7 @@ func (f *FailOverHA) SetURL(url *motan.URL) { func (f *FailOverHA) Call(request motan.Request, loadBalance motan.LoadBalance) motan.Response { retries := f.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), motan.RetriesKey, defaultRetries) var lastErr *motan.Exception + var response motan.Response for i := 0; i <= int(retries); i++ { ep := loadBalance.Select(request) if ep == nil { @@ -34,15 +35,23 @@ func (f *FailOverHA) Call(request motan.Request, loadBalance motan.LoadBalance) fmt.Sprintf("No refers for request, RequestID: %d, Request info: %+v", request.GetRequestID(), request.GetAttachments().RawMap())) } - response := ep.Call(request) - if response.GetException() == nil || response.GetException().ErrType == motan.BizException { - return response + response = ep.Call(request) + if response != nil { + if response.GetException() == nil || response.GetException().ErrType == motan.BizException { + return response + } + lastErr = response.GetException() } - lastErr = response.GetException() vlog.Warningf("FailOverHA call fail! url:%s, err:%+v", ep.GetURL().GetIdentity(), lastErr) } - errorResponse := getErrorResponse(request.GetRequestID(), fmt.Sprintf("FailOverHA call fail %d times. Exception: %s", retries+1, lastErr.ErrMsg)) - errorResponse.Exception.ErrCode = lastErr.ErrCode + if response != nil { // last response + return response + } + var errMsg string + if lastErr != nil { + errMsg = lastErr.ErrMsg + } + errorResponse := getErrorResponseWithCode(request.GetRequestID(), 500, fmt.Sprintf("FailOverHA call fail %d times. Exception: %s", retries+1, errMsg)) return errorResponse } diff --git a/protocol/motan1Protocol.go b/protocol/motan1Protocol.go new file mode 100644 index 00000000..de255584 --- /dev/null +++ b/protocol/motan1Protocol.go @@ -0,0 +1,588 @@ +package protocol + +import ( + "bufio" + "encoding/binary" + "errors" + motan "github.com/weibocom/motan-go/core" + vlog "github.com/weibocom/motan-go/log" + "io" + "time" +) + +const ( + InnerMotanMagic = 0xf0f0 + V1L1HeaderLength = 16 // first level header length + V1AllHeaderLength = 32 // two levels header length +) + +const ( + versionV1 = 0x01 + versionV1Compress = 0x02 // v1 压缩版本，已逐步废弃 +) + +// object stream type +const ( + OS_MAGIC uint16 = 0xaced + OS_VERSION uint16 = 0x0005 + GZIP_MAGIC = 0x1f8b + + TC_ARRAY = 0x75 + TC_CLASSDESC = 0x72 + TC_ENDBLOCKDATA = 0x78 + TC_NULL = 0x70 + TC_REFERENCE = 0x71 + TC_BLOCKDATA = 0x77 + TC_BLOCKDATALONG = 0x7A +) + +// hessian2 const +const ( + BC_STRING_DIRECT = 0x00 + STRING_DIRECT_MAX = 0x1f + BC_STRING_SHORT = 0x30 + STRING_SHORT_MAX = 0x3ff +) + +// v1 flags +const ( + FLAG_REQUEST = 0x00 + FLAG_RESPONSE = 0x01 + FLAG_RESPONSE_VOID = 0x03 + FLAG_RESPONSE_EXCEPTION = 0x05 + FLAG_RESPONSE_ATTACHMENT = 0x07 + FLAG_OTHER = 0xFF +) + +const ( + HEARTBEAT_INTERFACE_NAME = "com.weibo.api.motan.rpc.heartbeat" + HEARTBEAT_METHOD_NAME = "heartbeat" + HEARTBEAT_RESPONSE_STRING = HEARTBEAT_METHOD_NAME +) + +// base binary arrays +var ( + baseV1HeartbeatReq = []byte{241, 241, 0, 0, 24, 44, 223, 176, 126, 80, 0, 96, 0, 0, 0, 78, 240, 240, 1, 0, 24, 44, 223, 176, 126, 80, 0, 96, 0, 0, 0, 62, 172, 237, 0, 5, 119, 56, 0, 33, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 114, 112, 99, 46, 104, 101, 97, 114, 116, 98, 101, 97, 116, 0, 9, 104, 101, 97, 114, 116, 98, 101, 97, 116, 0, 4, 118, 111, 105, 100, 0, 0, 0, 0} + baseV1HeartbeatRes = []byte{241, 241, 0, 1, 24, 44, 226, 160, 201, 25, 2, 108, 0, 0, 0, 81, 240, 240, 1, 1, 24, 44, 226, 160, 201, 25, 2, 108, 0, 0, 0, 65, 172, 237, 0, 5, 119, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 10, 9, 104, 101, 97, 114, 116, 98, 101, 97, 116} + hessian2HeartbeatBytes = []byte{9, 104, 101, 97, 114, 116, 98, 101, 97, 116} + baseV1ExceptionRes = []byte{241, 241, 0, 1, 24, 46, 120, 216, 224, 128, 0, 1, 0, 0, 1, 70, 240, 240, 1, 5, 24, 46, 120, 216, 224, 128, 0, 1, 0, 0, 1, 54} + baseV1ResObjectStream = []byte{172, 237, 0, 5, 119, 61, 0, 0, 0, 0, 0, 0, 0, 2, 0, 51, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 101, 120, 99, 101, 112, 116, 105, 111, 110, 46, 77, 111, 116, 97, 110, 83, 101, 114, 118, 105, 99, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112} + h_exceptionDesc = []byte{67, 48, 51, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 101, 120, 99, 101, 112, 116, 105, 111, 110, 46, 77, 111, 116, 97, 110, 83, 101, 114, 118, 105, 99, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 150, 8, 101, 114, 114, 111, 114, 77, 115, 103, 13, 100, 101, 116, 97, 105, 108, 77, 101, 115, 115, 97, 103, 101, 5, 99, 97, 117, 115, 101, 13, 109, 111, 116, 97, 110, 69, 114, 114, 111, 114, 77, 115, 103, 10, 115, 116, 97, 99, 107, 84, 114, 97, 99, 101, 20, 115, 117, 112, 112, 114, 101, 115, 115, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 115, 96} + h_motanErrorMsgDesc = []byte{67, 48, 43, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 101, 120, 99, 101, 112, 116, 105, 111, 110, 46, 77, 111, 116, 97, 110, 69, 114, 114, 111, 114, 77, 115, 103, 147, 6, 115, 116, 97, 116, 117, 115, 9, 101, 114, 114, 111, 114, 99, 111, 100, 101, 7, 109, 101, 115, 115, 97, 103, 101, 97} +) + +var ( + ErrWrongMotanVersion = errors.New("unsupported motan version") + ErrOSVersion = errors.New("object stream magic number or version not correct") + ErrUnsupported = errors.New("unsupported object stream type") + ErrNotHasV1Msg = errors.New("not has MotanV1Message") +) + +type MotanV1Message struct { + OriginBytes []byte + V1InnerVersion byte + Flag byte + Rid uint64 + InnerLength int +} + +func DecodeMotanV1Request(msg *MotanV1Message) (motan.Request, error) { + objStream, err := newObjectStream(msg) + if err != nil { + return nil, err + } + err = objStream.parseReq() + if err != nil { + return nil, err + } + request := &motan.MotanRequest{ + RequestID: msg.Rid, + ServiceName: objStream.service, + Method: objStream.method, + MethodDesc: objStream.paramDesc, + Attachment: objStream.attachment} + request.GetRPCContext(true).OriginalMessage = msg + return request, nil +} + +func EncodeMotanV1Request(req motan.Request, sendId uint64) ([]byte, error) { + if msg, ok := req.GetRPCContext(true).OriginalMessage.(*MotanV1Message); ok { + writeV1Rid(msg.OriginBytes, sendId) // replace rid with sendId for send + return msg.OriginBytes, nil + } + return nil, ErrNotHasV1Msg +} + +func DecodeMotanV1Response(msg *MotanV1Message) (motan.Response, error) { + objStream, err := newObjectStream(msg) + if err != nil { + return nil, err + } + err = objStream.parseRes() + if err != nil { + return nil, err + } + response := &motan.MotanResponse{ + RequestID: msg.Rid, + ProcessTime: objStream.processTime, + Value: objStream.value, + Attachment: objStream.attachment} + if objStream.hasException { + if objStream.cName == "com.weibo.api.motan.exception.MotanBizException" { // biz exception + response.Exception = &motan.Exception{ErrCode: 500, ErrMsg: "v1: has biz exception", ErrType: motan.BizException} + } else { + response.Exception = &motan.Exception{ErrCode: 500, ErrMsg: "v1: has exception, class:" + objStream.cName, ErrType: motan.ServiceException} + } + } + response.GetRPCContext(true).OriginalMessage = msg + return response, nil +} + +func EncodeMotanV1Response(res motan.Response) ([]byte, error) { + if msg, ok := res.GetRPCContext(true).OriginalMessage.(*MotanV1Message); ok { + writeV1Rid(msg.OriginBytes, res.GetRequestID()) //replace sendId with rid + return msg.OriginBytes, nil + } else if res.GetException() != nil { + return BuildV1ExceptionResponse(res.GetRequestID(), "build v1 exception res. org err:"+res.GetException().ErrMsg), nil + } + return nil, ErrNotHasV1Msg +} + +func ReadV1Message(buf *bufio.Reader, maxContentLength int) (*MotanV1Message, time.Time, error) { + temp, err := buf.Peek(V1L1HeaderLength) + start := time.Now() // record time when starting to read data + if err != nil { + return nil, start, err + } + length := V1L1HeaderLength + int(binary.BigEndian.Uint32(temp[12:])) + if length < V1AllHeaderLength || length > V1L1HeaderLength+maxContentLength { + vlog.Errorf("content length over the limit. size:%d", length-V1L1HeaderLength) + return nil, start, ErrOverSize + } + ori := make([]byte, length, length) + _, err = io.ReadAtLeast(buf, ori, length) + if err != nil { + return nil, start, err + } + mn := binary.BigEndian.Uint16(ori[16:18]) + if mn != InnerMotanMagic { + vlog.Errorf("wrong v1 inner magic num:%d", mn) + return nil, start, ErrMagicNum + } + v1InnerVersion := ori[18] + flag := ori[19] + rid := binary.BigEndian.Uint64(ori[20:28]) + innerLength := int(binary.BigEndian.Uint32(ori[28:32])) + if innerLength+V1AllHeaderLength != length { + vlog.Errorf("inner content length not correct. size:%d, inner length:%d", length-V1L1HeaderLength, innerLength) + return nil, start, ErrWrongSize + } + msg := &MotanV1Message{OriginBytes: ori, V1InnerVersion: v1InnerVersion, + Flag: flag, Rid: rid, InnerLength: innerLength} + return msg, start, nil +} + +func IsV1HeartbeatReq(req motan.Request) bool { + if req != nil && req.GetServiceName() == HEARTBEAT_INTERFACE_NAME && + req.GetMethod() == HEARTBEAT_METHOD_NAME { + return true + } + return false +} + +func IsV1HeartbeatRes(res motan.Response) bool { + if res != nil { + if str, ok := res.GetValue().(string); ok { + return str == HEARTBEAT_RESPONSE_STRING + } + } + return false +} + +func BuildV1HeartbeatReq(rid uint64) []byte { + bytes := make([]byte, len(baseV1HeartbeatReq)) + copy(bytes, baseV1HeartbeatReq) + writeV1Rid(bytes, rid) + return bytes +} + +func BuildV1HeartbeatRes(rid uint64) []byte { + bytes := make([]byte, len(baseV1HeartbeatRes)) + copy(bytes, baseV1HeartbeatRes) + writeV1Rid(bytes, rid) + return bytes +} + +func BuildV1ExceptionResponse(rid uint64, errMsg string) []byte { + var result []byte + var byteArrayLengthPos int + if errMsg != "" { + buf := motan.NewBytesBuffer(350 + len(errMsg)) + // write v1 header + buf.Write(baseV1ExceptionRes) + writeV1Rid(buf.Bytes(), rid) + + // write object stream + buf.Write(baseV1ResObjectStream) + byteArrayLengthPos = buf.GetWPos() + buf.SetWPos(byteArrayLengthPos + 4) // skip byte array length + + // write hessian bytes of exception + buf.Write(h_exceptionDesc) + buf.Write([]byte("NNN")) // (three) null fields + buf.Write(h_motanErrorMsgDesc) + buf.Write([]byte{201, 247, 212, 39, 17}) // status && error code + writeHessianString(errMsg, buf) + buf.Write([]byte("NN")) // null fields + wpos := buf.GetWPos() + + // set byte array length + length := wpos - byteArrayLengthPos - 4 + buf.SetWPos(byteArrayLengthPos) + buf.WriteUint32(uint32(length)) + + // set l2 header length + length = wpos - 32 + buf.SetWPos(28) // l2 header length pos + buf.WriteUint32(uint32(length)) + + // set l1 header length + length = wpos - 16 + buf.SetWPos(12) // l1 header length pos + buf.WriteUint32(uint32(length)) + + buf.SetWPos(wpos) + return buf.Bytes() + } + return result +} + +func writeV1Rid(v1Msg []byte, rid uint64) { + if len(v1Msg) >= V1AllHeaderLength { + index := 4 // l1 header rid + binary.BigEndian.PutUint64(v1Msg[index:index+8], rid) + index = 20 // l2 header rid + binary.BigEndian.PutUint64(v1Msg[index:index+8], rid) + } +} + +type simpleObjectStream struct { + bytes []byte + pos int + len int + inBlock bool + blockRemain int + flag byte + parsed bool + maxContentLength int + v1InnerVersion byte + + // for request + service string + method string + paramDesc string + argSize int + attachment *motan.StringMap + + // for response + processTime int64 + hasException bool + cName string + value interface{} +} + +func newObjectStream(msg *MotanV1Message) (*simpleObjectStream, error) { + if msg.V1InnerVersion == 0x02 && len(msg.OriginBytes) > 34 && + binary.BigEndian.Uint16(msg.OriginBytes[32:34]) == GZIP_MAGIC { // motan v1compress协议版本 + decodedBytes, err := DecodeGzip(msg.OriginBytes[32:]) + if err != nil { + return nil, err + } + return &simpleObjectStream{bytes: decodedBytes, flag: msg.Flag, len: len(decodedBytes), v1InnerVersion: msg.V1InnerVersion}, nil + } + return &simpleObjectStream{bytes: msg.OriginBytes[V1AllHeaderLength:], flag: msg.Flag, len: len(msg.OriginBytes) - V1AllHeaderLength, v1InnerVersion: msg.V1InnerVersion}, nil +} + +func (s *simpleObjectStream) parseReq() error { + err := s.checkObjectStream() + if err != nil { + return err + } + blockStartPos := s.pos + // service infos + infos := make([]string, 3) + for i := 0; i < 3; i++ { + infos[i], err = s.readUtf() + if err != nil { + return err + } + if i == 0 && infos[0] == "1" { // v1 compress method info + var methodSign string + methodSign, err = s.readUtf() + if err != nil { + return err + } + infos[0] = "v1compressMethodSign" + infos[1] = methodSign + infos[2] = "" + break + } + } + s.service = infos[0] + s.method = infos[1] + s.paramDesc = infos[2] + s.blockRemain -= s.pos - blockStartPos + // arguments. + if s.blockRemain == 0 { //skip arguments bytes + s.inBlock = false // not block mode + checkNext := true + var t byte + for checkNext { + t, err = s.peekType() + if err != nil { + return err + } + switch t { + case TC_NULL: + s.pos++ + case TC_ARRAY: + _, err = s.skipByteArray(false) + case TC_BLOCKDATA, TC_BLOCKDATALONG: // arguments end + err = s.setBlock() + checkNext = false + default: + vlog.Errorf("unsupported object stream type, type:%d", t) + return ErrUnsupported + } + if err != nil { + return err + } + } + } + err = s.readAttachments() + if err != nil { + return err + } + s.parsed = true + return nil +} + +func (s *simpleObjectStream) parseRes() error { + err := s.checkObjectStream() + if err != nil { + return err + } + + s.processTime, err = s.readLong() + if err != nil { + return err + } + switch s.flag { + case FLAG_RESPONSE_EXCEPTION: + s.hasException = true + s.cName, _ = s.readUtf() // parse exception class name + case FLAG_RESPONSE: + s.cName, err = s.readUtf() + if err == nil && s.cName == "java.lang.String" { + var c []byte + c, err = s.skipByteArray(true) + if err != nil { + vlog.Warningf("parse string response value fail. err:%v", err) + } else if len(c) == len(hessian2HeartbeatBytes) { + //check hessian2 heartbeat string + isHeartbeat := true + for i := 0; i < len(hessian2HeartbeatBytes); i++ { + if c[i] != hessian2HeartbeatBytes[i] { + isHeartbeat = false + break + } + } + if isHeartbeat { + s.value = HEARTBEAT_RESPONSE_STRING + } + } + } + default: // No further parsing required + break + } + s.parsed = true + return nil +} + +func (s *simpleObjectStream) checkObjectStream() error { + if s.remain() < 4 { + vlog.Errorf("length is wrong v1 request type, flag:%d", s.flag) + return ErrOSVersion + } + // check stream magic number and version + if binary.BigEndian.Uint16(s.bytes[:2]) != OS_MAGIC || binary.BigEndian.Uint16(s.bytes[2:4]) != OS_VERSION { + vlog.Errorf("wrong v1 request type, flag:%d", s.flag) + return ErrOSVersion + } + s.pos += 4 + return s.setBlock() // init with block mode +} + +func (s *simpleObjectStream) readUtf() (string, error) { + l, err := s.readInt16() + if err != nil { + return "", err + } + var str string + if l > 0 { + if s.remain() < int(l) { + return str, io.EOF + } + str = string(s.bytes[s.pos : s.pos+int(l)]) + s.pos += int(l) + } + return str, nil +} + +func (s *simpleObjectStream) readInt16() (int16, error) { + if s.remain() < 2 { + return 0, io.EOF + } + i := int16(binary.BigEndian.Uint16(s.bytes[s.pos : s.pos+2])) + s.pos += 2 + return i, nil +} + +func (s *simpleObjectStream) readInt() (int, error) { + if s.remain() < 4 { + return 0, io.EOF + } + i := int(binary.BigEndian.Uint32(s.bytes[s.pos : s.pos+4])) + s.pos += 4 + return i, nil +} + +func (s *simpleObjectStream) readLong() (int64, error) { + if s.remain() < 8 { + return 0, io.EOF + } + i := int64(binary.BigEndian.Uint64(s.bytes[s.pos : s.pos+8])) + s.pos += 8 + return i, nil +} + +func (s *simpleObjectStream) readAttachments() (err error) { + // attachments. already in block mode + var size int + if s.v1InnerVersion == versionV1Compress { + var size16 int16 + size16, err = s.readInt16() + size = int(size16) + } else { + size, err = s.readInt() + } + if err != nil { + vlog.Errorf("read v1 attachment size fail. err:%v", err) + return err + } + attachments := motan.NewStringMap(DefaultMetaSize) + if size > 0 { // has attachments + for i := 0; i < size; i++ { + var k, v string + k, err = s.readUtf() + if err != nil { + vlog.Errorf("read v1 attachment key fail. err:%v", err) + return err + } + v, err = s.readUtf() + if err != nil { + vlog.Errorf("read v1 attachment value fail. err:%v", err) + return err + } + attachments.Store(k, v) + } + } + s.attachment = attachments + return nil +} + +// skip byte array(serialized object) +// the return content bytes will be nil if array length is 0, so check len(bytes) before use it +func (s *simpleObjectStream) skipByteArray(withContent bool) ([]byte, error) { + if s.remain() < 10 { + return nil, io.EOF + } + t := s.bytes[s.pos+1] // inner type + switch t { + case TC_CLASSDESC: + if s.remain() < 23 { // class desc (19) + array length(4) + vlog.Errorf("object stream: not enough bytes for parse TC_CLASSDESC, need size > 19, remain:%d", s.remain()) + return nil, io.EOF + } + s.pos += 19 // skip TC_CLASSDESC + case TC_REFERENCE: + s.pos += 6 // skip TC_REFERENCE + default: + vlog.Errorf("unsupported object stream type, type:%d", t) + return nil, ErrUnsupported + } + length, err := s.readInt() + if err != nil { + return nil, err + } + var content []byte + if length > 0 { + if withContent { + content = make([]byte, length, length) + copy(content, s.bytes[s.pos:s.pos+length]) + } + s.pos += length + } + return content, nil +} + +func (s *simpleObjectStream) setBlock() error { + if s.remain() < 2 { + return io.EOF + } + t := s.bytes[s.pos] + s.pos++ + switch t { + case TC_BLOCKDATA: + s.blockRemain = int(s.bytes[s.pos]) + s.pos++ + case TC_BLOCKDATALONG: + if s.remain() < 4 { + return io.EOF + } + s.blockRemain = int(binary.BigEndian.Uint32(s.bytes[s.pos : s.pos+4])) + s.pos += 4 + default: + vlog.Errorf("unsupported object stream type, type:%d", t) + return ErrUnsupported + } + s.inBlock = true + return nil +} + +func (s *simpleObjectStream) peekType() (byte, error) { + if s.remain() < 1 { + return 0, io.EOF + } + return s.bytes[s.pos], nil +} + +func (s *simpleObjectStream) remain() int { + return s.len - s.pos +} + +func writeHessianString(str string, buf *motan.BytesBuffer) { + // write length + length := len(str) + if length <= STRING_DIRECT_MAX { + buf.WriteByte(byte(BC_STRING_DIRECT + len(str))) + } else if length <= STRING_SHORT_MAX { + buf.WriteByte(byte(BC_STRING_SHORT + (length >> 8))) + buf.WriteByte(byte(length)) + } else { + buf.WriteByte('S') + buf.WriteByte(byte(length >> 8)) + buf.WriteByte(byte(length)) + } + buf.Write([]byte(str)) +} diff --git a/protocol/motan1Protocol_test.go b/protocol/motan1Protocol_test.go new file mode 100644 index 00000000..09a95a30 --- /dev/null +++ b/protocol/motan1Protocol_test.go @@ -0,0 +1,264 @@ +package protocol + +import ( + "bufio" + "bytes" + motan "github.com/weibocom/motan-go/core" + "testing" +) + +var ( + // req + testV1ReqBytes = []byte{241, 241, 0, 0, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 246, 240, 240, 1, 0, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 230, 172, 237, 0, 5, 119, 72, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 5, 104, 101, 108, 108, 111, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 119, 120, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1NoParamReqBytes = []byte{241, 241, 0, 0, 24, 50, 5, 245, 221, 16, 0, 2, 0, 0, 0, 204, 240, 240, 1, 0, 24, 50, 5, 245, 221, 16, 0, 2, 0, 0, 0, 188, 172, 237, 0, 5, 119, 182, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 7, 110, 111, 80, 97, 114, 97, 109, 0, 4, 118, 111, 105, 100, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1MultiParamReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 143, 123, 0, 0, 1, 0, 0, 1, 98, 240, 240, 1, 0, 24, 50, 6, 143, 123, 0, 0, 1, 0, 0, 1, 82, 172, 237, 0, 5, 119, 113, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 6, 114, 101, 110, 97, 109, 101, 0, 56, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 109, 111, 100, 101, 108, 46, 85, 115, 101, 114, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 59, 67, 48, 39, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 109, 111, 100, 101, 108, 46, 85, 115, 101, 114, 146, 2, 105, 100, 4, 110, 97, 109, 101, 96, 212, 8, 86, 3, 114, 97, 121, 117, 113, 0, 126, 0, 0, 0, 0, 0, 5, 4, 114, 97, 121, 48, 119, 120, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1CompressReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 224, 240, 240, 2, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 208, 172, 237, 0, 5, 119, 21, 0, 1, 49, 0, 16, 104, 101, 108, 108, 56, 49, 55, 102, 102, 51, 55, 51, 51, 50, 54, 57, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 119, 149, 0, 7, 0, 2, 95, 65, 0, 4, 52, 57, 57, 54, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 14, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 0, 3, 49, 57, 56, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1CompressGzipReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 202, 240, 240, 2, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 186, 31, 139, 8, 0, 0, 0, 0, 0, 0, 0, 91, 243, 150, 129, 181, 92, 148, 129, 209, 144, 65, 32, 35, 53, 39, 199, 194, 208, 60, 45, 205, 216, 220, 216, 216, 200, 204, 178, 180, 136, 129, 41, 218, 105, 205, 103, 241, 31, 108, 28, 33, 15, 152, 24, 24, 42, 10, 24, 24, 24, 216, 217, 114, 243, 75, 18, 243, 12, 202, 167, 50, 176, 51, 48, 197, 59, 50, 176, 152, 88, 90, 154, 49, 112, 39, 22, 20, 228, 100, 38, 39, 150, 100, 230, 231, 49, 112, 231, 86, 250, 130, 20, 185, 164, 230, 230, 51, 112, 39, 231, 100, 166, 230, 149, 184, 23, 229, 151, 22, 48, 240, 129, 53, 235, 166, 0, 37, 116, 139, 10, 146, 25, 128, 134, 165, 148, 230, 164, 98, 136, 243, 57, 231, 231, 149, 0, 117, 233, 250, 164, 230, 165, 151, 100, 48, 48, 27, 153, 24, 48, 176, 151, 165, 22, 21, 131, 204, 103, 54, 212, 51, 96, 96, 77, 199, 102, 34, 0, 179, 189, 51, 70, 208, 0, 0, 0} + + // res + testV1ResBytes = []byte{241, 241, 0, 1, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 85, 240, 240, 1, 1, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 69, 172, 237, 0, 5, 119, 26, 0, 0, 0, 0, 0, 0, 0, 1, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 14, 13, 72, 101, 108, 108, 111, 32, 109, 111, 116, 97, 110, 48, 33} + testV1VoidResBytes = []byte{241, 241, 0, 1, 24, 50, 6, 24, 127, 0, 0, 3, 0, 0, 0, 30, 240, 240, 1, 3, 24, 50, 6, 24, 127, 0, 0, 3, 0, 0, 0, 14, 172, 237, 0, 5, 119, 8, 0, 0, 0, 0, 0, 0, 0, 1} + testV1ExceptionResBytes = []byte{241, 241, 0, 1, 24, 50, 6, 67, 57, 48, 0, 4, 0, 0, 7, 184, 240, 240, 1, 5, 24, 50, 6, 67, 57, 48, 0, 4, 0, 0, 7, 168, 172, 237, 0, 5, 119, 57, 0, 0, 0, 0, 0, 0, 0, 24, 0, 47, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 101, 120, 99, 101, 112, 116, 105, 111, 110, 46, 77, 111, 116, 97, 110, 66, 105, 122, 69, 120, 99, 101, 112, 116, 105, 111, 110, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 7, 82, 67, 48, 47, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 101, 120, 99, 101, 112, 116, 105, 111, 110, 46, 77, 111, 116, 97, 110, 66, 105, 122, 69, 120, 99, 101, 112, 116, 105, 111, 110, 150, 8, 101, 114, 114, 111, 114, 77, 115, 103, 13, 100, 101, 116, 97, 105, 108, 77, 101, 115, 115, 97, 103, 101, 5, 99, 97, 117, 115, 101, 13, 109, 111, 116, 97, 110, 69, 114, 114, 111, 114, 77, 115, 103, 10, 115, 116, 97, 99, 107, 84, 114, 97, 99, 101, 20, 115, 117, 112, 112, 114, 101, 115, 115, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 115, 96, 27, 112, 114, 111, 118, 105, 100, 101, 114, 32, 99, 97, 108, 108, 32, 112, 114, 111, 99, 101, 115, 115, 32, 101, 114, 114, 111, 114, 27, 112, 114, 111, 118, 105, 100, 101, 114, 32, 99, 97, 108, 108, 32, 112, 114, 111, 99, 101, 115, 115, 32, 101, 114, 114, 111, 114, 67, 26, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 82, 117, 110, 116, 105, 109, 101, 69, 120, 99, 101, 112, 116, 105, 111, 110, 148, 13, 100, 101, 116, 97, 105, 108, 77, 101, 115, 115, 97, 103, 101, 5, 99, 97, 117, 115, 101, 10, 115, 116, 97, 99, 107, 84, 114, 97, 99, 101, 20, 115, 117, 112, 112, 114, 101, 115, 115, 101, 100, 69, 120, 99, 101, 112, 116, 105, 111, 110, 115, 97, 14, 106, 117, 115, 116, 32, 101, 120, 99, 101, 112, 116, 105, 111, 110, 81, 145, 113, 28, 91, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 97, 99, 107, 84, 114, 97, 99, 101, 69, 108, 101, 109, 101, 110, 116, 67, 27, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 97, 99, 107, 84, 114, 97, 99, 101, 69, 108, 101, 109, 101, 110, 116, 148, 14, 100, 101, 99, 108, 97, 114, 105, 110, 103, 67, 108, 97, 115, 115, 10, 109, 101, 116, 104, 111, 100, 78, 97, 109, 101, 8, 102, 105, 108, 101, 78, 97, 109, 101, 10, 108, 105, 110, 101, 78, 117, 109, 98, 101, 114, 98, 11, 114, 101, 109, 111, 116, 101, 67, 108, 97, 115, 115, 12, 114, 101, 109, 111, 116, 101, 77, 101, 116, 104, 111, 100, 10, 114, 101, 109, 111, 116, 101, 70, 105, 108, 101, 145, 112, 48, 38, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 67, 111, 108, 108, 101, 99, 116, 105, 111, 110, 115, 36, 85, 110, 109, 111, 100, 105, 102, 105, 97, 98, 108, 101, 76, 105, 115, 116, 67, 48, 43, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 101, 120, 99, 101, 112, 116, 105, 111, 110, 46, 77, 111, 116, 97, 110, 69, 114, 114, 111, 114, 77, 115, 103, 147, 6, 115, 116, 97, 116, 117, 115, 9, 101, 114, 114, 111, 114, 99, 111, 100, 101, 7, 109, 101, 115, 115, 97, 103, 101, 99, 201, 247, 212, 117, 49, 14, 112, 114, 111, 118, 105, 100, 101, 114, 32, 101, 114, 114, 111, 114, 86, 144, 159, 98, 48, 39, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 114, 112, 99, 46, 68, 101, 102, 97, 117, 108, 116, 80, 114, 111, 118, 105, 100, 101, 114, 6, 105, 110, 118, 111, 107, 101, 20, 68, 101, 102, 97, 117, 108, 116, 80, 114, 111, 118, 105, 100, 101, 114, 46, 106, 97, 118, 97, 200, 68, 98, 48, 40, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 114, 112, 99, 46, 65, 98, 115, 116, 114, 97, 99, 116, 80, 114, 111, 118, 105, 100, 101, 114, 4, 99, 97, 108, 108, 21, 65, 98, 115, 116, 114, 97, 99, 116, 80, 114, 111, 118, 105, 100, 101, 114, 46, 106, 97, 118, 97, 200, 52, 98, 48, 42, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 102, 105, 108, 116, 101, 114, 46, 65, 99, 99, 101, 115, 115, 76, 111, 103, 70, 105, 108, 116, 101, 114, 6, 102, 105, 108, 116, 101, 114, 20, 65, 99, 99, 101, 115, 115, 76, 111, 103, 70, 105, 108, 116, 101, 114, 46, 106, 97, 118, 97, 200, 73, 98, 48, 62, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 112, 114, 111, 116, 111, 99, 111, 108, 46, 115, 117, 112, 112, 111, 114, 116, 46, 80, 114, 111, 116, 111, 99, 111, 108, 70, 105, 108, 116, 101, 114, 68, 101, 99, 111, 114, 97, 116, 111, 114, 36, 50, 4, 99, 97, 108, 108, 28, 80, 114, 111, 116, 111, 99, 111, 108, 70, 105, 108, 116, 101, 114, 68, 101, 99, 111, 114, 97, 116, 111, 114, 46, 106, 97, 118, 97, 200, 150, 98, 48, 54, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 68, 101, 102, 97, 117, 108, 116, 80, 114, 111, 116, 101, 99, 116, 101, 100, 83, 116, 114, 97, 116, 101, 103, 121, 4, 99, 97, 108, 108, 29, 68, 101, 102, 97, 117, 108, 116, 80, 114, 111, 116, 101, 99, 116, 101, 100, 83, 116, 114, 97, 116, 101, 103, 121, 46, 106, 97, 118, 97, 200, 82, 98, 48, 51, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 80, 114, 111, 118, 105, 100, 101, 114, 77, 101, 115, 115, 97, 103, 101, 82, 111, 117, 116, 101, 114, 4, 99, 97, 108, 108, 26, 80, 114, 111, 118, 105, 100, 101, 114, 77, 101, 115, 115, 97, 103, 101, 82, 111, 117, 116, 101, 114, 46, 106, 97, 118, 97, 200, 114, 98, 48, 51, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 80, 114, 111, 118, 105, 100, 101, 114, 77, 101, 115, 115, 97, 103, 101, 82, 111, 117, 116, 101, 114, 6, 104, 97, 110, 100, 108, 101, 26, 80, 114, 111, 118, 105, 100, 101, 114, 77, 101, 115, 115, 97, 103, 101, 82, 111, 117, 116, 101, 114, 46, 106, 97, 118, 97, 200, 106, 98, 48, 90, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 115, 117, 112, 112, 111, 114, 116, 46, 68, 101, 102, 97, 117, 108, 116, 82, 112, 99, 72, 101, 97, 114, 116, 98, 101, 97, 116, 70, 97, 99, 116, 111, 114, 121, 36, 72, 101, 97, 114, 116, 77, 101, 115, 115, 97, 103, 101, 72, 97, 110, 100, 108, 101, 87, 114, 97, 112, 112, 101, 114, 6, 104, 97, 110, 100, 108, 101, 31, 68, 101, 102, 97, 117, 108, 116, 82, 112, 99, 72, 101, 97, 114, 116, 98, 101, 97, 116, 70, 97, 99, 116, 111, 114, 121, 46, 106, 97, 118, 97, 200, 100, 98, 48, 56, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 110, 101, 116, 116, 121, 52, 46, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 14, 112, 114, 111, 99, 101, 115, 115, 82, 101, 113, 117, 101, 115, 116, 24, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 46, 106, 97, 118, 97, 200, 152, 98, 48, 56, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 110, 101, 116, 116, 121, 52, 46, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 14, 112, 114, 111, 99, 101, 115, 115, 77, 101, 115, 115, 97, 103, 101, 24, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 46, 106, 97, 118, 97, 200, 135, 98, 48, 56, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 110, 101, 116, 116, 121, 52, 46, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 10, 97, 99, 99, 101, 115, 115, 36, 48, 48, 48, 24, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 46, 106, 97, 118, 97, 177, 98, 48, 58, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 116, 114, 97, 110, 115, 112, 111, 114, 116, 46, 110, 101, 116, 116, 121, 52, 46, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 36, 49, 3, 114, 117, 110, 24, 78, 101, 116, 116, 121, 67, 104, 97, 110, 110, 101, 108, 72, 97, 110, 100, 108, 101, 114, 46, 106, 97, 118, 97, 200, 74, 98, 48, 39, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 84, 104, 114, 101, 97, 100, 80, 111, 111, 108, 69, 120, 101, 99, 117, 116, 111, 114, 9, 114, 117, 110, 87, 111, 114, 107, 101, 114, 23, 84, 104, 114, 101, 97, 100, 80, 111, 111, 108, 69, 120, 101, 99, 117, 116, 111, 114, 46, 106, 97, 118, 97, 204, 118, 98, 48, 46, 106, 97, 118, 97, 46, 117, 116, 105, 108, 46, 99, 111, 110, 99, 117, 114, 114, 101, 110, 116, 46, 84, 104, 114, 101, 97, 100, 80, 111, 111, 108, 69, 120, 101, 99, 117, 116, 111, 114, 36, 87, 111, 114, 107, 101, 114, 3, 114, 117, 110, 23, 84, 104, 114, 101, 97, 100, 80, 111, 111, 108, 69, 120, 101, 99, 117, 116, 111, 114, 46, 106, 97, 118, 97, 202, 105, 98, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 84, 104, 114, 101, 97, 100, 3, 114, 117, 110, 11, 84, 104, 114, 101, 97, 100, 46, 106, 97, 118, 97, 202, 233, 81, 148} + testV1CompressResBytes = []byte{241, 241, 0, 1, 24, 50, 9, 120, 9, 240, 0, 1, 0, 0, 0, 89, 240, 240, 2, 7, 24, 50, 9, 120, 9, 240, 0, 1, 0, 0, 0, 73, 172, 237, 0, 5, 119, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 14, 13, 72, 101, 108, 108, 111, 32, 109, 111, 116, 97, 110, 48, 33, 119, 2, 0, 0} + testV1CompressGzipResBytes = []byte{241, 241, 0, 1, 24, 50, 9, 147, 104, 80, 0, 1, 0, 0, 0, 101, 240, 240, 2, 7, 24, 50, 9, 147, 104, 80, 0, 1, 0, 0, 0, 85, 31, 139, 8, 0, 0, 0, 0, 0, 0, 0, 91, 243, 150, 129, 181, 92, 138, 1, 6, 4, 178, 18, 203, 18, 245, 114, 18, 243, 210, 245, 130, 75, 138, 50, 243, 210, 75, 139, 24, 152, 162, 157, 214, 124, 22, 255, 193, 198, 17, 242, 128, 137, 129, 161, 162, 0, 168, 140, 143, 215, 35, 53, 39, 39, 95, 33, 55, 191, 36, 49, 207, 64, 177, 28, 40, 14, 0, 147, 111, 110, 27, 73, 0, 0, 0} + testV1CompressWithAttachmentResBytes = []byte{241, 241, 0, 1, 24, 50, 9, 213, 235, 80, 0, 1, 0, 0, 0, 129, 240, 240, 2, 7, 24, 50, 9, 213, 235, 80, 0, 1, 0, 0, 0, 113, 31, 139, 8, 0, 0, 0, 0, 0, 0, 0, 91, 243, 150, 129, 181, 92, 138, 1, 6, 4, 178, 18, 203, 18, 245, 114, 18, 243, 210, 245, 130, 75, 138, 50, 243, 210, 75, 139, 24, 152, 162, 157, 214, 124, 22, 255, 193, 198, 17, 242, 128, 137, 129, 161, 162, 0, 168, 140, 143, 215, 35, 53, 39, 39, 95, 33, 55, 191, 36, 49, 207, 64, 177, 92, 145, 129, 137, 129, 41, 222, 145, 129, 163, 36, 181, 184, 36, 56, 51, 61, 143, 129, 57, 62, 212, 145, 129, 11, 196, 13, 205, 3, 9, 0, 0, 125, 32, 68, 112, 104, 0, 0, 0} + + // heartbeat + testV1HeartbeatReq = []byte{241, 241, 0, 0, 24, 44, 223, 176, 126, 80, 0, 96, 0, 0, 0, 78, 240, 240, 1, 0, 24, 44, 223, 176, 126, 80, 0, 96, 0, 0, 0, 62, 172, 237, 0, 5, 119, 56, 0, 33, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 114, 112, 99, 46, 104, 101, 97, 114, 116, 98, 101, 97, 116, 0, 9, 104, 101, 97, 114, 116, 98, 101, 97, 116, 0, 4, 118, 111, 105, 100, 0, 0, 0, 0} + testV1HeartbeatRes = []byte{241, 241, 0, 1, 24, 44, 226, 160, 201, 25, 2, 108, 0, 0, 0, 81, 240, 240, 1, 1, 24, 44, 226, 160, 201, 25, 2, 108, 0, 0, 0, 65, 172, 237, 0, 5, 119, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 10, 9, 104, 101, 97, 114, 116, 98, 101, 97, 116} +) + +var ( + serviceName = "com.weibo.motan.demo.service.MotanDemoService" + method = "hello" + methodDesc = "java.lang.String" + group = "motan-demo-rpc" +) + +func TestEncodeMotanV1Request(t *testing.T) { + req := &motan.MotanRequest{RequestID: 3452345} + baseV1Msg := buildV1Msg(t, testV1ReqBytes) + req.GetRPCContext(true).OriginalMessage = baseV1Msg + rid := uint64(7348937488) + bs, err := EncodeMotanV1Request(req, rid) + if err != nil { + t.Fatalf("encode v1 req fail. err:%v", err) + } + assertTrue(len(bs) == len(testV1ReqBytes), "v1 encode req lens", t) + + v1Msg := buildV1Msg(t, bs) + req2, err := DecodeMotanV1Request(v1Msg) + if err != nil { + t.Fatalf("read v1 req msg fail. err:%v", err) + } + checkBaseReq(req2, rid, t) +} + +func checkBaseReq(req motan.Request, rid uint64, t *testing.T) { + assertTrue(req.GetRequestID() == rid, "v1req rid", t) + assertTrue(req.GetServiceName() == serviceName, "v1req service name", t) + assertTrue(req.GetMethod() == method, "v1req method", t) + assertTrue(req.GetMethodDesc() == methodDesc, "v1req method desc", t) + assertTrue(req.GetAttachment("group") == group, "v1req group", t) + assertTrue(req.GetAttachments().Len() == 5, "v1req attachment size", t) +} + +func TestEncodeMotanV1Response(t *testing.T) { + res := &motan.MotanResponse{RequestID: 36476555} + baseV1Msg := buildV1Msg(t, testV1ResBytes) + res.GetRPCContext(true).OriginalMessage = baseV1Msg + bs, err := EncodeMotanV1Response(res) + if err != nil { + t.Fatalf("encode v1 res fail. err:%v", err) + } + assertTrue(len(bs) == len(testV1ResBytes), "v1 encode res lens", t) + + v1Msg := buildV1Msg(t, bs) + res2, err := DecodeMotanV1Response(v1Msg) + if err != nil { + t.Fatalf("read v1 res msg fail. err:%v", err) + } + assertTrue(res2.GetRequestID() == res.RequestID, "v1 res rid", t) + assertTrue(res2.GetProcessTime() == 1, "v1 res process time", t) +} + +type reqTestInfo struct { + name string + expectMsg *MotanV1Message + serviceName string + method string + methodDesc string + attachmentSize int +} + +type resTestInfo struct { + name string + expectMsg *MotanV1Message + processTime int64 + hasException bool + attachmentSize int +} + +func TestParseV1ReqMessage(t *testing.T) { + tests := []*reqTestInfo{ + {"normal req", &MotanV1Message{testV1ReqBytes, 1, 0, 1743455988312178690, 230}, "com.weibo.motan.demo.service.MotanDemoService", "hello", "java.lang.String", 5}, + {"no param req", &MotanV1Message{testV1NoParamReqBytes, 1, 0, 1743462559279742978, 188}, "com.weibo.motan.demo.service.MotanDemoService", "noParam", "void", 5}, + {"multi params req", &MotanV1Message{testV1MultiParamReqBytes, 1, 0, 1743463219059490817, 338}, "com.weibo.motan.demo.service.MotanDemoService", "rename", "com.weibo.motan.demo.service.model.User,java.lang.String", 5}, + {"compress req", &MotanV1Message{testV1CompressReqBytes, 2, 0, 1743463667605700610, 208}, "v1compressMethodSign", "hell817ff3733269", "", 7}, + {"compress gzip req", &MotanV1Message{testV1CompressGzipReqBytes, 2, 0, 1743463667605700610, 186}, "v1compressMethodSign", "hell817ff3733269", "", 7}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + checkReq(t, tt) + }) + } +} + +func TestParseV1ResMessage(t *testing.T) { + tests := []*resTestInfo{ + {"normal res", &MotanV1Message{testV1ResBytes, 1, 1, 1743455988312178690, 69}, 1, false, 0}, + {"void res", &MotanV1Message{testV1VoidResBytes, 1, 3, 1743462708025491459, 14}, 1, false, 0}, + {"exception res", &MotanV1Message{testV1ExceptionResBytes, 1, 5, 1743462891537825796, 1960}, 24, true, 0}, + {"compress res", &MotanV1Message{testV1CompressResBytes, 2, 7, 1743466416913252353, 73}, 0, false, 0}, + {"compress gzip res", &MotanV1Message{testV1CompressGzipResBytes, 2, 7, 1743466534460719105, 85}, 0, false, 0}, + {"compress gzip attachment res", &MotanV1Message{testV1CompressWithAttachmentResBytes, 2, 7, 1743466820126375937, 113}, 0, false, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + checkRes(t, tt) + }) + } +} + +func checkReq(t *testing.T, info *reqTestInfo) { + msg := buildV1Msg(t, info.expectMsg.OriginBytes) + checkMsg(t, msg, info.expectMsg) + + req, err := DecodeMotanV1Request(msg) + if err != nil { + t.Fatalf("read v1 req msg fail. err:%v", err) + } + assertTrue(req.GetRequestID() == info.expectMsg.Rid, "req rid", t) + assertTrue(req.GetServiceName() == info.serviceName, "req service name", t) + assertTrue(req.GetMethod() == info.method, "req method", t) + assertTrue(req.GetMethodDesc() == info.methodDesc, "req method desc", t) + assertTrue(req.GetAttachments().Len() == info.attachmentSize, "req attachment size", t) +} + +func checkRes(t *testing.T, info *resTestInfo) { + msg := buildV1Msg(t, info.expectMsg.OriginBytes) + checkMsg(t, msg, info.expectMsg) + + res, err := DecodeMotanV1Response(msg) + if err != nil { + t.Fatalf("read v1 res msg fail. err:%v", err) + } + assertTrue(res.GetRequestID() == info.expectMsg.Rid, "res rid", t) + assertTrue(res.GetProcessTime() == info.processTime, "res process time", t) + assertTrue((res.GetException() != nil) == info.hasException, "res has exception", t) + size := 0 + if res.GetAttachments() != nil { + size = res.GetAttachments().Len() + } + assertTrue(size == info.attachmentSize, "res attachment size", t) +} + +func buildV1Msg(t *testing.T, bytes []byte) *MotanV1Message { + msg, _, err := ReadV1Message(bufio.NewReader(motan.CreateBytesBuffer(bytes)), DefaultMaxContentLength) + if err != nil { + t.Fatalf("read v1 msg fail. err:%v", err) + } + return msg +} + +func checkMsg(t *testing.T, msg *MotanV1Message, expectMsg *MotanV1Message) { + assertTrue(bytes.Equal(msg.OriginBytes, expectMsg.OriginBytes), "msg ori bytes", t) + assertTrue(msg.Rid == expectMsg.Rid, "msg rid", t) + assertTrue(msg.V1InnerVersion == expectMsg.V1InnerVersion, "msg inner version", t) + assertTrue(msg.Flag == expectMsg.Flag, "msg flag", t) + assertTrue(msg.InnerLength == expectMsg.InnerLength, "msg inner length", t) +} + +func TestHeartbeatReq(t *testing.T) { + // const heartbeat bytes validate + checkHeartbeatReq(t, testV1HeartbeatReq, 1742013105011949664) + + // build heartbeat req + rid := uint64(3479837409) + bs := BuildV1HeartbeatReq(rid) + checkHeartbeatReq(t, bs, rid) +} + +func TestHeartbeatRes(t *testing.T) { + // const heartbeat bytes validate + checkHeartbeatRes(t, testV1HeartbeatRes, 1742016336082043500) + + // build heartbeat res + rid := uint64(3479837409) + bs := BuildV1HeartbeatRes(rid) + checkHeartbeatRes(t, bs, rid) +} + +func checkHeartbeatReq(t *testing.T, bs []byte, rid uint64) { + msg := buildV1Msg(t, bs) + assertTrue(msg.Rid == rid, "heartbeat rid", t) + req, err := DecodeMotanV1Request(msg) + if err != nil { + t.Fatalf("read v1 req msg fail. err:%v", err) + } + assertTrue(IsV1HeartbeatReq(req), "is heartbeat req", t) +} + +func checkHeartbeatRes(t *testing.T, bs []byte, rid uint64) { + msg := buildV1Msg(t, bs) + assertTrue(msg.Rid == rid, "heartbeat rid", t) + res, err := DecodeMotanV1Response(msg) + if err != nil { + t.Fatalf("read v1 res msg fail. err:%v", err) + } + assertTrue(IsV1HeartbeatRes(res), "is heartbeat res", t) +} + +func TestBuildV1ExceptionResponse(t *testing.T) { + rid := uint64(84379387433) + errStr := "test error:slkdfjie&^*&3627%&^%^%$$#%$@*)(*ruoisu8" + bs := BuildV1ExceptionResponse(rid, errStr) + msg := buildV1Msg(t, bs) + res, err := DecodeMotanV1Response(msg) + if err != nil { + t.Fatalf("read v1 res msg fail. err:%v", err) + } + assertTrue(res.GetRequestID() == rid, "res rid", t) + assertTrue(res.GetException() != nil, "has exception", t) + assertTrue(res.GetException().ErrMsg == "v1: has exception, class:com.weibo.api.motan.exception.MotanServiceException", "error string", t) + assertTrue(res.GetException().ErrType == motan.ServiceException, "error type", t) +} + +func TestWriteHessianString(t *testing.T) { + shortStr := "oipauepriwo8&(^^$%30984--09-pioek;flks_)*)(&*&^$%^#jdkfei" // length = 57 (length > 31 && length < 1024) + var longStr string + for i := 0; i < 20; i++ { // length = 57*20 = 1140 (length > 1023) + longStr += shortStr + } + tests := []struct { + name string + str string + bs []byte + }{ + {"direct str", "sdfeuier", []byte{8, 115, 100, 102, 101, 117, 105, 101, 114}}, + {"short str", shortStr, []byte{48, 57, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105}}, + {"normal str", longStr, []byte{83, 4, 116, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105, 111, 105, 112, 97, 117, 101, 112, 114, 105, 119, 111, 56, 38, 40, 94, 94, 36, 37, 51, 48, 57, 56, 52, 45, 45, 48, 57, 45, 112, 105, 111, 101, 107, 59, 102, 108, 107, 115, 95, 41, 42, 41, 40, 38, 42, 38, 94, 36, 37, 94, 35, 106, 100, 107, 102, 101, 105}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + checkStrEncode(t, tt.str, tt.bs) + }) + } + +} + +func checkStrEncode(t *testing.T, str string, bs []byte) { + buf := motan.NewBytesBuffer(128) + writeHessianString(str, buf) + assertTrue(bytes.Equal(buf.Bytes(), bs), "string encode bytes", t) +} diff --git a/protocol/motanProtocol.go b/protocol/motanProtocol.go index 061a2785..0e1e413e 100644 --- a/protocol/motanProtocol.go +++ b/protocol/motanProtocol.go @@ -18,7 +18,8 @@ import ( ) const ( - DefaultMetaSize = 16 + DefaultMetaSize = 16 + DefaultMaxContentLength = 10 * 1024 * 1024 ) //message type @@ -31,9 +32,11 @@ const ( const ( MotanMagic = 0xf1f1 HeaderLength = 13 + Version1 = 0 Version2 = 1 defaultProtocol = "motan2" ) + const ( MPath = "M_p" MMethod = "M_m" @@ -112,6 +115,8 @@ var ( ErrSerializeNum = errors.New("message serialize number not correct") ErrSerializeNil = errors.New("message serialize not found") ErrSerializedData = errors.New("message serialized data not correct") + ErrOverSize = errors.New("content length over the limit") + ErrWrongSize = errors.New("content length not correct") ) // BuildRequestHeader build a proxy request header @@ -307,21 +312,35 @@ func (msg *Message) Clone() interface{} { return newMessage } +func CheckMotanVersion(buf *bufio.Reader) (version int, err error) { + var b []byte + b, err = buf.Peek(4) + if err != nil { + return -1, err + } + mn := binary.BigEndian.Uint16(b[:2]) + if mn != MotanMagic { + vlog.Errorf("wrong magic num:%d, err:%v", mn, err) + return -1, ErrMagicNum + } + return int(b[3] >> 3 & 0x1f), nil +} + func Decode(buf *bufio.Reader) (msg *Message, err error) { - msg, _, err = DecodeWithTime(buf) + msg, _, err = DecodeWithTime(buf, motan.DefaultMaxContentLength) return msg, err } -func DecodeWithTime(buf *bufio.Reader) (msg *Message, start time.Time, err error) { +func DecodeWithTime(buf *bufio.Reader, maxContentLength int) (msg *Message, start time.Time, err error) { temp := make([]byte, HeaderLength, HeaderLength) // decode header _, err = io.ReadAtLeast(buf, temp, HeaderLength) - start = time.Now() + start = time.Now() // record time when starting to read data if err != nil { return nil, start, err } - mn := binary.BigEndian.Uint16(temp[:2]) + mn := binary.BigEndian.Uint16(temp[:2]) // TODO 不再验证 if mn != MotanMagic { vlog.Errorf("wrong magic num:%d, err:%v", mn, err) return nil, start, ErrMagicNum @@ -331,7 +350,7 @@ func DecodeWithTime(buf *bufio.Reader) (msg *Message, start time.Time, err error header.MsgType = temp[2] header.VersionStatus = temp[3] version := header.GetVersion() - if version != Version2 { + if version != Version2 { // TODO 不再验证 vlog.Errorf("unsupported protocol version number: %d", version) return nil, start, ErrVersion } @@ -344,6 +363,10 @@ func DecodeWithTime(buf *bufio.Reader) (msg *Message, start time.Time, err error return nil, start, err } metasize := int(binary.BigEndian.Uint32(temp[:4])) + if metasize > maxContentLength { + vlog.Errorf("meta over size. meta size:%d, max size:%d", metasize, maxContentLength) + return nil, start, ErrOverSize + } metamap := motan.NewStringMap(DefaultMetaSize) if metasize > 0 { metadata, err := readBytes(buf, metasize) @@ -376,6 +399,10 @@ func DecodeWithTime(buf *bufio.Reader) (msg *Message, start time.Time, err error return nil, start, err } bodysize := int(binary.BigEndian.Uint32(temp[:4])) + if bodysize > maxContentLength { + vlog.Errorf("body over size. body size:%d, max size:%d", bodysize, maxContentLength) + return nil, start, ErrOverSize + } var body []byte if bodysize > 0 { body, err = readBytes(buf, bodysize) diff --git a/provider/motanProvider.go b/provider/motanProvider.go index 5661d922..c5cc8017 100644 --- a/provider/motanProvider.go +++ b/provider/motanProvider.go @@ -29,6 +29,9 @@ func (m *MotanProvider) Initialize() { vlog.Errorln("reverse proxy service port config error!") return } + if protocol == "motan2" || protocol == "motan" { // TODO temp compatible with motan1. remove if MotanCommonEndpoint as default Motan endpoint + protocol = "motanV1Compatible" + } host := m.url.GetParam(ProxyHostKey, DefaultHost) endpointURL := m.url.Copy() endpointURL.Protocol = protocol diff --git a/server/motanserver.go b/server/motanserver.go index 03d70650..8ad74721 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -34,12 +34,13 @@ func getConnections() int64 { } type MotanServer struct { - URL *motan.URL - handler motan.MessageHandler - listener net.Listener - extFactory motan.ExtensionFactory - proxy bool - isDestroyed chan bool + URL *motan.URL + handler motan.MessageHandler + listener net.Listener + extFactory motan.ExtensionFactory + proxy bool + isDestroyed chan bool + maxContextLength int } func (m *MotanServer) Open(block bool, proxy bool, handler motan.MessageHandler, extFactory motan.ExtensionFactory) error { @@ -74,6 +75,7 @@ func (m *MotanServer) Open(block bool, proxy bool, handler motan.MessageHandler, m.handler = handler m.extFactory = extFactory m.proxy = proxy + m.maxContextLength = int(m.URL.GetPositiveIntValue(motan.MaxContentLength, motan.DefaultMaxContentLength)) vlog.Infof("motan server is started. port:%d", m.URL.Port) if block { m.run() @@ -149,44 +151,59 @@ func (m *MotanServer) handleConn(conn net.Conn) { } for { - request, t, err := mpro.DecodeWithTime(buf) + v, err := mpro.CheckMotanVersion(buf) if err != nil { if err.Error() != "EOF" { - vlog.Warningf("decode motan message fail! con:%s, err:%s.", conn.RemoteAddr().String(), err.Error()) + vlog.Warningf("check motan version fail! con:%s, err:%s.", conn.RemoteAddr().String(), err.Error()) } break } - - request.Metadata.Store(motan.HostKey, ip) - var trace *motan.TraceContext - if !request.Header.IsHeartbeat() { - trace = motan.TracePolicy(request.Header.RequestID, request.Metadata) - if trace != nil { - trace.Addr = ip - trace.PutReqSpan(&motan.Span{Name: motan.Receive, Time: t}) - trace.PutReqSpan(&motan.Span{Name: motan.Decode, Time: time.Now()}) + if v == mpro.Version1 { + v1Msg, t, err := mpro.ReadV1Message(buf, m.maxContextLength) + if err != nil { + vlog.Warningf("decode motan v1 message fail! con:%s, err:%s.", conn.RemoteAddr().String(), err.Error()) + break + } + go m.processV1(v1Msg, t, ip, conn) + } else if v == mpro.Version2 { + msg, t, err := mpro.DecodeWithTime(buf, m.maxContextLength) + if err != nil { + vlog.Warningf("decode motan v2 message fail! con:%s, err:%s.", conn.RemoteAddr().String(), err.Error()) + break } + + go m.processV2(msg, t, ip, conn) + } else { + vlog.Warningf("unsupported motan version! version:%d con:%s", v, conn.RemoteAddr().String()) + break } - go m.processReq(t, request, trace, conn) } } -func (m *MotanServer) processReq(start time.Time, request *mpro.Message, tc *motan.TraceContext, conn net.Conn) { +func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, conn net.Conn) { defer motan.HandlePanic(nil) - request.Header.SetProxy(m.proxy) + msg.Metadata.Store(motan.HostKey, ip) + msg.Header.SetProxy(m.proxy) // TODO request , response reuse var mres motan.Response var mreq motan.Request var res *mpro.Message - lastRequestID := request.Header.RequestID - if request.Header.IsHeartbeat() { - res = mpro.BuildHeartbeat(request.Header.RequestID, mpro.Res) + var tc *motan.TraceContext + lastRequestID := msg.Header.RequestID + if msg.Header.IsHeartbeat() { + res = mpro.BuildHeartbeat(msg.Header.RequestID, mpro.Res) } else { - serialization := m.extFactory.GetSerialization("", request.Header.GetSerialize()) - req, err := mpro.ConvertToRequest(request, serialization) + tc = motan.TracePolicy(msg.Header.RequestID, msg.Metadata) + if tc != nil { + tc.Addr = ip + tc.PutReqSpan(&motan.Span{Name: motan.Receive, Time: start}) + tc.PutReqSpan(&motan.Span{Name: motan.Decode, Time: time.Now()}) + } + serialization := m.extFactory.GetSerialization("", msg.Header.GetSerialize()) + req, err := mpro.ConvertToRequest(msg, serialization) if err != nil { - vlog.Errorf("motan server convert to motan request fail. rid :%d, service: %s, method:%s,err:%s\n", request.Header.RequestID, request.Metadata.LoadOrEmpty(mpro.MPath), request.Metadata.LoadOrEmpty(mpro.MMethod), err.Error()) - res = mpro.BuildExceptionResponse(request.Header.RequestID, mpro.ExceptionToJSON(&motan.Exception{ErrCode: 500, ErrMsg: "deserialize fail. err:" + err.Error() + " method:" + request.Metadata.LoadOrEmpty(mpro.MMethod), ErrType: motan.ServiceException})) + vlog.Errorf("motan server convert to motan request fail. rid :%d, service: %s, method:%s,err:%s\n", msg.Header.RequestID, msg.Metadata.LoadOrEmpty(mpro.MPath), msg.Metadata.LoadOrEmpty(mpro.MMethod), err.Error()) + res = mpro.BuildExceptionResponse(msg.Header.RequestID, mpro.ExceptionToJSON(&motan.Exception{ErrCode: 500, ErrMsg: "deserialize fail. err:" + err.Error() + " method:" + msg.Metadata.LoadOrEmpty(mpro.MMethod), ErrType: motan.ServiceException})) } else { mreq = req reqCtx := req.GetRPCContext(true) @@ -217,7 +234,7 @@ func (m *MotanServer) processReq(start time.Time, request *mpro.Message, tc *mot } if err != nil { - res = mpro.BuildExceptionResponse(request.Header.RequestID, mpro.ExceptionToJSON(&motan.Exception{ErrCode: 500, ErrMsg: "convert to response fail. err:" + err.Error(), ErrType: motan.ServiceException})) + res = mpro.BuildExceptionResponse(msg.Header.RequestID, mpro.ExceptionToJSON(&motan.Exception{ErrCode: 500, ErrMsg: "convert to response fail. err:" + err.Error(), ErrType: motan.ServiceException})) } } } @@ -227,7 +244,6 @@ func (m *MotanServer) processReq(start time.Time, request *mpro.Message, tc *mot if tc != nil { tc.PutResSpan(&motan.Span{Name: motan.Encode, Time: time.Now()}) } - conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) _, err := conn.Write(resBuf.Bytes()) if err != nil { @@ -248,6 +264,59 @@ func (m *MotanServer) processReq(start time.Time, request *mpro.Message, tc *mot } } +func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip string, conn net.Conn) { + defer motan.HandlePanic(nil) + var res motan.Response + var result []byte + var reqCtx *motan.RPCContext + req, err := mpro.DecodeMotanV1Request(msg) + if err != nil { + vlog.Errorf("decode v1 request fail. conn: %s, err:%s", conn.RemoteAddr().String(), err.Error()) + result = mpro.BuildV1ExceptionResponse(msg.Rid, err.Error()) + } else { + reqCtx = req.GetRPCContext(true) + reqCtx.ExtFactory = m.extFactory + reqCtx.RequestReceiveTime = start + if mpro.IsV1HeartbeatReq(req) { + result = mpro.BuildV1HeartbeatRes(req.GetRequestID()) + } else { + // TraceContext Currently not supported in protocol v1 + callStart := time.Now() + res = m.handler.Call(req) + if res != nil { + resCtx := res.GetRPCContext(true) + resCtx.Proxy = m.proxy + if res.GetAttachment(mpro.MProcessTime) == "" { + res.SetAttachment(mpro.MProcessTime, strconv.FormatInt(int64(time.Now().Sub(callStart)/1e6), 10)) + } + result, err = mpro.EncodeMotanV1Response(res) + if err != nil { // not close connection when encode error occurs. + vlog.Errorf("encode v1 response fail. conn: %s, err:%s, res:%s", conn.RemoteAddr().String(), err.Error(), motan.GetResInfo(res)) + return + } + } + } + } + + if len(result) > 0 { + conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) + _, err = conn.Write(result) + if err != nil { + vlog.Errorf("connection will close. conn: %s, err:%s", conn.RemoteAddr().String(), err.Error()) + conn.Close() + } + } else { + vlog.Errorf("process v1 message fail: no result to send. conn: %s, req:%s, res:%s, result:%v, err: %v", conn.RemoteAddr().String(), motan.GetReqInfo(req), motan.GetResInfo(res), result, err) + } + if reqCtx != nil { + reqCtx.ResponseSendTime = time.Now() + } + if res != nil { + resCtx := res.GetRPCContext(true) + resCtx.OnFinish() + } +} + func getRemoteIP(address string) string { var ip string index := strings.Index(address, ":") From 87c764140d69a1b3b22a86b2a956dc3a28a32947 Mon Sep 17 00:00:00 2001 From: liangwei3 Date: Thu, 29 Sep 2022 16:23:32 +0800 Subject: [PATCH 28/75] =?UTF-8?q?agent=20=E5=90=AF=E5=8A=A8=E6=8F=90?= =?UTF-8?q?=E9=80=9F=201.=20=E5=B9=B6=E8=A1=8C=E5=88=9D=E5=A7=8B=E5=8C=96c?= =?UTF-8?q?luster=202.=20=E5=BB=BA=E8=BF=9E=E5=A2=9E=E5=8A=A0=E5=BB=B6?= =?UTF-8?q?=E8=BF=9F=E5=92=8C=E5=BC=82=E6=AD=A5=E7=9A=84=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E5=8F=AF=E9=85=8D=E7=BD=AE=203.=20=E8=A1=A5?= =?UTF-8?q?=E5=85=85=E5=8D=95=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent.go | 25 ++-- core/constants.go | 2 + core/map.go | 10 ++ core/url.go | 10 ++ endpoint/motanCommonEndpoint.go | 91 +++++++----- endpoint/motanCommonEndpoint_test.go | 209 +++++++++++++++++++++++++++ endpoint/motanEndpoint.go | 91 +++++++----- endpoint/motanEndpoint_test.go | 145 +++++++++++++++++-- 8 files changed, 494 insertions(+), 89 deletions(-) create mode 100644 endpoint/motanCommonEndpoint_test.go diff --git a/agent.go b/agent.go index 24b348c9..d08b128e 100644 --- a/agent.go +++ b/agent.go @@ -482,7 +482,11 @@ func (a *Agent) reloadClusters(ctx *motan.Context) { func (a *Agent) initClusters() { for _, url := range a.Context.RefersURLs { - a.initCluster(url) + // concurrently initialize cluster + go func(u *motan.URL) { + defer motan.HandlePanic(nil) + a.initCluster(u) + }(url) } } @@ -500,15 +504,16 @@ func (a *Agent) initCluster(url *motan.URL) { cluster: c, } service := url.Path - var serviceMapItemArr []serviceMapItem - if v, exists := a.serviceMap.Load(service); exists { - serviceMapItemArr = v.([]serviceMapItem) - serviceMapItemArr = append(serviceMapItemArr, item) - } else { - serviceMapItemArr = []serviceMapItem{item} - } - a.serviceMap.Store(url.Path, serviceMapItemArr) - + a.serviceMap.SafeDoFunc(func() { + var serviceMapItemArr []serviceMapItem + if v, exists := a.serviceMap.Load(service); exists { + serviceMapItemArr = v.([]serviceMapItem) + serviceMapItemArr = append(serviceMapItemArr, item) + } else { + serviceMapItemArr = []serviceMapItem{item} + } + a.serviceMap.UnsafeStore(url.Path, serviceMapItemArr) + }) mapKey := getClusterKey(url.Group, url.GetStringParamsWithDefault(motan.VersionKey, "0.1"), url.Protocol, url.Path) a.clusterMap.Store(mapKey, c) } diff --git a/core/constants.go b/core/constants.go index 309b531f..544a6c79 100644 --- a/core/constants.go +++ b/core/constants.go @@ -55,6 +55,8 @@ const ( ConnectTimeoutKey = "connectTimeout" ConnectRetryIntervalKey = "connectRetryInterval" ClientConnectionKey = "clientConnection" + LazyInit = "lazyInit" + AsyncInitConnection = "asyncInitConnection" ErrorCountThresholdKey = "errorCountThreshold" KeepaliveIntervalKey = "keepaliveInterval" UnixSockKey = "unixSock" diff --git a/core/map.go b/core/map.go index 2567a154..4aa2bcf3 100644 --- a/core/map.go +++ b/core/map.go @@ -122,6 +122,10 @@ func (m *CopyOnWriteMap) Range(f func(k, v interface{}) bool) { func (m *CopyOnWriteMap) Store(key, value interface{}) { m.mu.Lock() defer m.mu.Unlock() + m.UnsafeStore(key, value) +} + +func (m *CopyOnWriteMap) UnsafeStore(key, value interface{}) { lastMap := m.data() copiedMap := make(map[interface{}]interface{}, len(lastMap)+1) for k, v := range lastMap { @@ -161,3 +165,9 @@ func (m *CopyOnWriteMap) Swap(newMap map[interface{}]interface{}) map[interface{ m.innerMap.Store(newMap) return lastMap } + +func (m *CopyOnWriteMap) SafeDoFunc(f func()) { + m.mu.Lock() + defer m.mu.Unlock() + f() +} diff --git a/core/url.go b/core/url.go index 31aac261..f470b911 100644 --- a/core/url.go +++ b/core/url.go @@ -55,6 +55,16 @@ func (u *URL) GetPositiveIntValue(key string, defaultvalue int64) int64 { return intvalue } +func (u *URL) GetBoolValue(key string, defaultValue bool) bool { + if v, ok := u.Parameters[key]; ok { + boolValue, err := strconv.ParseBool(v) + if err == nil { + return boolValue + } + } + return defaultValue +} + func (u *URL) GetIntValue(key string, defaultValue int64) int64 { result, b := u.GetInt(key) if b { diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index daa14f16..51aa6b9f 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -29,6 +29,7 @@ type MotanCommonEndpoint struct { minRequestTimeoutMillisecond int64 maxRequestTimeoutMillisecond int64 clientConnection int + lazyInit bool maxContentLength int heartbeatVersion int @@ -61,40 +62,18 @@ func (m *MotanCommonEndpoint) Initialize() { m.maxRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MaxTimeOutKey) m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) + m.lazyInit = m.url.GetBoolValue(motan.LazyInit, false) + asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, false) m.heartbeatVersion = -1 m.DefaultVersion = mpro.Version2 factory := func() (net.Conn, error) { return net.DialTimeout("tcp", m.url.GetAddressStr(), connectTimeout) } config := &ChannelConfig{MaxContentLength: m.maxContentLength, Serialization: m.serialization} - channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization) - if err != nil { - vlog.Errorf("Channel pool init failed. url: %v, err:%s", m.url, err.Error()) - // retry connect - go func() { - defer motan.HandlePanic(nil) - // TODO: retry after 2^n * timeUnit - ticker := time.NewTicker(connectRetryInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization) - if err == nil { - m.channels = channels - m.setAvailable(true) - vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) - return - } - case <-m.destroyCh: - return - } - } - }() + if asyncInitConnection { + go m.initChannelPoolWithRetry(factory, config, connectRetryInterval) } else { - m.channels = channels - m.setAvailable(true) - vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + m.initChannelPoolWithRetry(factory, config, connectRetryInterval) } } @@ -192,6 +171,39 @@ func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { return response } +func (m *MotanCommonEndpoint) initChannelPoolWithRetry(factory ConnFactory, config *ChannelConfig, retryInterval time.Duration) { + defer motan.HandlePanic(nil) + channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) + if err != nil { + vlog.Errorf("Channel pool init failed. url: %v, err:%s", m.url, err.Error()) + // retry connect + go func() { + defer motan.HandlePanic(nil) + // TODO: retry after 2^n * timeUnit + ticker := time.NewTicker(retryInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) + if err == nil { + m.channels = channels + m.setAvailable(true) + vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + return + } + case <-m.destroyCh: + return + } + } + }() + } else { + m.channels = channels + m.setAvailable(true) + vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + } +} + func (m *MotanCommonEndpoint) recordErrAndKeepalive() { errCount := atomic.AddUint32(&m.errorCount, 1) // ensure trigger keepalive @@ -703,6 +715,7 @@ func (c *ChannelPool) Get() (*Channel, error) { if err != nil { vlog.Errorf("create channel failed. err:%s", err.Error()) } + _ = conn.(*net.TCPConn).SetNoDelay(true) channel = buildChannel(conn, c.config, c.serialization) } if err := retChannelPool(channels, channel); err != nil && channel != nil { @@ -746,7 +759,7 @@ func (c *ChannelPool) Close() error { return nil } -func NewChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, serialization motan.Serialization) (*ChannelPool, error) { +func NewChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, serialization motan.Serialization, lazyInit bool) (*ChannelPool, error) { if poolCap <= 0 { return nil, errors.New("invalid capacity settings") } @@ -756,14 +769,22 @@ func NewChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, ser config: config, serialization: serialization, } - for i := 0; i < poolCap; i++ { - conn, err := factory() - if err != nil { - channelPool.Close() - return nil, err + if lazyInit { + for i := 0; i < poolCap; i++ { + //delay logic just push nil into channelPool. when the first request comes in, + //endpoint will build a connection from factory + channelPool.channels <- nil + } + } else { + for i := 0; i < poolCap; i++ { + conn, err := factory() + if err != nil { + channelPool.Close() + return nil, err + } + _ = conn.(*net.TCPConn).SetNoDelay(true) + channelPool.channels <- buildChannel(conn, config, serialization) } - _ = conn.(*net.TCPConn).SetNoDelay(true) - channelPool.channels <- buildChannel(conn, config, serialization) } return channelPool, nil } diff --git a/endpoint/motanCommonEndpoint_test.go b/endpoint/motanCommonEndpoint_test.go new file mode 100644 index 00000000..05f75ad5 --- /dev/null +++ b/endpoint/motanCommonEndpoint_test.go @@ -0,0 +1,209 @@ +package endpoint + +import ( + "fmt" + "github.com/stretchr/testify/assert" + motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/protocol" + "github.com/weibocom/motan-go/serialize" + "net" + "runtime" + "testing" + "time" +) + +func TestGetV1Name(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "100") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, defaultChannelPoolSize, ep.clientConnection) + fmt.Printf("format\n") + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + //request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + fmt.Printf("res:%+v\n", res) + ep.Destroy() +} + +func TestV1RecordErrEmptyThreshold(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + for j := 0; j < 5; j++ { + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + ep.Call(request) + assert.True(t, ep.IsAvailable()) + } + ep.Destroy() +} + +func TestV1RecordErrWithErrThreshold(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "5") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + for j := 0; j < 10; j++ { + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + ep.Call(request) + if j < 4 { + assert.True(t, ep.IsAvailable()) + } else { + assert.False(t, ep.IsAvailable()) + } + } + <-ep.channels.channels + conn, err := ep.channels.factory() + assert.Nil(t, err) + _ = conn.(*net.TCPConn).SetNoDelay(true) + ep.channels.channels <- buildChannel(conn, ep.channels.config, ep.channels.serialization) + time.Sleep(time.Second * 2) + //assert.True(t, ep.IsAvailable()) + ep.Destroy() +} + +func TestMotanCommonEndpoint_SuccessCall(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "2000") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + assert.Nil(t, res.GetException()) + v := res.GetValue() + s, ok := v.(string) + assert.True(t, ok) + assert.Equal(t, s, "hello") +} + +func TestMotanCommonEndpoint_AsyncCall(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "2000") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + var resStr string + request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} + request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + assert.Nil(t, res.GetException()) + resp := <-request.GetRPCContext(false).Result.Done + assert.Nil(t, resp.Error) + assert.Equal(t, resStr, "hello") +} + +func TestMotanCommonEndpoint_ErrorCall(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + fmt.Println(res.GetException().ErrMsg) + assert.False(t, ep.IsAvailable()) + time.Sleep(1 * time.Millisecond) + beforeNGoroutine := runtime.NumGoroutine() + ep.Call(request) + time.Sleep(1 * time.Millisecond) + assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + ep.Destroy() +} + +func TestMotanCommonEndpoint_RequestTimeout(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + request.Attachment.Store(protocol.MTimeout, "150") + res := ep.Call(request) + fmt.Println(res.GetException().ErrMsg) + assert.False(t, ep.IsAvailable()) + time.Sleep(1 * time.Millisecond) + beforeNGoroutine := runtime.NumGoroutine() + ep.Call(request) + time.Sleep(1 * time.Millisecond) + assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + ep.Destroy() +} + +func TestV1LazyInit(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible", Parameters: map[string]string{"lazyInit": "true"}} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + c := <-ep.channels.channels + assert.Nil(t, c) + ep.channels.channels <- nil + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + fmt.Println(res.GetException().ErrMsg) + assert.False(t, ep.IsAvailable()) + time.Sleep(1 * time.Millisecond) + beforeNGoroutine := runtime.NumGoroutine() + ep.Call(request) + time.Sleep(1 * time.Millisecond) + assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + ep.Destroy() +} + +func TestV1AsyncInit(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible", Parameters: map[string]string{"asyncInitConnection": "true"}} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + time.Sleep(time.Second * 5) +} diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index e28dec22..58474767 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -47,6 +47,7 @@ type MotanEndpoint struct { minRequestTimeoutMillisecond int64 maxRequestTimeoutMillisecond int64 clientConnection int + lazyInit bool maxContentLength int // for heartbeat requestID @@ -78,38 +79,16 @@ func (m *MotanEndpoint) Initialize() { m.maxRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MaxTimeOutKey) m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) + m.lazyInit = m.url.GetBoolValue(motan.LazyInit, false) + asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, false) factory := func() (net.Conn, error) { return net.DialTimeout("tcp", m.url.GetAddressStr(), connectTimeout) } config := &ChannelConfig{MaxContentLength: m.maxContentLength} - channels, err := NewV2ChannelPool(m.clientConnection, factory, config, m.serialization) - if err != nil { - vlog.Errorf("Channel pool init failed. url: %v, err:%s", m.url, err.Error()) - // retry connect - go func() { - defer motan.HandlePanic(nil) - // TODO: retry after 2^n * timeUnit - ticker := time.NewTicker(connectRetryInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - channels, err := NewV2ChannelPool(m.clientConnection, factory, config, m.serialization) - if err == nil { - m.channels = channels - m.setAvailable(true) - vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) - return - } - case <-m.destroyCh: - return - } - } - }() + if asyncInitConnection { + go m.initChannelPoolWithRetry(factory, config, connectRetryInterval) } else { - m.channels = channels - m.setAvailable(true) - vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + m.initChannelPoolWithRetry(factory, config, connectRetryInterval) } } @@ -221,6 +200,39 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { return response } +func (m *MotanEndpoint) initChannelPoolWithRetry(factory ConnFactory, config *ChannelConfig, retryInterval time.Duration) { + defer motan.HandlePanic(nil) + channels, err := NewV2ChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) + if err != nil { + vlog.Errorf("Channel pool init failed. url: %v, err:%s", m.url, err.Error()) + // retry connect + go func() { + defer motan.HandlePanic(nil) + // TODO: retry after 2^n * timeUnit + ticker := time.NewTicker(retryInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + channels, err := NewV2ChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) + if err == nil { + m.channels = channels + m.setAvailable(true) + vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + return + } + case <-m.destroyCh: + return + } + } + }() + } else { + m.channels = channels + m.setAvailable(true) + vlog.Infof("Channel pool init success. url:%s", m.url.GetAddressStr()) + } +} + func (m *MotanEndpoint) recordErrAndKeepalive() { // errorCountThreshold <= 0 means not trigger keepalive if m.errorCountThreshold > 0 { @@ -658,6 +670,7 @@ func (c *V2ChannelPool) Get() (*V2Channel, error) { if err != nil { vlog.Errorf("create channel failed. err:%s", err.Error()) } + _ = conn.(*net.TCPConn).SetNoDelay(true) channel = buildV2Channel(conn, c.config, c.serialization) } if err := retV2ChannelPool(channels, channel); err != nil && channel != nil { @@ -701,7 +714,7 @@ func (c *V2ChannelPool) Close() error { return nil } -func NewV2ChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, serialization motan.Serialization) (*V2ChannelPool, error) { +func NewV2ChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, serialization motan.Serialization, lazyInit bool) (*V2ChannelPool, error) { if poolCap <= 0 { return nil, errors.New("invalid capacity settings") } @@ -711,14 +724,22 @@ func NewV2ChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, s config: config, serialization: serialization, } - for i := 0; i < poolCap; i++ { - conn, err := factory() - if err != nil { - channelPool.Close() - return nil, err + if lazyInit { + for i := 0; i < poolCap; i++ { + //delay logic just push nil into channelPool. when the first request comes in, + //endpoint will build a connection from factory + channelPool.channels <- nil + } + } else { + for i := 0; i < poolCap; i++ { + conn, err := factory() + if err != nil { + channelPool.Close() + return nil, err + } + _ = conn.(*net.TCPConn).SetNoDelay(true) + channelPool.channels <- buildV2Channel(conn, config, serialization) } - _ = conn.(*net.TCPConn).SetNoDelay(true) - channelPool.channels <- buildV2Channel(conn, config, serialization) } return channelPool, nil } diff --git a/endpoint/motanEndpoint_test.go b/endpoint/motanEndpoint_test.go index 1b64b4e3..b7d4fad9 100644 --- a/endpoint/motanEndpoint_test.go +++ b/endpoint/motanEndpoint_test.go @@ -1,18 +1,18 @@ package endpoint import ( + "bufio" "fmt" + "github.com/stretchr/testify/assert" + motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/log" "github.com/weibocom/motan-go/protocol" + "github.com/weibocom/motan-go/serialize" "net" "runtime" "strconv" "testing" "time" - - "github.com/stretchr/testify/assert" - motan "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/log" - "github.com/weibocom/motan-go/serialize" ) func TestMain(m *testing.M) { @@ -33,9 +33,10 @@ func TestGetName(t *testing.T) { assert.Equal(t, defaultChannelPoolSize, ep.clientConnection) fmt.Printf("format\n") request := &motan.MotanRequest{ServiceName: "test", Method: "test"} - request.Attachment = motan.NewStringMap(0) + //request.Attachment = motan.NewStringMap(0) res := ep.Call(request) fmt.Printf("res:%+v\n", res) + ep.Destroy() } func TestRecordErrEmptyThreshold(t *testing.T) { @@ -78,10 +79,59 @@ func TestRecordErrWithErrThreshold(t *testing.T) { assert.False(t, ep.IsAvailable()) } } + <-ep.channels.channels + conn, err := ep.channels.factory() + assert.Nil(t, err) + _ = conn.(*net.TCPConn).SetNoDelay(true) + ep.channels.channels <- buildV2Channel(conn, ep.channels.config, ep.channels.serialization) + time.Sleep(time.Second * 2) + //assert.True(t, ep.IsAvailable()) + ep.Destroy() +} + +func TestMotanEndpoint_SuccessCall(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan2"} + url.PutParam(motan.TimeOutKey, "2000") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanEndpoint{} + ep.SetURL(url) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + assert.Nil(t, res.GetException()) + v := res.GetValue() + s, ok := v.(string) + assert.True(t, ok) + assert.Equal(t, s, "hello") + ep.Destroy() +} + +func TestMotanEndpoint_AsyncCall(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan2"} + url.PutParam(motan.TimeOutKey, "2000") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanEndpoint{} + ep.SetURL(url) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + var resStr string + request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} + request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + assert.Nil(t, res.GetException()) + resp := <-request.GetRPCContext(false).Result.Done + assert.Nil(t, resp.Error) + assert.Equal(t, resStr, "hello") ep.Destroy() } -func TestMotanEndpoint_Call(t *testing.T) { +func TestMotanEndpoint_ErrorCall(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") @@ -130,6 +180,45 @@ func TestMotanEndpoint_RequestTimeout(t *testing.T) { ep.Destroy() } +func TestLazyInit(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan2", Parameters: map[string]string{"lazyInit": "true"}} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + c := <-ep.channels.channels + assert.Nil(t, c) + ep.channels.channels <- nil + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + res := ep.Call(request) + fmt.Println(res.GetException().ErrMsg) + assert.False(t, ep.IsAvailable()) + time.Sleep(1 * time.Millisecond) + beforeNGoroutine := runtime.NumGoroutine() + ep.Call(request) + time.Sleep(1 * time.Millisecond) + assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + ep.Destroy() +} + +func TestAsyncInit(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan2", Parameters: map[string]string{"asyncInitConnection": "true"}} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + ep := &MotanEndpoint{} + ep.SetURL(url) + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + time.Sleep(time.Second * 5) +} + func StartTestServer(port int) *MockServer { m := &MockServer{Port: port} m.Start() @@ -171,6 +260,44 @@ func handle(netListen net.Listener) { } func handleConnection(conn net.Conn, timeout int) { - time.Sleep(time.Millisecond * 1000) - conn.Close() + buf := bufio.NewReader(conn) + msg, _, err := protocol.DecodeWithTime(buf, 10*1024*1024) + if err != nil { + time.Sleep(time.Millisecond * 1000) + conn.Close() + return + } + processMsg(msg, conn) +} + +func processMsg(msg *protocol.Message, conn net.Conn) { + var res *protocol.Message + var tc *motan.TraceContext + var err error + lastRequestID := msg.Header.RequestID + if msg.Header.IsHeartbeat() { + res = protocol.BuildHeartbeat(msg.Header.RequestID, protocol.Res) + } else { + time.Sleep(time.Millisecond * 1000) + serialization := &serialize.SimpleSerialization{} + resp := &motan.MotanResponse{ + RequestID: lastRequestID, + Value: "hello", + ProcessTime: 1000, + } + res, err = protocol.ConvertToResMessage(resp, serialization) + if err != nil { + conn.Close() + } + } + res.Header.RequestID = lastRequestID + resBuf := res.Encode() + if tc != nil { + tc.PutResSpan(&motan.Span{Name: motan.Encode, Time: time.Now()}) + } + conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) + _, err = conn.Write(resBuf.Bytes()) + if err != nil { + conn.Close() + } } From fea2be0adc7eb536b2380128b905f7027c4ddb4c Mon Sep 17 00:00:00 2001 From: kyton <641862816@qq.com> Date: Tue, 11 Oct 2022 08:32:11 +0800 Subject: [PATCH 29/75] fix common endpoint recorderrandkeepalive (#288) --- endpoint/motanCommonEndpoint.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 51aa6b9f..64d8fb04 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -205,12 +205,15 @@ func (m *MotanCommonEndpoint) initChannelPoolWithRetry(factory ConnFactory, conf } func (m *MotanCommonEndpoint) recordErrAndKeepalive() { - errCount := atomic.AddUint32(&m.errorCount, 1) - // ensure trigger keepalive - if errCount >= uint32(m.errorCountThreshold) { - m.setAvailable(false) - vlog.Infoln("Referer disable:" + m.url.GetIdentity()) - go m.keepalive() + // errorCountThreshold <= 0 means not trigger keepalive + if m.errorCountThreshold > 0 { + errCount := atomic.AddUint32(&m.errorCount, 1) + // ensure trigger keepalive + if errCount >= uint32(m.errorCountThreshold) { + m.setAvailable(false) + vlog.Infoln("Referer disable:" + m.url.GetIdentity()) + go m.keepalive() + } } } From 10eaf1eab4cdfbddc5b44196fd304b0d147c7c31 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Mon, 17 Oct 2022 19:19:20 +0800 Subject: [PATCH 30/75] fix endpoint nil conn enter assertion bug (#289) --- endpoint/motanCommonEndpoint.go | 3 ++- endpoint/motanEndpoint.go | 3 ++- provider/httpxProvider.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 64d8fb04..a6200b7a 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -717,8 +717,9 @@ func (c *ChannelPool) Get() (*Channel, error) { conn, err := c.factory() if err != nil { vlog.Errorf("create channel failed. err:%s", err.Error()) + } else { + _ = conn.(*net.TCPConn).SetNoDelay(true) } - _ = conn.(*net.TCPConn).SetNoDelay(true) channel = buildChannel(conn, c.config, c.serialization) } if err := retChannelPool(channels, channel); err != nil && channel != nil { diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 58474767..d34058e2 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -669,8 +669,9 @@ func (c *V2ChannelPool) Get() (*V2Channel, error) { conn, err := c.factory() if err != nil { vlog.Errorf("create channel failed. err:%s", err.Error()) + } else { + _ = conn.(*net.TCPConn).SetNoDelay(true) } - _ = conn.(*net.TCPConn).SetNoDelay(true) channel = buildV2Channel(conn, c.config, c.serialization) } if err := retV2ChannelPool(channels, channel); err != nil && channel != nil { diff --git a/provider/httpxProvider.go b/provider/httpxProvider.go index bd554e80..8bb58f7d 100644 --- a/provider/httpxProvider.go +++ b/provider/httpxProvider.go @@ -52,7 +52,7 @@ func (h *HTTPXProvider) Initialize() { for _, method := range methodArr { sconf := make(sConfT) for k, v := range getSrvConf.(map[interface{}]interface{}) { - // @TODO gracful panic when got a conf err, like more %s in URL_FORMAT + // @TODO graceful panic when got a conf err, like more %s in URL_FORMAT sconf[k.(string)] = v.(string) } srvConf[method] = sconf From 4ccf6175d37e54ebbb750762a756ec8b7ed0b689 Mon Sep 17 00:00:00 2001 From: cocowh Date: Mon, 17 Oct 2022 19:20:12 +0800 Subject: [PATCH 31/75] bugfix:request clone originMessage (#291) --- core/motan.go | 2 +- protocol/motan1Protocol_test.go | 5 +++ protocol/motanProtocol_test.go | 59 +++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/core/motan.go b/core/motan.go index 1aa42b5a..68afc876 100644 --- a/core/motan.go +++ b/core/motan.go @@ -524,7 +524,7 @@ func (m *MotanRequest) Clone() interface{} { if oldMessage, ok := m.RPCContext.OriginalMessage.(Cloneable); ok { newRequest.RPCContext.OriginalMessage = oldMessage.Clone() } else { - newRequest.RPCContext.OriginalMessage = oldMessage + newRequest.RPCContext.OriginalMessage = m.RPCContext.OriginalMessage } } } diff --git a/protocol/motan1Protocol_test.go b/protocol/motan1Protocol_test.go index 09a95a30..47e1d737 100644 --- a/protocol/motan1Protocol_test.go +++ b/protocol/motan1Protocol_test.go @@ -52,6 +52,11 @@ func TestEncodeMotanV1Request(t *testing.T) { t.Fatalf("read v1 req msg fail. err:%v", err) } checkBaseReq(req2, rid, t) + + // test request clone + cloneReq := req2.Clone().(motan.Request) + checkBaseReq(cloneReq, rid, t) + assertTrue(cloneReq.GetRPCContext(true).OriginalMessage == v1Msg, "request originMessage", t) } func checkBaseReq(req motan.Request, rid uint64, t *testing.T) { diff --git a/protocol/motanProtocol_test.go b/protocol/motanProtocol_test.go index 6b1e6953..060ae0cb 100644 --- a/protocol/motanProtocol_test.go +++ b/protocol/motanProtocol_test.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "fmt" + "github.com/weibocom/motan-go/serialize" "math/rand" "sync" "sync/atomic" @@ -183,6 +184,64 @@ func assertTrue(b bool, msg string, t *testing.T) { } //TODO convert +func TestConvertToRequest(t *testing.T) { + h := &Header{} + h.SetVersion(Version2) + h.SetStatus(6) + h.SetOneWay(true) + h.SetSerialize(6) + h.SetGzip(true) + h.SetHeartbeat(true) + h.SetProxy(true) + h.SetRequest(true) + h.Magic = MotanMagic + h.RequestID = 2349789 + meta := core.NewStringMap(0) + meta.Store("k1", "v1") + meta.Store(MGroup, "group") + meta.Store(MMethod, "method") + meta.Store(MPath, "path") + body := []byte("testbody") + msg := &Message{Header: h, Metadata: meta, Body: body} + req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) + assertTrue(err == nil, "conver to request err", t) + assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) + assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) + assertTrue(req.GetAttachment(MPath) == "path", "request path", t) + + // test request clone + cloneReq := req.Clone().(core.Request) + assertTrue(err == nil, "conver to request err", t) + assertTrue(cloneReq.GetAttachment(MGroup) == "group", "clone request group", t) + assertTrue(cloneReq.GetAttachment(MMethod) == "method", "clone request method", t) + assertTrue(cloneReq.GetAttachment(MPath) == "path", "clone request path", t) + assertTrue(cloneReq.GetRPCContext(true).OriginalMessage.(*Message).Header.Serialize == msg.Header.Serialize, "clone request originMessage", t) + testCloneOriginMeta := []map[string]interface{}{ + { + "key": MMethod, + "expect": "method", + "msg": "clone originMessage meta method", + }, + { + "key": MGroup, + "expect": "group", + "msg": "clone originMessage meta group", + }, + { + "key": MPath, + "expect": "path", + "msg": "clone originMessage meta path", + }, + } + for _, m := range testCloneOriginMeta { + key := m["key"].(string) + expect := m["expect"].(string) + tips := m["msg"].(string) + value, ok := cloneReq.GetRPCContext(true).OriginalMessage.(*Message).Metadata.Load(key) + assertTrue(ok == true, "load clone originMessage meta "+key, t) + assertTrue(expect == value, tips, t) + } +} func BenchmarkEncodeGzip(b *testing.B) { DefaultGzipLevel = gzip.BestSpeed From 6181d23b015e564b44d5826b702cbb56caaee7c1 Mon Sep 17 00:00:00 2001 From: snail007 Date: Mon, 17 Oct 2022 19:22:42 +0800 Subject: [PATCH 32/75] Fix Heartbeat (#290) * fix heartbeat --- agent.go | 33 +++++++++++++++++++++++++++++++++ server/motanserver.go | 11 ++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/agent.go b/agent.go index d08b128e..9fd91ca0 100644 --- a/agent.go +++ b/agent.go @@ -14,6 +14,7 @@ import ( "time" cfg "github.com/weibocom/motan-go/config" + "github.com/weibocom/motan-go/provider" "gopkg.in/yaml.v2" "github.com/shirou/gopsutil/v3/process" @@ -195,6 +196,7 @@ func (a *Agent) StartMotanAgentFromConfig(config *cfg.Config) { a.configurer = NewDynamicConfigurer(a) go a.startMServer() go a.registerAgent() + go a.startAllServerStatusDaemon() f, err := os.Create(a.pidfile) if err != nil { vlog.Errorf("create file %s fail.", a.pidfile) @@ -1118,3 +1120,34 @@ func (a *Agent) UnexportService(url *motan.URL) error { } return nil } +func (a *Agent) startAllServerStatusDaemon() { + defer motan.HandlePanic(nil) + var addr string + var timeout = 2 * time.Second + timer := time.NewTicker(time.Second * 30) + defer timer.Stop() + for { + <-timer.C + if len(a.Context.ServiceURLs) == 0 { + mserver.HeartbeatDisabled = false + continue + } + var alive = false + for _, s := range a.Context.ServiceURLs { + if v := s.GetParam(mhttp.ProxyAddressKey, ""); v != "" { + addr = v + } else { + _, port, _ := motan.ParseExportInfo(s.GetParam(motan.ProxyKey, "")) + addr = net.JoinHostPort(s.GetParam(provider.ProxyHostKey, provider.DefaultHost), strconv.FormatInt(int64(port), 10)) + } + c, e := net.DialTimeout("tcp", addr, timeout) + if e != nil { + continue + } + c.Close() + alive = true + break + } + mserver.HeartbeatDisabled = !alive + } +} diff --git a/server/motanserver.go b/server/motanserver.go index 8ad74721..43ed7d6e 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -18,7 +18,8 @@ import ( ) var currentConnections int64 - +// heartbeat downgrade in case of all backend servers invalid when agent proxy mode +var HeartbeatDisabled bool var motanServerOnce sync.Once func incrConnections() { @@ -191,6 +192,10 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c var tc *motan.TraceContext lastRequestID := msg.Header.RequestID if msg.Header.IsHeartbeat() { + if HeartbeatDisabled { + conn.Close() + return + } res = mpro.BuildHeartbeat(msg.Header.RequestID, mpro.Res) } else { tc = motan.TracePolicy(msg.Header.RequestID, msg.Metadata) @@ -278,6 +283,10 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st reqCtx.ExtFactory = m.extFactory reqCtx.RequestReceiveTime = start if mpro.IsV1HeartbeatReq(req) { + if HeartbeatDisabled { + conn.Close() + return + } result = mpro.BuildV1HeartbeatRes(req.GetRequestID()) } else { // TraceContext Currently not supported in protocol v1 From 3bfb07bd7358904a89be9a9b7aaeb9ed57cde41d Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 18 Oct 2022 11:00:18 +0800 Subject: [PATCH 33/75] heartbeat --- agent.go | 44 +++++++++++-------------------------------- core/motan.go | 1 + server/motanserver.go | 12 ++++++++---- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/agent.go b/agent.go index 9fd91ca0..88c1d279 100644 --- a/agent.go +++ b/agent.go @@ -14,7 +14,6 @@ import ( "time" cfg "github.com/weibocom/motan-go/config" - "github.com/weibocom/motan-go/provider" "gopkg.in/yaml.v2" "github.com/shirou/gopsutil/v3/process" @@ -75,6 +74,8 @@ type Agent struct { configurer *DynamicConfigurer commandHandlers []CommandHandler + + heartbeatDowngrade bool } type CommandHandler interface { @@ -196,7 +197,6 @@ func (a *Agent) StartMotanAgentFromConfig(config *cfg.Config) { a.configurer = NewDynamicConfigurer(a) go a.startMServer() go a.registerAgent() - go a.startAllServerStatusDaemon() f, err := os.Create(a.pidfile) if err != nil { vlog.Errorf("create file %s fail.", a.pidfile) @@ -1074,6 +1074,15 @@ func (a *Agent) mhandle(k string, h http.Handler) { vlog.Infof("add manage server handle path:%s", k) } +func (a *Agent) HeartbeatDowngrade(b bool) { + if a.heartbeatDowngrade != b { + a.heartbeatDowngrade = b + for _, s := range a.agentPortServer { + s.SetHeartbeat(b) + } + } +} + func (a *Agent) getConfigData() []byte { data, err := yaml.Marshal(a.Context.Config.GetOriginMap()) if err != nil { @@ -1120,34 +1129,3 @@ func (a *Agent) UnexportService(url *motan.URL) error { } return nil } -func (a *Agent) startAllServerStatusDaemon() { - defer motan.HandlePanic(nil) - var addr string - var timeout = 2 * time.Second - timer := time.NewTicker(time.Second * 30) - defer timer.Stop() - for { - <-timer.C - if len(a.Context.ServiceURLs) == 0 { - mserver.HeartbeatDisabled = false - continue - } - var alive = false - for _, s := range a.Context.ServiceURLs { - if v := s.GetParam(mhttp.ProxyAddressKey, ""); v != "" { - addr = v - } else { - _, port, _ := motan.ParseExportInfo(s.GetParam(motan.ProxyKey, "")) - addr = net.JoinHostPort(s.GetParam(provider.ProxyHostKey, provider.DefaultHost), strconv.FormatInt(int64(port), 10)) - } - c, e := net.DialTimeout("tcp", addr, timeout) - if e != nil { - continue - } - c.Close() - alive = true - break - } - mserver.HeartbeatDisabled = !alive - } -} diff --git a/core/motan.go b/core/motan.go index 68afc876..7218517d 100644 --- a/core/motan.go +++ b/core/motan.go @@ -245,6 +245,7 @@ type Server interface { SetMessageHandler(mh MessageHandler) GetMessageHandler() MessageHandler Open(block bool, proxy bool, handler MessageHandler, extFactory ExtensionFactory) error + SetHeartbeat(b bool) } // Exporter : export and manage a service. one exporter bind with a service diff --git a/server/motanserver.go b/server/motanserver.go index 43ed7d6e..1ffcf9d2 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -18,8 +18,6 @@ import ( ) var currentConnections int64 -// heartbeat downgrade in case of all backend servers invalid when agent proxy mode -var HeartbeatDisabled bool var motanServerOnce sync.Once func incrConnections() { @@ -42,6 +40,8 @@ type MotanServer struct { proxy bool isDestroyed chan bool maxContextLength int + // heartbeat downgrade in case of all backend servers invalid when agent proxy mode + heartbeatDisabled bool } func (m *MotanServer) Open(block bool, proxy bool, handler motan.MessageHandler, extFactory motan.ExtensionFactory) error { @@ -192,7 +192,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c var tc *motan.TraceContext lastRequestID := msg.Header.RequestID if msg.Header.IsHeartbeat() { - if HeartbeatDisabled { + if m.heartbeatDisabled { conn.Close() return } @@ -283,7 +283,7 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st reqCtx.ExtFactory = m.extFactory reqCtx.RequestReceiveTime = start if mpro.IsV1HeartbeatReq(req) { - if HeartbeatDisabled { + if m.heartbeatDisabled { conn.Close() return } @@ -326,6 +326,10 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st } } +func (m *MotanServer) SetHeartbeat(b bool) { + m.heartbeatDisabled=b +} + func getRemoteIP(address string) string { var ip string index := strings.Index(address, ":") From 1795a28042e24b962bb9f534606c744df92f9fc7 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 18 Oct 2022 11:07:36 +0800 Subject: [PATCH 34/75] heartbeat --- agent.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/agent.go b/agent.go index 88c1d279..a34fe5ef 100644 --- a/agent.go +++ b/agent.go @@ -75,7 +75,7 @@ type Agent struct { commandHandlers []CommandHandler - heartbeatDowngrade bool + backendServerAliveStatus bool } type CommandHandler interface { @@ -1074,12 +1074,16 @@ func (a *Agent) mhandle(k string, h http.Handler) { vlog.Infof("add manage server handle path:%s", k) } -func (a *Agent) HeartbeatDowngrade(b bool) { - if a.heartbeatDowngrade != b { - a.heartbeatDowngrade = b - for _, s := range a.agentPortServer { - s.SetHeartbeat(b) - } +func (a *Agent) heartbeatDowngrade(b bool) { + for _, s := range a.agentPortServer { + s.SetHeartbeat(b) + } +} + +func (a *Agent) BackendStatusChanged(alive bool) { + if a.backendServerAliveStatus != alive { + a.backendServerAliveStatus = alive + a.heartbeatDowngrade(alive) } } From 69417fa418afc8e3de7d8537916cb5ab1641721c Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 18 Oct 2022 11:33:36 +0800 Subject: [PATCH 35/75] fix heartbeat --- agent.go | 12 ++++-------- server/motanserver.go | 14 ++++++++------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/agent.go b/agent.go index a34fe5ef..a441601f 100644 --- a/agent.go +++ b/agent.go @@ -74,8 +74,6 @@ type Agent struct { configurer *DynamicConfigurer commandHandlers []CommandHandler - - backendServerAliveStatus bool } type CommandHandler interface { @@ -1074,17 +1072,15 @@ func (a *Agent) mhandle(k string, h http.Handler) { vlog.Infof("add manage server handle path:%s", k) } -func (a *Agent) heartbeatDowngrade(b bool) { +// backend server heartbeat downgrade +func (a *Agent) setBackendServerHeartbeat(enable bool) { for _, s := range a.agentPortServer { - s.SetHeartbeat(b) + s.SetHeartbeat(enable) } } func (a *Agent) BackendStatusChanged(alive bool) { - if a.backendServerAliveStatus != alive { - a.backendServerAliveStatus = alive - a.heartbeatDowngrade(alive) - } + a.setBackendServerHeartbeat(alive) } func (a *Agent) getConfigData() []byte { diff --git a/server/motanserver.go b/server/motanserver.go index 1ffcf9d2..012a6ffd 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -40,12 +40,13 @@ type MotanServer struct { proxy bool isDestroyed chan bool maxContextLength int - // heartbeat downgrade in case of all backend servers invalid when agent proxy mode - heartbeatDisabled bool + + heartbeatEnabled bool } func (m *MotanServer) Open(block bool, proxy bool, handler motan.MessageHandler, extFactory motan.ExtensionFactory) error { m.isDestroyed = make(chan bool, 1) + m.heartbeatEnabled = true motanServerOnce.Do(func() { metrics.RegisterStatusSampleFunc("motan_server_connection_count", getConnections) @@ -192,7 +193,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c var tc *motan.TraceContext lastRequestID := msg.Header.RequestID if msg.Header.IsHeartbeat() { - if m.heartbeatDisabled { + if !m.heartbeatEnabled { conn.Close() return } @@ -283,7 +284,7 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st reqCtx.ExtFactory = m.extFactory reqCtx.RequestReceiveTime = start if mpro.IsV1HeartbeatReq(req) { - if m.heartbeatDisabled { + if !m.heartbeatEnabled { conn.Close() return } @@ -326,8 +327,9 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st } } -func (m *MotanServer) SetHeartbeat(b bool) { - m.heartbeatDisabled=b +// SetHeartbeat true: enable heartbeat, false: disable heartbeat +func (m *MotanServer) SetHeartbeat(enabled bool) { + m.heartbeatEnabled = enabled } func getRemoteIP(address string) string { From b65d7b4b2f9534511e3d861b184e3b792d5799ee Mon Sep 17 00:00:00 2001 From: cocowh Date: Sat, 22 Oct 2022 09:29:23 +0800 Subject: [PATCH 36/75] export log dir (#292) --- agent_test.go | 3 +++ log/log.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/agent_test.go b/agent_test.go index c96deae7..c145e343 100644 --- a/agent_test.go +++ b/agent_test.go @@ -4,6 +4,7 @@ import ( "bytes" "flag" "github.com/weibocom/motan-go/config" + vlog "github.com/weibocom/motan-go/log" "io/ioutil" "math/rand" "net/http" @@ -109,6 +110,8 @@ motan-agent: _ = flag.Set("log_dir", "./test/cdef") a.initParam() assert.Equal(a.logdir, "./test/cdef") + // test export log dir + assert.Equal(vlog.GetLogDir(), "./test/cdef") } func TestHTTPProxyBodySize(t *testing.T) { diff --git a/log/log.go b/log/log.go index 5cc49bd0..45f34a1a 100644 --- a/log/log.go +++ b/log/log.go @@ -244,6 +244,10 @@ func SetMetricsLogAvailable(status bool) { } } +func GetLogDir() string { + return *logDir +} + func newDefaultLog() Logger { encoderConfig := zapcore.EncoderConfig{ TimeKey: "time", From f27183d3ddc5ca331e4df96e78496d66f206f66d Mon Sep 17 00:00:00 2001 From: kyton <641862816@qq.com> Date: Fri, 11 Nov 2022 12:43:30 +0800 Subject: [PATCH 37/75] add environment setting for mport (#293) --- agent.go | 11 +++++++++-- agent_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/agent.go b/agent.go index a441601f..0e819472 100644 --- a/agent.go +++ b/agent.go @@ -325,9 +325,16 @@ func (a *Agent) initParam() { } mPort := *motan.Mport - if mPort == 0 && section != nil && section["mport"] != nil { - mPort = section["mport"].(int) + if mPort == 0 { + if envMPort, ok := os.LookupEnv("mport"); ok { + if envMPortInt, err := strconv.Atoi(envMPort); err == nil { + mPort = envMPortInt + } + } else if section != nil && section["mport"] != nil { + mPort = section["mport"].(int) + } } + if mPort == 0 { mPort = defaultManagementPort } diff --git a/agent_test.go b/agent_test.go index c145e343..be4d9a5e 100644 --- a/agent_test.go +++ b/agent_test.go @@ -112,6 +112,45 @@ motan-agent: assert.Equal(a.logdir, "./test/cdef") // test export log dir assert.Equal(vlog.GetLogDir(), "./test/cdef") + + + mportConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + mport: 8003 +`))) + assert.Nil(err) + conf.Merge(mportConfig) + section, err = conf.GetSection("motan-agent") + assert.Nil(err) + a.initParam() + assert.Equal(a.mport, 8003) + + mportConfigENV, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + mport: 8003 +`))) + assert.Nil(err) + conf.Merge(mportConfigENV) + section, err = conf.GetSection("motan-agent") + assert.Nil(err) + err = os.Setenv("mport", "8006") + a.initParam() + assert.Equal(a.mport, 8006) + + mportConfigENVParam, err := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + mport: 8003 +`))) + assert.Nil(err) + conf.Merge(mportConfigENVParam) + section, err = conf.GetSection("motan-agent") + assert.Nil(err) + err = os.Setenv("mport", "8006") + _ = flag.Set("mport", "8007") + a.initParam() + assert.Equal(a.mport, 8007) + + } func TestHTTPProxyBodySize(t *testing.T) { From 661fbc5d73633ee7bb5aee15187c2877fd4d36ca Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Fri, 9 Dec 2022 09:55:20 +0800 Subject: [PATCH 38/75] update dynamicConfig, support multi registry (#294) * update dynamicConfig, support multi registry * add dynamic config error log * change subscribe back --- agent.go | 8 ++--- core/url.go | 6 ++++ dynamicConfig.go | 19 +++++++---- dynamicConfig_test.go | 74 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/agent.go b/agent.go index 0e819472..76ea4fce 100644 --- a/agent.go +++ b/agent.go @@ -835,10 +835,10 @@ func (a *Agent) doExportService(url *motan.URL) { return } - a.serviceExporters.Store(url.GetIdentity(), exporter) + a.serviceExporters.Store(url.GetIdentityWithRegistry(), exporter) vlog.Infof("service export success. url:%v", url) for _, r := range exporter.Registries { - rid := r.GetURL().GetIdentity() + rid := r.GetURL().GetIdentityWithRegistry() if _, ok := a.serviceRegistries.Load(rid); !ok { a.serviceRegistries.Store(rid, r) } @@ -1109,7 +1109,7 @@ func urlExist(url *motan.URL, urls map[string]*motan.URL) bool { func (a *Agent) SubscribeService(url *motan.URL) error { if urlExist(url, a.Context.RefersURLs) { - return nil + return fmt.Errorf("url exist, ignore subscribe, url: %s", url.GetIdentity()) } a.initCluster(url) return nil @@ -1117,7 +1117,7 @@ func (a *Agent) SubscribeService(url *motan.URL) error { func (a *Agent) ExportService(url *motan.URL) error { if urlExist(url, a.Context.ServiceURLs) { - return nil + return fmt.Errorf("url exist, ignore export. url: %s", url.GetIdentityWithRegistry()) } a.doExportService(url) return nil diff --git a/core/url.go b/core/url.go index f470b911..b906de90 100644 --- a/core/url.go +++ b/core/url.go @@ -42,6 +42,12 @@ func (u *URL) GetIdentity() string { return u.identity } +func (u *URL) GetIdentityWithRegistry() string { + id := u.GetIdentity() + registryId := u.GetParam(RegistryKey, "") + return id + "®istry=" + registryId +} + func (u *URL) ClearCachedInfo() { u.address = "" u.identity = "" diff --git a/dynamicConfig.go b/dynamicConfig.go index 86d9d2eb..3c21c3f5 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -91,11 +91,15 @@ func (c *DynamicConfigurer) Register(url *core.URL) error { func (c *DynamicConfigurer) doRegister(url *core.URL) error { c.regLock.Lock() defer c.regLock.Unlock() - if _, ok := c.registerNodes[url.GetIdentity()]; ok { + if _, ok := c.registerNodes[url.GetIdentityWithRegistry()]; ok { return nil } - c.agent.ExportService(url) - c.registerNodes[url.GetIdentity()] = url + err := c.agent.ExportService(url) + if err != nil { + vlog.Warningf("dynamic register failed, error: %s", err.Error()) + } else { + c.registerNodes[url.GetIdentityWithRegistry()] = url + } return nil } @@ -112,9 +116,9 @@ func (c *DynamicConfigurer) doUnregister(url *core.URL) error { c.regLock.Lock() defer c.regLock.Unlock() - if _, ok := c.registerNodes[url.GetIdentity()]; ok { + if _, ok := c.registerNodes[url.GetIdentityWithRegistry()]; ok { c.agent.UnexportService(url) - delete(c.registerNodes, url.GetIdentity()) + delete(c.registerNodes, url.GetIdentityWithRegistry()) } return nil } @@ -135,7 +139,10 @@ func (c *DynamicConfigurer) doSubscribe(url *core.URL) error { return nil } c.subscribeNodes[url.GetIdentity()] = url - c.agent.SubscribeService(url) + err := c.agent.SubscribeService(url) + if err != nil { + vlog.Warningf("dynamic subscribe failed, error: %s", err.Error()) + } return nil } diff --git a/dynamicConfig_test.go b/dynamicConfig_test.go index 477af59e..cddd8b9f 100644 --- a/dynamicConfig_test.go +++ b/dynamicConfig_test.go @@ -2,6 +2,7 @@ package motan import ( "bytes" + motan "github.com/weibocom/motan-go/core" "net/http/httptest" "testing" @@ -27,3 +28,76 @@ func TestDynamicConfigurerHandler_readURLs(t *testing.T) { _, err = d.readURLsFromRequest(req3) assert.NotNil(t, err) } + +func TestDynamicConfigurerMultiRegistry(t *testing.T) { + a := NewAgent(nil) + a.Context = &motan.Context{ + RegistryURLs: make(map[string]*motan.URL), + RefersURLs: make(map[string]*motan.URL), + BasicReferURLs: make(map[string]*motan.URL), + ServiceURLs: make(map[string]*motan.URL), + BasicServiceURLs: make(map[string]*motan.URL), + } + configurer := &DynamicConfigurer{ + agent: a, + registerNodes: make(map[string]*motan.URL), + subscribeNodes: make(map[string]*motan.URL), + } + urls := []*motan.URL{ + { + Protocol: "motan2", + Host: "127.0.0.1", + Port: 1910, + Path: "test_path", + Group: "test_group", + Parameters: map[string]string{ + "registry": "r1", + }, + }, + { + Protocol: "motan2", + Host: "127.0.0.1", + Port: 1910, + Path: "test_path", + Group: "test_group", + Parameters: map[string]string{ + "registry": "r2", + }, + }, + { + Protocol: "motan2", + Host: "127.0.0.1", + Port: 1910, + Path: "test_path", + Group: "test_group1", + Parameters: map[string]string{ + "registry": "r1", + }, + }, + { + Protocol: "motan2", + Host: "127.0.0.1", + Port: 1910, + Path: "test_path", + Group: "test_group1", + Parameters: map[string]string{ + "registry": "r2", + }, + }, + // 增加一个重复的 + { + Protocol: "motan2", + Host: "127.0.0.1", + Port: 1910, + Path: "test_path", + Group: "test_group1", + Parameters: map[string]string{ + "registry": "r2", + }, + }, + } + for _, j := range urls { + configurer.doRegister(j) + } + assert.Equal(t, len(configurer.registerNodes), 4) +} From 22d4d63fd28ba0a64e1859c5873262d6e11c627b Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Wed, 14 Dec 2022 16:33:20 +0800 Subject: [PATCH 39/75] Dev (#295) * update rpc request x-forwarded-for logic, fit multi agent --- provider/httpProvider.go | 13 +++++++++---- provider/httpxProvider.go | 5 +++-- provider/motanProvider.go | 4 +++- provider/motanProvider_test.go | 31 +++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/provider/httpProvider.go b/provider/httpProvider.go index 65a74876..cd334dc0 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -272,7 +272,9 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { return true }) httpReq.Header.Del("Connection") - httpReq.Header.Set("X-Forwarded-For", ip) + if httpReq.Header.Peek("X-Forwarded-For") == nil { + httpReq.Header.Set("X-Forwarded-For", ip) + } if len(bodyBytes) != 0 { httpReq.BodyWriter().Write(bodyBytes) } @@ -328,7 +330,9 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { if len(httpReq.Header.Host()) == 0 { httpReq.Header.SetHost(h.domain) } - httpReq.Header.Set("X-Forwarded-For", ip) + if httpReq.Header.Peek("X-Forwarded-For") == nil { + httpReq.Header.Set("X-Forwarded-For", ip) + } err = h.fastClient.Do(httpReq, httpRes) if err != nil { fillExceptionWithCode(resp, http.StatusServiceUnavailable, t, err) @@ -372,8 +376,9 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { req.Header.Add(k, v) return true }) - - req.Header.Add("x-forwarded-for", ip) + if req.Header.Get("x-forwarded-for") == "" { + req.Header.Add("x-forwarded-for", ip) + } timeout := h.url.GetTimeDuration(motan.TimeOutKey, time.Millisecond, DefaultRequestTimeout) c := http.Client{ diff --git a/provider/httpxProvider.go b/provider/httpxProvider.go index 8bb58f7d..adf29e88 100644 --- a/provider/httpxProvider.go +++ b/provider/httpxProvider.go @@ -174,8 +174,9 @@ func (h *HTTPXProvider) Call(request motan.Request) motan.Response { } else { ip = request.GetAttachment(motan.HostKey) } - req.Header.Add("x-forwarded-for", ip) - + if req.Header.Peek("x-forwarded-for") == nil { + req.Header.Add("x-forwarded-for", ip) + } err = h.httpClient.Do(req, httpResp) if err != nil { vlog.Errorf("new HTTP Provider Do HTTP Call, request:%+v, err: %v", req, err) diff --git a/provider/motanProvider.go b/provider/motanProvider.go index c5cc8017..01d5e0c7 100644 --- a/provider/motanProvider.go +++ b/provider/motanProvider.go @@ -59,7 +59,9 @@ func (m *MotanProvider) Initialize() { func (m *MotanProvider) Call(request motan.Request) motan.Response { if m.IsAvailable() { // x-forwared-for - request.SetAttachment("x-forwarded-for", request.GetAttachment(motan.HostKey)) + if request.GetAttachment("x-forwarded-for") == "" && request.GetAttachment("X-Forwarded-For") == "" { + request.SetAttachment("x-forwarded-for", request.GetAttachment(motan.HostKey)) + } return m.ep.Call(request) } t := time.Now().UnixNano() diff --git a/provider/motanProvider_test.go b/provider/motanProvider_test.go index 216e8d1f..38e74761 100644 --- a/provider/motanProvider_test.go +++ b/provider/motanProvider_test.go @@ -1,6 +1,7 @@ package provider import ( + "github.com/stretchr/testify/assert" "testing" motan "github.com/weibocom/motan-go/core" @@ -44,3 +45,33 @@ func TestGetName(t *testing.T) { } } + +func TestXForwardedFor(t *testing.T) { + //init factory + factory := &motan.DefaultExtensionFactory{} + factory.Initialize() + endpoint.RegistDefaultEndpoint(factory) + RegistDefaultProvider(factory) + + //init motanProvider + mContext := motan.Context{} + mContext.ConfigFile = confFilePath + mContext.Initialize() + request := &motan.MotanRequest{} + request.SetAttachment("x-forwarded-for", "test") + url := mContext.ServiceURLs[serviceName] + + //call correct + providerCorr := MotanProvider{url: url, extFactory: factory} + providerCorr.Initialize() + providerCorr.Call(request) + assert.Equal(t, request.GetAttachment("x-forwarded-for"), "test") + request = &motan.MotanRequest{} + request.SetAttachment("X-Forwarded-For", "test") + providerCorr.Call(request) + assert.Equal(t, request.GetAttachment("x-forwarded-for"), "") + request = &motan.MotanRequest{} + request.SetAttachment("x-Forwarded-For", "test") + providerCorr.Call(request) + assert.NotEqual(t, request.GetAttachment("x-forwarded-for"), "test") +} From 5da53025d3247d1049dd07739b4fbd8e188c9349 Mon Sep 17 00:00:00 2001 From: liangwei3 Date: Mon, 19 Dec 2022 13:11:43 +0800 Subject: [PATCH 40/75] add motan provider to fit motan1 pressure --- provider/provider.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/provider/provider.go b/provider/provider.go index bc373d8c..37862de8 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -13,6 +13,7 @@ const ( HTTP = "http" HTTPX = "httpx" MOTAN2 = "motan2" + MOTAN = "motan" Mock = "mockProvider" Default = "default" ) @@ -35,6 +36,10 @@ func RegistDefaultProvider(extFactory motan.ExtensionFactory) { return &MotanProvider{url: url, extFactory: extFactory} }) + extFactory.RegistExtProvider(MOTAN, func(url *motan.URL) motan.Provider { + return &MotanProvider{url: url, extFactory: extFactory} + }) + extFactory.RegistExtProvider(Mock, func(url *motan.URL) motan.Provider { return &MockProvider{URL: url} }) From 578fb8e3ce2ccf24fbc785943e58506508cc53ab Mon Sep 17 00:00:00 2001 From: liangwei3 Date: Mon, 19 Dec 2022 14:42:42 +0800 Subject: [PATCH 41/75] add motan server to fit motan1 pressure --- server/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/server.go b/server/server.go index b7958f89..001757cb 100644 --- a/server/server.go +++ b/server/server.go @@ -11,6 +11,7 @@ import ( const ( Motan2 = "motan2" + Motan = "motan" CGI = "cgi" ) @@ -22,6 +23,9 @@ func RegistDefaultServers(extFactory motan.ExtensionFactory) { extFactory.RegistExtServer(Motan2, func(url *motan.URL) motan.Server { return &MotanServer{URL: url} }) + extFactory.RegistExtServer(Motan, func(url *motan.URL) motan.Server { + return &MotanServer{URL: url} + }) extFactory.RegistExtServer(CGI, func(url *motan.URL) motan.Server { return &MotanServer{URL: url} }) From 036de11dd80d21370bf77f78eb8d439cf033908a Mon Sep 17 00:00:00 2001 From: zhanglei28 Date: Thu, 22 Dec 2022 13:42:23 +0800 Subject: [PATCH 42/75] log filter init info --- agent_test.go | 2 -- cluster/motanCluster.go | 2 ++ core/util.go | 17 +++++++++++++++++ core/util_test.go | 10 ++++++++++ ha/backupRequestHA_test.go | 4 ++-- server/server.go | 4 +++- 6 files changed, 34 insertions(+), 5 deletions(-) diff --git a/agent_test.go b/agent_test.go index be4d9a5e..f0afb320 100644 --- a/agent_test.go +++ b/agent_test.go @@ -113,7 +113,6 @@ motan-agent: // test export log dir assert.Equal(vlog.GetLogDir(), "./test/cdef") - mportConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: mport: 8003 @@ -150,7 +149,6 @@ motan-agent: a.initParam() assert.Equal(a.mport, 8007) - } func TestHTTPProxyBodySize(t *testing.T) { diff --git a/cluster/motanCluster.go b/cluster/motanCluster.go index 6d742531..b8a596fa 100644 --- a/cluster/motanCluster.go +++ b/cluster/motanCluster.go @@ -234,6 +234,7 @@ func (m *MotanCluster) addFilter(ep motan.EndPoint, filters []motan.Filter) mota } fep.StatusFilters = statusFilters fep.Filter = lastf + vlog.Infof("MotanCluster add ep filters. url:%+v, filters:%s", ep.GetURL(), motan.GetEPFilterInfo(fep.Filter)) return fep } @@ -312,6 +313,7 @@ func (m *MotanCluster) initFilters() { if len(endpointFilters) > 0 { m.Filters = endpointFilters } + vlog.Infof("MotanCluster init filter. url:%+v, cluster filter:%#v, ep filter size:%d, ep filters:%#v", m.GetURL(), m.clusterFilter, len(m.Filters), m.Filters) } func (m *MotanCluster) NotifyAgentCommand(commandInfo string) { diff --git a/core/util.go b/core/util.go index b20d3040..e25dba77 100644 --- a/core/util.go +++ b/core/util.go @@ -137,6 +137,23 @@ func GetResInfo(response Response) string { return "" } +func GetEPFilterInfo(filter EndPointFilter) string { + if filter != nil { + var buffer bytes.Buffer + writeEPFilter(filter, &buffer) + return buffer.String() + } + return "" +} + +func writeEPFilter(filter EndPointFilter, buffer *bytes.Buffer) { + buffer.WriteString(filter.GetName()) + if filter.GetNext() != nil { + buffer.WriteString("->") + writeEPFilter(filter.GetNext(), buffer) + } +} + func HandlePanic(f func()) { if err := recover(); err != nil { vlog.Errorf("recover panic. error:%v, stack: %s", err, debug.Stack()) diff --git a/core/util_test.go b/core/util_test.go index 18a76a01..92a14bf5 100644 --- a/core/util_test.go +++ b/core/util_test.go @@ -124,3 +124,13 @@ func TestGetResInfo(t *testing.T) { assert.Equal(t, "res{374867809809,}", GetResInfo(res)) assert.Equal(t, "res{374867809809,testErrMsg}", GetResInfo(resE)) } + +func TestGetEPFilterInfo(t *testing.T) { + filter1 := &TestEndPointFilter{Index: 1} + filter2 := &TestEndPointFilter{Index: 2} + filter3 := &TestEndPointFilter{Index: 3} + filter1.SetNext(filter2) + filter2.SetNext(filter3) + str := GetEPFilterInfo(filter1) + assert.Equal(t, "TestEndPointFilter->TestEndPointFilter->TestEndPointFilter", str) +} diff --git a/ha/backupRequestHA_test.go b/ha/backupRequestHA_test.go index c9cb5fa8..1a774be7 100644 --- a/ha/backupRequestHA_test.go +++ b/ha/backupRequestHA_test.go @@ -80,7 +80,7 @@ func TestBackupRequestHA_Call2(t *testing.T) { nlb.OnRefresh([]motan.EndPoint{getEP(1)}) res := ha.Call(request, nlb) time.Sleep(10*time.Millisecond + 1*time.Second) - ep1 := getEP(1) // third round + ep1 := getEP(1) // third round testEndpoints := []motan.EndPoint{ep1} nlb.OnRefresh(testEndpoints) res = ha.Call(request, nlb) @@ -111,7 +111,7 @@ func TestBackupRequestHA_Call3(t *testing.T) { nlb.OnRefresh([]motan.EndPoint{getEP(1)}) res := ha.Call(request, nlb) time.Sleep(10*time.Millisecond + 1*time.Second) - ep1 := getEP(1) // third round + ep1 := getEP(1) // third round testEndpoints := []motan.EndPoint{ep1} nlb.OnRefresh(testEndpoints) res = ha.Call(request, nlb) diff --git a/server/server.go b/server/server.go index 001757cb..78f60db8 100644 --- a/server/server.go +++ b/server/server.go @@ -226,5 +226,7 @@ func WrapWithFilter(provider motan.Provider, extFactory motan.ExtensionFactory, } } } - return &FilterProviderWrapper{provider: provider, filter: lastf} + fpw := &FilterProviderWrapper{provider: provider, filter: lastf} + vlog.Infof("FilterProviderWrapper url: %+v, filter size:%d, filters:%s", provider.GetURL(), len(filters), motan.GetEPFilterInfo(fpw.filter)) + return fpw } From bac2b3218e769ff8fb8455f5acde085f73e2285a Mon Sep 17 00:00:00 2001 From: liangwei3 Date: Tue, 27 Dec 2022 10:51:58 +0800 Subject: [PATCH 43/75] change fast http client max connection from 512 to 1024 --- provider/httpProvider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/httpProvider.go b/provider/httpProvider.go index cd334dc0..58af13e9 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -87,7 +87,7 @@ func (h *HTTPProvider) Initialize() { h.locationMatcher = mhttp.NewLocationMatcherFromContext(h.domain, h.gctx) h.proxyAddr = h.url.GetParam(mhttp.ProxyAddressKey, "") h.proxySchema = h.url.GetParam(mhttp.ProxySchemaKey, "http") - h.maxConnections = int(h.url.GetPositiveIntValue(mhttp.MaxConnectionsKey, 512)) + h.maxConnections = int(h.url.GetPositiveIntValue(mhttp.MaxConnectionsKey, 1024)) h.enableRewrite = true enableRewriteStr := h.url.GetParam(mhttp.EnableRewriteKey, "true") if enableRewrite, err := strconv.ParseBool(enableRewriteStr); err != nil { From 8ad361248a28041d69924131d8eff31985bede8d Mon Sep 17 00:00:00 2001 From: snail007 Date: Fri, 30 Dec 2022 11:04:28 +0800 Subject: [PATCH 44/75] reload cluster compat dynamic refers (#300) * reload cluster compat dynamic refers --- agent.go | 12 +++++++++++- agent_test.go | 10 +++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/agent.go b/agent.go index 76ea4fce..0dde8c04 100644 --- a/agent.go +++ b/agent.go @@ -430,7 +430,17 @@ func (a *Agent) reloadClusters(ctx *motan.Context) { serviceItemKeep := make(map[string]bool) clusterMap := make(map[interface{}]interface{}) serviceMap := make(map[interface{}]interface{}) - for _, url := range a.Context.RefersURLs { + var allRefersURLs = []*motan.URL{} + if a.configurer != nil { + //keep all dynamic refers + for _, url := range a.configurer.subscribeNodes { + allRefersURLs = append(allRefersURLs, url) + } + } + for _, v := range a.Context.RefersURLs { + allRefersURLs = append(allRefersURLs, v) + } + for _, url := range allRefersURLs { if url.Parameters[motan.ApplicationKey] == "" { url.Parameters[motan.ApplicationKey] = a.agentURL.Parameters[motan.ApplicationKey] } diff --git a/agent_test.go b/agent_test.go index f0afb320..c52d82c2 100644 --- a/agent_test.go +++ b/agent_test.go @@ -339,9 +339,17 @@ func TestAgent_InitCall(t *testing.T) { "test4-1": {Parameters: map[string]string{core.VersionKey: ""}, Path: "test4", Group: "g2", Protocol: ""}, "test5": {Parameters: map[string]string{core.VersionKey: "1.0"}, Path: "test5", Group: "g1", Protocol: "motan"}, } + dynamicURLs := map[string]*core.URL{ + "test6": {Parameters: map[string]string{core.VersionKey: ""}, Path: "test6", Group: "g1", Protocol: ""}, + } + agent.serviceMap.Store("test6", []serviceMapItem{ + {url: dynamicURLs["test6"], cluster: nil}, + }) + agent.configurer = NewDynamicConfigurer(agent) + agent.configurer.subscribeNodes = dynamicURLs ctx.RefersURLs = reloadUrls agent.reloadClusters(ctx) - assert.Equal(t, agent.serviceMap.Len(), 2, "hot-load serviceMap except length error") + assert.Equal(t, agent.serviceMap.Len(), 3, "hot-load serviceMap except length error") for _, v := range []struct { service string From cf14bbb0da2689708ca77d468f4b4fda17b5f0da Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 3 Jan 2023 11:22:06 +0800 Subject: [PATCH 45/75] update gopkg.in/yaml.v2 v2.2.4 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a93b245d..b5d6bfce 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect google.golang.org/grpc v1.15.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/yaml.v2 v2.2.1 + gopkg.in/yaml.v2 v2.2.4 ) replace ( From 231e9048d0b128e9975686b3b06d1909ba4f7928 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 3 Jan 2023 11:25:55 +0800 Subject: [PATCH 46/75] update testing go version 1.18.x 1.19.x --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f4f55301..a80f57d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: testing: strategy: matrix: - go-version: [1.12.x,1.13.x,1.14.x,1.15.x,1.16.x,1.17.x] + go-version: [1.12.x,1.13.x,1.14.x,1.15.x,1.16.x,1.17.x,1.18.x,1.19.x] platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} steps: From 4973fc82f7bf58eea8e17cdcbf80e079ea399d8d Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 3 Jan 2023 11:30:07 +0800 Subject: [PATCH 47/75] update actions/setup-go@v2 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a80f57d4..76af20b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.15 - uses: actions/setup-go@v1 + uses: actions/setup-go@v2 with: go-version: 1.15.x id: go From a6a67ff1befd1720644fcb5e0597032e45269c83 Mon Sep 17 00:00:00 2001 From: snail007 Date: Tue, 10 Jan 2023 17:57:16 +0800 Subject: [PATCH 48/75] ep unix (#302) * unix sock support for endpoint, mesh client, http provider, motan provider --- .github/workflows/test.yml | 28 +++-- agent.go | 9 ++ agent_test.go | 212 +++++++++++++++++++++++++++++++- core/constants.go | 1 + core/url.go | 4 + dynamicConfig.go | 3 + endpoint/motanCommonEndpoint.go | 15 ++- endpoint/motanEndpoint.go | 16 ++- go.mod | 21 ++-- provider/httpProvider.go | 3 + provider/httpProvider_test.go | 9 +- registry/directRegistry.go | 18 ++- registry/registry.go | 6 + server.go | 24 +++- server_test.go | 11 +- 15 files changed, 339 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76af20b0..e48ca001 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,11 @@ on: - dev name: build + +env: + GOPATH: /home/runner/go + GO111MODULE: on + jobs: testing: strategy: @@ -27,12 +32,15 @@ jobs: with: go-version: ${{ matrix.go-version }} - - name: Install Go Dependencies - run: | - go get -d -v -t $(go list ./... | grep -v main) - - name: Run tests - run: go test -v -race $(go list ./... | grep -v main) + run: | + echo "GOPATH >>> $GOPATH" + echo "pwd >>> $PWD" + mkdir -p $GOPATH/src/github.com/weibocom/ + cp -R ../motan-go $GOPATH/src/github.com/weibocom/ + cd $GOPATH/src/github.com/weibocom/motan-go + go mod tidy + go test -v -race $(go list ./... | grep -v main) codecov: name: codecov runs-on: ubuntu-latest @@ -46,12 +54,14 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Get dependencies - run: | - go get -d -v -t $(go list ./... | grep -v main) - - name: Generate coverage report run: | + echo "GOPATH >>> $GOPATH" + echo "pwd >>> $PWD" + mkdir -p $GOPATH/src/github.com/weibocom/ + cp -R ../motan-go $GOPATH/src/github.com/weibocom/ + cd $GOPATH/src/github.com/weibocom/motan-go + go mod tidy go test -v -race -coverprofile=coverage.txt -covermode=atomic $(go list ./... | grep -v main) - name: Upload coverage report diff --git a/agent.go b/agent.go index 0dde8c04..f54f1c91 100644 --- a/agent.go +++ b/agent.go @@ -39,6 +39,11 @@ const ( defaultStatusSnap = "status" ) +var ( + initParamLock sync.Mutex + setAgentLock sync.Mutex +) + type Agent struct { ConfigFile string extFactory motan.ExtensionFactory @@ -272,6 +277,8 @@ func (a *Agent) recoverStatus() { } func (a *Agent) initParam() { + initParamLock.Lock() + defer initParamLock.Unlock() section, err := a.Context.Config.GetSection("motan-agent") if err != nil { fmt.Println("get config of \"motan-agent\" fail! err " + err.Error()) @@ -1072,7 +1079,9 @@ func (a *Agent) mhandle(k string, h http.Handler) { } }() if sa, ok := h.(SetAgent); ok { + setAgentLock.Lock() sa.SetAgent(a) + setAgentLock.Unlock() } http.HandleFunc(k, func(w http.ResponseWriter, r *http.Request) { if !PermissionCheck(r) { diff --git a/agent_test.go b/agent_test.go index c52d82c2..f4c8ea74 100644 --- a/agent_test.go +++ b/agent_test.go @@ -3,10 +3,14 @@ package motan import ( "bytes" "flag" + _ "fmt" "github.com/weibocom/motan-go/config" vlog "github.com/weibocom/motan-go/log" + _ "github.com/weibocom/motan-go/server" + _ "golang.org/x/net/context" "io/ioutil" "math/rand" + "net" "net/http" "net/url" "os" @@ -32,6 +36,109 @@ var proxyClient *http.Client var meshClient *MeshClient var agent *Agent +func Test_unixClientCall1(t *testing.T) { + t.Parallel() + startServer(t, "helloService", 22991) + time.Sleep(time.Second * 3) + // start client mesh + ext := GetDefaultExtFactory() + os.Remove("agent.sock") + config, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + mport: 12500 + port: 13821 + eport: 13281 + htport: 24282 + unixSock: agent.sock + log_dir: "stdout" + snapshot_dir: "./snapshot" + application: "testing" + +motan-registry: + direct: + protocol: direct + address: 127.0.0.1:22991 + +motan-refer: + recom-engine-refer: + group: hello + path: helloService + protocol: motan2 + registry: direct + serialization: breeze`))) + agent := NewAgent(ext) + go agent.StartMotanAgentFromConfig(config) + time.Sleep(time.Second * 3) + c1 := NewMeshClient() + c1.SetAddress("unix://./agent.sock") + c1.Initialize() + req := c1.BuildRequestWithGroup("helloService", "Hello", []interface{}{"jack"}, "hello") + resp := c1.BaseCall(req, nil) + assert.Nil(t, resp.GetException()) + assert.Equal(t, "Hello jack from motan server", resp.GetValue()) +} +func Test_unixClientCall2(t *testing.T) { + t.Parallel() + startServer(t, "helloService", 22992) + time.Sleep(time.Second * 3) + // start client mesh + ext := GetDefaultExtFactory() + os.Remove("agent2.sock") + config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + mport: 12501 + port: 12921 + mport: 12903 + eport: 12981 + htport: 23982 + unixSock: ./agent2.sock + log_dir: "stdout" + snapshot_dir: "./snapshot" + application: "testing" + +motan-registry: + direct: + protocol: direct + address: 127.0.0.1:22992 + +motan-refer: + test-refer: + group: hello + path: helloService + protocol: motanV1Compatible + registry: direct + serialization: breeze +`))) + agent := NewAgent(ext) + go agent.StartMotanAgentFromConfig(config1) + time.Sleep(time.Second * 3) + + ext1 := GetDefaultExtFactory() + cfg, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-client: + log_dir: stdout + application: client-test +motan-registry: + local: + protocol: direct + address: unix://./agent2.sock +motan-refer: + test-refer: + registry: local + serialization: breeze + protocol: motanV1Compatible + group: hello + path: helloService + requestTimeout: 3000 +`))) + mccontext := NewClientContextFromConfig(cfg) + mccontext.Start(ext1) + mclient := mccontext.GetClient("test-refer") + var reply string + err := mclient.Call("Hello", []interface{}{"jack"}, &reply) + assert.Nil(t, err) + assert.Equal(t, "Hello jack from motan server", reply) +} func TestMain(m *testing.M) { core.RegistLocalProvider("LocalTestService", &LocalTestServiceProvider{}) cfgFile := filepath.Join("testdata", "agent.yaml") @@ -148,7 +255,6 @@ motan-agent: _ = flag.Set("mport", "8007") a.initParam() assert.Equal(a.mport, 8007) - } func TestHTTPProxyBodySize(t *testing.T) { @@ -428,3 +534,107 @@ func (l *LocalTestServiceProvider) Destroy() { func (l *LocalTestServiceProvider) GetPath() string { return l.url.Path } + +func Test_unixHTTPClientCall(t *testing.T) { + t.Parallel() + go func() { + http.HandleFunc("/unixclient", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("okay")) + }) + os.Remove("http2.sock") + addr, _ := net.ResolveUnixAddr("unix", "http2.sock") + l, err := net.ListenUnix("unix", addr) + if err != nil { + panic(err) + } + err = http.Serve(l, nil) + if err != nil { + panic(err) + } + }() + // start unix server mesh + ext := GetDefaultExtFactory() + config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + port: 12822 + mport: 12504 + eport: 23282 + htport: 23283 + log_dir: "stdout" + snapshot_dir: "./snapshot" + application: "testing" + +motan-registry: + direct: + protocol: direct + +motan-service: + test01: + protocol: motan2 + provider: http + proxyAddress: unix://./http2.sock + group: hello + path: helloService + registry: direct + serialization: simple + enableRewrite: false + export: motan2:23282 +`))) + agent := NewAgent(ext) + go agent.StartMotanAgentFromConfig(config1) + time.Sleep(time.Second * 3) + c1 := NewMeshClient() + c1.SetAddress("127.0.0.1:23282") + c1.Initialize() + var reply []byte + req := c1.BuildRequestWithGroup("helloService", "/unixclient", []interface{}{}, "hello") + req.SetAttachment("HTTP_HOST", "test.com") + resp := c1.BaseCall(req, &reply) + assert.Nil(t, resp.GetException()) + assert.Equal(t, "okay", string(reply)) +} +func Test_unixRPCClientCall(t *testing.T) { + t.Parallel() + os.Remove("server.sock") + startServer(t, "helloService", 0, "server.sock") + time.Sleep(time.Second * 3) + // start client mesh + // start unix server mesh + ext := GetDefaultExtFactory() + config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + port: 12821 + mport: 12503 + eport: 12281 + htport: 23282 + log_dir: "stdout" + snapshot_dir: "./snapshot" + application: "testing" + +motan-registry: + direct: + protocol: direct + +motan-service: + test01: + protocol: motan2 + provider: motan2 + group: hello + path: helloService + registry: direct + serialization: simple + proxy.host: unix://./server.sock + export: motan2:12281 +`))) + agent := NewAgent(ext) + go agent.StartMotanAgentFromConfig(config1) + time.Sleep(time.Second * 3) + c1 := NewMeshClient() + c1.SetAddress("127.0.0.1:12281") + c1.Initialize() + var reply []byte + req := c1.BuildRequestWithGroup("helloService", "Hello", []interface{}{"jack"}, "hello") + resp := c1.BaseCall(req, &reply) + assert.Nil(t, resp.GetException()) + assert.Equal(t, "Hello jack from motan server", string(reply)) +} diff --git a/core/constants.go b/core/constants.go index 544a6c79..a5d489e9 100644 --- a/core/constants.go +++ b/core/constants.go @@ -65,6 +65,7 @@ const ( HTTPProxyUnixSockKey = "httpProxyUnixSock" MixGroups = "mixGroups" MaxContentLength = "maxContentLength" + UnixSockProtocolFlag = "unix://" ) // nodeType diff --git a/core/url.go b/core/url.go index b906de90..f61acde9 100644 --- a/core/url.go +++ b/core/url.go @@ -216,6 +216,10 @@ func (u *URL) GetAddressStr() string { if u.address != "" { return u.address } + if strings.HasPrefix(u.Host, UnixSockProtocolFlag) { + u.address = u.Host + return u.address + } u.address = u.Host + ":" + u.GetPortStr() return u.address } diff --git a/dynamicConfig.go b/dynamicConfig.go index 3c21c3f5..98ca0538 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -186,9 +186,12 @@ func (c *DynamicConfigurer) getRegistryInfo() *registrySnapInfoStorage { type DynamicConfigurerHandler struct { agent *Agent + lock sync.Mutex } func (h *DynamicConfigurerHandler) SetAgent(agent *Agent) { + h.lock.Lock() + defer h.lock.Unlock() h.agent = agent } diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index a6200b7a..ff2d0fe5 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -8,6 +8,7 @@ import ( mpro "github.com/weibocom/motan-go/protocol" "net" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -67,7 +68,11 @@ func (m *MotanCommonEndpoint) Initialize() { m.heartbeatVersion = -1 m.DefaultVersion = mpro.Version2 factory := func() (net.Conn, error) { - return net.DialTimeout("tcp", m.url.GetAddressStr(), connectTimeout) + address := m.url.GetAddressStr() + if strings.HasPrefix(address, motan.UnixSockProtocolFlag) { + return net.DialTimeout("unix", address[len(motan.UnixSockProtocolFlag):], connectTimeout) + } + return net.DialTimeout("tcp", address, connectTimeout) } config := &ChannelConfig{MaxContentLength: m.maxContentLength, Serialization: m.serialization} if asyncInitConnection { @@ -718,7 +723,9 @@ func (c *ChannelPool) Get() (*Channel, error) { if err != nil { vlog.Errorf("create channel failed. err:%s", err.Error()) } else { - _ = conn.(*net.TCPConn).SetNoDelay(true) + if c, ok := conn.(*net.TCPConn); ok { + c.SetNoDelay(true) + } } channel = buildChannel(conn, c.config, c.serialization) } @@ -786,7 +793,9 @@ func NewChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, ser channelPool.Close() return nil, err } - _ = conn.(*net.TCPConn).SetNoDelay(true) + if c, ok := conn.(*net.TCPConn); ok { + c.SetNoDelay(true) + } channelPool.channels <- buildChannel(conn, config, serialization) } } diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index d34058e2..203f3074 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -82,7 +83,11 @@ func (m *MotanEndpoint) Initialize() { m.lazyInit = m.url.GetBoolValue(motan.LazyInit, false) asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, false) factory := func() (net.Conn, error) { - return net.DialTimeout("tcp", m.url.GetAddressStr(), connectTimeout) + address := m.url.GetAddressStr() + if strings.HasPrefix(address, motan.UnixSockProtocolFlag) { + return net.DialTimeout("unix", address[len(motan.UnixSockProtocolFlag):], connectTimeout) + } + return net.DialTimeout("tcp", address, connectTimeout) } config := &ChannelConfig{MaxContentLength: m.maxContentLength} if asyncInitConnection { @@ -670,7 +675,9 @@ func (c *V2ChannelPool) Get() (*V2Channel, error) { if err != nil { vlog.Errorf("create channel failed. err:%s", err.Error()) } else { - _ = conn.(*net.TCPConn).SetNoDelay(true) + if c, ok := conn.(*net.TCPConn); ok { + c.SetNoDelay(true) + } } channel = buildV2Channel(conn, c.config, c.serialization) } @@ -738,7 +745,10 @@ func NewV2ChannelPool(poolCap int, factory ConnFactory, config *ChannelConfig, s channelPool.Close() return nil, err } - _ = conn.(*net.TCPConn).SetNoDelay(true) + if c, ok := conn.(*net.TCPConn); ok { + c.SetNoDelay(true) + } + channelPool.channels <- buildV2Channel(conn, config, serialization) } } diff --git a/go.mod b/go.mod index b5d6bfce..a83fbb4b 100644 --- a/go.mod +++ b/go.mod @@ -6,26 +6,29 @@ require ( github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 github.com/beberlei/fastcgi-serve v0.0.0-20151230120321-4676005f65b7 github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.2.0 + github.com/golang/protobuf v1.3.2 github.com/juju/ratelimit v1.0.1 - github.com/kr/pretty v0.1.0 // indirect + github.com/klauspost/compress v1.4.1 // indirect + github.com/klauspost/cpuid v1.2.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/mitchellh/mapstructure v1.1.2 github.com/opentracing/opentracing-go v1.0.2 github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec github.com/shirou/gopsutil/v3 v3.21.9 - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/stretchr/testify v1.7.0 github.com/valyala/fasthttp v1.2.0 github.com/weibreeze/breeze-go v0.1.1 - go.uber.org/atomic v1.3.2 // indirect + go.uber.org/atomic v1.4.0 // indirect go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.9.1 - golang.org/x/net v0.0.0-20181005035420-146acd28ed58 - golang.org/x/time v0.0.0-00010101000000-000000000000 - google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect - google.golang.org/grpc v1.15.0 + go.uber.org/zap v1.10.0 + golang.org/x/net v0.0.0-20201224014010-6772e930b67b + golang.org/x/text v0.3.3 // indirect + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a // indirect + google.golang.org/grpc v1.21.1 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.2.4 ) diff --git a/provider/httpProvider.go b/provider/httpProvider.go index 58af13e9..eb12b045 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -99,6 +99,9 @@ func (h *HTTPProvider) Initialize() { Name: "motan", Addr: h.proxyAddr, Dial: func(addr string) (net.Conn, error) { + if strings.HasPrefix(addr, motan.UnixSockProtocolFlag) { + return net.DialTimeout("unix", addr[len(motan.UnixSockProtocolFlag):], timeout) + } c, err := fasthttp.DialTimeout(addr, timeout) if err != nil { return c, err diff --git a/provider/httpProvider_test.go b/provider/httpProvider_test.go index fe83e19f..1e10d955 100644 --- a/provider/httpProvider_test.go +++ b/provider/httpProvider_test.go @@ -44,8 +44,8 @@ func TestHTTPProvider_Call(t *testing.T) { context.Config, _ = config.NewConfigFromReader(bytes.NewReader([]byte(httpProviderTestData))) providerURL := &core.URL{Protocol: "http", Path: "test4"} providerURL.PutParam(mhttp.DomainKey, "test.domain") - providerURL.PutParam("proxyAddress", "localhost:9090") - + providerURL.PutParam("requestTimeout", "2000") + providerURL.PutParam("proxyAddress", "localhost:8090") provider := &HTTPProvider{url: providerURL, gctx: context} provider.Initialize() req := &core.MotanRequest{} @@ -53,6 +53,7 @@ func TestHTTPProvider_Call(t *testing.T) { req.Method = "/p1/test" req.SetAttachment("Host", "test.domain") req.SetAttachment(mhttp.QueryString, "a=b") + assert.Nil(t, provider.Call(req).GetException()) assert.Equal(t, "/2/p1/test?a=b", string(provider.Call(req).GetValue().([]byte))) req.SetAttachment(mhttp.Proxy, "true") @@ -71,7 +72,7 @@ func TestHTTPProvider_Call(t *testing.T) { func TestMain(m *testing.M) { go func() { - var addr = ":9090" + var addr = ":8090" handler := &http.ServeMux{} handler.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { request.ParseForm() @@ -79,6 +80,6 @@ func TestMain(m *testing.M) { }) http.ListenAndServe(addr, handler) }() - time.Sleep(time.Second * 2) + time.Sleep(time.Second * 3) os.Exit(m.Run()) } diff --git a/registry/directRegistry.go b/registry/directRegistry.go index e3721eb2..b960eaf6 100644 --- a/registry/directRegistry.go +++ b/registry/directRegistry.go @@ -66,12 +66,18 @@ func parseURLs(url *motan.URL) []*motan.URL { urls = append(urls, url) } else if address, exist := url.Parameters[motan.AddressKey]; exist { for _, add := range strings.Split(address, ",") { - hostport := motan.TrimSplit(add, ":") - if len(hostport) == 2 { - port, err := strconv.Atoi(hostport[1]) - if err == nil { - u := &motan.URL{Host: hostport[0], Port: port} - urls = append(urls, u) + if strings.HasPrefix(add, "unix://") { + u := &motan.URL{Host: add, Port: 0} + u.PutParam(motan.AddressKey, add) + urls = append(urls, u) + } else { + hostport := motan.TrimSplit(add, ":") + if len(hostport) == 2 { + port, err := strconv.Atoi(hostport[1]) + if err == nil { + u := &motan.URL{Host: hostport[0], Port: port} + urls = append(urls, u) + } } } } diff --git a/registry/registry.go b/registry/registry.go index ca6ab6a1..5d79171f 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -29,6 +29,10 @@ const ( Mesh = "mesh" ) +var ( + setSnapshotConfLock sync.Mutex +) + type SnapshotNodeInfo struct { ExtInfo string `json:"extInfo"` Addr string `json:"address"` @@ -87,6 +91,8 @@ func flushSnapshot() { } func SetSnapshotConf(snapshotInterval time.Duration, snapshotDir string) { + setSnapshotConfLock.Lock() + defer setSnapshotConfLock.Unlock() snapshotConf.SnapshotDir = snapshotDir snapshotConf.SnapshotInterval = snapshotInterval } diff --git a/server.go b/server.go index e93013bf..8cb81e28 100644 --- a/server.go +++ b/server.go @@ -5,8 +5,11 @@ import ( "flag" "fmt" "github.com/weibocom/motan-go/config" + "github.com/weibocom/motan-go/provider" + "hash/fnv" "reflect" "strconv" + "strings" "sync" motan "github.com/weibocom/motan-go/core" @@ -122,6 +125,11 @@ func (m *MSContext) Start(extfactory motan.ExtensionFactory) { } } +func (m *MSContext) hashInt(s string) int { + h := fnv.New32a() + h.Write([]byte(s)) + return int(h.Sum32()) +} func (m *MSContext) export(url *motan.URL) { defer motan.HandlePanic(nil) service := m.serviceImpls[url.Parameters[motan.RefKey]] @@ -142,10 +150,18 @@ func (m *MSContext) export(url *motan.URL) { } } url.Protocol = protocol - porti, err := strconv.Atoi(port) - if err != nil { - vlog.Errorf("export port not int. port:%s, url:%+v", port, url) - return + var porti int + var err error + if v := url.GetParam(provider.ProxyHostKey, ""); strings.HasPrefix(v, motan.UnixSockProtocolFlag) { + porti = m.hashInt(v) + } else if v := url.GetParam(motan.UnixSockKey, ""); v != "" { + porti = m.hashInt(v) + } else { + porti, err = strconv.Atoi(port) + if err != nil { + vlog.Errorf("export port not int. port:%s, url:%+v", port, url) + return + } } url.Port = porti if url.Host == "" { diff --git a/server_test.go b/server_test.go index 2ac08611..7f713fc2 100644 --- a/server_test.go +++ b/server_test.go @@ -113,7 +113,7 @@ func TestNewMotanServerContextFromConfig(t *testing.T) { assert.Equal("Hello Ray from motan server", resp.GetValue()) } -func startServer(t *testing.T, path string, port int) motan.ExtensionFactory { +func startServer(t *testing.T, path string, port int, unixSock ...string) motan.ExtensionFactory { cfgText := ` motan-server: log_dir: "stdout" @@ -129,12 +129,19 @@ motan-service: path: %s group: bj protocol: motan2 + provider: default registry: direct serialization: simple ref : "serviceID" export: "motan2:%d" + unixSock: "%s" ` - cfgText = fmt.Sprintf(cfgText, path, port) + unixSock0 := "" + if len(unixSock) > 0 && unixSock[0] != "" { + unixSock0 = unixSock[0] + port = 0 + } + cfgText = fmt.Sprintf(cfgText, path, port, unixSock0) assert := assert2.New(t) conf, err := config.NewConfigFromReader(bytes.NewReader([]byte(cfgText))) assert.Nil(err) From 88394ea454f8892a40dc03f8a76ecc3b2f69e830 Mon Sep 17 00:00:00 2001 From: snail007 Date: Tue, 10 Jan 2023 19:35:13 +0800 Subject: [PATCH 49/75] modify serverkey --- server.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server.go b/server.go index 8cb81e28..240d0f20 100644 --- a/server.go +++ b/server.go @@ -23,7 +23,7 @@ type MSContext struct { context *motan.Context extFactory motan.ExtensionFactory portService map[int]motan.Exporter - portServer map[int]motan.Server + portServer map[string]motan.Server serviceImpls map[string]interface{} registries map[string]motan.Registry // all registries used for services @@ -151,17 +151,21 @@ func (m *MSContext) export(url *motan.URL) { } url.Protocol = protocol var porti int + var serverKey string var err error if v := url.GetParam(provider.ProxyHostKey, ""); strings.HasPrefix(v, motan.UnixSockProtocolFlag) { - porti = m.hashInt(v) + porti = 0 + serverKey = v } else if v := url.GetParam(motan.UnixSockKey, ""); v != "" { - porti = m.hashInt(v) + porti = 0 + serverKey = v } else { porti, err = strconv.Atoi(port) if err != nil { vlog.Errorf("export port not int. port:%s, url:%+v", port, url) return } + serverKey = port } url.Port = porti if url.Host == "" { @@ -176,7 +180,7 @@ func (m *MSContext) export(url *motan.URL) { exporter := &mserver.DefaultExporter{} exporter.SetProvider(provider) - server := m.portServer[url.Port] + server := m.portServer[serverKey] if server == nil { server = m.extFactory.GetServer(url) @@ -184,7 +188,7 @@ func (m *MSContext) export(url *motan.URL) { motan.Initialize(handler) handler.AddProvider(provider) server.Open(false, false, handler, m.extFactory) - m.portServer[url.Port] = server + m.portServer[serverKey] = server } else if canShareChannel(*url, *server.GetURL()) { server.GetMessageHandler().AddProvider(provider) } else { @@ -212,7 +216,7 @@ func (m *MSContext) Initialize() { if !m.inited { m.context = motan.NewContextFromConfig(m.config, "", "") m.portService = make(map[int]motan.Exporter, 32) - m.portServer = make(map[int]motan.Server, 32) + m.portServer = make(map[string]motan.Server, 32) m.serviceImpls = make(map[string]interface{}, 32) m.registries = make(map[string]motan.Registry) m.inited = true From bac6f039f2742e219cd849725aabe72213200a0c Mon Sep 17 00:00:00 2001 From: Ray Date: Mon, 16 Jan 2023 17:19:21 +0800 Subject: [PATCH 50/75] re-encode attachment in motan1 protocol (#303) --- protocol/motan1Protocol.go | 339 ++++++++++++++++++++++++++++---- protocol/motan1Protocol_test.go | 86 ++++++-- 2 files changed, 362 insertions(+), 63 deletions(-) diff --git a/protocol/motan1Protocol.go b/protocol/motan1Protocol.go index de255584..d9a48e8c 100644 --- a/protocol/motan1Protocol.go +++ b/protocol/motan1Protocol.go @@ -60,6 +60,8 @@ const ( HEARTBEAT_RESPONSE_STRING = HEARTBEAT_METHOD_NAME ) +const MAX_BLOCK_SIZE = 1024 + // base binary arrays var ( baseV1HeartbeatReq = []byte{241, 241, 0, 0, 24, 44, 223, 176, 126, 80, 0, 96, 0, 0, 0, 78, 240, 240, 1, 0, 24, 44, 223, 176, 126, 80, 0, 96, 0, 0, 0, 62, 172, 237, 0, 5, 119, 56, 0, 33, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 97, 112, 105, 46, 109, 111, 116, 97, 110, 46, 114, 112, 99, 46, 104, 101, 97, 114, 116, 98, 101, 97, 116, 0, 9, 104, 101, 97, 114, 116, 98, 101, 97, 116, 0, 4, 118, 111, 105, 100, 0, 0, 0, 0} @@ -72,10 +74,11 @@ var ( ) var ( - ErrWrongMotanVersion = errors.New("unsupported motan version") - ErrOSVersion = errors.New("object stream magic number or version not correct") - ErrUnsupported = errors.New("unsupported object stream type") - ErrNotHasV1Msg = errors.New("not has MotanV1Message") + ErrWrongMotanVersion = errors.New("unsupported motan version") + ErrOSVersion = errors.New("object stream magic number or version not correct") + ErrUnsupported = errors.New("unsupported object stream type") + ErrNotHasV1Msg = errors.New("not has MotanV1Message") + ErrParseV1MsgAttachment = errors.New("parse MotanV1Message attachment fail") ) type MotanV1Message struct { @@ -84,6 +87,7 @@ type MotanV1Message struct { Flag byte Rid uint64 InnerLength int + objStream *simpleObjectStream //only for request encode } func DecodeMotanV1Request(msg *MotanV1Message) (motan.Request, error) { @@ -95,6 +99,7 @@ func DecodeMotanV1Request(msg *MotanV1Message) (motan.Request, error) { if err != nil { return nil, err } + msg.objStream = objStream request := &motan.MotanRequest{ RequestID: msg.Rid, ServiceName: objStream.service, @@ -107,8 +112,7 @@ func DecodeMotanV1Request(msg *MotanV1Message) (motan.Request, error) { func EncodeMotanV1Request(req motan.Request, sendId uint64) ([]byte, error) { if msg, ok := req.GetRPCContext(true).OriginalMessage.(*MotanV1Message); ok { - writeV1Rid(msg.OriginBytes, sendId) // replace rid with sendId for send - return msg.OriginBytes, nil + return encodeMotanV1(msg, req.GetAttachments(), sendId), nil } return nil, ErrNotHasV1Msg } @@ -257,6 +261,63 @@ func BuildV1ExceptionResponse(rid uint64, errMsg string) []byte { return result } +func encodeMotanV1(msg *MotanV1Message, att *motan.StringMap, rid uint64) []byte { + stream := msg.objStream + // re-encode attachment + attBytes := encodeMotanV1Attachment(att, msg.V1InnerVersion) + var preSize int + if stream.isGzip { + preSize = len(stream.bytes) + len(attBytes) + } else { + preSize = len(msg.OriginBytes) + len(attBytes) + } + buf := motan.NewBytesBuffer(preSize) + // write header + buf.Write(msg.OriginBytes[:V1AllHeaderLength]) + buf.Write(stream.bytes[:4]) // object stream magic num + + // write body + if stream.argSize == 0 { // no params + writeBlock(buf, stream.sBlock.bytes[:stream.attachmentPos], attBytes) // write service info && attachments + } else { + writeBlock(buf, stream.sBlock.bytes) // write service info + buf.Write(stream.bytes[stream.argStart:stream.argEnd]) // write params + writeBlock(buf, attBytes) // write attachments + } + + // re-write rid and length + ret := buf.Bytes() + writeV1Rid(ret, rid) // replace rid with sendId for send + writeV1InnerLength(ret, uint32(len(ret)-V1AllHeaderLength)) + return ret +} + +// att should not nil +func encodeMotanV1Attachment(att *motan.StringMap, v1InnerVersion byte) []byte { + size := att.Len() + buf := motan.NewBytesBuffer(25 * size) + // write size + if v1InnerVersion == versionV1Compress { + buf.WriteUint16(uint16(size)) + } else { + buf.WriteUint32(uint32(size)) + } + if size > 0 { + // write KV + att.Range(func(k, v string) bool { + writeUtfStr(k, buf) + writeUtfStr(v, buf) + return true + }) + } + return buf.Bytes() +} + +func writeUtfStr(str string, buf *motan.BytesBuffer) { + buf.WriteUint16(uint16(len(str))) + buf.Write([]byte(str)) +} + func writeV1Rid(v1Msg []byte, rid uint64) { if len(v1Msg) >= V1AllHeaderLength { index := 4 // l1 header rid @@ -266,16 +327,85 @@ func writeV1Rid(v1Msg []byte, rid uint64) { } } +func writeV1InnerLength(v1Msg []byte, innerLength uint32) { + if len(v1Msg) >= V1AllHeaderLength { + index := 12 // l1 header length + binary.BigEndian.PutUint32(v1Msg[index:index+4], innerLength+16) + index = 28 // l2 header length + binary.BigEndian.PutUint32(v1Msg[index:index+4], innerLength) + } +} + +func writeBlock(buf *motan.BytesBuffer, bs ...[]byte) { + var tRemain int // total remain + for _, b := range bs { + tRemain += len(b) + } + first := true + lastBlock := false + blockPos := 0 // block pos + wPos := 0 // bytes pos + var ws int // write size + for _, b := range bs { + if lastBlock { // last block + buf.Write(b) + tRemain -= len(b) + } else { + r := len(b) // current bytes remain + for r > 0 { + if first || blockPos >= MAX_BLOCK_SIZE { // new block + writeBlockHeader(buf, minInt(tRemain, MAX_BLOCK_SIZE)) + blockPos = 0 // reset block pos for new block + if tRemain <= MAX_BLOCK_SIZE { + lastBlock = true + } + if first { + first = false + } + } + ws = minInt(r, MAX_BLOCK_SIZE-blockPos) // should write size + if ws > 0 { + wPos = len(b) - r + buf.Write(b[wPos : wPos+ws]) + r -= ws + blockPos += ws + tRemain -= ws + } + } + } + } +} + +func writeBlockHeader(buf *motan.BytesBuffer, l int) { + if l <= 0xFF { + buf.WriteByte(TC_BLOCKDATA) + buf.WriteByte(byte(l)) + } else { + buf.WriteByte(TC_BLOCKDATALONG) + buf.WriteUint32(uint32(l)) + } +} + +func minInt(a, b int) int { + if a <= b { + return a + } + return b +} + type simpleObjectStream struct { - bytes []byte - pos int - len int - inBlock bool - blockRemain int - flag byte - parsed bool - maxContentLength int - v1InnerVersion byte + bytes []byte + pos int + len int + flag byte + parsed bool + v1InnerVersion byte + isGzip bool + attachmentPos int // attachment position in sBlock if no params + sBlock *block // service info block + aBlock *block // attachment block. maybe nil + argStart int // args start index in bytes + argEnd int // args end index in bytes // for request service string @@ -298,7 +428,7 @@ func newObjectStream(msg *MotanV1Message) (*simpleObjectStream, error) { if err != nil { return nil, err } - return &simpleObjectStream{bytes: decodedBytes, flag: msg.Flag, len: len(decodedBytes), v1InnerVersion: msg.V1InnerVersion}, nil + return &simpleObjectStream{bytes: decodedBytes, flag: msg.Flag, len: len(decodedBytes), v1InnerVersion: msg.V1InnerVersion, isGzip: true}, nil } return &simpleObjectStream{bytes: msg.OriginBytes[V1AllHeaderLength:], flag: msg.Flag, len: len(msg.OriginBytes) - V1AllHeaderLength, v1InnerVersion: msg.V1InnerVersion}, nil } @@ -308,17 +438,16 @@ func (s *simpleObjectStream) parseReq() error { if err != nil { return err } - blockStartPos := s.pos // service infos infos := make([]string, 3) for i := 0; i < 3; i++ { - infos[i], err = s.readUtf() + infos[i], err = s.readUtfFromBlock(s.sBlock) if err != nil { return err } if i == 0 && infos[0] == "1" { // v1 compress method info var methodSign string - methodSign, err = s.readUtf() + methodSign, err = s.readUtfFromBlock(s.sBlock) if err != nil { return err } @@ -331,13 +460,14 @@ func (s *simpleObjectStream) parseReq() error { s.service = infos[0] s.method = infos[1] s.paramDesc = infos[2] - s.blockRemain -= s.pos - blockStartPos + // arguments. - if s.blockRemain == 0 { //skip arguments bytes - s.inBlock = false // not block mode + if s.sBlock.remain() == 0 { //has args, should skip arguments bytes + s.argStart = s.pos checkNext := true var t byte for checkNext { + s.argSize++ t, err = s.peekType() if err != nil { return err @@ -348,8 +478,10 @@ func (s *simpleObjectStream) parseReq() error { case TC_ARRAY: _, err = s.skipByteArray(false) case TC_BLOCKDATA, TC_BLOCKDATALONG: // arguments end - err = s.setBlock() + s.argEnd = s.pos + s.aBlock, err = s.getBlock() // attachment block checkNext = false + s.argSize-- default: vlog.Errorf("unsupported object stream type, type:%d", t) return ErrUnsupported @@ -358,8 +490,16 @@ func (s *simpleObjectStream) parseReq() error { return err } } + } else { + s.attachmentPos = s.sBlock.pos + } + if s.sBlock.remain() > 0 { + err = s.readAttachments(s.sBlock) + } else if s.aBlock != nil { + err = s.readAttachments(s.aBlock) + } else { + return ErrParseV1MsgAttachment } - err = s.readAttachments() if err != nil { return err } @@ -373,16 +513,16 @@ func (s *simpleObjectStream) parseRes() error { return err } - s.processTime, err = s.readLong() + s.processTime, err = s.readLongFromBlock(s.sBlock) if err != nil { return err } switch s.flag { case FLAG_RESPONSE_EXCEPTION: s.hasException = true - s.cName, _ = s.readUtf() // parse exception class name + s.cName, _ = s.readUtfFromBlock(s.sBlock) // parse exception class name case FLAG_RESPONSE: - s.cName, err = s.readUtf() + s.cName, err = s.readUtfFromBlock(s.sBlock) if err == nil && s.cName == "java.lang.String" { var c []byte c, err = s.skipByteArray(true) @@ -420,7 +560,12 @@ func (s *simpleObjectStream) checkObjectStream() error { return ErrOSVersion } s.pos += 4 - return s.setBlock() // init with block mode + block, err := s.getBlock() + if err != nil { + return err + } + s.sBlock = block + return nil } func (s *simpleObjectStream) readUtf() (string, error) { @@ -439,6 +584,22 @@ func (s *simpleObjectStream) readUtf() (string, error) { return str, nil } +func (s *simpleObjectStream) readUtfFromBlock(block *block) (string, error) { + l, err := s.readInt16FromBlock(block) + if err != nil { + return "", err + } + var str string + if l > 0 { + if block.remain() < int(l) { + return str, io.EOF + } + str = string(block.bytes[block.pos : block.pos+int(l)]) + block.pos += int(l) + } + return str, nil +} + func (s *simpleObjectStream) readInt16() (int16, error) { if s.remain() < 2 { return 0, io.EOF @@ -448,6 +609,15 @@ func (s *simpleObjectStream) readInt16() (int16, error) { return i, nil } +func (s *simpleObjectStream) readInt16FromBlock(block *block) (int16, error) { + if block.remain() < 2 { + return 0, io.EOF + } + i := int16(binary.BigEndian.Uint16(block.bytes[block.pos : block.pos+2])) + block.pos += 2 + return i, nil +} + func (s *simpleObjectStream) readInt() (int, error) { if s.remain() < 4 { return 0, io.EOF @@ -457,6 +627,15 @@ func (s *simpleObjectStream) readInt() (int, error) { return i, nil } +func (s *simpleObjectStream) readIntFromBlock(block *block) (int, error) { + if block.remain() < 4 { + return 0, io.EOF + } + i := int(binary.BigEndian.Uint32(block.bytes[block.pos : block.pos+4])) + block.pos += 4 + return i, nil +} + func (s *simpleObjectStream) readLong() (int64, error) { if s.remain() < 8 { return 0, io.EOF @@ -466,15 +645,24 @@ func (s *simpleObjectStream) readLong() (int64, error) { return i, nil } -func (s *simpleObjectStream) readAttachments() (err error) { - // attachments. already in block mode +func (s *simpleObjectStream) readLongFromBlock(block *block) (int64, error) { + if block.remain() < 8 { + return 0, io.EOF + } + i := int64(binary.BigEndian.Uint64(block.bytes[block.pos : block.pos+8])) + block.pos += 8 + return i, nil +} + +// read attachment from block。 +func (s *simpleObjectStream) readAttachments(block *block) (err error) { var size int if s.v1InnerVersion == versionV1Compress { var size16 int16 - size16, err = s.readInt16() + size16, err = s.readInt16FromBlock(block) size = int(size16) } else { - size, err = s.readInt() + size, err = s.readIntFromBlock(block) } if err != nil { vlog.Errorf("read v1 attachment size fail. err:%v", err) @@ -484,12 +672,12 @@ func (s *simpleObjectStream) readAttachments() (err error) { if size > 0 { // has attachments for i := 0; i < size; i++ { var k, v string - k, err = s.readUtf() + k, err = s.readUtfFromBlock(block) if err != nil { vlog.Errorf("read v1 attachment key fail. err:%v", err) return err } - v, err = s.readUtf() + v, err = s.readUtfFromBlock(block) if err != nil { vlog.Errorf("read v1 attachment value fail. err:%v", err) return err @@ -518,7 +706,7 @@ func (s *simpleObjectStream) skipByteArray(withContent bool) ([]byte, error) { case TC_REFERENCE: s.pos += 6 // skip TC_REFERENCE default: - vlog.Errorf("unsupported object stream type, type:%d", t) + vlog.Errorf("unsupported object stream type in skipByteArray, type:%d", t) return nil, ErrUnsupported } length, err := s.readInt() @@ -536,27 +724,81 @@ func (s *simpleObjectStream) skipByteArray(withContent bool) ([]byte, error) { return content, nil } -func (s *simpleObjectStream) setBlock() error { +// get data bytes from blocks +func (s *simpleObjectStream) getBlock() (*block, error) { if s.remain() < 2 { - return io.EOF + return nil, io.EOF } t := s.bytes[s.pos] s.pos++ + b := &block{} + var l int switch t { case TC_BLOCKDATA: - s.blockRemain = int(s.bytes[s.pos]) + l = int(s.bytes[s.pos]) s.pos++ case TC_BLOCKDATALONG: if s.remain() < 4 { - return io.EOF + return nil, io.EOF } - s.blockRemain = int(binary.BigEndian.Uint32(s.bytes[s.pos : s.pos+4])) + l = int(binary.BigEndian.Uint32(s.bytes[s.pos : s.pos+4])) s.pos += 4 default: vlog.Errorf("unsupported object stream type, type:%d", t) - return ErrUnsupported + return nil, ErrUnsupported + } + + if s.remain() < l { + return nil, io.EOF + } + b.bytes = s.bytes[s.pos:(s.pos + l)] + s.pos += l + if l == MAX_BLOCK_SIZE { + nt, err := s.peekType() + if err == nil && (nt == TC_BLOCKDATA || nt == TC_BLOCKDATALONG) { // has next block + buf := motan.NewBytesBuffer(MAX_BLOCK_SIZE * 2) + buf.Write(b.bytes) // first block + err = s.appendBlockBytes(buf) + if err != nil { + return nil, err + } + b.bytes = buf.Bytes() + } + } + return b, nil +} + +func (s *simpleObjectStream) appendBlockBytes(buf *motan.BytesBuffer) error { + checkNext := true + var l int + for checkNext { + if s.remain() < 1 { // no more data + return nil + } + t := s.bytes[s.pos] + s.pos++ + switch t { + case TC_BLOCKDATA: + l = int(s.bytes[s.pos]) + s.pos++ + case TC_BLOCKDATALONG: + if s.remain() < 4 { + return io.EOF + } + l = int(binary.BigEndian.Uint32(s.bytes[s.pos : s.pos+4])) + s.pos += 4 + default: // no more blocks + return nil + } + if s.remain() < l { + return io.EOF + } + buf.Write(s.bytes[s.pos:(s.pos + l)]) + s.pos += l + if l != MAX_BLOCK_SIZE { + checkNext = false + } } - s.inBlock = true return nil } @@ -571,6 +813,19 @@ func (s *simpleObjectStream) remain() int { return s.len - s.pos } +type block struct { + bytes []byte + pos int +} + +func (b block) len() int { + return len(b.bytes) +} + +func (b block) remain() int { + return len(b.bytes) - b.pos +} + func writeHessianString(str string, buf *motan.BytesBuffer) { // write length length := len(str) diff --git a/protocol/motan1Protocol_test.go b/protocol/motan1Protocol_test.go index 47e1d737..f097c530 100644 --- a/protocol/motan1Protocol_test.go +++ b/protocol/motan1Protocol_test.go @@ -4,16 +4,20 @@ import ( "bufio" "bytes" motan "github.com/weibocom/motan-go/core" + "strconv" + "strings" "testing" ) var ( // req - testV1ReqBytes = []byte{241, 241, 0, 0, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 246, 240, 240, 1, 0, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 230, 172, 237, 0, 5, 119, 72, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 5, 104, 101, 108, 108, 111, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 119, 120, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} - testV1NoParamReqBytes = []byte{241, 241, 0, 0, 24, 50, 5, 245, 221, 16, 0, 2, 0, 0, 0, 204, 240, 240, 1, 0, 24, 50, 5, 245, 221, 16, 0, 2, 0, 0, 0, 188, 172, 237, 0, 5, 119, 182, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 7, 110, 111, 80, 97, 114, 97, 109, 0, 4, 118, 111, 105, 100, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} - testV1MultiParamReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 143, 123, 0, 0, 1, 0, 0, 1, 98, 240, 240, 1, 0, 24, 50, 6, 143, 123, 0, 0, 1, 0, 0, 1, 82, 172, 237, 0, 5, 119, 113, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 6, 114, 101, 110, 97, 109, 101, 0, 56, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 109, 111, 100, 101, 108, 46, 85, 115, 101, 114, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 59, 67, 48, 39, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 109, 111, 100, 101, 108, 46, 85, 115, 101, 114, 146, 2, 105, 100, 4, 110, 97, 109, 101, 96, 212, 8, 86, 3, 114, 97, 121, 117, 113, 0, 126, 0, 0, 0, 0, 0, 5, 4, 114, 97, 121, 48, 119, 120, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} - testV1CompressReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 224, 240, 240, 2, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 208, 172, 237, 0, 5, 119, 21, 0, 1, 49, 0, 16, 104, 101, 108, 108, 56, 49, 55, 102, 102, 51, 55, 51, 51, 50, 54, 57, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 119, 149, 0, 7, 0, 2, 95, 65, 0, 4, 52, 57, 57, 54, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 14, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 0, 3, 49, 57, 56, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} - testV1CompressGzipReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 202, 240, 240, 2, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 186, 31, 139, 8, 0, 0, 0, 0, 0, 0, 0, 91, 243, 150, 129, 181, 92, 148, 129, 209, 144, 65, 32, 35, 53, 39, 199, 194, 208, 60, 45, 205, 216, 220, 216, 216, 200, 204, 178, 180, 136, 129, 41, 218, 105, 205, 103, 241, 31, 108, 28, 33, 15, 152, 24, 24, 42, 10, 24, 24, 24, 216, 217, 114, 243, 75, 18, 243, 12, 202, 167, 50, 176, 51, 48, 197, 59, 50, 176, 152, 88, 90, 154, 49, 112, 39, 22, 20, 228, 100, 38, 39, 150, 100, 230, 231, 49, 112, 231, 86, 250, 130, 20, 185, 164, 230, 230, 51, 112, 39, 231, 100, 166, 230, 149, 184, 23, 229, 151, 22, 48, 240, 129, 53, 235, 166, 0, 37, 116, 139, 10, 146, 25, 128, 134, 165, 148, 230, 164, 98, 136, 243, 57, 231, 231, 149, 0, 117, 233, 250, 164, 230, 165, 151, 100, 48, 48, 27, 153, 24, 48, 176, 151, 165, 22, 21, 131, 204, 103, 54, 212, 51, 96, 96, 77, 199, 102, 34, 0, 179, 189, 51, 70, 208, 0, 0, 0} + testV1ReqBytes = []byte{241, 241, 0, 0, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 246, 240, 240, 1, 0, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 230, 172, 237, 0, 5, 119, 72, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 5, 104, 101, 108, 108, 111, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 119, 120, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1NoParamReqBytes = []byte{241, 241, 0, 0, 24, 50, 5, 245, 221, 16, 0, 2, 0, 0, 0, 204, 240, 240, 1, 0, 24, 50, 5, 245, 221, 16, 0, 2, 0, 0, 0, 188, 172, 237, 0, 5, 119, 182, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 7, 110, 111, 80, 97, 114, 97, 109, 0, 4, 118, 111, 105, 100, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1MultiParamReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 143, 123, 0, 0, 1, 0, 0, 1, 98, 240, 240, 1, 0, 24, 50, 6, 143, 123, 0, 0, 1, 0, 0, 1, 82, 172, 237, 0, 5, 119, 113, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 6, 114, 101, 110, 97, 109, 101, 0, 56, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 109, 111, 100, 101, 108, 46, 85, 115, 101, 114, 44, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 59, 67, 48, 39, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 109, 111, 100, 101, 108, 46, 85, 115, 101, 114, 146, 2, 105, 100, 4, 110, 97, 109, 101, 96, 212, 8, 86, 3, 114, 97, 121, 117, 113, 0, 126, 0, 0, 0, 0, 0, 5, 4, 114, 97, 121, 48, 119, 120, 0, 0, 0, 5, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1CompressReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 224, 240, 240, 2, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 208, 172, 237, 0, 5, 119, 21, 0, 1, 49, 0, 16, 104, 101, 108, 108, 56, 49, 55, 102, 102, 51, 55, 51, 51, 50, 54, 57, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 119, 149, 0, 7, 0, 2, 95, 65, 0, 4, 52, 57, 57, 54, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 14, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 0, 3, 49, 57, 56, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1CompressGzipReqBytes = []byte{241, 241, 0, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 202, 240, 240, 2, 0, 24, 50, 6, 247, 234, 112, 0, 2, 0, 0, 0, 186, 31, 139, 8, 0, 0, 0, 0, 0, 0, 0, 91, 243, 150, 129, 181, 92, 148, 129, 209, 144, 65, 32, 35, 53, 39, 199, 194, 208, 60, 45, 205, 216, 220, 216, 216, 200, 204, 178, 180, 136, 129, 41, 218, 105, 205, 103, 241, 31, 108, 28, 33, 15, 152, 24, 24, 42, 10, 24, 24, 24, 216, 217, 114, 243, 75, 18, 243, 12, 202, 167, 50, 176, 51, 48, 197, 59, 50, 176, 152, 88, 90, 154, 49, 112, 39, 22, 20, 228, 100, 38, 39, 150, 100, 230, 231, 49, 112, 231, 86, 250, 130, 20, 185, 164, 230, 230, 51, 112, 39, 231, 100, 166, 230, 149, 184, 23, 229, 151, 22, 48, 240, 129, 53, 235, 166, 0, 37, 116, 139, 10, 146, 25, 128, 134, 165, 148, 230, 164, 98, 136, 243, 57, 231, 231, 149, 0, 117, 233, 250, 164, 230, 165, 151, 100, 48, 48, 27, 153, 24, 48, 176, 151, 165, 22, 21, 131, 204, 103, 54, 212, 51, 96, 96, 77, 199, 102, 34, 0, 179, 189, 51, 70, 208, 0, 0, 0} + testV1LongAttachmentReqBytes = []byte{241, 241, 0, 0, 24, 90, 11, 154, 159, 176, 0, 1, 0, 0, 5, 185, 240, 240, 1, 0, 24, 90, 11, 154, 159, 176, 0, 1, 0, 0, 5, 169, 172, 237, 0, 5, 119, 72, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 5, 104, 101, 108, 108, 111, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 122, 0, 0, 4, 0, 0, 0, 0, 6, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 116, 101, 109, 112, 107, 101, 121, 4, 176, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 0, 0, 1, 51, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 122, 120, 99, 118, 98, 97, 115, 100, 102, 103, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} + testV1ChAttachmentReqBytes = []byte{241, 241, 0, 0, 24, 90, 165, 145, 27, 0, 0, 1, 0, 0, 1, 16, 240, 240, 1, 0, 24, 90, 165, 145, 27, 0, 0, 1, 0, 0, 1, 0, 172, 237, 0, 5, 119, 72, 0, 45, 99, 111, 109, 46, 119, 101, 105, 98, 111, 46, 109, 111, 116, 97, 110, 46, 100, 101, 109, 111, 46, 115, 101, 114, 118, 105, 99, 101, 46, 77, 111, 116, 97, 110, 68, 101, 109, 111, 83, 101, 114, 118, 105, 99, 101, 0, 5, 104, 101, 108, 108, 111, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 7, 6, 109, 111, 116, 97, 110, 48, 119, 146, 0, 0, 0, 6, 0, 11, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 0, 11, 109, 121, 77, 111, 116, 97, 110, 68, 101, 109, 111, 0, 11, 99, 108, 105, 101, 110, 116, 71, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 6, 109, 111, 100, 117, 108, 101, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99, 0, 7, 118, 101, 114, 115, 105, 111, 110, 0, 3, 49, 46, 48, 0, 10, 107, 101, 121, 45, 228, 184, 173, 230, 150, 135, 0, 12, 118, 97, 108, 117, 101, 45, 228, 184, 173, 230, 150, 135, 0, 5, 103, 114, 111, 117, 112, 0, 14, 109, 111, 116, 97, 110, 45, 100, 101, 109, 111, 45, 114, 112, 99} // res testV1ResBytes = []byte{241, 241, 0, 1, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 85, 240, 240, 1, 1, 24, 49, 255, 251, 240, 224, 0, 2, 0, 0, 0, 69, 172, 237, 0, 5, 119, 26, 0, 0, 0, 0, 0, 0, 0, 1, 0, 16, 106, 97, 118, 97, 46, 108, 97, 110, 103, 46, 83, 116, 114, 105, 110, 103, 117, 114, 0, 2, 91, 66, 172, 243, 23, 248, 6, 8, 84, 224, 2, 0, 0, 120, 112, 0, 0, 0, 14, 13, 72, 101, 108, 108, 111, 32, 109, 111, 116, 97, 110, 48, 33} @@ -36,9 +40,12 @@ var ( ) func TestEncodeMotanV1Request(t *testing.T) { - req := &motan.MotanRequest{RequestID: 3452345} baseV1Msg := buildV1Msg(t, testV1ReqBytes) - req.GetRPCContext(true).OriginalMessage = baseV1Msg + req, err := DecodeMotanV1Request(baseV1Msg) + if err != nil { + t.Fatalf("encode v1 req fail. err:%v", err) + } + rid := uint64(7348937488) bs, err := EncodeMotanV1Request(req, rid) if err != nil { @@ -69,9 +76,12 @@ func checkBaseReq(req motan.Request, rid uint64, t *testing.T) { } func TestEncodeMotanV1Response(t *testing.T) { - res := &motan.MotanResponse{RequestID: 36476555} baseV1Msg := buildV1Msg(t, testV1ResBytes) - res.GetRPCContext(true).OriginalMessage = baseV1Msg + res, err := DecodeMotanV1Response(baseV1Msg) + if err != nil { + t.Fatalf("encode v1 res fail. err:%v", err) + } + bs, err := EncodeMotanV1Response(res) if err != nil { t.Fatalf("encode v1 res fail. err:%v", err) @@ -83,7 +93,7 @@ func TestEncodeMotanV1Response(t *testing.T) { if err != nil { t.Fatalf("read v1 res msg fail. err:%v", err) } - assertTrue(res2.GetRequestID() == res.RequestID, "v1 res rid", t) + assertTrue(res2.GetRequestID() == res.GetRequestID(), "v1 res rid", t) assertTrue(res2.GetProcessTime() == 1, "v1 res process time", t) } @@ -106,11 +116,13 @@ type resTestInfo struct { func TestParseV1ReqMessage(t *testing.T) { tests := []*reqTestInfo{ - {"normal req", &MotanV1Message{testV1ReqBytes, 1, 0, 1743455988312178690, 230}, "com.weibo.motan.demo.service.MotanDemoService", "hello", "java.lang.String", 5}, - {"no param req", &MotanV1Message{testV1NoParamReqBytes, 1, 0, 1743462559279742978, 188}, "com.weibo.motan.demo.service.MotanDemoService", "noParam", "void", 5}, - {"multi params req", &MotanV1Message{testV1MultiParamReqBytes, 1, 0, 1743463219059490817, 338}, "com.weibo.motan.demo.service.MotanDemoService", "rename", "com.weibo.motan.demo.service.model.User,java.lang.String", 5}, - {"compress req", &MotanV1Message{testV1CompressReqBytes, 2, 0, 1743463667605700610, 208}, "v1compressMethodSign", "hell817ff3733269", "", 7}, - {"compress gzip req", &MotanV1Message{testV1CompressGzipReqBytes, 2, 0, 1743463667605700610, 186}, "v1compressMethodSign", "hell817ff3733269", "", 7}, + {"normal req", &MotanV1Message{OriginBytes: testV1ReqBytes, V1InnerVersion: 1, Flag: 0, Rid: 1743455988312178690, InnerLength: 230}, "com.weibo.motan.demo.service.MotanDemoService", "hello", "java.lang.String", 5}, + {"no param req", &MotanV1Message{OriginBytes: testV1NoParamReqBytes, V1InnerVersion: 1, Flag: 0, Rid: 1743462559279742978, InnerLength: 188}, "com.weibo.motan.demo.service.MotanDemoService", "noParam", "void", 5}, + {"multi params req", &MotanV1Message{OriginBytes: testV1MultiParamReqBytes, V1InnerVersion: 1, Flag: 0, Rid: 1743463219059490817, InnerLength: 338}, "com.weibo.motan.demo.service.MotanDemoService", "rename", "com.weibo.motan.demo.service.model.User,java.lang.String", 5}, + {"compress req", &MotanV1Message{OriginBytes: testV1CompressReqBytes, V1InnerVersion: 2, Flag: 0, Rid: 1743463667605700610, InnerLength: 208}, "v1compressMethodSign", "hell817ff3733269", "", 7}, + {"compress gzip req", &MotanV1Message{OriginBytes: testV1CompressGzipReqBytes, V1InnerVersion: 2, Flag: 0, Rid: 1743463667605700610, InnerLength: 186}, "v1compressMethodSign", "hell817ff3733269", "", 7}, + {"long attachment req", &MotanV1Message{OriginBytes: testV1LongAttachmentReqBytes, V1InnerVersion: 1, Flag: 0, Rid: 1754727763546210305, InnerLength: 1449}, "com.weibo.motan.demo.service.MotanDemoService", "hello", "java.lang.String", 6}, + {"Chinese attachment req", &MotanV1Message{OriginBytes: testV1ChAttachmentReqBytes, V1InnerVersion: 1, Flag: 0, Rid: 1754897047456055297, InnerLength: 256}, "com.weibo.motan.demo.service.MotanDemoService", "hello", "java.lang.String", 6}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -121,12 +133,12 @@ func TestParseV1ReqMessage(t *testing.T) { func TestParseV1ResMessage(t *testing.T) { tests := []*resTestInfo{ - {"normal res", &MotanV1Message{testV1ResBytes, 1, 1, 1743455988312178690, 69}, 1, false, 0}, - {"void res", &MotanV1Message{testV1VoidResBytes, 1, 3, 1743462708025491459, 14}, 1, false, 0}, - {"exception res", &MotanV1Message{testV1ExceptionResBytes, 1, 5, 1743462891537825796, 1960}, 24, true, 0}, - {"compress res", &MotanV1Message{testV1CompressResBytes, 2, 7, 1743466416913252353, 73}, 0, false, 0}, - {"compress gzip res", &MotanV1Message{testV1CompressGzipResBytes, 2, 7, 1743466534460719105, 85}, 0, false, 0}, - {"compress gzip attachment res", &MotanV1Message{testV1CompressWithAttachmentResBytes, 2, 7, 1743466820126375937, 113}, 0, false, 0}, + {"normal res", &MotanV1Message{OriginBytes: testV1ResBytes, V1InnerVersion: 1, Flag: 1, Rid: 1743455988312178690, InnerLength: 69}, 1, false, 0}, + {"void res", &MotanV1Message{OriginBytes: testV1VoidResBytes, V1InnerVersion: 1, Flag: 3, Rid: 1743462708025491459, InnerLength: 14}, 1, false, 0}, + {"exception res", &MotanV1Message{OriginBytes: testV1ExceptionResBytes, V1InnerVersion: 1, Flag: 5, Rid: 1743462891537825796, InnerLength: 1960}, 24, true, 0}, + {"compress res", &MotanV1Message{OriginBytes: testV1CompressResBytes, V1InnerVersion: 2, Flag: 7, Rid: 1743466416913252353, InnerLength: 73}, 0, false, 0}, + {"compress gzip res", &MotanV1Message{OriginBytes: testV1CompressGzipResBytes, V1InnerVersion: 2, Flag: 7, Rid: 1743466534460719105, InnerLength: 85}, 0, false, 0}, + {"compress gzip attachment res", &MotanV1Message{OriginBytes: testV1CompressWithAttachmentResBytes, V1InnerVersion: 2, Flag: 7, Rid: 1743466820126375937, InnerLength: 113}, 0, false, 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -148,6 +160,37 @@ func checkReq(t *testing.T, info *reqTestInfo) { assertTrue(req.GetMethod() == info.method, "req method", t) assertTrue(req.GetMethodDesc() == info.methodDesc, "req method desc", t) assertTrue(req.GetAttachments().Len() == info.attachmentSize, "req attachment size", t) + + // check attachment change + keyPrefix := "newKey" + valuePrefix := "newValue#j349中文2jd9872340jdjk-" + num := 10 + for i := 0; i < num; i++ { + req.SetAttachment(keyPrefix+strconv.Itoa(i), valuePrefix+strconv.Itoa(i)) + } + newRid := uint64(273948723984) + encBytes, err := EncodeMotanV1Request(req, newRid) + msg2 := buildV1Msg(t, encBytes) + req2, err := DecodeMotanV1Request(msg2) + if err != nil { + t.Fatalf("read v1 req msg fail. err:%v", err) + } + assertTrue(req2.GetRequestID() == newRid, "req rid", t) + assertTrue(req2.GetServiceName() == info.serviceName, "req service name", t) + assertTrue(req2.GetMethod() == info.method, "req method", t) + assertTrue(req2.GetMethodDesc() == info.methodDesc, "req method desc", t) + assertTrue(req2.GetAttachments().Len() == info.attachmentSize+num, "req attachment size", t) + cnt := 0 + req2.GetAttachments().Range(func(k, v string) bool { + if strings.HasPrefix(k, keyPrefix) { + cnt++ + if !strings.HasPrefix(v, valuePrefix) { + return false + } + } + return true + }) + assertTrue(cnt == num, "add attachment check", t) } func checkRes(t *testing.T, info *resTestInfo) { @@ -182,6 +225,7 @@ func checkMsg(t *testing.T, msg *MotanV1Message, expectMsg *MotanV1Message) { assertTrue(msg.V1InnerVersion == expectMsg.V1InnerVersion, "msg inner version", t) assertTrue(msg.Flag == expectMsg.Flag, "msg flag", t) assertTrue(msg.InnerLength == expectMsg.InnerLength, "msg inner length", t) + // TODO } func TestHeartbeatReq(t *testing.T) { From 62ef27a2aebc072d93b72056c3d2b556a6e26c30 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 17 Jan 2023 14:20:04 +0800 Subject: [PATCH 51/75] fix processV1 --- endpoint/motanCommonEndpoint.go | 4 ++-- endpoint/motanEndpoint.go | 1 - server/motanserver.go | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index ff2d0fe5..29c55f5c 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -74,7 +74,7 @@ func (m *MotanCommonEndpoint) Initialize() { } return net.DialTimeout("tcp", address, connectTimeout) } - config := &ChannelConfig{MaxContentLength: m.maxContentLength, Serialization: m.serialization} + config := &ChannelConfig{MaxContentLength: m.maxContentLength} if asyncInitConnection { go m.initChannelPoolWithRetry(factory, config, connectRetryInterval) } else { @@ -361,7 +361,7 @@ func (s *Stream) Send() (err error) { return err } } else { // encode motan v2 - msg, err = mpro.ConvertToReqMessage(s.req, s.channel.config.Serialization) + msg, err = mpro.ConvertToReqMessage(s.req, s.channel.serialization) if err != nil { vlog.Errorf("convert motan request fail! ep: %s, req: %s, err:%s", s.channel.address, motan.GetReqInfo(s.req), err.Error()) return err diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 203f3074..736b840b 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -333,7 +333,6 @@ func (m *MotanEndpoint) IsAvailable() bool { // ChannelConfig : ChannelConfig type ChannelConfig struct { MaxContentLength int - Serialization motan.Serialization } func DefaultConfig() *ChannelConfig { diff --git a/server/motanserver.go b/server/motanserver.go index 012a6ffd..29fbfc04 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -280,6 +280,7 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st vlog.Errorf("decode v1 request fail. conn: %s, err:%s", conn.RemoteAddr().String(), err.Error()) result = mpro.BuildV1ExceptionResponse(msg.Rid, err.Error()) } else { + req.SetAttachment(motan.HostKey, ip) reqCtx = req.GetRPCContext(true) reqCtx.ExtFactory = m.extFactory reqCtx.RequestReceiveTime = start From 7858bc105bf2a0b58dd7d4572053ab3e3b883252 Mon Sep 17 00:00:00 2001 From: cocowh Date: Wed, 18 Jan 2023 09:05:39 +0800 Subject: [PATCH 52/75] not found provider count (#304) --- agent.go | 6 ++++++ agent_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/agent.go b/agent.go index f54f1c91..90953a51 100644 --- a/agent.go +++ b/agent.go @@ -11,6 +11,7 @@ import ( "runtime" "strconv" "sync" + "sync/atomic" "time" cfg "github.com/weibocom/motan-go/config" @@ -42,6 +43,7 @@ const ( var ( initParamLock sync.Mutex setAgentLock sync.Mutex + notFoundProviderCount int64 = 0 ) type Agent struct { @@ -235,6 +237,9 @@ func (a *Agent) registerStatusSampler() { metrics.RegisterStatusSampleFunc("goroutine_count", func() int64 { return int64(runtime.NumGoroutine()) }) + metrics.RegisterStatusSampleFunc("not_found_provider_count", func() int64 { + return atomic.SwapInt64(¬FoundProviderCount, 0) + }) } func (a *Agent) initStatus() { @@ -892,6 +897,7 @@ func (sa *serverAgentMessageHandler) Call(request motan.Request) (res motan.Resp return res } vlog.Errorf("not found provider for %s", motan.GetReqInfo(request)) + atomic.AddInt64(¬FoundProviderCount, 1) return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "not found provider for " + serviceKey, ErrType: motan.ServiceException}) } diff --git a/agent_test.go b/agent_test.go index f4c8ea74..146ad281 100644 --- a/agent_test.go +++ b/agent_test.go @@ -6,6 +6,7 @@ import ( _ "fmt" "github.com/weibocom/motan-go/config" vlog "github.com/weibocom/motan-go/log" + "github.com/weibocom/motan-go/serialize" _ "github.com/weibocom/motan-go/server" _ "golang.org/x/net/context" "io/ioutil" @@ -18,6 +19,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -478,6 +480,35 @@ func TestAgent_InitCall(t *testing.T) { } } +func TestNotFoundProvider(t *testing.T) { + notFoundService := "notFoundService" + request := meshClient.BuildRequest(notFoundService, "test", []interface{}{}) + epUrl := &core.URL{ + Protocol: "motanV1Compatible", + Host: "127.0.0.1", + Port: 9982, + Path: "notFoundService", + Group: "test", + Parameters: map[string]string{ + core.TimeOutKey: "3000", + core.ApplicationKey: "testep", + core.ConnectRetryIntervalKey: "5000", + core.ErrorCountThresholdKey: "0", + }, + } + ext := GetDefaultExtFactory() + ep := ext.GetEndPoint(epUrl) + assert.NotNil(t, ep) + ep.SetSerialization(ext.GetSerialization(serialize.Simple, serialize.SimpleNumber)) + core.Initialize(ep) + ep.SetProxy(true) + resp := ep.Call(request) + assert.NotNil(t, resp.GetException()) + assert.Contains(t, "not found provider for test_notFoundService", resp.GetException().ErrMsg) + assert.Equal(t, int64(1), atomic.LoadInt64(¬FoundProviderCount)) + ep.Destroy() +} + func newRequest(serviceName string, method string, argments ...interface{}) *core.MotanRequest { request := &core.MotanRequest{ RequestID: rand.Uint64(), From 4e7195736a19bcd2327030de72cd2653d64e50a5 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:22:55 +0800 Subject: [PATCH 53/75] add access log discard ability (#306) * add access log discard ability --- agent.go | 80 +++++++++++++++++++------------------- client.go | 24 +----------- log/log.go | 42 ++++++++++++-------- log/log_test.go | 11 ++++++ metrics/metrics.go | 36 +++-------------- metrics/sampler/sampler.go | 46 ++++++++++++++++++++++ server.go | 23 +---------- 7 files changed, 130 insertions(+), 132 deletions(-) create mode 100644 metrics/sampler/sampler.go diff --git a/agent.go b/agent.go index 90953a51..7184f2c4 100644 --- a/agent.go +++ b/agent.go @@ -41,8 +41,8 @@ const ( ) var ( - initParamLock sync.Mutex - setAgentLock sync.Mutex + initParamLock sync.Mutex + setAgentLock sync.Mutex notFoundProviderCount int64 = 0 ) @@ -304,28 +304,7 @@ func (a *Agent) initParam() { if logDir == "" { logDir = "." } - logAsync := "" - if section != nil && section["log_async"] != nil { - logAsync = strconv.FormatBool(section["log_async"].(bool)) - } - logStructured := "" - if section != nil && section["log_structured"] != nil { - logStructured = strconv.FormatBool(section["log_structured"].(bool)) - } - rotatePerHour := "" - if section != nil && section["rotate_per_hour"] != nil { - rotatePerHour = strconv.FormatBool(section["rotate_per_hour"].(bool)) - } - logLevel := "" - if section != nil && section["log_level"] != nil { - logLevel = section["log_level"].(string) - } - logFilterCaller := "" - if section != nil && section["log_filter_caller"] != nil { - logFilterCaller = strconv.FormatBool(section["log_filter_caller"].(bool)) - } - - initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel, logFilterCaller) + initLog(logDir, section) registerSwitchers(a.Context) port := *motan.Port @@ -922,28 +901,49 @@ func getClusterKey(group, version, protocol, path string) string { return group + "_" + version + "_" + protocol + "_" + path } -func initLog(logDir, logAsync, logStructured, rotatePerHour string, logLevel string, logFilterCaller string) { - // TODO: remove after a better handle - if logDir == "stdout" { - return +func initLog(logDir string, section map[interface{}]interface{}) { + if section != nil && section["log_async"] != nil { + logAsync := strconv.FormatBool(section["log_async"].(bool)) + if logAsync != "" { + _ = flag.Set("log_async", logAsync) + } } - fmt.Printf("use log dir:%s\n", logDir) - _ = flag.Set("log_dir", logDir) - if logAsync != "" { - _ = flag.Set("log_async", logAsync) + + if section != nil && section["log_structured"] != nil { + logStructured := strconv.FormatBool(section["log_structured"].(bool)) + if logStructured != "" { + _ = flag.Set("log_structured", logStructured) + } } - if logStructured != "" { - _ = flag.Set("log_structured", logStructured) + if section != nil && section["rotate_per_hour"] != nil { + rotatePerHour := strconv.FormatBool(section["rotate_per_hour"].(bool)) + if rotatePerHour != "" { + _ = flag.Set("rotate_per_hour", rotatePerHour) + } + } + if section != nil && section["log_level"] != nil { + logLevel := section["log_level"].(string) + if logLevel != "" { + _ = flag.Set("log_level", logLevel) + } } - if rotatePerHour != "" { - _ = flag.Set("rotate_per_hour", rotatePerHour) + if section != nil && section["log_filter_caller"] != nil { + logFilterCaller := strconv.FormatBool(section["log_filter_caller"].(bool)) + if logFilterCaller != "" { + _ = flag.Set("log_filter_caller", logFilterCaller) + } } - if logLevel != "" { - _ = flag.Set("log_level", logLevel) + if section != nil && section["log_buffer_size"] != nil { + logBufferSize := strconv.Itoa(section["log_buffer_size"].(int)) + if logBufferSize != "" { + _ = flag.Set("log_buffer_size", logBufferSize) + } } - if logFilterCaller != "" { - _ = flag.Set("log_filter_caller", logFilterCaller) + if logDir == "stdout" { + return } + fmt.Printf("use log dir:%s\n", logDir) + _ = flag.Set("log_dir", logDir) vlog.LogInit(nil) } diff --git a/client.go b/client.go index cc880eec..b00a3c59 100644 --- a/client.go +++ b/client.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/weibocom/motan-go/config" vlog "github.com/weibocom/motan-go/log" - "strconv" "sync" "github.com/weibocom/motan-go/cluster" @@ -114,28 +113,7 @@ func NewClientContextFromConfig(conf *config.Config) (mc *MCContext) { if logDir == "" { logDir = "." } - logAsync := "" - if section != nil && section["log_async"] != nil { - logAsync = strconv.FormatBool(section["log_async"].(bool)) - } - logStructured := "" - if section != nil && section["log_structured"] != nil { - logStructured = strconv.FormatBool(section["log_structured"].(bool)) - } - rotatePerHour := "" - if section != nil && section["rotate_per_hour"] != nil { - rotatePerHour = strconv.FormatBool(section["rotate_per_hour"].(bool)) - } - logLevel := "" - if section != nil && section["log_level"] != nil { - logLevel = section["log_level"].(string) - } - logFilterCaller := "" - if section != nil && section["log_filter_caller"] != nil { - logFilterCaller = strconv.FormatBool(section["log_filter_caller"].(bool)) - } - - initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel, logFilterCaller) + initLog(logDir, section) registerSwitchers(mc.context) return mc } diff --git a/log/log.go b/log/log.go index 45f34a1a..594bb0b7 100644 --- a/log/log.go +++ b/log/log.go @@ -3,12 +3,14 @@ package vlog import ( "bytes" "flag" + "github.com/weibocom/motan-go/metrics/sampler" "log" "os" "path/filepath" "runtime/debug" "strconv" "sync" + "sync/atomic" "time" "go.uber.org/zap" @@ -20,6 +22,7 @@ var ( once sync.Once logDir = flag.String("log_dir", ".", "If non-empty, write log files in this directory") logAsync = flag.Bool("log_async", true, "If false, write log sync, default is true") + logBufferSize = flag.Int("log_buffer_size", defaultAsyncLogLen, "If in async logging mode, this variable change the logger's buffer size. default value is 5000") logLevel = flag.String("log_level", "info", "Init log level, default is info.") logStructured = flag.Bool("log_structured", false, "If true, write accessLog structured, default is false") rotatePerHour = flag.Bool("rotate_per_hour", true, "") @@ -27,8 +30,7 @@ var ( ) const ( - defaultAsyncLogLen = 5000 - + defaultAsyncLogLen = 5000 defaultLogSuffix = ".log" defaultAccessLogName = "access" defaultMetricsLogName = "metrics" @@ -311,15 +313,16 @@ func newZapLogger(logName string) (*zap.Logger, zap.AtomicLevel) { } type defaultLogger struct { - async bool - accessStructured bool - outputChan chan *AccessLogEntity - logger *zap.SugaredLogger - accessLogger *zap.Logger - metricsLogger *zap.Logger - loggerLevel zap.AtomicLevel - accessLevel zap.AtomicLevel - metricsLevel zap.AtomicLevel + async bool + accessStructured bool + accessDiscardCount int64 + outputChan chan *AccessLogEntity + logger *zap.SugaredLogger + accessLogger *zap.Logger + metricsLogger *zap.Logger + loggerLevel zap.AtomicLevel + accessLevel zap.AtomicLevel + metricsLevel zap.AtomicLevel } func (d *defaultLogger) Infoln(fields ...interface{}) { @@ -356,7 +359,12 @@ func (d *defaultLogger) Fatalf(format string, fields ...interface{}) { func (d *defaultLogger) AccessLog(logObject *AccessLogEntity) { if d.async { - d.outputChan <- logObject + select { + case d.outputChan <- logObject: + default: + atomic.AddInt64(&d.accessDiscardCount, 1) + } + } else { d.doAccessLog(logObject) } @@ -425,7 +433,7 @@ func (d *defaultLogger) Flush() { func (d *defaultLogger) SetAsync(value bool) { d.async = value if value { - d.outputChan = make(chan *AccessLogEntity, defaultAsyncLogLen) + d.outputChan = make(chan *AccessLogEntity, *logBufferSize) go func() { defer func() { if err := recover(); err != nil { @@ -434,12 +442,12 @@ func (d *defaultLogger) SetAsync(value bool) { } }() for { - select { - case l := <-d.outputChan: - d.doAccessLog(l) - } + d.doAccessLog(<-d.outputChan) } }() + sampler.RegisterStatusSampleFunc("accesslog_discard_count", func() int64 { + return atomic.SwapInt64(&d.accessDiscardCount, 0) + }) } } diff --git a/log/log_test.go b/log/log_test.go index da600c25..fe025fbc 100644 --- a/log/log_test.go +++ b/log/log_test.go @@ -2,6 +2,7 @@ package vlog import ( "bytes" + "flag" "fmt" "strconv" "testing" @@ -135,3 +136,13 @@ func TestAccessLog(t *testing.T) { buffer.WriteString(logObject.Exception) assert.Equal(t, buffer.String(), expectString) } + +func TestChangeLogBufferSize(t *testing.T) { + testLogger := newDefaultLog() + testLogger.SetAsync(true) + assert.Equal(t, cap(testLogger.(*defaultLogger).outputChan), 5000) + _ = flag.Set("log_buffer_size", "1") + testLogger = newDefaultLog() + testLogger.SetAsync(true) + assert.Equal(t, cap(testLogger.(*defaultLogger).outputChan), 1) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index 11216fbd..574917b1 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,6 +1,7 @@ package metrics import ( + "github.com/weibocom/motan-go/metrics/sampler" "strings" "sync" "sync/atomic" @@ -53,8 +54,6 @@ var ( writers: make(map[string]StatWriter), evtBuf: &sync.Pool{New: func() interface{} { return new(event) }}, } - statusSamplerRegisterLock sync.Mutex - statusSamplers = make(map[string]StatusSampler) ) type StatItem interface { @@ -98,16 +97,6 @@ type StatWriter interface { Write(snapshots []Snapshot) error } -type StatusSampler interface { - Sample() int64 -} - -type StatusSampleFunc func() int64 - -func (f StatusSampleFunc) Sample() int64 { - return f() -} - func GetOrRegisterStatItem(group string, service string) StatItem { itemsLock.RLock() item := items[group+service] @@ -228,28 +217,15 @@ func ElapseTimeSuffix(t int64) string { } func RegisterStatusSampleFunc(key string, sf func() int64) { - RegisterStatusSampler(key, StatusSampleFunc(sf)) -} - -func RegisterStatusSampler(key string, sampler StatusSampler) { - statusSamplerRegisterLock.Lock() - defer statusSamplerRegisterLock.Unlock() - statusSamplers[key] = sampler -} - -func UnregisterStatusSampler(key string) { - statusSamplerRegisterLock.Lock() - defer statusSamplerRegisterLock.Unlock() - delete(statusSamplers, key) + sampler.RegisterStatusSampleFunc(key, sf) } func sampleStatus(application string) { - statusSamplerRegisterLock.Lock() - defer statusSamplerRegisterLock.Unlock() defer motan.HandlePanic(nil) - for key, s := range statusSamplers { - AddGauge(DefaultStatGroup, DefaultStatService, DefaultStatRole+KeyDelimiter+application+KeyDelimiter+key, s.Sample()) - } + sampler.RangeDo(func(key string, value sampler.StatusSampler) bool { + AddGauge(DefaultStatGroup, DefaultStatService, DefaultStatRole+KeyDelimiter+application+KeyDelimiter+key, value.Sample()) + return true + }) } func startSampleStatus(application string) { diff --git a/metrics/sampler/sampler.go b/metrics/sampler/sampler.go new file mode 100644 index 00000000..fd19f6ea --- /dev/null +++ b/metrics/sampler/sampler.go @@ -0,0 +1,46 @@ +package sampler + +import ( + "sync" +) + +var ( + statusSamplerRegisterLock sync.Mutex + statusSamplers = make(map[string]StatusSampler) +) + +type StatusSampler interface { + Sample() int64 +} + +type StatusSampleFunc func() int64 + +func (f StatusSampleFunc) Sample() int64 { + return f() +} + +func RegisterStatusSampleFunc(key string, sf func() int64) { + RegisterStatusSampler(key, StatusSampleFunc(sf)) +} + +func RegisterStatusSampler(key string, sampler StatusSampler) { + statusSamplerRegisterLock.Lock() + defer statusSamplerRegisterLock.Unlock() + statusSamplers[key] = sampler +} + +func UnregisterStatusSampler(key string) { + statusSamplerRegisterLock.Lock() + defer statusSamplerRegisterLock.Unlock() + delete(statusSamplers, key) +} + +func RangeDo(f func(key string, value StatusSampler) bool) { + statusSamplerRegisterLock.Lock() + defer statusSamplerRegisterLock.Unlock() + for k, e := range statusSamplers { + if !f(k, e) { + break + } + } +} diff --git a/server.go b/server.go index 240d0f20..f36436a5 100644 --- a/server.go +++ b/server.go @@ -57,28 +57,7 @@ func NewMotanServerContextFromConfig(conf *config.Config) (ms *MSContext) { if logDir == "" { logDir = "." } - logAsync := "" - if section != nil && section["log_async"] != nil { - logAsync = strconv.FormatBool(section["log_async"].(bool)) - } - logStructured := "" - if section != nil && section["log_structured"] != nil { - logStructured = strconv.FormatBool(section["log_structured"].(bool)) - } - rotatePerHour := "" - if section != nil && section["rotate_per_hour"] != nil { - rotatePerHour = strconv.FormatBool(section["rotate_per_hour"].(bool)) - } - logLevel := "" - if section != nil && section["log_level"] != nil { - logLevel = section["log_level"].(string) - } - logFilterCaller := "" - if section != nil && section["log_filter_caller"] != nil { - logFilterCaller = strconv.FormatBool(section["log_filter_caller"].(bool)) - } - - initLog(logDir, logAsync, logStructured, rotatePerHour, logLevel, logFilterCaller) + initLog(logDir, section) registerSwitchers(ms.context) return ms From 27ed50d89348b8973c71f56b33b0b0fa3a456a08 Mon Sep 17 00:00:00 2001 From: Ray Date: Tue, 28 Feb 2023 16:57:04 +0800 Subject: [PATCH 54/75] fill motan v1 default request info (#307) --- .gitignore | 1 + agent.go | 47 ++++++++++++++++++---------- agent_test.go | 5 +++ client_test.go | 1 + cluster/motanCluster.go | 11 +------ core/motan.go | 3 ++ core/switcher.go | 3 +- core/url.go | 35 ++++++++++++--------- endpoint/motanCommonEndpoint.go | 24 ++++++++++---- endpoint/motanCommonEndpoint_test.go | 8 +++++ endpoint/motanEndpoint.go | 13 +++++--- endpoint/motanEndpoint_test.go | 8 +++++ http/httpRpc_test.go | 3 ++ meshClient.go | 1 + protocol/motan1Protocol.go | 8 +++-- server_test.go | 2 ++ testdata/agent-reload.yaml | 1 + 17 files changed, 117 insertions(+), 57 deletions(-) diff --git a/.gitignore b/.gitignore index 6160bf02..b3c37487 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ main/magent* log/log.test* go.sum agent_runtime +test/ diff --git a/agent.go b/agent.go index 7184f2c4..3fabd135 100644 --- a/agent.go +++ b/agent.go @@ -499,9 +499,6 @@ func (a *Agent) initClusters() { } func (a *Agent) initCluster(url *motan.URL) { - a.clsLock.Lock() - defer a.clsLock.Unlock() - if url.Parameters[motan.ApplicationKey] == "" { url.Parameters[motan.ApplicationKey] = a.agentURL.Parameters[motan.ApplicationKey] } @@ -523,6 +520,8 @@ func (a *Agent) initCluster(url *motan.URL) { a.serviceMap.UnsafeStore(url.Path, serviceMapItemArr) }) mapKey := getClusterKey(url.Group, url.GetStringParamsWithDefault(motan.VersionKey, "0.1"), url.Protocol, url.Path) + a.clsLock.Lock() // Mutually exclusive with the reloadClusters method + defer a.clsLock.Unlock() a.clusterMap.Store(mapKey, c) } @@ -617,13 +616,8 @@ type agentMessageHandler struct { } func (a *agentMessageHandler) clusterCall(request motan.Request, ck string, motanCluster *cluster.MotanCluster) (res motan.Response) { - if request.GetAttachment(mpro.MSource) == "" { - application := motanCluster.GetURL().GetParam(motan.ApplicationKey, "") - if application == "" { - application = a.agent.agentURL.GetParam(motan.ApplicationKey, "") - } - request.SetAttachment(mpro.MSource, application) - } + // fill default request info + fillDefaultReqInfo(request, motanCluster.GetURL()) res = motanCluster.Call(request) if res == nil { vlog.Warningf("motanCluster Call return nil. cluster:%s", ck) @@ -647,13 +641,7 @@ func (a *agentMessageHandler) httpCall(request motan.Request, ck string, httpClu }() if service, ok := httpCluster.CanServe(request.GetMethod()); ok { // if the client can use rpc, do it - if request.GetAttachment(mpro.MSource) == "" { - application := httpCluster.GetURL().GetParam(motan.ApplicationKey, "") - if application == "" { - application = a.agent.agentURL.GetParam(motan.ApplicationKey, "") - } - request.SetAttachment(mpro.MSource, application) - } + fillDefaultReqInfo(request, httpCluster.GetURL()) request.SetAttachment(mpro.MPath, service) if motanRequest, ok := request.(*motan.MotanRequest); ok { motanRequest.ServiceName = service @@ -695,6 +683,31 @@ func (a *agentMessageHandler) httpCall(request motan.Request, ck string, httpClu return res } +// fill default reqeust info such as application, group.. +func fillDefaultReqInfo(r motan.Request, url *motan.URL) { + if r.GetRPCContext(true).IsMotanV1 { + if r.GetAttachment(motan.ApplicationKey) == "" { + application := url.GetParam(motan.ApplicationKey, "") + if application != "" { + r.SetAttachment(motan.ApplicationKey, application) + } + } + if r.GetAttachment(motan.GroupKey) == "" { + r.SetAttachment(motan.GroupKey, url.Group) + } + } else { + if r.GetAttachment(mpro.MSource) == "" { + application := url.GetParam(motan.ApplicationKey, "") + if application != "" { + r.SetAttachment(mpro.MSource, application) + } + } + if r.GetAttachment(mpro.MGroup) == "" { + r.SetAttachment(mpro.MGroup, url.Group) + } + } +} + func (a *agentMessageHandler) Call(request motan.Request) motan.Response { c, ck, err := a.findCluster(request) if err == nil { diff --git a/agent_test.go b/agent_test.go index 146ad281..39859c8b 100644 --- a/agent_test.go +++ b/agent_test.go @@ -67,6 +67,7 @@ motan-refer: path: helloService protocol: motan2 registry: direct + asyncInitConnection: false serialization: breeze`))) agent := NewAgent(ext) go agent.StartMotanAgentFromConfig(config) @@ -110,6 +111,7 @@ motan-refer: protocol: motanV1Compatible registry: direct serialization: breeze + asyncInitConnection: false `))) agent := NewAgent(ext) go agent.StartMotanAgentFromConfig(config1) @@ -132,6 +134,7 @@ motan-refer: group: hello path: helloService requestTimeout: 3000 + asyncInitConnection: false `))) mccontext := NewClientContextFromConfig(cfg) mccontext.Start(ext1) @@ -355,6 +358,7 @@ func TestAgent_InitCall(t *testing.T) { agent.agentURL = &core.URL{Parameters: make(map[string]string)} urlTest := &core.URL{Parameters: make(map[string]string)} urlTest.Group = "test1" + urlTest.Parameters[core.AsyncInitConnection] = "false" agent.initCluster(urlTest) agentHandler := &agentMessageHandler{agent: agent} @@ -494,6 +498,7 @@ func TestNotFoundProvider(t *testing.T) { core.ApplicationKey: "testep", core.ConnectRetryIntervalKey: "5000", core.ErrorCountThresholdKey: "0", + core.AsyncInitConnection: "false", }, } ext := GetDefaultExtFactory() diff --git a/client_test.go b/client_test.go index ddc0d60e..eff481e0 100644 --- a/client_test.go +++ b/client_test.go @@ -28,6 +28,7 @@ motan-refer: protocol: motan2 registry: direct serialization: simple + asyncInitConnection: false ` conf, err := config.NewConfigFromReader(bytes.NewReader([]byte(cfgText))) assert.Nil(err) diff --git a/cluster/motanCluster.go b/cluster/motanCluster.go index b8a596fa..bb036dc9 100644 --- a/cluster/motanCluster.go +++ b/cluster/motanCluster.go @@ -11,9 +11,7 @@ import ( "time" motan "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/endpoint" "github.com/weibocom/motan-go/log" - "github.com/weibocom/motan-go/protocol" ) type MotanCluster struct { @@ -68,13 +66,6 @@ func (m *MotanCluster) Call(request motan.Request) (res motan.Response) { vlog.Errorf("cluster call panic. req:%s", motan.GetReqInfo(request)) }) if m.available { - if m.proxy { - // TODO: for case client has no group or protocol convert - // we should have the same entry to set all necessary attachments - if endpoint.GetRequestGroup(request) == "" { - request.SetAttachment(protocol.MGroup, m.url.Group) - } - } return m.clusterFilter.Filter(m.HaStrategy, m.LoadBalance, request) } vlog.Infoln("cluster:" + m.GetIdentity() + "is not available!") @@ -234,7 +225,7 @@ func (m *MotanCluster) addFilter(ep motan.EndPoint, filters []motan.Filter) mota } fep.StatusFilters = statusFilters fep.Filter = lastf - vlog.Infof("MotanCluster add ep filters. url:%+v, filters:%s", ep.GetURL(), motan.GetEPFilterInfo(fep.Filter)) + vlog.Infof("MotanCluster add ep filters. url:%s, filters:%s", ep.GetURL().GetIdentity(), motan.GetEPFilterInfo(fep.Filter)) return fep } diff --git a/core/motan.go b/core/motan.go index 7218517d..9567e3fe 100644 --- a/core/motan.go +++ b/core/motan.go @@ -373,6 +373,9 @@ type RPCContext struct { // trace context Tc *TraceContext + + // ---- internal vars ---- + IsMotanV1 bool } func (c *RPCContext) AddFinishHandler(handler FinishHandler) { diff --git a/core/switcher.go b/core/switcher.go index dd955c5a..9b71c643 100644 --- a/core/switcher.go +++ b/core/switcher.go @@ -30,8 +30,7 @@ func (s *SwitcherManager) Register(name string, value bool, listeners ...Switche } s.switcherLock.Lock() defer s.switcherLock.Unlock() - if _, ok := s.switcherMap[name]; ok { - vlog.Warningf("[switcher] register failed: %s has been registered", name) + if _, ok := s.switcherMap[name]; ok { // just ignore if already registered return } newSwitcher := &Switcher{name: name, value: value, listeners: []SwitcherListener{}} diff --git a/core/url.go b/core/url.go index f61acde9..ec94faa6 100644 --- a/core/url.go +++ b/core/url.go @@ -5,6 +5,7 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "time" "github.com/weibocom/motan-go/log" @@ -19,8 +20,8 @@ type URL struct { Parameters map[string]string `json:"parameters"` // cached info - address string - identity string + address atomic.Value + identity atomic.Value } var ( @@ -32,14 +33,16 @@ var ( // GetIdentity return the identity of url. identity info includes protocol, host, port, path, group // the identity will cached, so must clear cached info after update above info by calling ClearCachedInfo() func (u *URL) GetIdentity() string { - if u.identity != "" { - return u.identity + temp := u.identity.Load() + if temp != nil && temp != "" { + return temp.(string) } - u.identity = u.Protocol + "://" + u.Host + ":" + u.GetPortStr() + "/" + u.Path + "?group=" + u.Group + idt := u.Protocol + "://" + u.Host + ":" + u.GetPortStr() + "/" + u.Path + "?group=" + u.Group if u.Protocol == "direct" { - u.identity += "&address=" + u.GetParam("address", "") + idt += "&address=" + u.GetParam("address", "") } - return u.identity + u.identity.Store(idt) + return idt } func (u *URL) GetIdentityWithRegistry() string { @@ -49,8 +52,8 @@ func (u *URL) GetIdentityWithRegistry() string { } func (u *URL) ClearCachedInfo() { - u.address = "" - u.identity = "" + u.address.Store("") + u.identity.Store("") } func (u *URL) GetPositiveIntValue(key string, defaultvalue int64) int64 { @@ -213,15 +216,17 @@ func (u *URL) GetPortStr() string { } func (u *URL) GetAddressStr() string { - if u.address != "" { - return u.address + temp := u.address.Load() + if temp != nil && temp != "" { + return temp.(string) } if strings.HasPrefix(u.Host, UnixSockProtocolFlag) { - u.address = u.Host - return u.address + temp = u.Host + } else { + temp = u.Host + ":" + u.GetPortStr() } - u.address = u.Host + ":" + u.GetPortStr() - return u.address + u.address.Store(temp) + return temp.(string) } func (u *URL) Copy() *URL { diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 29c55f5c..be4ee94c 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -63,8 +63,8 @@ func (m *MotanCommonEndpoint) Initialize() { m.maxRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MaxTimeOutKey) m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) - m.lazyInit = m.url.GetBoolValue(motan.LazyInit, false) - asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, false) + m.lazyInit = m.url.GetBoolValue(motan.LazyInit, defaultLazyInit) + asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, defaultAsyncInitConnection) m.heartbeatVersion = -1 m.DefaultVersion = mpro.Version2 factory := func() (net.Conn, error) { @@ -141,10 +141,7 @@ func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { } deadline := m.GetRequestTimeout(request) // do call - group := GetRequestGroup(request) - if group != m.url.Group && m.url.Group != "" { - request.SetAttachment(mpro.MGroup, m.url.Group) - } + m.fitGroup(request) response, err := channel.Call(request, deadline, rc) if err != nil { vlog.Errorf("motanEndpoint call fail. ep:%s, req:%s, error: %s", m.url.GetAddressStr(), motan.GetReqInfo(request), err.Error()) @@ -176,6 +173,21 @@ func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { return response } +// Due to traffic switching, the group of the cluster may not same as the endpoint's +func (m *MotanCommonEndpoint) fitGroup(r motan.Request) { + if r.GetRPCContext(true).IsMotanV1 { // motan v1 use group key + group := r.GetAttachment(motan.GroupKey) + if group != m.url.Group && m.url.Group != "" { + r.SetAttachment(motan.GroupKey, m.url.Group) + } + } else { + group := GetRequestGroup(r) + if group != m.url.Group && m.url.Group != "" { + r.SetAttachment(mpro.MGroup, m.url.Group) + } + } +} + func (m *MotanCommonEndpoint) initChannelPoolWithRetry(factory ConnFactory, config *ChannelConfig, retryInterval time.Duration) { defer motan.HandlePanic(nil) channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) diff --git a/endpoint/motanCommonEndpoint_test.go b/endpoint/motanCommonEndpoint_test.go index 05f75ad5..3cd2715f 100644 --- a/endpoint/motanCommonEndpoint_test.go +++ b/endpoint/motanCommonEndpoint_test.go @@ -15,6 +15,7 @@ import ( func TestGetV1Name(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) @@ -34,6 +35,7 @@ func TestV1RecordErrEmptyThreshold(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -54,6 +56,7 @@ func TestV1RecordErrWithErrThreshold(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "5") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -85,6 +88,7 @@ func TestMotanCommonEndpoint_SuccessCall(t *testing.T) { url.PutParam(motan.TimeOutKey, "2000") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetSerialization(&serialize.SimpleSerialization{}) @@ -105,6 +109,7 @@ func TestMotanCommonEndpoint_AsyncCall(t *testing.T) { url.PutParam(motan.TimeOutKey, "2000") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetSerialization(&serialize.SimpleSerialization{}) @@ -125,6 +130,7 @@ func TestMotanCommonEndpoint_ErrorCall(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -149,6 +155,7 @@ func TestMotanCommonEndpoint_RequestTimeout(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -174,6 +181,7 @@ func TestV1LazyInit(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetProxy(true) diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 736b840b..39a1f5d9 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -23,6 +23,8 @@ var ( defaultKeepaliveInterval = 1000 * time.Millisecond defaultConnectRetryInterval = 60 * time.Second defaultErrorCountThreshold = 10 + defaultLazyInit = false + defaultAsyncInitConnection = true ErrChannelShutdown = fmt.Errorf("the channel has been shutdown") ErrSendRequestTimeout = fmt.Errorf("timeout err: send request timeout") ErrRecvRequestTimeout = fmt.Errorf("timeout err: receive request timeout") @@ -39,7 +41,7 @@ type MotanEndpoint struct { channels *V2ChannelPool destroyed bool destroyCh chan struct{} - available bool + available atomic.Value errorCount uint32 proxy bool errorCountThreshold int64 @@ -58,7 +60,7 @@ type MotanEndpoint struct { } func (m *MotanEndpoint) setAvailable(available bool) { - m.available = available + m.available.Store(available) } func (m *MotanEndpoint) SetSerialization(s motan.Serialization) { @@ -71,6 +73,7 @@ func (m *MotanEndpoint) SetProxy(proxy bool) { func (m *MotanEndpoint) Initialize() { m.destroyCh = make(chan struct{}, 1) + m.available.Store(false) // init value connectTimeout := m.url.GetTimeDuration(motan.ConnectTimeoutKey, time.Millisecond, defaultConnectTimeout) connectRetryInterval := m.url.GetTimeDuration(motan.ConnectRetryIntervalKey, time.Millisecond, defaultConnectRetryInterval) m.errorCountThreshold = m.url.GetIntValue(motan.ErrorCountThresholdKey, int64(defaultErrorCountThreshold)) @@ -80,8 +83,8 @@ func (m *MotanEndpoint) Initialize() { m.maxRequestTimeoutMillisecond, _ = m.url.GetInt(motan.MaxTimeOutKey) m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) - m.lazyInit = m.url.GetBoolValue(motan.LazyInit, false) - asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, false) + m.lazyInit = m.url.GetBoolValue(motan.LazyInit, defaultLazyInit) + asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, defaultAsyncInitConnection) factory := func() (net.Conn, error) { address := m.url.GetAddressStr() if strings.HasPrefix(address, motan.UnixSockProtocolFlag) { @@ -327,7 +330,7 @@ func (m *MotanEndpoint) SetURL(url *motan.URL) { } func (m *MotanEndpoint) IsAvailable() bool { - return m.available + return m.available.Load().(bool) } // ChannelConfig : ChannelConfig diff --git a/endpoint/motanEndpoint_test.go b/endpoint/motanEndpoint_test.go index b7d4fad9..9bb0efe6 100644 --- a/endpoint/motanEndpoint_test.go +++ b/endpoint/motanEndpoint_test.go @@ -25,6 +25,7 @@ func TestMain(m *testing.M) { func TestGetName(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -43,6 +44,7 @@ func TestRecordErrEmptyThreshold(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -63,6 +65,7 @@ func TestRecordErrWithErrThreshold(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "5") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -94,6 +97,7 @@ func TestMotanEndpoint_SuccessCall(t *testing.T) { url.PutParam(motan.TimeOutKey, "2000") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetSerialization(&serialize.SimpleSerialization{}) @@ -115,6 +119,7 @@ func TestMotanEndpoint_AsyncCall(t *testing.T) { url.PutParam(motan.TimeOutKey, "2000") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetSerialization(&serialize.SimpleSerialization{}) @@ -136,6 +141,7 @@ func TestMotanEndpoint_ErrorCall(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -160,6 +166,7 @@ func TestMotanEndpoint_RequestTimeout(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetProxy(true) @@ -185,6 +192,7 @@ func TestLazyInit(t *testing.T) { url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanEndpoint{} ep.SetURL(url) ep.SetProxy(true) diff --git a/http/httpRpc_test.go b/http/httpRpc_test.go index 6f99da2e..bf33aaf2 100644 --- a/http/httpRpc_test.go +++ b/http/httpRpc_test.go @@ -113,6 +113,7 @@ func callTimeOut(address string, port string, path string, group string, timeout clientExt := motan.GetDefaultExtFactory() info := fmt.Sprintf("motan2://%s:%s/%s?serialization=simple&maxRequestTimeout=30000&group=%s", address, port, path, group) u := motancore.FromExtInfo(info) + u.PutParam(motancore.AsyncInitConnection, "false") ep = clientExt.GetEndPoint(u) if ep == nil { return nil, fmt.Errorf("get end point error") @@ -150,6 +151,7 @@ func callTimeOutWithAttachment(address string, port string, path string, group s clientExt := motan.GetDefaultExtFactory() info := fmt.Sprintf("motan2://%s:%s/%s?serialization=simple&maxRequestTimeout=30000&group=%s", address, port, path, group) u := motancore.FromExtInfo(info) + u.PutParam(motancore.AsyncInitConnection, "false") ep = clientExt.GetEndPoint(u) if ep == nil { return nil, fmt.Errorf("get end point error") @@ -190,6 +192,7 @@ func callTimeOutWrongArgumentCount(address string, port string, path string, gro clientExt := motan.GetDefaultExtFactory() info := fmt.Sprintf("motan2://%s:%s/%s?serialization=simple&maxRequestTimeout=30000&group=%s", address, port, path, group) u := motancore.FromExtInfo(info) + u.PutParam(motancore.AsyncInitConnection, "false") ep = clientExt.GetEndPoint(u) if ep == nil { return nil, fmt.Errorf("get end point error") diff --git a/meshClient.go b/meshClient.go index 589f4e0e..88e93a46 100644 --- a/meshClient.go +++ b/meshClient.go @@ -69,6 +69,7 @@ func (c *MeshClient) Initialize() { clusterURL.PutParam(core.RegistryKey, meshDirectRegistryKey) clusterURL.PutParam(core.ConnectRetryIntervalKey, "5000") clusterURL.PutParam(core.SerializationKey, c.serialization) + clusterURL.PutParam(core.AsyncInitConnection, "false") meshRegistryURL := &core.URL{} meshRegistryURL.Protocol = "direct" meshRegistryURL.PutParam(core.AddressKey, c.address) diff --git a/protocol/motan1Protocol.go b/protocol/motan1Protocol.go index d9a48e8c..d2845569 100644 --- a/protocol/motan1Protocol.go +++ b/protocol/motan1Protocol.go @@ -106,7 +106,9 @@ func DecodeMotanV1Request(msg *MotanV1Message) (motan.Request, error) { Method: objStream.method, MethodDesc: objStream.paramDesc, Attachment: objStream.attachment} - request.GetRPCContext(true).OriginalMessage = msg + ctx := request.GetRPCContext(true) + ctx.OriginalMessage = msg + ctx.IsMotanV1 = true return request, nil } @@ -138,7 +140,9 @@ func DecodeMotanV1Response(msg *MotanV1Message) (motan.Response, error) { response.Exception = &motan.Exception{ErrCode: 500, ErrMsg: "v1: has exception, class:" + objStream.cName, ErrType: motan.ServiceException} } } - response.GetRPCContext(true).OriginalMessage = msg + ctx := response.GetRPCContext(true) + ctx.OriginalMessage = msg + ctx.IsMotanV1 = true return response, nil } diff --git a/server_test.go b/server_test.go index 7f713fc2..b028d4de 100644 --- a/server_test.go +++ b/server_test.go @@ -52,6 +52,7 @@ motan-server: clientExt := GetDefaultExtFactory() u := motan.FromExtInfo("motan2://127.0.0.1:64332/helloService?serialization=simple") assert.NotNil(u) + u.Parameters["asyncInitConnection"] = "false" ep := clientExt.GetEndPoint(u) assert.NotNil(ep) ep.SetSerialization(motan.GetSerialization(u, ext)) @@ -98,6 +99,7 @@ func TestNewMotanServerContextFromConfig(t *testing.T) { ext := startServer(t, "helloService", 64532) clientExt := GetDefaultExtFactory() u := motan.FromExtInfo("motan2://127.0.0.1:64532/helloService?serialization=simple") + u.Parameters["asyncInitConnection"] = "false" assert.NotNil(u) ep := clientExt.GetEndPoint(u) assert.NotNil(ep) diff --git a/testdata/agent-reload.yaml b/testdata/agent-reload.yaml index fa9dd1ff..4af3381e 100644 --- a/testdata/agent-reload.yaml +++ b/testdata/agent-reload.yaml @@ -15,6 +15,7 @@ motan-basicRefer: protocol: motan2 registry: direct serialization: simple + asyncInitConnection: false #conf of refers motan-refer: From 206fcb3cc6fc0878d40366378997853bc239f8b3 Mon Sep 17 00:00:00 2001 From: Ray Date: Thu, 9 Mar 2023 16:34:02 +0800 Subject: [PATCH 55/75] configure direct services through environment variables (#308) --- cluster/motanCluster.go | 41 +++++++++++----- cluster/motanCluster_test.go | 24 +++++++++- core/constants.go | 7 ++- core/util.go | 80 ++++++++++++++++++++++++++----- core/util_test.go | 85 ++++++++++++++++++++++++++++++--- protocol/motan1Protocol.go | 4 ++ registry/directRegistry.go | 16 +++++-- registry/directRegistry_test.go | 24 +++++++++- registry/zkRegistry.go | 3 +- 9 files changed, 243 insertions(+), 41 deletions(-) diff --git a/cluster/motanCluster.go b/cluster/motanCluster.go index bb036dc9..3819941a 100644 --- a/cluster/motanCluster.go +++ b/cluster/motanCluster.go @@ -91,10 +91,12 @@ func (m *MotanCluster) initCluster() bool { m.closed = false // parse registry and subscribe - m.parseRegistry() - + err := m.parseRegistry() + if err != nil { + vlog.Errorf("init MotanCluster fail. cluster:%s, err:%s", m.GetIdentity(), err.Error()) + return false + } vlog.Infof("init MotanCluster %s", m.GetIdentity()) - return true } func (m *MotanCluster) SetLoadBalance(loadBalance motan.LoadBalance) { @@ -130,7 +132,7 @@ func (m *MotanCluster) Notify(registryURL *motan.URL, urls []*motan.URL) { vlog.Infof("cluster %s receive notify size %d. ", m.GetIdentity(), len(urls)) m.notifyLock.Lock() defer m.notifyLock.Unlock() - // process weight if has + // process weight if any urls = processWeight(m, urls) endpoints := make([]motan.EndPoint, 0, len(urls)) endpointMap := make(map[string]motan.EndPoint) @@ -181,7 +183,7 @@ func (m *MotanCluster) Notify(registryURL *motan.URL, urls []*motan.URL) { if len(m.registryRefers) > 1 { delete(m.registryRefers, registryURL.GetIdentity()) } else { - // notify will ignored if endpoints size is 0 in single regisry mode + // ignored if endpoints size is 0 in single registry mode vlog.Infof("cluster %s notify endpoint is 0. notify ignored.", m.GetIdentity()) return } @@ -209,14 +211,14 @@ func processWeight(m *MotanCluster, urls []*motan.URL) []*motan.URL { func (m *MotanCluster) addFilter(ep motan.EndPoint, filters []motan.Filter) motan.EndPoint { fep := &motan.FilterEndPoint{URL: ep.GetURL(), Caller: ep} statusFilters := make([]motan.Status, 0, len(filters)) - var lastf motan.EndPointFilter - lastf = motan.GetLastEndPointFilter() + var lf motan.EndPointFilter + lf = motan.GetLastEndPointFilter() for _, f := range filters { if filter := f.NewFilter(ep.GetURL()); filter != nil { if ef, ok := filter.(motan.EndPointFilter); ok { motan.CanSetContext(ef, m.Context) - ef.SetNext(lastf) - lastf = ef + ef.SetNext(lf) + lf = ef if sf, ok := ef.(motan.Status); ok { statusFilters = append(statusFilters, sf) } @@ -224,7 +226,7 @@ func (m *MotanCluster) addFilter(ep motan.EndPoint, filters []motan.Filter) mota } } fep.StatusFilters = statusFilters - fep.Filter = lastf + fep.Filter = lf vlog.Infof("MotanCluster add ep filters. url:%s, filters:%s", ep.GetURL().GetIdentity(), motan.GetEPFilterInfo(fep.Filter)) return fep } @@ -262,6 +264,10 @@ func (m *MotanCluster) SetExtFactory(factory motan.ExtensionFactory) { } func (m *MotanCluster) parseRegistry() (err error) { + envReg := motan.GetDirectEnvRegistry(m.url) + if envReg != nil { // If the direct url is specified by the env variable, other registries are ignored + return m.parseFromEnvRegistry(envReg) + } regs, ok := m.url.Parameters[motan.RegistryKey] if !ok { errInfo := fmt.Sprintf("registry not found! url %+v", m.url) @@ -296,6 +302,19 @@ func (m *MotanCluster) parseRegistry() (err error) { return err } +func (m *MotanCluster) parseFromEnvRegistry(reg *motan.URL) error { + vlog.Infof("direct registry is found from env, will replace registry with %+v, cluster:%s", reg, m.url.GetIdentity()) + registry := m.extFactory.GetRegistry(reg) + if registry == nil { + return errors.New("env direct registry is nil. reg url: " + reg.GetIdentity()) + } + registries := make([]motan.Registry, 0, 1) + m.Registries = append(registries, registry) + urls := registry.Discover(m.url) + m.Notify(reg, urls) + return nil +} + func (m *MotanCluster) initFilters() { clusterFilter, endpointFilters := motan.GetURLFilters(m.url, m.extFactory) if clusterFilter != nil { @@ -438,7 +457,7 @@ func getBestGroup(groupNodes map[string][]*motan.URL) string { var bestGroup string for group, rtt := range groupRtt { if minRtt == 0 || minRtt > rtt { - // First rtt group or the rtt is less then former + // First rtt group or the rtt is less than former minRtt = rtt bestGroup = group } diff --git a/cluster/motanCluster_test.go b/cluster/motanCluster_test.go index 9ef995e6..9fa3d18b 100644 --- a/cluster/motanCluster_test.go +++ b/cluster/motanCluster_test.go @@ -2,6 +2,9 @@ package cluster import ( "fmt" + "github.com/stretchr/testify/assert" + "github.com/weibocom/motan-go/registry" + "os" "testing" motan "github.com/weibocom/motan-go/core" @@ -82,7 +85,13 @@ func TestCall(t *testing.T) { } func initCluster() *MotanCluster { + return initClusterWithPathGroup("", "") +} + +func initClusterWithPathGroup(path string, group string) *MotanCluster { url := &motan.URL{Parameters: make(map[string]string)} + url.Path = path + url.Group = group url.Protocol = "test" url.Parameters[motan.Hakey] = "failover" url.Parameters[motan.RegistryKey] = "vintage,consul,direct" @@ -102,6 +111,18 @@ func TestDestroy(t *testing.T) { } } +func TestParseRegistryFromEnv(t *testing.T) { + motan.ClearDirectEnvRegistry() + os.Setenv(motan.DirectRPCEnvironmentName, "helloService>change_group@127.0.0.1:8005,127.0.0.1:8006;unknownService>127.0.0.1:8888,127.0.0.1:9999") + cluster := initClusterWithPathGroup("com.weibo.helloService", "group1") + assert.Equal(t, 2, len(cluster.Refers)) + for _, r := range cluster.GetRefers() { + assert.Equal(t, "change_group", r.GetURL().Group) + assert.Equal(t, "127.0.0.1", r.GetURL().Host) + assert.True(t, r.GetURL().Port == 8005 || r.GetURL().Port == 8006) + } +} + //-------------test struct-------------------- func getCustomExt() motan.ExtensionFactory { ext := &motan.DefaultExtensionFactory{} @@ -133,9 +154,10 @@ func getCustomExt() motan.ExtensionFactory { ext.RegistExtRegistry("test", func(url *motan.URL) motan.Registry { return &motan.TestRegistry{URL: url} }) - ext.RegistExtEndpoint("test", func(url *motan.URL) motan.EndPoint { return &motan.TestEndPoint{URL: url} }) + + registry.RegistDefaultRegistry(ext) return ext } diff --git a/core/constants.go b/core/constants.go index a5d489e9..be9a64ff 100644 --- a/core/constants.go +++ b/core/constants.go @@ -91,7 +91,12 @@ const ( DefaultWriteTimeout = 5 * time.Second DefaultMaxContentLength = 10 * 1024 * 1024 GroupNameSeparator = "," - GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" +) + +// env variables +const ( + GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" + DirectRPCEnvironmentName = "MESH_DIRECT_RPC" ) // meta keys diff --git a/core/util.go b/core/util.go index e25dba77..0438ca69 100644 --- a/core/util.go +++ b/core/util.go @@ -8,6 +8,7 @@ import ( "runtime/debug" "strconv" "strings" + "sync" "unicode" "github.com/weibocom/motan-go/log" @@ -21,7 +22,9 @@ const ( var ( PanicStatFunc func() - localIPs = make([]string, 0) + localIPs = make([]string, 0) + initDirectEnv sync.Once + directRpc map[string]*URL ) func ParseExportInfo(export string) (string, int, error) { @@ -36,12 +39,12 @@ func ParseExportInfo(export string) (string, int, error) { } port = s[1] } - porti, err := strconv.Atoi(port) + pi, err := strconv.Atoi(port) if err != nil { vlog.Errorf("export port not int. port:%s", port) - return protocol, porti, err + return protocol, pi, err } - return protocol, porti, err + return protocol, pi, err } func InterfaceToString(in interface{}) string { @@ -59,17 +62,17 @@ func InterfaceToString(in interface{}) string { return rs } -// GetLocalIPs ip from ipnet +// GetLocalIPs ip from ip net func GetLocalIPs() []string { if len(localIPs) == 0 { - addrs, err := net.InterfaceAddrs() + addr, err := net.InterfaceAddrs() if err != nil { vlog.Warningf("get local ip fail. %s", err.Error()) } else { - for _, address := range addrs { - if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil { - localIPs = append(localIPs, ipnet.IP.String()) + for _, address := range addr { + if ipNet, ok := address.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { + if ipNet.IP.To4() != nil { + localIPs = append(localIPs, ipNet.IP.String()) } } } @@ -78,7 +81,7 @@ func GetLocalIPs() []string { return localIPs } -// GetLocalIP falg of localIP > ipnet +// GetLocalIP flag of localIP > ip net func GetLocalIP() string { if *LocalIP != "" { return *LocalIP @@ -218,7 +221,7 @@ func ListenUnixSock(unixSockAddr string) (net.Listener, error) { return listener, nil } -// c convert slices to set slices +// SlicesUnique deduplicate the values of the slice func SlicesUnique(src []string) []string { var dst []string set := map[string]bool{} @@ -230,3 +233,56 @@ func SlicesUnique(src []string) []string { } return dst } + +// GetDirectEnvRegistry get the direct registry from the environment variable. +// return registry urls if url match, or return nil +func GetDirectEnvRegistry(url *URL) *URL { + initDirectEnv.Do(func() { + str := os.Getenv(DirectRPCEnvironmentName) + if str != "" { + vlog.Infof("find env " + DirectRPCEnvironmentName + ":" + str) + ss := strings.Split(str, ";") // multi services + for _, s := range ss { + sInfo := strings.Split(s, ">") + if len(sInfo) == 2 { + group := "" + addr := sInfo[1] + gIndex := strings.Index(sInfo[1], "@") + if gIndex > -1 { // has group info + group = sInfo[1][:gIndex] + addr = sInfo[1][gIndex+1:] + } + reg := &URL{Protocol: "direct", Group: strings.TrimSpace(group)} + reg.PutParam(AddressKey, strings.TrimSpace(addr)) + key := strings.TrimSpace(sInfo[0]) + vlog.Infof("add direct registry info, key:"+key+", url: %+v", reg) + if directRpc == nil { + directRpc = make(map[string]*URL, 16) + } + directRpc[key] = reg + } + } + } + }) + if directRpc != nil { // have direct rpc + p := url.Path + for k, u := range directRpc { + if strings.HasSuffix(k, "*") { // prefix match + if strings.HasPrefix(p, k[:len(k)-1]) { + return u + } + } else { // suffix match + if strings.HasSuffix(p, k) { + return u + } + } + } + } + return nil +} + +// ClearDirectEnvRegistry is only for unit test +func ClearDirectEnvRegistry() { + directRpc = nil + initDirectEnv = sync.Once{} +} diff --git a/core/util_test.go b/core/util_test.go index 92a14bf5..2f9ab0f9 100644 --- a/core/util_test.go +++ b/core/util_test.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "os" "strconv" "testing" @@ -22,7 +23,7 @@ func TestGetLocalIP(t *testing.T) { if ip == "" { t.Errorf("get local ip fail. ip:%s", ip) } - hostname := "testhostname" + hostname := "testHostname" *LocalIP = hostname ip = GetLocalIP() if ip != hostname { @@ -42,19 +43,19 @@ func TestSliceShuffle(t *testing.T) { if len(ns) != len(s) || len(ns) != 32 { t.Errorf("slice shuffle fail. size not correct. size:%d", len(ns)) } - diffcount := 0 + diffCount := 0 for i := 0; i < size; i++ { if ns[i] != s[i] { - diffcount++ + diffCount++ } } - fmt.Printf("shuffle diff count:%d\n", diffcount) - if diffcount < 2 { - t.Errorf("shuffle fail. diff count:%d", diffcount) + fmt.Printf("shuffle diff count:%d\n", diffCount) + if diffCount < 2 { + t.Errorf("shuffle fail. diff count:%d", diffCount) } } -func TestFisrtUpper(t *testing.T) { +func TestFirstUpper(t *testing.T) { s := "test" ns := FirstUpper(s) if ns != "Test" { @@ -134,3 +135,73 @@ func TestGetEPFilterInfo(t *testing.T) { str := GetEPFilterInfo(filter1) assert.Equal(t, "TestEndPointFilter->TestEndPointFilter->TestEndPointFilter", str) } + +func TestGetDirectEnvRegistry(t *testing.T) { + os.Unsetenv(DirectRPCEnvironmentName) + ClearDirectEnvRegistry() + // test init value + assert.Nil(t, directRpc) + + u := &URL{Path: "com.weibo.helloService"} + u1 := &URL{Path: "com.weibo.testService"} + u2 := &URL{Path: "com.weibo.tempService"} + + // test not set env + reg := GetDirectEnvRegistry(u) + assert.Nil(t, directRpc) + assert.Nil(t, reg) + + // test parse + ClearDirectEnvRegistry() + os.Setenv(DirectRPCEnvironmentName, "helloService>127.0.0.1:8005") + reg = GetDirectEnvRegistry(u) + assert.True(t, directRpc != nil && len(directRpc) == 1) + checkReg(reg, t, "127.0.0.1:8005", "") + + // test parse multi + ClearDirectEnvRegistry() + os.Setenv(DirectRPCEnvironmentName, "helloService>127.0.0.1:8005;testService>10.123.123.123:7777,10.123.123.123:8888,10.123.123.123:9999;tempService>10.111.111.111:7777") + reg = GetDirectEnvRegistry(u) + assert.Equal(t, 3, len(directRpc)) + checkReg(reg, t, "127.0.0.1:8005", "") + + reg = GetDirectEnvRegistry(u1) + checkReg(reg, t, "10.123.123.123:7777,10.123.123.123:8888,10.123.123.123:9999", "") + + reg = GetDirectEnvRegistry(u2) + checkReg(reg, t, "10.111.111.111:7777", "") + + // test parse group + ClearDirectEnvRegistry() + os.Setenv(DirectRPCEnvironmentName, "helloService>change_group@127.0.0.1:8005;testService>temp-group@10.123.123.123:7777,10.123.123.123:8888,10.123.123.123:9999;") + reg = GetDirectEnvRegistry(u) + assert.Equal(t, 2, len(directRpc)) + checkReg(reg, t, "127.0.0.1:8005", "change_group") + + reg = GetDirectEnvRegistry(u1) + checkReg(reg, t, "10.123.123.123:7777,10.123.123.123:8888,10.123.123.123:9999", "temp-group") + + // test prefix match + ClearDirectEnvRegistry() + os.Setenv(DirectRPCEnvironmentName, "com.weibo.t*>newGroup@127.0.0.1:8005,127.0.0.1:8006") + reg = GetDirectEnvRegistry(u1) + assert.Equal(t, 1, len(directRpc)) + checkReg(reg, t, "127.0.0.1:8005,127.0.0.1:8006", "newGroup") + + reg = GetDirectEnvRegistry(u2) + checkReg(reg, t, "127.0.0.1:8005,127.0.0.1:8006", "newGroup") + + // test not match + reg = GetDirectEnvRegistry(u) + assert.Nil(t, reg) + + os.Unsetenv(DirectRPCEnvironmentName) + ClearDirectEnvRegistry() +} + +func checkReg(reg *URL, t *testing.T, addr string, group string) { + assert.NotNil(t, reg) + assert.Equal(t, "direct", reg.Protocol) + assert.Equal(t, addr, reg.GetParam(AddressKey, "")) + assert.Equal(t, group, reg.Group) +} diff --git a/protocol/motan1Protocol.go b/protocol/motan1Protocol.go index d2845569..136c525f 100644 --- a/protocol/motan1Protocol.go +++ b/protocol/motan1Protocol.go @@ -139,6 +139,7 @@ func DecodeMotanV1Response(msg *MotanV1Message) (motan.Response, error) { } else { response.Exception = &motan.Exception{ErrCode: 500, ErrMsg: "v1: has exception, class:" + objStream.cName, ErrType: motan.ServiceException} } + vlog.Warningf("v1 exception message: %s", objStream.errMsg) } ctx := response.GetRPCContext(true) ctx.OriginalMessage = msg @@ -422,6 +423,7 @@ type simpleObjectStream struct { processTime int64 hasException bool cName string + errMsg string value interface{} } @@ -525,6 +527,8 @@ func (s *simpleObjectStream) parseRes() error { case FLAG_RESPONSE_EXCEPTION: s.hasException = true s.cName, _ = s.readUtfFromBlock(s.sBlock) // parse exception class name + //It cannot be fully deserialized, so try to convert it into a string to provide more information + s.errMsg = string(s.bytes) case FLAG_RESPONSE: s.cName, err = s.readUtfFromBlock(s.sBlock) if err == nil && s.cName == "java.lang.String" { diff --git a/registry/directRegistry.go b/registry/directRegistry.go index b960eaf6..fcdc6876 100644 --- a/registry/directRegistry.go +++ b/registry/directRegistry.go @@ -37,10 +37,13 @@ func (d *DirectRegistry) Discover(url *motan.URL) []*motan.URL { } result := make([]*motan.URL, 0, len(d.urls)) for _, u := range d.urls { - newURL := *url + newURL := url.Copy() newURL.Host = u.Host newURL.Port = u.Port - result = append(result, &newURL) + if u.Group != "" { // specify group + newURL.Group = u.Group + } + result = append(result, newURL) } return result } @@ -63,11 +66,13 @@ func (d *DirectRegistry) StartSnapshot(conf *motan.SnapshotConf) {} func parseURLs(url *motan.URL) []*motan.URL { urls := make([]*motan.URL, 0) if len(url.Host) > 0 && url.Port > 0 { - urls = append(urls, url) + urls = append(urls, url.Copy()) } else if address, exist := url.Parameters[motan.AddressKey]; exist { for _, add := range strings.Split(address, ",") { + u := url.Copy() if strings.HasPrefix(add, "unix://") { - u := &motan.URL{Host: add, Port: 0} + u.Host = add + u.Port = 0 u.PutParam(motan.AddressKey, add) urls = append(urls, u) } else { @@ -75,7 +80,8 @@ func parseURLs(url *motan.URL) []*motan.URL { if len(hostport) == 2 { port, err := strconv.Atoi(hostport[1]) if err == nil { - u := &motan.URL{Host: hostport[0], Port: port} + u.Host = hostport[0] + u.Port = port urls = append(urls, u) } } diff --git a/registry/directRegistry_test.go b/registry/directRegistry_test.go index 5b93d81d..ce82b573 100644 --- a/registry/directRegistry_test.go +++ b/registry/directRegistry_test.go @@ -46,7 +46,7 @@ func TestDirectDiscover(t *testing.T) { func TestMultiAddress(t *testing.T) { params := make(map[string]string) - params["address"] = "127.0.0.1:8002;10.210.235.1:8003;127.0.0.1:8005" + params["address"] = "127.0.0.1:8002,10.210.235.1:8003,127.0.0.1:8005" regURL := &motan.URL{Parameters: params} registry := &DirectRegistry{url: regURL} urlParams := make(map[string]string) @@ -54,8 +54,10 @@ func TestMultiAddress(t *testing.T) { urlParams["group"] = "test" u1 := &motan.URL{Protocol: "motan", Host: "10.210.230.10", Port: 8999, Parameters: urlParams} urls := registry.Discover(u1) + if len(urls) != 3 { + t.Fatalf("discover multi address size not correct. size:%d", len(urls)) + } for i, u := range urls { - fmt.Printf("u: %v \n", u) if u.Host != registry.urls[i].Host || u.Port != registry.urls[i].Port { t.Fatalf("discover not correct. url: %+v", u) } @@ -66,3 +68,21 @@ func TestMultiAddress(t *testing.T) { } } } + +func TestSpecifyGroup(t *testing.T) { + params := make(map[string]string) + params["address"] = "127.0.0.1:8002,127.0.0.1:8003" + gName := "test-group" + registry := &DirectRegistry{} + registry.SetURL(&motan.URL{Group: gName, Parameters: params}) + su := &motan.URL{Protocol: "motan", Group: "another-group", Host: "10.210.230.10", Port: 8999} + urls := registry.Discover(su) + if len(urls) != 2 { + t.Fatalf("SpecifyGroup discover size not correct. size:%d", len(urls)) + } + for _, u := range urls { + if u.Group != gName { + t.Fatalf("SpecifyGroup check group fail. url:%+v", u) + } + } +} diff --git a/registry/zkRegistry.go b/registry/zkRegistry.go index 37ba5807..7fd239cf 100644 --- a/registry/zkRegistry.go +++ b/registry/zkRegistry.go @@ -8,7 +8,6 @@ import ( "time" "github.com/samuel/go-zookeeper/zk" - "github.com/weibocom/motan-go/cluster" motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/log" ) @@ -343,7 +342,7 @@ func (z *ZkRegistry) doSubscribeCommand(url *motan.URL) { if listeners, ok := z.subscribedCommandMap[commandPath]; ok && len(data) > 0 { cmdInfo := getNodeInfo(data) for lis := range listeners { - lis.NotifyCommand(url, cluster.ServiceCmd, cmdInfo) + lis.NotifyCommand(url, 1, cmdInfo) // service command vlog.Infof("[ZkRegistry] command changed, path:%s, cmdInfo:%s", commandPath, cmdInfo) } } From 16427704635869b02615bcc6414e88fc2e8eb510 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:34:38 +0800 Subject: [PATCH 56/75] concurrent init cluster and async init ep change (#309) 1. add access log discard ability in async mode. 2. log buffer size can be modified. 3. output discard access log count into metrics log every 5s. --- agent.go | 36 +++++++++++++++++++++++--- agent_test.go | 46 +++++++++++++++++++++++++++++++++ core/constants.go | 4 ++- endpoint/motanCommonEndpoint.go | 5 ++-- endpoint/motanEndpoint.go | 16 ++++++++++-- 5 files changed, 98 insertions(+), 9 deletions(-) diff --git a/agent.go b/agent.go index 3fabd135..a1d7b7f0 100644 --- a/agent.go +++ b/agent.go @@ -3,6 +3,7 @@ package motan import ( "flag" "fmt" + "github.com/weibocom/motan-go/endpoint" "io/ioutil" "net" "net/http" @@ -41,9 +42,10 @@ const ( ) var ( - initParamLock sync.Mutex - setAgentLock sync.Mutex - notFoundProviderCount int64 = 0 + initParamLock sync.Mutex + setAgentLock sync.Mutex + notFoundProviderCount int64 = 0 + defaultInitClusterTimeout int64 = 10000 //ms ) type Agent struct { @@ -366,7 +368,16 @@ func (a *Agent) initParam() { if err != nil { panic("Init runtime directory error: " + err.Error()) } - + asyncInit := true + if section != nil && section[motan.MotanEpAsyncInit] != nil { + if ai, ok := section[motan.MotanEpAsyncInit].(bool); ok { + asyncInit = ai + vlog.Infof("%s is set to %s", motan.MotanEpAsyncInit, strconv.FormatBool(ai)) + } else { + vlog.Warningf("illegal %s input, input should be bool", motan.MotanEpAsyncInit) + } + } + endpoint.SetMotanEPDefaultAsynInit(asyncInit) vlog.Infof("agent port:%d, manage port:%d, pidfile:%s, logdir:%s, runtimedir:%s", port, mPort, pidFile, logDir, runtimeDir) a.logdir = logDir a.port = port @@ -489,13 +500,30 @@ func (a *Agent) reloadClusters(ctx *motan.Context) { } func (a *Agent) initClusters() { + initTimeout := a.Context.AgentURL.GetIntValue(motan.InitClusterTimeoutKey, defaultInitClusterTimeout) + timer := time.NewTimer(time.Millisecond * time.Duration(initTimeout)) + wg := sync.WaitGroup{} + wg.Add(len(a.Context.RefersURLs)) for _, url := range a.Context.RefersURLs { // concurrently initialize cluster go func(u *motan.URL) { + defer wg.Done() defer motan.HandlePanic(nil) a.initCluster(u) }(url) } + finishChan := make(chan struct{}) + go func() { + wg.Wait() + finishChan <- struct{}{} + }() + select { + case <-timer.C: + vlog.Infof("agent init cluster timeout(%dms), do not wait(rest cluster keep doing initialization backend)", initTimeout) + case <-finishChan: + defer timer.Stop() + vlog.Infoln("agent cluster init complete") + } } func (a *Agent) initCluster(url *motan.URL) { diff --git a/agent_test.go b/agent_test.go index 39859c8b..5d0921b4 100644 --- a/agent_test.go +++ b/agent_test.go @@ -3,8 +3,10 @@ package motan import ( "bytes" "flag" + "fmt" _ "fmt" "github.com/weibocom/motan-go/config" + "github.com/weibocom/motan-go/endpoint" vlog "github.com/weibocom/motan-go/log" "github.com/weibocom/motan-go/serialize" _ "github.com/weibocom/motan-go/server" @@ -646,6 +648,7 @@ motan-agent: log_dir: "stdout" snapshot_dir: "./snapshot" application: "testing" + motanEpAsyncInit: false motan-registry: direct: @@ -674,3 +677,46 @@ motan-service: assert.Nil(t, resp.GetException()) assert.Equal(t, "Hello jack from motan server", string(reply)) } + +func Test_changeDefaultMotanEpAsyncInit(t *testing.T) { + template := ` +motan-agent: + port: 12821 + mport: 12503 + eport: 12281 + htport: 23282 + log_dir: "stdout" + motanEpAsyncInit: %s + snapshot_dir: "./snapshot" + application: "testing" + +motan-registry: + direct: + protocol: direct + +motan-service: + test01: + protocol: motan2 + provider: motan2 + group: hello + path: helloService + registry: direct + serialization: simple + proxy.host: unix://./server.sock + export: motan2:12281 +` + ext := GetDefaultExtFactory() + config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(fmt.Sprintf(template, "false")))) + a := NewAgent(ext) + a.Context = &core.Context{Config: config1} + a.Context.Initialize() + a.initParam() + assert.Equal(t, endpoint.GetDefaultMotanEPAsynInit(), false) + + config2, _ := config.NewConfigFromReader(bytes.NewReader([]byte(fmt.Sprintf(template, "true")))) + a = NewAgent(ext) + a.Context = &core.Context{Config: config2} + a.Context.Initialize() + a.initParam() + assert.Equal(t, endpoint.GetDefaultMotanEPAsynInit(), true) +} diff --git a/core/constants.go b/core/constants.go index be9a64ff..101a04fa 100644 --- a/core/constants.go +++ b/core/constants.go @@ -2,7 +2,7 @@ package core import "time" -//--------------all global public constants-------------- +// --------------all global public constants-------------- // exception type const ( FrameworkException = iota @@ -37,6 +37,7 @@ const ( DisableGlobalFilter = "disableGlobalFilter" DefaultFilter = "defaultFilter" DisableDefaultFilter = "disableDefaultFilter" + MotanEpAsyncInit = "motanEpAsyncInit" RegistryKey = "registry" WeightKey = "weight" SerializationKey = "serialization" @@ -52,6 +53,7 @@ const ( RemoteIPKey = "remoteIP" ProxyRegistryKey = "proxyRegistry" ProxyRegistryUrlString = "proxyRegistryUrlString" + InitClusterTimeoutKey = "initClusterTimeout" ConnectTimeoutKey = "connectTimeout" ConnectRetryIntervalKey = "connectRetryInterval" ClientConnectionKey = "clientConnection" diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index be4ee94c..a17b11f8 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -64,7 +64,7 @@ func (m *MotanCommonEndpoint) Initialize() { m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) m.lazyInit = m.url.GetBoolValue(motan.LazyInit, defaultLazyInit) - asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, defaultAsyncInitConnection) + asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, GetDefaultMotanEPAsynInit()) m.heartbeatVersion = -1 m.DefaultVersion = mpro.Version2 factory := func() (net.Conn, error) { @@ -556,7 +556,8 @@ func (s *Stream) Close() { } // Call send request to the server. -// about return: exception in response will record error count, err will not. +// +// about return: exception in response will record error count, err will not. func (c *Channel) Call(req motan.Request, deadline time.Duration, rc *motan.RPCContext) (motan.Response, error) { stream, err := c.NewStream(req, rc) if err != nil { diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 39a1f5d9..2a26e438 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -24,7 +24,7 @@ var ( defaultConnectRetryInterval = 60 * time.Second defaultErrorCountThreshold = 10 defaultLazyInit = false - defaultAsyncInitConnection = true + defaultAsyncInitConnection atomic.Value ErrChannelShutdown = fmt.Errorf("the channel has been shutdown") ErrSendRequestTimeout = fmt.Errorf("timeout err: send request timeout") ErrRecvRequestTimeout = fmt.Errorf("timeout err: receive request timeout") @@ -84,7 +84,7 @@ func (m *MotanEndpoint) Initialize() { m.clientConnection = int(m.url.GetPositiveIntValue(motan.ClientConnectionKey, int64(defaultChannelPoolSize))) m.maxContentLength = int(m.url.GetPositiveIntValue(motan.MaxContentLength, int64(mpro.DefaultMaxContentLength))) m.lazyInit = m.url.GetBoolValue(motan.LazyInit, defaultLazyInit) - asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, defaultAsyncInitConnection) + asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, GetDefaultMotanEPAsynInit()) factory := func() (net.Conn, error) { address := m.url.GetAddressStr() if strings.HasPrefix(address, motan.UnixSockProtocolFlag) { @@ -786,3 +786,15 @@ func buildV2Channel(conn net.Conn, config *ChannelConfig, serialization motan.Se return channel } + +func SetMotanEPDefaultAsynInit(ai bool) { + defaultAsyncInitConnection.Store(ai) +} + +func GetDefaultMotanEPAsynInit() bool { + res := defaultAsyncInitConnection.Load() + if res == nil { + return true + } + return res.(bool) +} From 514ce404222f5553c121a0dd1868545eb7950271 Mon Sep 17 00:00:00 2001 From: wuhua3 Date: Thu, 27 Apr 2023 18:08:58 +0800 Subject: [PATCH 57/75] support http_provider upstream_code --- provider/httpProvider.go | 1 + 1 file changed, 1 insertion(+) diff --git a/provider/httpProvider.go b/provider/httpProvider.go index eb12b045..73a16c28 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -483,5 +483,6 @@ func fillException(resp *motan.MotanResponse, start int64, err error) { func updateUpstreamStatusCode(resp *motan.MotanResponse, statusCode int) { resCtx := resp.GetRPCContext(true) + resp.SetAttachment(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) resCtx.Meta.Store(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) } From f8b84ba74b7f9d77d8efc839e072812288808e3e Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 12 May 2023 09:48:05 +0800 Subject: [PATCH 58/75] set remote address in response in proxy mode (#310) --- agent.go | 19 +++++++++++-------- core/constants.go | 2 ++ core/motan.go | 3 ++- endpoint/motanCommonEndpoint.go | 1 + endpoint/motanEndpoint.go | 1 + provider/httpProvider.go | 12 ++++++------ provider/httpxProvider.go | 4 ++-- provider/motanProvider.go | 4 ++-- provider/motanProvider_test.go | 10 +++++----- 9 files changed, 32 insertions(+), 24 deletions(-) diff --git a/agent.go b/agent.go index a1d7b7f0..f688ebeb 100644 --- a/agent.go +++ b/agent.go @@ -736,18 +736,21 @@ func fillDefaultReqInfo(r motan.Request, url *motan.URL) { } } -func (a *agentMessageHandler) Call(request motan.Request) motan.Response { +func (a *agentMessageHandler) Call(request motan.Request) (res motan.Response) { c, ck, err := a.findCluster(request) if err == nil { - return a.clusterCall(request, ck, c) + res = a.clusterCall(request, ck, c) + } else if httpCluster := a.agent.httpClusterMap.LoadOrNil(request.GetServiceName()); httpCluster != nil { + // if normal cluster not found we try http cluster, here service of request represent domain + res = a.httpCall(request, ck, httpCluster.(*cluster.HTTPCluster)) + } else { + vlog.Warningf("cluster not found. cluster: %s, request id:%d", err.Error(), request.GetRequestID()) + res = getDefaultResponse(request.GetRequestID(), "cluster not found. cluster: "+err.Error()) } - - // if normal cluster not found we try http cluster, here service of request represent domain - if httpCluster := a.agent.httpClusterMap.LoadOrNil(request.GetServiceName()); httpCluster != nil { - return a.httpCall(request, ck, httpCluster.(*cluster.HTTPCluster)) + if res.GetRPCContext(true).RemoteAddr != "" { // set response remote addr + res.SetAttachment(motan.XForwardedForLower, res.GetRPCContext(true).RemoteAddr) } - vlog.Warningf("cluster not found. cluster: %s, request id:%d", err.Error(), request.GetRequestID()) - return getDefaultResponse(request.GetRequestID(), "cluster not found. cluster: "+err.Error()) + return res } func (a *agentMessageHandler) matchRule(typ, cond, key string, data []serviceMapItem, f func(u *motan.URL) string) (foundClusters []serviceMapItem, err error) { if cond == "" { diff --git a/core/constants.go b/core/constants.go index 101a04fa..aa16fdd0 100644 --- a/core/constants.go +++ b/core/constants.go @@ -68,6 +68,8 @@ const ( MixGroups = "mixGroups" MaxContentLength = "maxContentLength" UnixSockProtocolFlag = "unix://" + XForwardedForLower = "x-forwarded-for" // used as motan default proxy key + XForwardedFor = "X-Forwarded-For" ) // nodeType diff --git a/core/motan.go b/core/motan.go index 9567e3fe..d52b5de7 100644 --- a/core/motan.go +++ b/core/motan.go @@ -375,7 +375,8 @@ type RPCContext struct { Tc *TraceContext // ---- internal vars ---- - IsMotanV1 bool + IsMotanV1 bool + RemoteAddr string // remote address } func (c *RPCContext) AddFinishHandler(handler FinishHandler) { diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index a17b11f8..71b063a2 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -170,6 +170,7 @@ func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { return m.defaultErrMotanResponse(request, err.Error()) } } + response.GetRPCContext(true).RemoteAddr = channel.address return response } diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 2a26e438..e2e3d42f 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -205,6 +205,7 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { return m.defaultErrMotanResponse(request, err.Error()) } } + response.GetRPCContext(true).RemoteAddr = channel.address return response } diff --git a/provider/httpProvider.go b/provider/httpProvider.go index 73a16c28..7f45f8f6 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -275,8 +275,8 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { return true }) httpReq.Header.Del("Connection") - if httpReq.Header.Peek("X-Forwarded-For") == nil { - httpReq.Header.Set("X-Forwarded-For", ip) + if httpReq.Header.Peek(motan.XForwardedFor) == nil { + httpReq.Header.Set(motan.XForwardedFor, ip) } if len(bodyBytes) != 0 { httpReq.BodyWriter().Write(bodyBytes) @@ -333,8 +333,8 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { if len(httpReq.Header.Host()) == 0 { httpReq.Header.SetHost(h.domain) } - if httpReq.Header.Peek("X-Forwarded-For") == nil { - httpReq.Header.Set("X-Forwarded-For", ip) + if httpReq.Header.Peek(motan.XForwardedFor) == nil { + httpReq.Header.Set(motan.XForwardedFor, ip) } err = h.fastClient.Do(httpReq, httpRes) if err != nil { @@ -379,8 +379,8 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { req.Header.Add(k, v) return true }) - if req.Header.Get("x-forwarded-for") == "" { - req.Header.Add("x-forwarded-for", ip) + if req.Header.Get(motan.XForwardedFor) == "" { + req.Header.Add(motan.XForwardedFor, ip) } timeout := h.url.GetTimeDuration(motan.TimeOutKey, time.Millisecond, DefaultRequestTimeout) diff --git a/provider/httpxProvider.go b/provider/httpxProvider.go index adf29e88..dee3a26d 100644 --- a/provider/httpxProvider.go +++ b/provider/httpxProvider.go @@ -174,8 +174,8 @@ func (h *HTTPXProvider) Call(request motan.Request) motan.Response { } else { ip = request.GetAttachment(motan.HostKey) } - if req.Header.Peek("x-forwarded-for") == nil { - req.Header.Add("x-forwarded-for", ip) + if req.Header.Peek(motan.XForwardedFor) == nil { + req.Header.Add(motan.XForwardedFor, ip) } err = h.httpClient.Do(req, httpResp) if err != nil { diff --git a/provider/motanProvider.go b/provider/motanProvider.go index 01d5e0c7..c885d216 100644 --- a/provider/motanProvider.go +++ b/provider/motanProvider.go @@ -59,8 +59,8 @@ func (m *MotanProvider) Initialize() { func (m *MotanProvider) Call(request motan.Request) motan.Response { if m.IsAvailable() { // x-forwared-for - if request.GetAttachment("x-forwarded-for") == "" && request.GetAttachment("X-Forwarded-For") == "" { - request.SetAttachment("x-forwarded-for", request.GetAttachment(motan.HostKey)) + if request.GetAttachment(motan.XForwardedForLower) == "" && request.GetAttachment(motan.XForwardedFor) == "" { + request.SetAttachment(motan.XForwardedForLower, request.GetAttachment(motan.HostKey)) } return m.ep.Call(request) } diff --git a/provider/motanProvider_test.go b/provider/motanProvider_test.go index 38e74761..2a086518 100644 --- a/provider/motanProvider_test.go +++ b/provider/motanProvider_test.go @@ -58,20 +58,20 @@ func TestXForwardedFor(t *testing.T) { mContext.ConfigFile = confFilePath mContext.Initialize() request := &motan.MotanRequest{} - request.SetAttachment("x-forwarded-for", "test") + request.SetAttachment(motan.XForwardedForLower, "test") url := mContext.ServiceURLs[serviceName] //call correct providerCorr := MotanProvider{url: url, extFactory: factory} providerCorr.Initialize() providerCorr.Call(request) - assert.Equal(t, request.GetAttachment("x-forwarded-for"), "test") + assert.Equal(t, request.GetAttachment(motan.XForwardedForLower), "test") request = &motan.MotanRequest{} - request.SetAttachment("X-Forwarded-For", "test") + request.SetAttachment(motan.XForwardedFor, "test") providerCorr.Call(request) - assert.Equal(t, request.GetAttachment("x-forwarded-for"), "") + assert.Equal(t, request.GetAttachment(motan.XForwardedForLower), "") request = &motan.MotanRequest{} request.SetAttachment("x-Forwarded-For", "test") providerCorr.Call(request) - assert.NotEqual(t, request.GetAttachment("x-forwarded-for"), "test") + assert.NotEqual(t, request.GetAttachment(motan.XForwardedForLower), "test") } From bb7b7c8f3540a78aa7211a959c2f8a11b69a7169 Mon Sep 17 00:00:00 2001 From: snail007 Date: Thu, 1 Jun 2023 14:17:30 +0800 Subject: [PATCH 59/75] fix pb panic (#313) --- serialize/pb.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/serialize/pb.go b/serialize/pb.go index 24a2696c..2ae604ff 100644 --- a/serialize/pb.go +++ b/serialize/pb.go @@ -147,7 +147,11 @@ func (p *PbSerialization) serializeBuf(buf *proto.Buffer, v interface{}) (err er for i := 0; i < rv.Len(); i++ { iv := rv.Index(i) if _, ok := iv.Interface().(proto.Message); !ok { - return errors.New("not support other types in pb-array in PbSerialization. type: " + iv.Elem().Kind().String()) + kind := iv.Type().Kind() + if iv.Type().Kind() == reflect.Ptr { + kind = iv.Elem().Kind() + } + return errors.New("not support other types in pb-array in PbSerialization. type: " + kind.String()) } if err = p.serializeBuf(buf, iv.Elem().Interface()); err != nil { return err From 2e1a4c3237dbac92eac2fb90c7c50bff2c33e64d Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 6 Jun 2023 12:23:55 +0800 Subject: [PATCH 60/75] fix endpoint can not recordErrAndKeepalive correctly --- endpoint/motanCommonEndpoint.go | 2 +- endpoint/motanEndpoint.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 71b063a2..220587a3 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -152,7 +152,7 @@ func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { return defaultAsyncResponse } excep := response.GetException() - if excep != nil && excep.ErrCode == 503 { + if excep != nil && excep.ErrType != motan.BizException { m.recordErrAndKeepalive() } else { // reset errorCount diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index e2e3d42f..44c0ee50 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -194,7 +194,7 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "convert response fail!" + err.Error(), ErrType: motan.ServiceException}) } excep := response.GetException() - if excep != nil && excep.ErrCode == 503 { + if excep != nil && excep.ErrType != motan.BizException { m.recordErrAndKeepalive() } else { // reset errorCount From e37e10f049a0b1682301e898c0c4f6689a204e6e Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 6 Jun 2023 12:32:04 +0800 Subject: [PATCH 61/75] update ci --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e48ca001..dd5251b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Go if: success() @@ -52,7 +52,7 @@ jobs: id: go - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Generate coverage report run: | From f40934852576e2f5b30cf842dce0f2acc79ef3d1 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Tue, 6 Jun 2023 12:38:22 +0800 Subject: [PATCH 62/75] update ci --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dd5251b3..df431f1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: - name: Install Go if: success() - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ matrix.go-version }} @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.15 - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.15.x id: go From 6b1745671e6a52c0be16ca10ce8347e547e7defc Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Thu, 6 Jul 2023 15:20:43 +0800 Subject: [PATCH 63/75] update: (#314) after notify, offline endpoints will be destroyed asynchronously after 2s delay --- cluster/motanCluster.go | 10 +++++++--- cluster/motanCluster_test.go | 25 +++++++++++++++++++++++-- core/test.go | 16 ++++++++++++++-- ha/backupRequestHA_test.go | 4 +++- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/cluster/motanCluster.go b/cluster/motanCluster.go index 3819941a..b0180ac4 100644 --- a/cluster/motanCluster.go +++ b/cluster/motanCluster.go @@ -191,9 +191,13 @@ func (m *MotanCluster) Notify(registryURL *motan.URL, urls []*motan.URL) { m.registryRefers[registryURL.GetIdentity()] = endpoints } m.refresh() - for _, ep := range endpointMap { - ep.Destroy() - } + go func() { + defer motan.HandlePanic(nil) + time.Sleep(time.Second * 2) + for _, ep := range endpointMap { + ep.Destroy() + } + }() } // remove rule protocol && set weight diff --git a/cluster/motanCluster_test.go b/cluster/motanCluster_test.go index 9fa3d18b..20702af1 100644 --- a/cluster/motanCluster_test.go +++ b/cluster/motanCluster_test.go @@ -6,6 +6,7 @@ import ( "github.com/weibocom/motan-go/registry" "os" "testing" + "time" motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/ha" @@ -72,7 +73,27 @@ func TestNotify(t *testing.T) { if len(cluster.Refers) == 0 { t.Fatalf("cluster notify-refers size not correct. expect :2, refers size:%d", len(cluster.Refers)) } - + urls = append(urls, &motan.URL{Host: "127.0.0.1", Port: 8001, Protocol: "test"}) + var destroyEndpoint motan.EndPoint + for _, j := range cluster.Refers { + if j.GetURL().Port == 8002 { + destroyEndpoint = j + } + } + if destroyEndpoint == nil { + t.Fatalf("cluster endpoint is nil") + } + if !destroyEndpoint.IsAvailable() { + t.Fatalf("cluster endpoint should be not available") + } + cluster.Notify(RegistryURL, urls) + time.Sleep(time.Second * 3) + if len(cluster.Refers) != 1 { + t.Fatalf("cluster notify-refers size not correct. expect :2, refers size:%d", len(cluster.Refers)) + } + if destroyEndpoint.IsAvailable() { + t.Fatalf("cluster endpoint should not be available") + } } func TestCall(t *testing.T) { @@ -123,7 +144,7 @@ func TestParseRegistryFromEnv(t *testing.T) { } } -//-------------test struct-------------------- +// -------------test struct-------------------- func getCustomExt() motan.ExtensionFactory { ext := &motan.DefaultExtensionFactory{} ext.Initialize() diff --git a/core/test.go b/core/test.go index f1e2a6e7..af3d857d 100644 --- a/core/test.go +++ b/core/test.go @@ -3,6 +3,7 @@ package core import ( "errors" "fmt" + "sync/atomic" "time" ) @@ -116,6 +117,7 @@ func (t *TestProvider) GetPath() string { type TestEndPoint struct { URL *URL ProcessTime int64 + available atomic.Value } func (t *TestEndPoint) GetURL() *URL { @@ -137,10 +139,20 @@ func (t *TestEndPoint) Call(request Request) Response { } func (t *TestEndPoint) IsAvailable() bool { - return true + return t.available.Load().(bool) +} + +func (t *TestEndPoint) Initialize() { + t.SetAvailable(true) } -func (t *TestEndPoint) Destroy() {} +func (t *TestEndPoint) Destroy() { + t.SetAvailable(false) +} + +func (t *TestEndPoint) SetAvailable(a bool) { + t.available.Store(a) +} func (t *TestEndPoint) SetProxy(proxy bool) {} diff --git a/ha/backupRequestHA_test.go b/ha/backupRequestHA_test.go index 1a774be7..e7563fbe 100644 --- a/ha/backupRequestHA_test.go +++ b/ha/backupRequestHA_test.go @@ -121,7 +121,9 @@ func TestBackupRequestHA_Call3(t *testing.T) { } func getEP(processTime int64) motan.EndPoint { - fep := &motan.FilterEndPoint{Caller: &motan.TestEndPoint{ProcessTime: processTime}} + caller := &motan.TestEndPoint{ProcessTime: processTime} + motan.Initialize(caller) + fep := &motan.FilterEndPoint{Caller: caller} mf := &filter.MetricsFilter{} mf.SetContext(&motan.Context{Config: config.NewConfig()}) mf.SetNext(motan.GetLastEndPointFilter()) From 6bd0adc36029b04a807a1eebaaa45c8ca5370f4e Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:37:03 +0800 Subject: [PATCH 64/75] Dev (#315) * update: http provider add new feature: response with wrong status code(4xx,5xx) will return motan exception if enableHttpException configuration is set. --- http/httpProxy.go | 8 +++--- provider/httpProvider.go | 48 ++++++++++++++++++++++++++------- provider/httpProvider_test.go | 50 +++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/http/httpProxy.go b/http/httpProxy.go index 779ace69..dc048722 100644 --- a/http/httpProxy.go +++ b/http/httpProxy.go @@ -33,6 +33,7 @@ const ( ProxySchemaKey = "proxySchema" MaxConnectionsKey = "maxConnections" EnableRewriteKey = "enableRewrite" + EnableHttpExceptionKey = "enableHttpException" ) const ( @@ -415,9 +416,10 @@ func (m *LocationMatcher) NeedURLQueryString() bool { // We use meta element HTTP_Method as http method, HTTP_QueryString as query string // Request method as request uri // Body will transform to a http body with following rules: -// if body is a map[string]string we transform it as a form data -// if body is a string or []byte just use it -// else is unsupported +// +// if body is a map[string]string we transform it as a form data +// if body is a string or []byte just use it +// else is unsupported func MotanRequestToFasthttpRequest(motanRequest core.Request, fasthttpRequest *fasthttp.Request, defaultHTTPMethod string) error { httpMethod := motanRequest.GetAttachment(Method) if httpMethod == "" { diff --git a/provider/httpProvider.go b/provider/httpProvider.go index 7f45f8f6..f19cfe69 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -32,14 +32,15 @@ type HTTPProvider struct { gctx *motan.Context mixVars []string // for transparent http proxy - fastClient *fasthttp.HostClient - proxyAddr string - proxySchema string - locationMatcher *mhttp.LocationMatcher - maxConnections int - domain string - defaultHTTPMethod string - enableRewrite bool + fastClient *fasthttp.HostClient + proxyAddr string + proxySchema string + locationMatcher *mhttp.LocationMatcher + maxConnections int + domain string + defaultHTTPMethod string + enableRewrite bool + enableHttpException bool } const ( @@ -95,6 +96,13 @@ func (h *HTTPProvider) Initialize() { } else { h.enableRewrite = enableRewrite } + h.enableHttpException = false + enableHttpExceptionStr := h.url.GetParam(mhttp.EnableHttpExceptionKey, "false") + if enableHttpException, err := strconv.ParseBool(enableHttpExceptionStr); err != nil { + vlog.Errorf("%s should be a bool value, but got: %s", mhttp.EnableHttpExceptionKey, enableHttpExceptionStr) + } else { + h.enableHttpException = enableHttpException + } h.fastClient = &fasthttp.HostClient{ Name: "motan", Addr: h.proxyAddr, @@ -286,6 +294,12 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { fillExceptionWithCode(resp, http.StatusServiceUnavailable, t, err) return resp } + if h.enableHttpException { + if httpRes.StatusCode() >= 400 { + fillHttpException(resp, httpRes.StatusCode(), t, httpRes.Body()) + return resp + } + } headerBuffer := &bytes.Buffer{} httpRes.Header.Del("Connection") httpRes.Header.WriteTo(headerBuffer) @@ -341,6 +355,12 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { fillExceptionWithCode(resp, http.StatusServiceUnavailable, t, err) return resp } + if h.enableHttpException { + if httpRes.StatusCode() >= 400 { + fillHttpException(resp, httpRes.StatusCode(), t, httpRes.Body()) + return resp + } + } mhttp.FasthttpResponseToMotanResponse(resp, httpRes) resp.ProcessTime = (time.Now().UnixNano() - t) / 1e6 updateUpstreamStatusCode(resp, httpRes.StatusCode()) @@ -407,7 +427,6 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { defer httpResp.Body.Close() headers := httpResp.Header statusCode := httpResp.StatusCode - body, err := ioutil.ReadAll(httpResp.Body) l := len(body) if l == 0 { @@ -420,6 +439,12 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { ErrMsg: fmt.Sprintf("%s", err), ErrType: http.StatusServiceUnavailable} return resp } + if h.enableHttpException { + if statusCode >= 400 { + fillHttpException(resp, statusCode, t, body) + return resp + } + } request.GetAttachments().Range(func(k, v string) bool { resp.SetAttachment(k, v) return true @@ -477,6 +502,11 @@ func fillExceptionWithCode(resp *motan.MotanResponse, code int, start int64, err resp.Exception = &motan.Exception{ErrCode: code, ErrMsg: fmt.Sprintf("%s", err), ErrType: code} } +func fillHttpException(resp *motan.MotanResponse, statusCode int, start int64, body []byte) { + resp.ProcessTime = int64((time.Now().UnixNano() - start) / 1e6) + resp.Exception = &motan.Exception{ErrCode: statusCode, ErrMsg: string(body), ErrType: motan.BizException} +} + func fillException(resp *motan.MotanResponse, start int64, err error) { fillExceptionWithCode(resp, http.StatusServiceUnavailable, start, err) } diff --git a/provider/httpProvider_test.go b/provider/httpProvider_test.go index 1e10d955..cf587864 100644 --- a/provider/httpProvider_test.go +++ b/provider/httpProvider_test.go @@ -70,6 +70,45 @@ func TestHTTPProvider_Call(t *testing.T) { assert.Equal(t, "/2/p1/test?a=b", string(provider.Call(req).GetValue().([]interface{})[1].([]byte))) } +func TestHTTPProvider_Http_Exception(t *testing.T) { + context := &core.Context{} + context.Config, _ = config.NewConfigFromReader(bytes.NewReader([]byte(httpProviderTestData))) + providerURL := &core.URL{Protocol: "http", Path: "test4"} + providerURL.PutParam(mhttp.DomainKey, "test.domain") + providerURL.PutParam("requestTimeout", "2000") + providerURL.PutParam("proxyAddress", "localhost:8091") + provider := &HTTPProvider{url: providerURL, gctx: context} + provider.Initialize() + req := &core.MotanRequest{} + req.ServiceName = "test4" + req.Method = "/p1/test" + req.SetAttachment("Host", "test.domain") + req.SetAttachment(mhttp.QueryString, "a=b") + assert.Nil(t, provider.Call(req).GetException()) +} + +func TestHTTPProvider_Http_EnableException(t *testing.T) { + context := &core.Context{} + context.Config, _ = config.NewConfigFromReader(bytes.NewReader([]byte(httpProviderTestData))) + providerURL := &core.URL{Protocol: "http", Path: "test4"} + providerURL.PutParam(mhttp.DomainKey, "test.domain") + providerURL.PutParam("requestTimeout", "2000") + providerURL.PutParam("proxyAddress", "localhost:8091") + providerURL.PutParam(mhttp.EnableHttpExceptionKey, "true") + provider := &HTTPProvider{url: providerURL, gctx: context} + provider.Initialize() + req := &core.MotanRequest{} + req.ServiceName = "test4" + req.Method = "/p1/test" + req.SetAttachment("Host", "test.domain") + req.SetAttachment(mhttp.QueryString, "a=b") + exception := provider.Call(req).GetException() + assert.NotNil(t, exception) + assert.Equal(t, exception.ErrCode, 500) + assert.Equal(t, exception.ErrType, core.BizException) + assert.Equal(t, exception.ErrMsg, "request failed") +} + func TestMain(m *testing.M) { go func() { var addr = ":8090" @@ -80,6 +119,17 @@ func TestMain(m *testing.M) { }) http.ListenAndServe(addr, handler) }() + go func() { + // 返回500的server + var addr = ":8091" + handler := &http.ServeMux{} + handler.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { + request.ParseForm() + writer.WriteHeader(500) + writer.Write([]byte("request failed")) + }) + http.ListenAndServe(addr, handler) + }() time.Sleep(time.Second * 3) os.Exit(m.Run()) } From baa661ad0a7477b86a1e09fc488f8aa4b9eb8c1e Mon Sep 17 00:00:00 2001 From: arraykeys Date: Mon, 16 Oct 2023 15:11:12 +0800 Subject: [PATCH 65/75] access log add column upstream code --- filter/accessLog.go | 5 ++++- log/log.go | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/filter/accessLog.go b/filter/accessLog.go index 9e73a79c..de750d72 100644 --- a/filter/accessLog.go +++ b/filter/accessLog.go @@ -80,6 +80,7 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 resCtx := response.GetRPCContext(true) // response code should be same as upstream responseCode := "" + metaUpstreamCode := resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) if resCtx.Meta != nil { responseCode = resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) } @@ -107,5 +108,7 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 TotalTime: totalTime, //ms ResponseCode: responseCode, Success: exception == nil, - Exception: string(exceptionData)}) + Exception: string(exceptionData), + UpstreamCode: metaUpstreamCode, + }) } diff --git a/log/log.go b/log/log.go index 594bb0b7..d1d484bb 100644 --- a/log/log.go +++ b/log/log.go @@ -54,6 +54,7 @@ type AccessLogEntity struct { Success bool `json:"success"` ResponseCode string `json:"responseCode"` Exception string `json:"exception"` + UpstreamCode string `json:"upstream_code"` } type Logger interface { @@ -386,7 +387,8 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { zap.Int64("totalTime", logObject.TotalTime), zap.Bool("success", logObject.Success), zap.String("responseCode", logObject.ResponseCode), - zap.String("exception", logObject.Exception)) + zap.String("exception", logObject.Exception), + zap.String("upstreamCode", logObject.UpstreamCode)) } else { var buffer bytes.Buffer buffer.WriteString(logObject.FilterName) @@ -416,6 +418,8 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { buffer.WriteString(logObject.ResponseCode) buffer.WriteString("|") buffer.WriteString(logObject.Exception) + buffer.WriteString("|") + buffer.WriteString(logObject.UpstreamCode) d.accessLogger.Info(buffer.String()) } } From f4d38e3aaa624f8981ccc8cf5390da452b561c57 Mon Sep 17 00:00:00 2001 From: arraykeys Date: Mon, 16 Oct 2023 15:23:07 +0800 Subject: [PATCH 66/75] access log add column upstream code --- filter/accessLog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter/accessLog.go b/filter/accessLog.go index de750d72..06c1e6e7 100644 --- a/filter/accessLog.go +++ b/filter/accessLog.go @@ -80,7 +80,7 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 resCtx := response.GetRPCContext(true) // response code should be same as upstream responseCode := "" - metaUpstreamCode := resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) + metaUpstreamCode, _ := response.GetAttachments().Load(motan.MetaUpstreamCode) if resCtx.Meta != nil { responseCode = resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) } From 9f8bdaec16af135b8fa15c17507e23c45ddb5bfc Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 10 Nov 2023 11:29:03 +0800 Subject: [PATCH 67/75] add consistentHashKey load balance (#317) add consistentHashKey load balance (#317) --- core/constants.go | 9 +- go.mod | 2 + lb/consistentHashKeyLb.go | 183 +++++++++++++++++++++++++++ lb/consistentHashKeyLb_test.go | 220 +++++++++++++++++++++++++++++++++ lb/lb.go | 8 +- 5 files changed, 418 insertions(+), 4 deletions(-) create mode 100644 lb/consistentHashKeyLb.go create mode 100644 lb/consistentHashKeyLb_test.go diff --git a/core/constants.go b/core/constants.go index aa16fdd0..2876ea27 100644 --- a/core/constants.go +++ b/core/constants.go @@ -68,8 +68,13 @@ const ( MixGroups = "mixGroups" MaxContentLength = "maxContentLength" UnixSockProtocolFlag = "unix://" - XForwardedForLower = "x-forwarded-for" // used as motan default proxy key - XForwardedFor = "X-Forwarded-For" +) + +// attachment keys +const ( + XForwardedForLower = "x-forwarded-for" // used as motan default proxy key + XForwardedFor = "X-Forwarded-For" + ConsistentHashKey = "consistentHashKey" //string used to calculate consistent hash ) // nodeType diff --git a/go.mod b/go.mod index a83fbb4b..989f8b55 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.11 require ( github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 github.com/beberlei/fastcgi-serve v0.0.0-20151230120321-4676005f65b7 + github.com/buraksezer/consistent v0.10.0 + github.com/cespare/xxhash/v2 v2.2.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.3.2 github.com/juju/ratelimit v1.0.1 diff --git a/lb/consistentHashKeyLb.go b/lb/consistentHashKeyLb.go new file mode 100644 index 00000000..28767561 --- /dev/null +++ b/lb/consistentHashKeyLb.go @@ -0,0 +1,183 @@ +package lb + +import ( + "errors" + "github.com/buraksezer/consistent" + "github.com/cespare/xxhash/v2" + motan "github.com/weibocom/motan-go/core" + vlog "github.com/weibocom/motan-go/log" + "math/rand" + "strconv" +) + +const ( + // config keys + PartSizeKey = "consistentHashKey.partSize" + LoadKey = "consistentHashKey.load" + ReplicaKey = "consistentHashKey.replica" + + // default values + DefaultPartSizeAddend = 271 + DefaultReplica = 10 + DefaultMinLoad = 1.1 +) + +var BuildConsistentHashFailError = errors.New("build consistent hash fail") + +type ConsistentHashLB struct { + url *motan.URL + endpoints []motan.EndPoint + cHash *consistent.Consistent + lastPartSize int + load float64 + replica int +} + +type member struct { + Key string // Key needs to be generated at build time + Endpoint motan.EndPoint +} + +func (m *member) String() string { + return m.Key +} + +type hasher struct{} + +func (h hasher) Sum64(data []byte) uint64 { + return xxhash.Sum64(data) +} + +func (c *ConsistentHashLB) OnRefresh(endpoints []motan.EndPoint) { + if len(endpoints) == 1 { + c.endpoints = endpoints + c.cHash = nil + return + } + ok := c.buildConsistent(endpoints) + if !ok { + vlog.Errorf("ConsistentHashLB OnRefresh failed, endpoints not update. endpoints size:%d\n", len(endpoints)) + return + } + c.endpoints = endpoints +} + +func (c *ConsistentHashLB) Select(request motan.Request) motan.EndPoint { + if len(c.endpoints) == 1 { + return c.endpoints[0] + } + key := request.GetAttachment(motan.ConsistentHashKey) + var endpoint motan.EndPoint + if key != "" { // Use consistent hashing when hash key is not empty + endpoint = c.cHash.LocateKey([]byte(key)).(*member).Endpoint + } + if endpoint == nil || !endpoint.IsAvailable() { // When the hash key is empty or the hash endpoint is unavailable, the endpoint is randomly selected. + _, endpoint = SelectOneAtRandom(c.endpoints) + } + return endpoint +} + +func (c *ConsistentHashLB) SelectArray(request motan.Request) []motan.EndPoint { + if len(c.endpoints) > MaxSelectArraySize { + key := request.GetAttachment(motan.ConsistentHashKey) + if key != "" { + members, err := c.cHash.GetClosestN([]byte(key), MaxSelectArraySize) + if err != nil { + vlog.Warningf("ConsistentHashLB SelectArray failed, key:%s, err:%v\n", key, err) + } else { + endpoints := make([]motan.EndPoint, 0, len(members)) + for _, m := range members { + if m.(*member).Endpoint.IsAvailable() { + endpoints = append(endpoints, m.(*member).Endpoint) + } + } + return endpoints + } + } + } + return SelectArrayFromIndex(c.endpoints, rand.Intn(len(c.endpoints))) +} + +func (c *ConsistentHashLB) SetWeight(weight string) { +} + +func (c *ConsistentHashLB) buildConsistent(endpoints []motan.EndPoint) bool { + //Calculate partSize + partSize := 0 + if c.lastPartSize == 0 { // first build + partSize = int(c.url.GetIntValue(PartSizeKey, 0)) + } else { // Try not to change the size when building again + partSize = c.lastPartSize + } + if partSize < len(endpoints)*2 { // Recalculate size when conditions are not met + partSize = len(endpoints)*5 + DefaultPartSizeAddend + } + + // Calculate load on first build + if c.load == 0 { + load, err := strconv.ParseFloat(c.url.GetParam(LoadKey, "0"), 64) + if err != nil || load < DefaultMinLoad { + load = DefaultMinLoad + } + c.load = load + } + + // Calculate replica on first build + if c.replica == 0 { + replica := int(c.url.GetIntValue(ReplicaKey, 0)) + if replica <= 0 { + if len(endpoints) < 100 { + replica = DefaultReplica * 2 + } else { + replica = DefaultReplica + } + } + c.replica = replica + } + // Build member list + members := make([]consistent.Member, len(endpoints)) + for i := 0; i < len(endpoints); i++ { + members[i] = &member{Key: endpoints[i].GetURL().GetAddressStr(), Endpoint: endpoints[i]} + } + var cHash *consistent.Consistent + var err error + count := 0 + load := c.load + for { + cHash, err = c.buildConsistent0(members, partSize, c.replica, load) + if err == nil { + break + } + // Increase load when build failed + load += 0.1 + count++ + if count > 20 { // Try 20 times at most, if it still fails, give up + vlog.Errorf("build consistent hash ring failed after maximum retries. part size:%d, load:%.2f, replica:%d, member:%d\n", partSize, load, c.replica, len(endpoints)) + return false + } + } + c.cHash = cHash + c.lastPartSize = partSize + if load != c.load { + c.load = load + } + vlog.Infof("build consistent hash ring, part size:%d, load:%.2f, replica:%d, member:%d", c.lastPartSize, c.load, c.replica, len(endpoints)) + return true +} + +func (c *ConsistentHashLB) buildConsistent0(members []consistent.Member, partSize int, replica int, load float64) (cHash *consistent.Consistent, err error) { + defer func() { + if r := recover(); r != nil { + vlog.Warningf("build consistent hash ring failed with part size:%d, load:%.2f, replica:%d, member:%d, err:%v\n", partSize, load, replica, len(members), r) + err = BuildConsistentHashFailError + } + }() + cfg := consistent.Config{ + PartitionCount: partSize, + ReplicationFactor: replica, + Load: load, + Hasher: hasher{}, + } + cHash = consistent.New(members, cfg) + return cHash, nil +} diff --git a/lb/consistentHashKeyLb_test.go b/lb/consistentHashKeyLb_test.go new file mode 100644 index 00000000..11737af9 --- /dev/null +++ b/lb/consistentHashKeyLb_test.go @@ -0,0 +1,220 @@ +package lb + +import ( + "fmt" + "github.com/stretchr/testify/assert" + motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/endpoint" + "testing" +) + +func TestConsistentHashLB_OnRefresh(t *testing.T) { + lb := &ConsistentHashLB{} + + // first refresh + resetLB(lb) + checkDefault(t, lb, 10) + + // refresh multiple times + sizeArray := []int{2, 8, 12, 51, 100, 123, 500, 1000} + for _, s := range sizeArray { + if s >= 500 { + lb.replica = 10 // To reduce testing costs + } + checkDefault(t, lb, s) + } + + // specified part size, loadFactor, replica + resetLB(lb) + lb.url.PutParam(PartSizeKey, "100") + lb.url.PutParam(LoadKey, "1.35") + lb.url.PutParam(ReplicaKey, "5") + checkDefault(t, lb, 50) + assert.Equal(t, 100, lb.lastPartSize) + assert.Equal(t, 1.35, lb.load) + assert.Equal(t, 5, lb.replica) + + // inappropriate value specified + resetLB(lb) + lb.url.PutParam(PartSizeKey, "70") // < 2 * len(eps) + lb.url.PutParam(LoadKey, "1.0") // < 1.1 + lb.url.PutParam(ReplicaKey, "-5") // < 0 + checkDefault(t, lb, 50) + assert.NotEqual(t, 70, lb.lastPartSize) + assert.NotEqual(t, 1.0, lb.load) + assert.NotEqual(t, -5, lb.replica) + + // change part size + resetLB(lb) + lb.url.PutParam(PartSizeKey, "100") + checkDefault(t, lb, 30) + assert.Equal(t, 100, lb.lastPartSize) + checkDefault(t, lb, 55) + assert.NotEqual(t, 100, lb.lastPartSize) + + // change loadFactor + resetLB(lb) + lb.load = 1.0 + checkDefault(t, lb, 30) +} + +func TestConsistentHashLB_Select(t *testing.T) { + lb := &ConsistentHashLB{} + resetLB(lb) + req := &motan.MotanRequest{Method: "test"} + + // one endpoint + lb.OnRefresh(buildEndpoint(1)) + checkHashEP(t, lb, req, lb.endpoints[0]) + + // use hash key + req.SetAttachment(motan.ConsistentHashKey, "234234") + checkHashEP(t, lb, req, lb.endpoints[0]) + + // repeat select + lb.OnRefresh(buildEndpoint(30)) + keys := []string{"234", "1", "sjide", "Y&^(U23j49", "skd9f0i9*(RK3erp3oi29kf"} + for _, k := range keys { + checkConsistent(t, lb, k, 20) + } + + // not use hash key(random) + req.SetAttachment(motan.ConsistentHashKey, "") + ep := lb.Select(req) + ep2 := lb.Select(req) + assert.NotEqual(t, ep, ep2) + + // change part size + resetLB(lb) + lb.url.PutParam(PartSizeKey, "100") + lb.OnRefresh(buildEndpoint(49)) + key := "sjide123" + checkConsistent(t, lb, key, 20) + req.SetAttachment(motan.ConsistentHashKey, key) + ep = lb.Select(req) + lb.OnRefresh(buildEndpoint(51)) + checkConsistent(t, lb, key, 20) + ep2 = lb.Select(req) + assert.NotEqual(t, ep, ep2) + + // endpoint unavailable + resetLB(lb) + lb.OnRefresh(buildEndpoint(30)) + ep = lb.Select(req) + ep.(*lbTestMockEndpoint).isAvail = false + ep2 = lb.Select(req) + assert.NotEqual(t, ep, ep2) +} + +func TestConsistentHashLB_SelectArray(t *testing.T) { + lb := &ConsistentHashLB{} + resetLB(lb) + req := &motan.MotanRequest{Method: "test"} + + // less endpoint + lb.OnRefresh(buildEndpoint(MaxSelectArraySize)) + eps := lb.SelectArray(req) + assert.Equal(t, MaxSelectArraySize, len(eps)) + + // use hash key + resetLB(lb) + lb.OnRefresh(buildEndpoint(30)) + req.SetAttachment(motan.ConsistentHashKey, "234234") + ep := lb.Select(req) + eps = lb.SelectArray(req) + assert.Equal(t, ep, eps[0]) + assert.Equal(t, MaxSelectArraySize, len(eps)) + + // repeat select + var eps2 []motan.EndPoint + for i := 0; i < 20; i++ { + eps2 = lb.SelectArray(req) + assert.Equal(t, eps, eps2) + } + + // test random + req.SetAttachment(motan.ConsistentHashKey, "") + eps = lb.SelectArray(req) + eps2 = lb.SelectArray(req) + assert.NotEqual(t, eps, eps2) +} + +func TestConsistentMigration(t *testing.T) { + lb := &ConsistentHashLB{} + resetLB(lb) + + // distribution migration under default configuration + // endpoint changes slightly + diffDistribution(lb, 49, 50) + diffDistribution(lb, 49, 51) + diffDistribution(lb, 49, 48) + diffDistribution(lb, 49, 47) + + // endpoint changes significantly + diffDistribution(lb, 49, 70) + diffDistribution(lb, 49, 99) + diffDistribution(lb, 49, 31) + diffDistribution(lb, 49, 27) + diffDistribution(lb, 500, 1000) +} + +func checkDefault(t *testing.T, lb *ConsistentHashLB, epsSize int) { + eps := buildEndpoint(epsSize) + lb.OnRefresh(eps) + assert.True(t, lb.lastPartSize >= len(eps)*2) + assert.True(t, lb.load >= DefaultMinLoad) + assert.True(t, lb.replica > 0) + assert.NotNil(t, lb.cHash) + assert.Equal(t, eps, lb.endpoints) +} + +func checkHashEP(t *testing.T, lb *ConsistentHashLB, req motan.Request, expectEP motan.EndPoint) { + ep := lb.Select(req) + assert.Equal(t, expectEP, ep) +} + +func checkConsistent(t *testing.T, lb *ConsistentHashLB, hashKey string, times int) { + req := &motan.MotanRequest{Method: "test"} + req.SetAttachment(motan.ConsistentHashKey, hashKey) + ep := lb.Select(req) + for i := 0; i < times; i++ { + assert.Equal(t, ep, lb.Select(req)) + } +} + +func diffDistribution(lb *ConsistentHashLB, epSize1 int, epSize2 int) int { + lb.OnRefresh(buildEndpoint(epSize1)) + cHash1 := lb.cHash + partSize := lb.lastPartSize + lb.OnRefresh(buildEndpoint(epSize2)) + cHash2 := lb.cHash + if lb.lastPartSize < partSize { + partSize = lb.lastPartSize + } + diff := 0 + for i := 0; i < partSize; i++ { + if cHash1.GetPartitionOwner(i).String() != cHash2.GetPartitionOwner(i).String() { + diff++ + } + } + fmt.Printf("=== ep size %d -> %d, diff distribution: %f\n", epSize1, epSize2, float64(diff)/float64(partSize)) + return diff +} + +func resetLB(lb *ConsistentHashLB) { + lb.endpoints = nil + lb.lastPartSize = 0 + lb.load = 0 + lb.replica = 0 + lb.cHash = nil + lb.url = &motan.URL{Host: "localhost", Port: 0, Protocol: "motan2"} +} + +func buildEndpoint(size int) []motan.EndPoint { + endpoints := make([]motan.EndPoint, 0, size) + for i := 0; i < size; i++ { + url := &motan.URL{Host: "10.10.10." + fmt.Sprintf("%d", i), Port: 8000 + i, Protocol: "motan2"} + endpoints = append(endpoints, &lbTestMockEndpoint{MockEndpoint: &endpoint.MockEndpoint{URL: url}, index: i, isAvail: true}) + } + return endpoints +} diff --git a/lb/lb.go b/lb/lb.go index d275ee70..062c5333 100644 --- a/lb/lb.go +++ b/lb/lb.go @@ -13,8 +13,9 @@ import ( // ext name const ( - Random = "random" - Roundrobin = "roundrobin" + Random = "random" + Roundrobin = "roundrobin" + ConsistentHashKey = "consistentHashKey" ) const ( @@ -35,6 +36,9 @@ func RegistDefaultLb(extFactory motan.ExtensionFactory) { extFactory.RegistExtLb(Roundrobin, NewWeightLbFunc(func(url *motan.URL) motan.LoadBalance { return &RoundrobinLB{url: url} })) + extFactory.RegistExtLb(ConsistentHashKey, NewWeightLbFunc(func(url *motan.URL) motan.LoadBalance { + return &ConsistentHashLB{url: url} + })) } // WeightedLbWrapper support multi group weighted LB From 890edb9c7d699c1d8de4c28417c35d0c45e94193 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:27:18 +0800 Subject: [PATCH 68/75] Update (#316) after notify, offline endpoints will be destroyed asynchronously after 2s delay --- agent.go | 68 ++++++++++-- agent_test.go | 255 ++++++++++++++++++++++++++++++++++++++++++- cluster/command.go | 7 ++ core/constants.go | 9 ++ core/motan.go | 12 ++ default.go | 6 + manageHandler.go | 78 ++++++++++++- registry/registry.go | 16 ++- server.go | 64 +++++++++-- server_test.go | 83 ++++++++++++++ 10 files changed, 571 insertions(+), 27 deletions(-) diff --git a/agent.go b/agent.go index f688ebeb..17884ad4 100644 --- a/agent.go +++ b/agent.go @@ -4,6 +4,8 @@ import ( "flag" "fmt" "github.com/weibocom/motan-go/endpoint" + vlog "github.com/weibocom/motan-go/log" + "gopkg.in/yaml.v2" "io/ioutil" "net" "net/http" @@ -15,15 +17,12 @@ import ( "sync/atomic" "time" - cfg "github.com/weibocom/motan-go/config" - "gopkg.in/yaml.v2" - "github.com/shirou/gopsutil/v3/process" "github.com/valyala/fasthttp" "github.com/weibocom/motan-go/cluster" + cfg "github.com/weibocom/motan-go/config" motan "github.com/weibocom/motan-go/core" mhttp "github.com/weibocom/motan-go/http" - "github.com/weibocom/motan-go/log" "github.com/weibocom/motan-go/metrics" mpro "github.com/weibocom/motan-go/protocol" "github.com/weibocom/motan-go/registry" @@ -59,7 +58,7 @@ type Agent struct { clusterMap *motan.CopyOnWriteMap httpClusterMap *motan.CopyOnWriteMap - status int + status int64 agentURL *motan.URL logdir string port int @@ -77,8 +76,9 @@ type Agent struct { manageHandlers map[string]http.Handler - svcLock sync.Mutex - clsLock sync.Mutex + svcLock sync.Mutex + clsLock sync.Mutex + registryLock sync.Mutex configurer *DynamicConfigurer @@ -157,13 +157,13 @@ func (a *Agent) GetAgentServer() motan.Server { func (a *Agent) SetAllServicesAvailable() { a.availableAllServices() - a.status = http.StatusOK + atomic.StoreInt64(&a.status, http.StatusOK) a.saveStatus() } func (a *Agent) SetAllServicesUnavailable() { a.unavailableAllServices() - a.status = http.StatusServiceUnavailable + atomic.StoreInt64(&a.status, http.StatusServiceUnavailable) a.saveStatus() } @@ -204,6 +204,7 @@ func (a *Agent) StartMotanAgentFromConfig(config *cfg.Config) { a.configurer = NewDynamicConfigurer(a) go a.startMServer() go a.registerAgent() + go a.startRegistryFailback() f, err := os.Create(a.pidfile) if err != nil { vlog.Errorf("create file %s fail.", a.pidfile) @@ -211,7 +212,7 @@ func (a *Agent) StartMotanAgentFromConfig(config *cfg.Config) { defer f.Close() f.WriteString(strconv.Itoa(os.Getpid())) } - if a.status == http.StatusOK { + if atomic.LoadInt64(&a.status) == http.StatusOK { // recover form a unexpected case a.availableAllServices() } @@ -219,6 +220,47 @@ func (a *Agent) StartMotanAgentFromConfig(config *cfg.Config) { a.startAgent() } +func (a *Agent) startRegistryFailback() { + vlog.Infoln("start agent failback") + ticker := time.NewTicker(registry.DefaultFailbackInterval * time.Millisecond) + defer ticker.Stop() + for range ticker.C { + a.registryLock.Lock() + a.serviceRegistries.Range(func(k, v interface{}) bool { + if vv, ok := v.(motan.RegistryStatusManager); ok { + statusMap := vv.GetRegistryStatus() + for _, j := range statusMap { + curStatus := atomic.LoadInt64(&a.status) + if curStatus == http.StatusOK && j.Status == motan.RegisterFailed { + vlog.Infoln(fmt.Sprintf("detect agent register fail, do register again, service: %s", j.Service.GetIdentity())) + v.(motan.Registry).Available(j.Service) + } else if curStatus == http.StatusServiceUnavailable && j.Status == motan.UnregisterFailed { + vlog.Infoln(fmt.Sprintf("detect agent unregister fail, do unregister again, service: %s", j.Service.GetIdentity())) + v.(motan.Registry).Unavailable(j.Service) + } + } + } + return true + }) + a.registryLock.Unlock() + } + +} + +func (a *Agent) GetRegistryStatus() []map[string]*motan.RegistryStatus { + a.registryLock.Lock() + defer a.registryLock.Unlock() + var res []map[string]*motan.RegistryStatus + a.serviceRegistries.Range(func(k, v interface{}) bool { + if vv, ok := v.(motan.RegistryStatusManager); ok { + statusMap := vv.GetRegistryStatus() + res = append(res, statusMap) + } + return true + }) + return res +} + func (a *Agent) registerStatusSampler() { metrics.RegisterStatusSampleFunc("memory", func() int64 { p, _ := process.NewProcess(int32(os.Getpid())) @@ -252,7 +294,7 @@ func (a *Agent) initStatus() { key := metrics.DefaultStatRole + metrics.KeyDelimiter + application + metrics.KeyDelimiter + "abnormal_exit.total_count" metrics.AddCounter(metrics.DefaultStatGroup, metrics.DefaultStatService, key, 1) } else { - a.status = http.StatusServiceUnavailable + atomic.StoreInt64(&a.status, http.StatusServiceUnavailable) } } @@ -829,6 +871,8 @@ func (a *Agent) startServerAgent() { } func (a *Agent) availableAllServices() { + a.registryLock.Lock() + defer a.registryLock.Unlock() a.serviceRegistries.Range(func(k, v interface{}) bool { v.(motan.Registry).Available(nil) return true @@ -836,6 +880,8 @@ func (a *Agent) availableAllServices() { } func (a *Agent) unavailableAllServices() { + a.registryLock.Lock() + defer a.registryLock.Unlock() a.serviceRegistries.Range(func(k, v interface{}) bool { v.(motan.Registry).Unavailable(nil) return true diff --git a/agent_test.go b/agent_test.go index 5d0921b4..55b8929d 100644 --- a/agent_test.go +++ b/agent_test.go @@ -2,12 +2,14 @@ package motan import ( "bytes" + "encoding/json" "flag" "fmt" _ "fmt" "github.com/weibocom/motan-go/config" "github.com/weibocom/motan-go/endpoint" vlog "github.com/weibocom/motan-go/log" + "github.com/weibocom/motan-go/registry" "github.com/weibocom/motan-go/serialize" _ "github.com/weibocom/motan-go/server" _ "golang.org/x/net/context" @@ -40,6 +42,10 @@ var proxyClient *http.Client var meshClient *MeshClient var agent *Agent +var ( + testRegistryFailSwitcher int64 = 0 +) + func Test_unixClientCall1(t *testing.T) { t.Parallel() startServer(t, "helloService", 22991) @@ -258,10 +264,11 @@ motan-agent: conf.Merge(mportConfigENVParam) section, err = conf.GetSection("motan-agent") assert.Nil(err) - err = os.Setenv("mport", "8006") - _ = flag.Set("mport", "8007") - a.initParam() - assert.Equal(a.mport, 8007) + //err = os.Setenv("mport", "8006") + //_ = flag.Set("mport", "8007") + //a.initParam() + //assert.Equal(a.mport, 8007) + os.Unsetenv("mport") } func TestHTTPProxyBodySize(t *testing.T) { @@ -720,3 +727,243 @@ motan-service: a.initParam() assert.Equal(t, endpoint.GetDefaultMotanEPAsynInit(), true) } + +func Test_agentRegistryFailback(t *testing.T) { + template := ` +motan-agent: + port: 12829 + mport: 12604 + eport: 12282 + hport: 23283 + +motan-registry: + direct: + protocol: direct + test: + protocol: test-registry + +motan-service: + test01: + protocol: motan2 + provider: motan2 + group: hello + path: helloService + registry: test + serialization: simple + export: motan2:12282 + check: true +` + extFactory := GetDefaultExtFactory() + extFactory.RegistExtRegistry("test-registry", func(url *core.URL) core.Registry { + return &testRegistry{url: url} + }) + config1, err := config.NewConfigFromReader(bytes.NewReader([]byte(template))) + assert.Nil(t, err) + agent := NewAgent(extFactory) + go agent.StartMotanAgentFromConfig(config1) + time.Sleep(time.Second * 10) + + setRegistryFailSwitcher(true) + m := agent.GetRegistryStatus() + assert.Equal(t, len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "helloService" { + assert.Equal(t, mm.Status, core.NotRegister) + } + } + agentStatus := getCurAgentStatus(12604) + assert.Equal(t, agentStatus, core.NotRegister) + agent.SetAllServicesAvailable() + m = agent.GetRegistryStatus() + for _, mm := range m[0] { + if mm.Service.Path == "helloService" { + assert.Equal(t, mm.Status, core.RegisterFailed) + } + } + agentStatus = getCurAgentStatus(12604) + assert.Equal(t, agentStatus, core.RegisterFailed) + setRegistryFailSwitcher(false) + time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) + m = agent.GetRegistryStatus() + assert.Equal(t, len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "helloService" { + assert.Equal(t, mm.Status, core.RegisterSuccess) + } + } + agentStatus = getCurAgentStatus(12604) + assert.Equal(t, agentStatus, core.RegisterSuccess) + setRegistryFailSwitcher(true) + agent.SetAllServicesUnavailable() + m = agent.GetRegistryStatus() + assert.Equal(t, len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "helloService" { + assert.Equal(t, mm.Status, core.UnregisterFailed) + } + } + agentStatus = getCurAgentStatus(12604) + assert.Equal(t, agentStatus, core.UnregisterFailed) + setRegistryFailSwitcher(false) + time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) + m = agent.GetRegistryStatus() + assert.Equal(t, len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "helloService" { + assert.Equal(t, mm.Status, core.UnregisterSuccess) + } + } + agentStatus = getCurAgentStatus(12604) + assert.Equal(t, agentStatus, core.UnregisterSuccess) +} + +type testRegistry struct { + url *core.URL + namingServiceStatus *core.CopyOnWriteMap + registeredServices map[string]*core.URL +} + +func (t *testRegistry) Initialize() { + t.registeredServices = make(map[string]*core.URL) + t.namingServiceStatus = core.NewCopyOnWriteMap() +} + +func (t *testRegistry) GetName() string { + return "test-registry" +} + +func (t *testRegistry) GetURL() *core.URL { + return t.url +} + +func (t *testRegistry) SetURL(url *core.URL) { + t.url = url +} + +func (t *testRegistry) Subscribe(url *core.URL, listener core.NotifyListener) { +} + +func (t *testRegistry) Unsubscribe(url *core.URL, listener core.NotifyListener) { +} + +func (t *testRegistry) Discover(url *core.URL) []*core.URL { + return nil +} + +func (t *testRegistry) Register(serverURL *core.URL) { + t.registeredServices[serverURL.GetIdentity()] = serverURL + t.namingServiceStatus.Store(serverURL.GetIdentity(), &core.RegistryStatus{ + Status: core.NotRegister, + Service: serverURL, + Registry: t, + IsCheck: registry.IsCheck(serverURL), + }) + +} + +func (t *testRegistry) UnRegister(serverURL *core.URL) { + delete(t.registeredServices, serverURL.GetIdentity()) + t.namingServiceStatus.Delete(serverURL.GetIdentity()) +} + +func (t *testRegistry) Available(serverURL *core.URL) { + if getRegistryFailSwitcher() { + for _, u := range t.registeredServices { + t.namingServiceStatus.Store(u.GetIdentity(), &core.RegistryStatus{ + Status: core.RegisterFailed, + Registry: t, + Service: u, + ErrMsg: "error", + IsCheck: registry.IsCheck(u), + }) + } + } else { + for _, u := range t.registeredServices { + t.namingServiceStatus.Store(u.GetIdentity(), &core.RegistryStatus{ + Status: core.RegisterSuccess, + Registry: t, + Service: u, + IsCheck: registry.IsCheck(u), + }) + } + } +} + +func (t *testRegistry) Unavailable(serverURL *core.URL) { + if getRegistryFailSwitcher() { + for _, u := range t.registeredServices { + t.namingServiceStatus.Store(u.GetIdentity(), &core.RegistryStatus{ + Status: core.UnregisterFailed, + Registry: t, + Service: u, + ErrMsg: "error", + IsCheck: registry.IsCheck(u), + }) + } + } else { + for _, u := range t.registeredServices { + t.namingServiceStatus.Store(u.GetIdentity(), &core.RegistryStatus{ + Status: core.UnregisterSuccess, + Registry: t, + Service: u, + IsCheck: registry.IsCheck(u), + }) + } + } +} + +func (t *testRegistry) GetRegisteredServices() []*core.URL { + return nil +} + +func (t *testRegistry) StartSnapshot(conf *core.SnapshotConf) { +} + +func (t *testRegistry) GetRegistryStatus() map[string]*core.RegistryStatus { + res := make(map[string]*core.RegistryStatus) + t.namingServiceStatus.Range(func(k, v interface{}) bool { + res[k.(string)] = v.(*core.RegistryStatus) + return true + }) + return res +} + +func setRegistryFailSwitcher(b bool) { + if b { + atomic.StoreInt64(&testRegistryFailSwitcher, 1) + } else { + atomic.StoreInt64(&testRegistryFailSwitcher, 0) + } + +} + +func getRegistryFailSwitcher() bool { + return atomic.LoadInt64(&testRegistryFailSwitcher) == 1 +} + +func getCurAgentStatus(port int64) string { + type ( + Result struct { + Status string `json:"status"` + RegistryStatus interface{} `json:"registryStatus"` + } + ) + client := http.Client{ + Timeout: time.Second * 3, + } + resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/registry/status", port)) + if err != nil { + return err.Error() + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err.Error() + } + res := Result{} + err = json.Unmarshal(b, &res) + if err != nil { + return err.Error() + } + return res.Status +} diff --git a/cluster/command.go b/cluster/command.go index 5cd7da20..f49ad7af 100644 --- a/cluster/command.go +++ b/cluster/command.go @@ -166,6 +166,13 @@ func GetCommandRegistryWrapper(cluster *MotanCluster, registry motan.Registry) m return cmdRegistry } +func (c *CommandRegistryWrapper) GetRegistryStatus() map[string]*motan.RegistryStatus { + if v, ok := c.registry.(motan.RegistryStatusManager); ok { + return v.GetRegistryStatus() + } + return nil +} + func (c *CommandRegistryWrapper) Register(serverURL *motan.URL) { c.registry.Register(serverURL) } diff --git a/core/constants.go b/core/constants.go index 2876ea27..20ec7e76 100644 --- a/core/constants.go +++ b/core/constants.go @@ -77,6 +77,15 @@ const ( ConsistentHashKey = "consistentHashKey" //string used to calculate consistent hash ) +// registryStatus +const ( + RegisterSuccess = "register-success" + RegisterFailed = "register-failed" + UnregisterSuccess = "unregister-success" + UnregisterFailed = "unregister-failed" + NotRegister = "not-register" +) + // nodeType const ( NodeTypeService = "service" diff --git a/core/motan.go b/core/motan.go index d52b5de7..adfd9016 100644 --- a/core/motan.go +++ b/core/motan.go @@ -197,6 +197,18 @@ type Registry interface { SnapshotService } +type RegistryStatusManager interface { + GetRegistryStatus() map[string]*RegistryStatus +} + +type RegistryStatus struct { + Status string + Service *URL + Registry RegisterService + ErrMsg string + IsCheck bool +} + // NotifyListener : NotifyListener type NotifyListener interface { Identity diff --git a/default.go b/default.go index e681a818..52e41cd7 100644 --- a/default.go +++ b/default.go @@ -41,6 +41,7 @@ func GetDefaultManageHandlers() map[string]http.Handler { defaultManageHandlers["/503"] = status defaultManageHandlers["/version"] = status defaultManageHandlers["/status"] = status + defaultManageHandlers["/registry/status"] = status info := &InfoHandler{} defaultManageHandlers["/getConfig"] = info @@ -59,6 +60,11 @@ func GetDefaultManageHandlers() map[string]http.Handler { defaultManageHandlers["/debug/stat/process"] = debug defaultManageHandlers["/debug/stat/openFiles"] = debug defaultManageHandlers["/debug/stat/connections"] = debug + defaultManageHandlers["/debug/pprof/allocs"] = debug + defaultManageHandlers["/debug/pprof/block"] = debug + defaultManageHandlers["/debug/pprof/goroutine"] = debug + defaultManageHandlers["/debug/pprof/mutex"] = debug + defaultManageHandlers["/debug/pprof/heap"] = debug switcher := &SwitcherHandler{} defaultManageHandlers["/switcher/set"] = switcher diff --git a/manageHandler.go b/manageHandler.go index 4a4a8365..98fd4208 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -11,12 +11,14 @@ import ( "math" "math/rand" "net/http" + nppf "net/http/pprof" "os" "runtime" "runtime/pprof" "runtime/trace" "strconv" "strings" + "sync/atomic" "time" "github.com/shirou/gopsutil/v3/cpu" @@ -65,12 +67,72 @@ func (s *StatusHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rw.Write([]byte(Version)) case "/status": rw.Write(s.getStatus()) + case "/registry/status": + rw.Write(s.getRegistryStatus()) default: - rw.WriteHeader(s.a.status) - rw.Write([]byte(http.StatusText(s.a.status))) + rw.WriteHeader(int(s.a.status)) + rw.Write([]byte(http.StatusText(int(s.a.status)))) } } +func (s *StatusHandler) getRegistryStatus() []byte { + type ( + ResultStatus struct { + Group string `json:"group"` + Service string `json:"service"` + Registry string `json:"registry"` + Status string `json:"status"` + ErrMsg string `json:"errMsg"` + IsCheck bool `json:"isCheck"` + } + Result struct { + Status string `json:"status"` + RegistryStatus []ResultStatus `json:"registryStatus"` + } + ) + statuses := s.a.GetRegistryStatus() + var res []ResultStatus + curAgentStatus := atomic.LoadInt64(&s.a.status) + var resStatus string + if curAgentStatus == http.StatusOK { + resStatus = motan.RegisterSuccess + } else { + resStatus = motan.UnregisterSuccess + } + for _, j := range statuses { + for _, k := range j { + res = append(res, ResultStatus{ + Group: k.Service.Group, + Service: k.Service.Path, + Registry: k.Service.GetParam(motan.RegistryKey, ""), + Status: k.Status, + ErrMsg: k.ErrMsg, + IsCheck: k.IsCheck, + }) + if k.IsCheck { + if curAgentStatus == http.StatusOK { + if k.Status == motan.RegisterFailed { + resStatus = k.Status + } else if k.Status == motan.NotRegister && resStatus != motan.RegisterFailed { + resStatus = k.Status + } + } else { + if k.Status == motan.UnregisterFailed { + resStatus = k.Status + } else if k.Status == motan.NotRegister && resStatus != motan.UnregisterFailed { + resStatus = k.Status + } + } + } + } + } + resByte, _ := json.Marshal(Result{ + Status: resStatus, + RegistryStatus: res, + }) + return resByte +} + func (s *StatusHandler) getStatus() []byte { type ( MethodStatus struct { @@ -89,7 +151,7 @@ func (s *StatusHandler) getStatus() []byte { } ) result := Result{ - Status: s.a.status, + Status: int(s.a.status), Services: make([]ServiceStatus, 0, 16), } s.a.serviceExporters.Range(func(k, v interface{}) bool { @@ -223,6 +285,16 @@ func (d *DebugHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { Symbol(rw, req) case "/debug/pprof/trace": Trace(rw, req) + case "/debug/pprof/allocs": + nppf.Handler("allocs").ServeHTTP(rw, req) + case "/debug/pprof/block": + nppf.Handler("block").ServeHTTP(rw, req) + case "/debug/pprof/goroutine": + nppf.Handler("goroutine").ServeHTTP(rw, req) + case "/debug/pprof/mutex": + nppf.Handler("mutex").ServeHTTP(rw, req) + case "/debug/pprof/heap": + nppf.Handler("heap").ServeHTTP(rw, req) case "/debug/mesh/trace": MeshTrace(rw, req) case "/debug/stat/system": diff --git a/registry/registry.go b/registry/registry.go index 5d79171f..0dca1d32 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "sync" "time" "unsafe" @@ -18,9 +19,10 @@ const ( DefaultHeartbeatInterval = 10 * 1000 //ms DefaultTimeout = 3 * 1000 //ms DefaultSnapshotDir = "./snapshot" + DefaultFailbackInterval = 30 * 1000 //ms ) -//ext name +// ext name const ( Direct = "direct" Local = "local" @@ -133,6 +135,18 @@ func IsAgent(url *motan.URL) bool { return isAgent } +func IsCheck(url *motan.URL) bool { + isCheck := false + var err error + if t, ok := url.Parameters["check"]; ok { + isCheck, err = strconv.ParseBool(t) + if err != nil { + return false + } + } + return isCheck +} + func GetSubKey(url *motan.URL) string { return url.Group + "/" + url.Path + "/service" } diff --git a/server.go b/server.go index f36436a5..808b8e82 100644 --- a/server.go +++ b/server.go @@ -5,16 +5,18 @@ import ( "flag" "fmt" "github.com/weibocom/motan-go/config" + motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/log" "github.com/weibocom/motan-go/provider" + "github.com/weibocom/motan-go/registry" + mserver "github.com/weibocom/motan-go/server" "hash/fnv" + "net/http" "reflect" "strconv" "strings" "sync" - - motan "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/log" - mserver "github.com/weibocom/motan-go/server" + "time" ) // MSContext is Motan Server Context @@ -26,9 +28,10 @@ type MSContext struct { portServer map[string]motan.Server serviceImpls map[string]interface{} registries map[string]motan.Registry // all registries used for services - - csync sync.Mutex - inited bool + registryLock sync.Mutex + status int + csync sync.Mutex + inited bool } const ( @@ -102,6 +105,7 @@ func (m *MSContext) Start(extfactory motan.ExtensionFactory) { for _, url := range m.context.ServiceURLs { m.export(url) } + go m.startRegistryFailback() } func (m *MSContext) hashInt(s string) int { @@ -238,14 +242,58 @@ func (m *MSContext) RegisterService(s interface{}, sid string) error { // ServicesAvailable will enable all service registed in registries func (m *MSContext) ServicesAvailable() { // TODO: same as agent + m.registryLock.Lock() + defer m.registryLock.Unlock() + m.status = http.StatusOK availableService(m.registries) } -// ServicesUnavailable will enable all service registed in registries +// ServicesUnavailable will enable all service registered in registries func (m *MSContext) ServicesUnavailable() { + m.registryLock.Lock() + defer m.registryLock.Unlock() + m.status = http.StatusServiceUnavailable unavailableService(m.registries) } +func (m *MSContext) GetRegistryStatus() []map[string]*motan.RegistryStatus { + m.registryLock.Lock() + defer m.registryLock.Unlock() + var res []map[string]*motan.RegistryStatus + for _, v := range m.registries { + if vv, ok := v.(motan.RegistryStatusManager); ok { + res = append(res, vv.GetRegistryStatus()) + } + } + return res +} + +func (m *MSContext) startRegistryFailback() { + vlog.Infoln("start MSContext registry failback") + ticker := time.NewTicker(registry.DefaultFailbackInterval * time.Millisecond) + defer ticker.Stop() + for range ticker.C { + m.registryLock.Lock() + for _, v := range m.registries { + if vv, ok := v.(motan.RegistryStatusManager); ok { + statusMap := vv.GetRegistryStatus() + for _, j := range statusMap { + if m.status == http.StatusOK && j.Status == motan.RegisterFailed { + vlog.Infoln(fmt.Sprintf("detect register fail, do register again, service: %s", j.Service.GetIdentity())) + v.(motan.Registry).Available(j.Service) + } else if m.status == http.StatusServiceUnavailable && j.Status == motan.UnregisterFailed { + vlog.Infoln(fmt.Sprintf("detect unregister fail, do unregister again, service: %s", j.Service.GetIdentity())) + v.(motan.Registry).Unavailable(j.Service) + } + } + } + + } + m.registryLock.Unlock() + } + +} + func canShareChannel(u1 motan.URL, u2 motan.URL) bool { if u1.Protocol != u2.Protocol { return false diff --git a/server_test.go b/server_test.go index b028d4de..3ee7b110 100644 --- a/server_test.go +++ b/server_test.go @@ -6,6 +6,7 @@ import ( assert2 "github.com/stretchr/testify/assert" "github.com/weibocom/motan-go/config" motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/registry" "testing" "time" ) @@ -93,6 +94,88 @@ motan-server: request.Attachment = motan.NewStringMap(motan.DefaultAttachmentSize) } +func TestServerRegisterFailBack(t *testing.T) { + assert := assert2.New(t) + cfgText := ` +motan-server: + log_dir: "stdout" + application: "app-golang" # server identify. + +motan-registry: + direct: + protocol: direct + test: + protocol: test-registry + +#conf of services +motan-service: + mytest-motan2: + path: testpath + group: testgroup + protocol: motan2 + registry: test + serialization: simple + ref : "serviceID" + check: true + export: "motan2:14564" +` + conf, err := config.NewConfigFromReader(bytes.NewReader([]byte(cfgText))) + assert.Nil(err) + + extFactory := GetDefaultExtFactory() + extFactory.RegistExtRegistry("test-registry", func(url *motan.URL) motan.Registry { + return &testRegistry{url: url} + }) + mscontext := NewMotanServerContextFromConfig(conf) + err = mscontext.RegisterService(&HelloService{}, "serviceID") + assert.Nil(err) + mscontext.Start(extFactory) + time.Sleep(time.Second * 3) + m := mscontext.GetRegistryStatus() + assert.Equal(len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "testpath" { + assert.Equal(mm.Status, motan.NotRegister) + } + } + setRegistryFailSwitcher(true) + mscontext.ServicesAvailable() + m = mscontext.GetRegistryStatus() + assert.Equal(len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "testpath" { + assert.Equal(mm.Status, motan.RegisterFailed) + } + } + setRegistryFailSwitcher(false) + time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) + m = mscontext.GetRegistryStatus() + assert.Equal(len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "testpath" { + assert.Equal(mm.Status, motan.RegisterSuccess) + } + } + setRegistryFailSwitcher(true) + mscontext.ServicesUnavailable() + m = mscontext.GetRegistryStatus() + assert.Equal(len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "testpath" { + assert.Equal(mm.Status, motan.UnregisterFailed) + } + } + setRegistryFailSwitcher(false) + time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) + m = mscontext.GetRegistryStatus() + assert.Equal(len(m), 1) + for _, mm := range m[0] { + if mm.Service.Path == "testpath" { + assert.Equal(mm.Status, motan.UnregisterSuccess) + } + } +} + func TestNewMotanServerContextFromConfig(t *testing.T) { assert := assert2.New(t) From 3cf5d101d799da17f2e7cb50408ce58e582fd4f9 Mon Sep 17 00:00:00 2001 From: snail007 Date: Wed, 6 Dec 2023 17:09:23 +0800 Subject: [PATCH 69/75] Profile base to dev (#350) * preview to base (#333) * Profile base bytesbuffer (#326) * add WriteString to BytesBuffer * Encode WriteString * Profile base object reuse (#327) * reuse bytesbuffer & stream * Optimise StringMap Range (#328) * Profile base object reuse (#332) * reuse bytesbuffer & stream * Profile preview to base (#349) * Profile base add metric (#338) * Optimise filter addMetric * Profile base findcluster (#340) * Profile base findcluster (#337) * Profile base do acccesslog (#339) * bugfix: BytesBuffer.Reset (#334) * Access log (#336) * Profile base do accesslog (#343) Accesslogv2 (#342) * Profile base preview escape (#345) * optimise Escape * Profile base timenow (#344) (#347) * Profile base do accesslog (#343) * update: log/bytesbuffer (#341) * bytesBuffer release * Accesslogv2 (#346) * update: log/bytesbuffer --- agent.go | 51 ++++++++---- core/bytes.go | 59 ++++++++++++- core/constants.go | 4 + core/map.go | 22 ++--- core/motan.go | 124 +++++++++++++++++++++++---- endpoint/motanCommonEndpoint.go | 134 +++++++++++++++++++++++------- endpoint/motanEndpoint.go | 119 +++++++++++++++++++------- endpoint/motanEndpoint_test.go | 5 +- filter/accessLog.go | 48 +++++------ filter/clusterMetrics.go | 13 ++- filter/metrics.go | 39 +++++---- filter/metrics_test.go | 25 +++--- ha/backupRequestHA.go | 2 +- log/bytes.go | 82 ++++++++++++++++++ log/bytes_test.go | 104 +++++++++++++++++++++++ log/log.go | 37 +++++++-- manageHandler.go | 2 +- metrics/graphite.go | 10 ++- metrics/metrics.go | 112 +++++++++++++++++++++---- protocol/motanProtocol.go | 143 ++++++++++++++++++++++---------- protocol/motanProtocol_test.go | 122 +++++++++++++++++++++++++-- provider/httpProvider.go | 2 - server/motanserver.go | 20 ++++- 23 files changed, 1027 insertions(+), 252 deletions(-) create mode 100644 log/bytes.go create mode 100644 log/bytes_test.go diff --git a/agent.go b/agent.go index 17884ad4..541d8365 100644 --- a/agent.go +++ b/agent.go @@ -45,6 +45,11 @@ var ( setAgentLock sync.Mutex notFoundProviderCount int64 = 0 defaultInitClusterTimeout int64 = 10000 //ms + clusterSlicePool = sync.Pool{ + New: func() interface{} { + return make([]serviceMapItem, 0, 5) + }, + } ) type Agent struct { @@ -794,32 +799,28 @@ func (a *agentMessageHandler) Call(request motan.Request) (res motan.Response) { } return res } -func (a *agentMessageHandler) matchRule(typ, cond, key string, data []serviceMapItem, f func(u *motan.URL) string) (foundClusters []serviceMapItem, err error) { +func (a *agentMessageHandler) fillMatch(typ, cond, key string, data []serviceMapItem, f func(u *motan.URL) string, match *[]serviceMapItem) error { if cond == "" { - err = fmt.Errorf("empty %s is not supported", typ) - return + return fmt.Errorf("empty %s is not supported", typ) } for _, item := range data { if f(item.url) == cond { - foundClusters = append(foundClusters, item) + *match = append(*match, item) } } - if len(foundClusters) == 0 { - err = fmt.Errorf("cluster not found. cluster:%s", key) - return + if len(*match) == 0 { + return fmt.Errorf("cluster not found. cluster:%s", key) } - return + return nil } func (a *agentMessageHandler) findCluster(request motan.Request) (c *cluster.MotanCluster, key string, err error) { service := request.GetServiceName() group := request.GetAttachment(mpro.MGroup) version := request.GetAttachment(mpro.MVersion) protocol := request.GetAttachment(mpro.MProxyProtocol) - reqInfo := fmt.Sprintf("request information: {service: %s, group: %s, protocol: %s, version: %s}", - service, group, protocol, version) serviceItemArrI, exists := a.agent.serviceMap.Load(service) if !exists { - err = fmt.Errorf("cluster not found. cluster:%s, %s", service, reqInfo) + err = fmt.Errorf("cluster not found. cluster:%s, %s", service, getReqInfo(service, group, protocol, version)) return } search := []struct { @@ -832,23 +833,31 @@ func (a *agentMessageHandler) findCluster(request motan.Request) (c *cluster.Mot {"protocol", protocol, func(u *motan.URL) string { return u.Protocol }}, {"version", version, func(u *motan.URL) string { return u.GetParam(motan.VersionKey, "") }}, } - foundClusters := serviceItemArrI.([]serviceMapItem) + clusters := serviceItemArrI.([]serviceMapItem) + matched := clusterSlicePool.Get().([]serviceMapItem) + if cap(matched) < len(clusters) { + matched = make([]serviceMapItem, 0, len(clusters)) + } for i, rule := range search { if i == 0 { key = rule.cond } else { key += "_" + rule.cond } - foundClusters, err = a.matchRule(rule.tip, rule.cond, key, foundClusters, rule.condFn) + err = a.fillMatch(rule.tip, rule.cond, key, clusters, rule.condFn, &matched) if err != nil { + putBackClusterSlice(matched) return } - if len(foundClusters) == 1 { - c = foundClusters[0].cluster + if len(matched) == 1 { + c = matched[0].cluster + putBackClusterSlice(matched) return } + matched = matched[:0] } - err = fmt.Errorf("less condition to select cluster, maybe this service belongs to multiple group, protocol, version; cluster: %s, %s", key, reqInfo) + err = fmt.Errorf("less condition to select cluster, maybe this service belongs to multiple group, protocol, version; cluster: %s, %s", key, getReqInfo(service, group, protocol, version)) + putBackClusterSlice(matched) return } @@ -1222,6 +1231,11 @@ func urlExist(url *motan.URL, urls map[string]*motan.URL) bool { return false } +func getReqInfo(service, group, protocol, version string) string { + return fmt.Sprintf("request information: {service: %s, group: %s, protocol: %s, version: %s}", + service, group, protocol, version) +} + func (a *Agent) SubscribeService(url *motan.URL) error { if urlExist(url, a.Context.RefersURLs) { return fmt.Errorf("url exist, ignore subscribe, url: %s", url.GetIdentity()) @@ -1251,3 +1265,8 @@ func (a *Agent) UnexportService(url *motan.URL) error { } return nil } + +func putBackClusterSlice(s []serviceMapItem) { + s = s[:0] + clusterSlicePool.Put(s) +} diff --git a/core/bytes.go b/core/bytes.go index 604c6d57..457ff2d3 100644 --- a/core/bytes.go +++ b/core/bytes.go @@ -4,6 +4,17 @@ import ( "encoding/binary" "errors" "io" + "math/rand" + "sync" + "time" +) + +var ( + maxReuseBufSize = 204800 + discardRatio = 0.1 + bytesBufferPool = sync.Pool{New: func() interface{} { + return new(BytesBuffer) + }} ) // BytesBuffer is a variable-sized buffer of bytes with Read and Write methods. @@ -26,10 +37,16 @@ func NewBytesBuffer(initsize int) *BytesBuffer { // NewBytesBufferWithOrder create a empty BytesBuffer with initial size and byte order func NewBytesBufferWithOrder(initsize int, order binary.ByteOrder) *BytesBuffer { - return &BytesBuffer{buf: make([]byte, initsize), - order: order, - temp: make([]byte, 8), + bb := AcquireBytesBuffer() + if bb.buf == nil { + bb.buf = make([]byte, initsize) + } + if bb.temp == nil { + bb.temp = make([]byte, 8) } + bb.order = order + + return bb } // CreateBytesBuffer create a BytesBuffer from data bytes @@ -78,6 +95,16 @@ func (b *BytesBuffer) WriteByte(c byte) { b.wpos++ } +// WriteString write a str string append the BytesBuffer, and the wpos will increase len(str) +func (b *BytesBuffer) WriteString(str string) { + l := len(str) + if len(b.buf) < b.wpos+l { + b.grow(l) + } + copy(b.buf[b.wpos:], str) + b.wpos += l +} + // Write write a byte array append the BytesBuffer, and the wpos will increase len(bytes) func (b *BytesBuffer) Write(bytes []byte) { l := len(bytes) @@ -262,3 +289,29 @@ func (b *BytesBuffer) Remain() int { return b.wpos - b.rpos } func (b *BytesBuffer) Len() int { return b.wpos - 0 } func (b *BytesBuffer) Cap() int { return cap(b.buf) } + +func hitDiscard() bool { + r := rand.New(rand.NewSource(time.Now().UnixNano())).Intn(100) + if float64(r)/100 < discardRatio { + return true + } + return false +} + +func AcquireBytesBuffer() *BytesBuffer { + b := bytesBufferPool.Get() + if b == nil { + return &BytesBuffer{} + } + return b.(*BytesBuffer) +} + +func ReleaseBytesBuffer(b *BytesBuffer) { + if b != nil { + //if cap(b.buf) > maxReuseBufSize && hitDiscard() { + // return + //} + b.Reset() + bytesBufferPool.Put(b) + } +} diff --git a/core/constants.go b/core/constants.go index 20ec7e76..c873cb63 100644 --- a/core/constants.go +++ b/core/constants.go @@ -129,3 +129,7 @@ const ( EUnkonwnMsg = 1003 EConvertMsg = 1004 ) + +const ( + DefaultDecodeLength = 100 +) diff --git a/core/map.go b/core/map.go index 4aa2bcf3..7434eb89 100644 --- a/core/map.go +++ b/core/map.go @@ -26,6 +26,15 @@ func (m *StringMap) Store(key, value string) { m.mu.Unlock() } +func (m *StringMap) Reset() { + //TODO: 这个地方是否应该加锁呢？ + m.mu.Lock() + for k := range m.innerMap { + delete(m.innerMap, k) + } + m.mu.Unlock() +} + func (m *StringMap) Delete(key string) { m.mu.Lock() delete(m.innerMap, key) @@ -48,17 +57,8 @@ func (m *StringMap) LoadOrEmpty(key string) string { // If f returns false, range stops the iteration func (m *StringMap) Range(f func(k, v string) bool) { m.mu.RLock() - keys := make([]string, 0, len(m.innerMap)) - for k := range m.innerMap { - keys = append(keys, k) - } - m.mu.RUnlock() - - for _, k := range keys { - v, ok := m.Load(k) - if !ok { - continue - } + defer m.mu.RUnlock() + for k, v := range m.innerMap { if !f(k, v) { break } diff --git a/core/motan.go b/core/motan.go index adfd9016..8c0903aa 100644 --- a/core/motan.go +++ b/core/motan.go @@ -14,7 +14,20 @@ import ( type taskHandler func() -var refreshTaskPool = make(chan taskHandler, 100) +var ( + refreshTaskPool = make(chan taskHandler, 100) + requestPool = sync.Pool{New: func() interface{} { + return &MotanRequest{ + RPCContext: &RPCContext{}, + Arguments: []interface{}{}, + } + }} + responsePool = sync.Pool{New: func() interface{} { + return &MotanResponse{ + RPCContext: &RPCContext{}, + } + }} +) func init() { go func() { @@ -28,10 +41,8 @@ func init() { } const ( - DefaultAttachmentSize = 16 - DefaultRPCContextMetaSize = 8 - - ProtocolLocal = "local" + DefaultAttachmentSize = 16 + ProtocolLocal = "local" ) var ( @@ -373,8 +384,6 @@ type RPCContext struct { AsyncCall bool Result *AsyncResult Reply interface{} - - Meta *StringMap // various time, it's owned by motan request context RequestSendTime time.Time RequestReceiveTime time.Time @@ -391,6 +400,44 @@ type RPCContext struct { RemoteAddr string // remote address } +func ResetRPCContext(c *RPCContext) { + if c != nil { + c.ExtFactory = nil + c.OriginalMessage = nil + c.Oneway = false + c.Proxy = false + c.GzipSize = 0 + c.BodySize = 0 + c.SerializeNum = 0 + c.Serialized = false + c.AsyncCall = false + c.Result = nil + c.Reply = nil + c.FinishHandlers = c.FinishHandlers[:0] + c.Tc = nil + c.IsMotanV1 = false + c.RemoteAddr = "" + } +} + +func (c *RPCContext) Reset() { + c.ExtFactory = nil + c.OriginalMessage = nil + c.Oneway = false + c.Proxy = false + c.GzipSize = 0 + c.BodySize = 0 + c.SerializeNum = 0 + c.Serialized = false + c.AsyncCall = false + c.Result = nil + c.Reply = nil + c.FinishHandlers = c.FinishHandlers[:0] + c.Tc = nil + c.IsMotanV1 = false + c.RemoteAddr = "" +} + func (c *RPCContext) AddFinishHandler(handler FinishHandler) { c.FinishHandlers = append(c.FinishHandlers, handler) } @@ -443,6 +490,34 @@ type MotanRequest struct { mu sync.Mutex } +func GetMotanRequestFromPool() *MotanRequest { + return requestPool.Get().(*MotanRequest) +} + +func PutMotanRequestBackPool(req *MotanRequest) { + if req != nil { + req.Method = "" + req.RequestID = 0 + req.ServiceName = "" + req.MethodDesc = "" + ResetRPCContext(req.RPCContext) + req.Attachment = nil + req.Arguments = req.Arguments[:0] + requestPool.Put(req) + } +} + +// Reset: Reset +func (m *MotanRequest) Reset() { + m.Method = "" + m.RequestID = 0 + m.ServiceName = "" + m.MethodDesc = "" + m.RPCContext.Reset() + m.Attachment = nil + m.Arguments = m.Arguments[:0] +} + // GetAttachment GetAttachment func (m *MotanRequest) GetAttachment(key string) string { if m.Attachment == nil { @@ -500,9 +575,7 @@ func (m *MotanRequest) GetAttachments() *StringMap { func (m *MotanRequest) GetRPCContext(canCreate bool) *RPCContext { if m.RPCContext == nil && canCreate { - m.RPCContext = &RPCContext{ - Meta: NewStringMap(DefaultRPCContextMetaSize), - } + m.RPCContext = &RPCContext{} } return m.RPCContext } @@ -529,7 +602,6 @@ func (m *MotanRequest) Clone() interface{} { AsyncCall: m.RPCContext.AsyncCall, Result: m.RPCContext.Result, Reply: m.RPCContext.Reply, - Meta: m.RPCContext.Meta, RequestSendTime: m.RPCContext.RequestSendTime, RequestReceiveTime: m.RPCContext.RequestReceiveTime, ResponseSendTime: m.RPCContext.ResponseSendTime, @@ -573,6 +645,32 @@ type MotanResponse struct { mu sync.Mutex } +func GetMotanResponseFromPool() *MotanResponse { + return responsePool.Get().(*MotanResponse) +} + +func PutMotanResponseBackPool(resp *MotanResponse) { + if resp != nil { + //resp.Reset() + resp.RequestID = 0 + resp.Value = nil + resp.Exception = nil + resp.ProcessTime = 0 + resp.Attachment = nil + ResetRPCContext(resp.RPCContext) + responsePool.Put(resp) + } +} + +func (m *MotanResponse) Reset() { + m.RequestID = 0 + m.Value = nil + m.Exception = nil + m.ProcessTime = 0 + m.Attachment = nil + m.RPCContext.Reset() +} + func (m *MotanResponse) GetAttachment(key string) string { if m.Attachment == nil { return "" @@ -618,9 +716,7 @@ func (m *MotanResponse) GetAttachments() *StringMap { func (m *MotanResponse) GetRPCContext(canCreate bool) *RPCContext { if m.RPCContext == nil && canCreate { - m.RPCContext = &RPCContext{ - Meta: NewStringMap(DefaultRPCContextMetaSize), - } + m.RPCContext = &RPCContext{} } return m.RPCContext } diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 220587a3..0c857bed 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -14,6 +14,12 @@ import ( "time" ) +var ( + streamPool = sync.Pool{New: func() interface{} { + return new(Stream) + }} +) + // MotanCommonEndpoint supports motan v1, v2 protocols type MotanCommonEndpoint struct { url *motan.URL @@ -351,11 +357,25 @@ type Stream struct { isClose atomic.Value // bool isHeartbeat bool // for heartbeat heartbeatVersion int // for heartbeat + sendTimer *time.Timer + recvTimer *time.Timer + release bool // concurrency issues for stream timeout and channel recv msg +} + +func (s *Stream) Reset() { + s.channel = nil + s.req = nil + s.res = nil + s.rc = nil } func (s *Stream) Send() (err error) { - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.sendTimer == nil { + s.sendTimer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.sendTimer.Reset(s.deadline.Sub(time.Now())) + } + defer s.sendTimer.Stop() var bytes []byte var msg *mpro.Message @@ -386,13 +406,15 @@ func (s *Stream) Send() (err error) { } } + ready := sendReady{} if msg != nil { // encode v2 message - bytes = msg.Encode().Bytes() + ready.bytesBuffer = msg.Encode() + bytes = ready.bytesBuffer.Bytes() } + ready.data = bytes if s.rc != nil && s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Encode, Addr: s.channel.address, Time: time.Now()}) } - ready := sendReady{data: bytes} select { case s.channel.sendCh <- ready: if s.rc != nil { @@ -403,7 +425,7 @@ func (s *Stream) Send() (err error) { } } return nil - case <-timer.C: + case <-s.sendTimer.C: return ErrSendRequestTimeout case <-s.channel.shutdownCh: return ErrChannelShutdown @@ -413,10 +435,16 @@ func (s *Stream) Send() (err error) { // Recv sync recv func (s *Stream) Recv() (motan.Response, error) { defer func() { - s.Close() + if s.Close() { + s.release = true + } }() - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.recvTimer == nil { + s.recvTimer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.recvTimer.Reset(s.deadline.Sub(time.Now())) + } + defer s.recvTimer.Stop() select { case <-s.recvNotifyCh: msg := s.res @@ -424,17 +452,16 @@ func (s *Stream) Recv() (motan.Response, error) { return nil, errors.New("recv err: recvMsg is nil") } return msg, nil - case <-timer.C: + case <-s.recvTimer.C: + s.release = false return nil, ErrRecvRequestTimeout case <-s.channel.shutdownCh: + s.release = false return nil, ErrChannelShutdown } } func (s *Stream) notify(msg interface{}, t time.Time) { - defer func() { - s.Close() - }() decodeTime := time.Now() var res motan.Response var v2Msg *mpro.Message @@ -481,6 +508,9 @@ func (s *Stream) notify(msg interface{}, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) } if s.rc.AsyncCall { + defer func() { + s.Close() + }() result := s.rc.Result if err != nil { result.Error = err @@ -507,15 +537,18 @@ func (c *Channel) NewStream(req motan.Request, rc *motan.RPCContext) (*Stream, e if c.IsClosed() { return nil, ErrChannelShutdown } - s := &Stream{ - streamId: GenerateRequestID(), - channel: c, - req: req, - recvNotifyCh: make(chan struct{}, 1), - deadline: time.Now().Add(defaultRequestTimeout), // default deadline - rc: rc, + s := AcquireStream() + s.streamId = GenerateRequestID() + s.channel = c + s.isHeartbeat = false + s.req = req + if s.recvNotifyCh == nil { + s.recvNotifyCh = make(chan struct{}, 1) } + s.deadline = time.Now().Add(defaultRequestTimeout) + s.rc = rc s.isClose.Store(false) + s.release = true c.streamLock.Lock() c.streams[s.streamId] = s c.streamLock.Unlock() @@ -526,34 +559,41 @@ func (c *Channel) NewHeartbeatStream(heartbeatVersion int) (*Stream, error) { if c.IsClosed() { return nil, ErrChannelShutdown } - s := &Stream{ - streamId: GenerateRequestID(), - channel: c, - isHeartbeat: true, - heartbeatVersion: heartbeatVersion, - recvNotifyCh: make(chan struct{}, 1), - deadline: time.Now().Add(defaultRequestTimeout), + s := AcquireStream() + s.streamId = GenerateRequestID() + s.channel = c + s.isHeartbeat = true + s.heartbeatVersion = heartbeatVersion + if s.recvNotifyCh == nil { + s.recvNotifyCh = make(chan struct{}, 1) } + s.deadline = time.Now().Add(defaultRequestTimeout) s.isClose.Store(false) + s.release = true c.heartbeatLock.Lock() c.heartbeats[s.streamId] = s c.heartbeatLock.Unlock() return s, nil } -func (s *Stream) Close() { - if !s.isClose.Load().(bool) { +func (s *Stream) Close() bool { + var exist bool + if !s.isClose.Swap(true).(bool) { if s.isHeartbeat { s.channel.heartbeatLock.Lock() - delete(s.channel.heartbeats, s.streamId) + if _, exist = s.channel.heartbeats[s.streamId]; exist { + delete(s.channel.heartbeats, s.streamId) + } s.channel.heartbeatLock.Unlock() } else { s.channel.streamLock.Lock() - delete(s.channel.streams, s.streamId) + if _, exist = s.channel.heartbeats[s.streamId]; exist { + delete(s.channel.streams, s.streamId) + } s.channel.streamLock.Unlock() } - s.isClose.Store(true) } + return exist } // Call send request to the server. @@ -564,6 +604,12 @@ func (c *Channel) Call(req motan.Request, deadline time.Duration, rc *motan.RPCC if err != nil { return nil, err } + defer func() { + if rc == nil || !rc.AsyncCall { + ReleaseStream(stream) + } + }() + stream.SetDeadline(deadline) err = stream.Send() if err != nil { @@ -580,6 +626,9 @@ func (c *Channel) HeartBeat(heartbeatVersion int) (motan.Response, error) { if err != nil { return nil, err } + defer func() { + ReleaseStream(stream) + }() err = stream.Send() if err != nil { return nil, err @@ -601,6 +650,7 @@ func (c *Channel) recv() { } func (c *Channel) recvLoop() error { + decodeBuf := make([]byte, motan.DefaultDecodeLength) for { v, err := mpro.CheckMotanVersion(c.bufRead) if err != nil { @@ -611,7 +661,7 @@ func (c *Channel) recvLoop() error { if v == mpro.Version1 { msg, t, err = mpro.ReadV1Message(c.bufRead, c.config.MaxContentLength) } else if v == mpro.Version2 { - msg, t, err = mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) + msg, t, err = mpro.DecodeWithTime(c.bufRead, &decodeBuf, c.config.MaxContentLength) } else { vlog.Warningf("unsupported motan version! version:%d con:%s.", v, c.conn.RemoteAddr().String()) err = mpro.ErrVersion @@ -647,10 +697,12 @@ func (c *Channel) handleMsg(msg interface{}, t time.Time) { if isHeartbeat { c.heartbeatLock.Lock() stream = c.heartbeats[rid] + delete(c.heartbeats, rid) c.heartbeatLock.Unlock() } else { c.streamLock.Lock() stream = c.streams[rid] + delete(c.streams, rid) c.streamLock.Unlock() } if stream == nil { @@ -680,6 +732,9 @@ func (c *Channel) send() { sent += n } } + if ready.bytesBuffer != nil { + motan.ReleaseBytesBuffer(ready.bytesBuffer) + } case <-c.shutdownCh: return } @@ -845,3 +900,18 @@ func buildChannel(conn net.Conn, config *ChannelConfig, serialization motan.Seri return channel } + +func AcquireStream() *Stream { + v := streamPool.Get() + if v == nil { + return &Stream{} + } + return v.(*Stream) +} + +func ReleaseStream(stream *Stream) { + if stream != nil && stream.release { + stream.Reset() + streamPool.Put(stream) + } +} diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 44c0ee50..90082b85 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -32,7 +32,10 @@ var ( defaultAsyncResponse = &motan.MotanResponse{Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), RPCContext: &motan.RPCContext{AsyncCall: true}} - errPanic = errors.New("panic error") + errPanic = errors.New("panic error") + v2StreamPool = sync.Pool{New: func() interface{} { + return new(V2Stream) + }} ) type MotanEndpoint struct { @@ -142,9 +145,8 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { m.recordErrAndKeepalive() return m.defaultErrMotanResponse(request, "motanEndpoint error: channels is null") } - startTime := time.Now().UnixNano() if rc.AsyncCall { - rc.Result.StartTime = startTime + rc.Result.StartTime = time.Now().UnixNano() } // get a channel channel, err := m.channels.Get() @@ -380,8 +382,9 @@ type V2Channel struct { } type V2Stream struct { - channel *V2Channel - sendMsg *mpro.Message + channel *V2Channel + sendMsg *mpro.Message + streamId uint64 // recv msg recvMsg *mpro.Message recvNotifyCh chan struct{} @@ -391,17 +394,31 @@ type V2Stream struct { rc *motan.RPCContext isClose atomic.Value // bool isHeartBeat bool + sendTimer *time.Timer + recvTimer *time.Timer + release bool // concurrency issues for stream timeout and channel recv msg +} + +func (s *V2Stream) Reset() { + s.channel = nil + s.sendMsg = nil + s.recvMsg = nil + s.rc = nil } func (s *V2Stream) Send() error { - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.sendTimer == nil { + s.sendTimer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.sendTimer.Reset(s.deadline.Sub(time.Now())) + } + defer s.sendTimer.Stop() buf := s.sendMsg.Encode() if s.rc != nil && s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Encode, Addr: s.channel.address, Time: time.Now()}) } - ready := sendReady{data: buf.Bytes()} + ready := sendReady{data: buf.Bytes(), bytesBuffer: buf} select { case s.channel.sendCh <- ready: if s.rc != nil { @@ -412,7 +429,7 @@ func (s *V2Stream) Send() error { } } return nil - case <-timer.C: + case <-s.sendTimer.C: return ErrSendRequestTimeout case <-s.channel.shutdownCh: return ErrChannelShutdown @@ -422,10 +439,16 @@ func (s *V2Stream) Send() error { // Recv sync recv func (s *V2Stream) Recv() (*mpro.Message, error) { defer func() { - s.Close() + if s.Close() { + s.release = true + } }() - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.recvTimer == nil { + s.recvTimer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.recvTimer.Reset(s.deadline.Sub(time.Now())) + } + defer s.recvTimer.Stop() select { case <-s.recvNotifyCh: msg := s.recvMsg @@ -433,17 +456,16 @@ func (s *V2Stream) Recv() (*mpro.Message, error) { return nil, errors.New("recv err: recvMsg is nil") } return msg, nil - case <-timer.C: + case <-s.recvTimer.C: + s.release = false return nil, ErrRecvRequestTimeout case <-s.channel.shutdownCh: + s.release = false return nil, ErrChannelShutdown } } func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { - defer func() { - s.Close() - }() if s.rc != nil { s.rc.ResponseReceiveTime = t if s.rc.Tc != nil { @@ -451,6 +473,9 @@ func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Decode, Time: time.Now()}) } if s.rc.AsyncCall { + defer func() { + s.Close() + }() msg.Header.SetProxy(s.rc.Proxy) result := s.rc.Result response, err := mpro.ConvertToResponse(msg, s.channel.serialization) @@ -487,16 +512,19 @@ func (c *V2Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Strea if c.IsClosed() { return nil, ErrChannelShutdown } - s := &V2Stream{ - channel: c, - sendMsg: msg, - recvNotifyCh: make(chan struct{}, 1), - deadline: time.Now().Add(1 * time.Second), - rc: rc, + + s := AcquireV2Stream() + s.channel = c + s.sendMsg = msg + if s.recvNotifyCh == nil { + s.recvNotifyCh = make(chan struct{}, 1) } + s.deadline = time.Now().Add(1 * time.Second) + s.rc = rc s.isClose.Store(false) // RequestID is communication identifier, it is own by channel msg.Header.RequestID = GenerateRequestID() + s.streamId = msg.Header.RequestID if msg.Header.IsHeartbeat() { c.heartbeatLock.Lock() c.heartbeats[msg.Header.RequestID] = s @@ -506,27 +534,35 @@ func (c *V2Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Strea c.streamLock.Lock() c.streams[msg.Header.RequestID] = s c.streamLock.Unlock() + s.isHeartBeat = false } + s.release = true return s, nil } -func (s *V2Stream) Close() { - if !s.isClose.Load().(bool) { +func (s *V2Stream) Close() bool { + var exist bool + if !s.isClose.Swap(true).(bool) { if s.isHeartBeat { s.channel.heartbeatLock.Lock() - delete(s.channel.heartbeats, s.sendMsg.Header.RequestID) + if _, exist = s.channel.heartbeats[s.streamId]; exist { + delete(s.channel.heartbeats, s.streamId) + } s.channel.heartbeatLock.Unlock() } else { s.channel.streamLock.Lock() - delete(s.channel.streams, s.sendMsg.Header.RequestID) + if _, exist = s.channel.heartbeats[s.streamId]; exist { + delete(s.channel.streams, s.streamId) + } s.channel.streamLock.Unlock() } - s.isClose.Store(true) } + return exist } type sendReady struct { - data []byte + data []byte + bytesBuffer *motan.BytesBuffer } func (c *V2Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RPCContext) (*mpro.Message, error) { @@ -534,6 +570,12 @@ func (c *V2Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RP if err != nil { return nil, err } + defer func() { + if rc == nil || !rc.AsyncCall { + ReleaseV2Stream(stream) + } + }() + stream.SetDeadline(deadline) if err := stream.Send(); err != nil { return nil, err @@ -558,8 +600,9 @@ func (c *V2Channel) recv() { } func (c *V2Channel) recvLoop() error { + readSlice := make([]byte, motan.DefaultDecodeLength) for { - res, t, err := mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) + res, t, err := mpro.DecodeWithTime(c.bufRead, &readSlice, c.config.MaxContentLength) if err != nil { return err } @@ -597,6 +640,7 @@ func (c *V2Channel) send() { sent += n } } + motan.ReleaseBytesBuffer(ready.bytesBuffer) case <-c.shutdownCh: return } @@ -606,6 +650,7 @@ func (c *V2Channel) send() { func (c *V2Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { c.heartbeatLock.Lock() stream := c.heartbeats[msg.Header.RequestID] + delete(c.heartbeats, msg.Header.RequestID) c.heartbeatLock.Unlock() if stream == nil { vlog.Warningf("handle heartbeat message, missing stream: %d, ep:%s", msg.Header.RequestID, c.address) @@ -618,6 +663,7 @@ func (c *V2Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { func (c *V2Channel) handleMessage(msg *mpro.Message, t time.Time) error { c.streamLock.Lock() stream := c.streams[msg.Header.RequestID] + delete(c.streams, msg.Header.RequestID) c.streamLock.Unlock() if stream == nil { vlog.Warningf("handle recv message, missing stream: %d, ep:%s", msg.Header.RequestID, c.address) @@ -799,3 +845,18 @@ func GetDefaultMotanEPAsynInit() bool { } return res.(bool) } + +func AcquireV2Stream() *V2Stream { + v := v2StreamPool.Get() + if v == nil { + return &V2Stream{} + } + return v.(*V2Stream) +} + +func ReleaseV2Stream(stream *V2Stream) { + if stream != nil && stream.release { + stream.Reset() + v2StreamPool.Put(stream) + } +} diff --git a/endpoint/motanEndpoint_test.go b/endpoint/motanEndpoint_test.go index 9bb0efe6..1fd71939 100644 --- a/endpoint/motanEndpoint_test.go +++ b/endpoint/motanEndpoint_test.go @@ -21,7 +21,7 @@ func TestMain(m *testing.M) { m.Run() } -//TODO more UT +// TODO more UT func TestGetName(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") @@ -269,7 +269,8 @@ func handle(netListen net.Listener) { func handleConnection(conn net.Conn, timeout int) { buf := bufio.NewReader(conn) - msg, _, err := protocol.DecodeWithTime(buf, 10*1024*1024) + readSlice := make([]byte, 100) + msg, _, err := protocol.DecodeWithTime(buf, &readSlice, 10*1024*1024) if err != nil { time.Sleep(time.Millisecond * 1000) conn.Close() diff --git a/filter/accessLog.go b/filter/accessLog.go index 06c1e6e7..ee980087 100644 --- a/filter/accessLog.go +++ b/filter/accessLog.go @@ -2,11 +2,10 @@ package filter import ( "encoding/json" - "strconv" - "time" - motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/log" + "strconv" + "time" ) const ( @@ -34,15 +33,17 @@ func (t *AccessLogFilter) NewFilter(url *motan.URL) motan.Filter { func (t *AccessLogFilter) Filter(caller motan.Caller, request motan.Request) motan.Response { role := defaultRole var ip string + var start time.Time switch caller.(type) { case motan.Provider: role = serverAgentRole ip = request.GetAttachment(motan.HostKey) + start = request.GetRPCContext(true).RequestReceiveTime case motan.EndPoint: role = clientAgentRole ip = caller.GetURL().Host + start = time.Now() } - start := time.Now() response := t.GetNext().Filter(caller, request) address := ip + ":" + caller.GetURL().GetPortStr() if _, ok := caller.(motan.Provider); ok { @@ -81,9 +82,6 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 // response code should be same as upstream responseCode := "" metaUpstreamCode, _ := response.GetAttachments().Load(motan.MetaUpstreamCode) - if resCtx.Meta != nil { - responseCode = resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) - } var exceptionData []byte if exception != nil { exceptionData, _ = json.Marshal(exception) @@ -94,21 +92,23 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 responseCode = "200" } } - vlog.AccessLog(&vlog.AccessLogEntity{ - FilterName: filterName, - Role: role, - RequestID: response.GetRequestID(), - Service: request.GetServiceName(), - Method: request.GetMethod(), - RemoteAddress: address, - Desc: request.GetMethodDesc(), - ReqSize: reqCtx.BodySize, - ResSize: resCtx.BodySize, - BizTime: response.GetProcessTime(), //ms - TotalTime: totalTime, //ms - ResponseCode: responseCode, - Success: exception == nil, - Exception: string(exceptionData), - UpstreamCode: metaUpstreamCode, - }) + + logEntity := vlog.AcquireAccessLogEntity() + logEntity.FilterName = filterName + logEntity.Role = role + logEntity.RequestID = response.GetRequestID() + logEntity.Service = request.GetServiceName() + logEntity.Method = request.GetMethod() + logEntity.RemoteAddress = address + logEntity.Desc = request.GetMethodDesc() + logEntity.ReqSize = reqCtx.BodySize + logEntity.ResSize = resCtx.BodySize + logEntity.BizTime = response.GetProcessTime() //ms + logEntity.TotalTime = totalTime //ms + logEntity.ResponseCode = responseCode + logEntity.Success = exception == nil + logEntity.Exception = string(exceptionData) + logEntity.UpstreamCode = metaUpstreamCode + + vlog.AccessLog(logEntity) } diff --git a/filter/clusterMetrics.go b/filter/clusterMetrics.go index bbbbcfbe..75a6e435 100644 --- a/filter/clusterMetrics.go +++ b/filter/clusterMetrics.go @@ -4,7 +4,6 @@ import ( "time" motan "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/metrics" "github.com/weibocom/motan-go/protocol" ) @@ -57,11 +56,11 @@ func (c *ClusterMetricsFilter) Filter(haStrategy motan.HaStrategy, loadBalance m if ctx != nil && ctx.Proxy { role = "motan-client-agent" } - key := metrics.Escape(role) + - ":" + metrics.Escape(request.GetAttachment(protocol.MSource)) + - ":" + metrics.Escape(request.GetMethod()) - addMetric(metrics.Escape(request.GetAttachment(protocol.MGroup))+".cluster", - metrics.Escape(request.GetAttachment(protocol.MPath)), - key, time.Since(start).Nanoseconds()/1e6, response) + //key := metrics.Escape(role) + + // ":" + metrics.Escape(request.GetAttachment(protocol.MSource)) + + // ":" + metrics.Escape(request.GetMethod()) + keys := []string{role, request.GetAttachment(protocol.MSource), request.GetMethod()} + addMetricWithKeys(request.GetAttachment(protocol.MGroup), ".cluster", + request.GetAttachment(protocol.MPath), keys, time.Since(start).Nanoseconds()/1e6, response) return response } diff --git a/filter/metrics.go b/filter/metrics.go index 848eac88..1bb0fdb4 100644 --- a/filter/metrics.go +++ b/filter/metrics.go @@ -1,11 +1,10 @@ package filter import ( - "time" - motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/metrics" "github.com/weibocom/motan-go/protocol" + "time" ) const ( @@ -57,7 +56,13 @@ func (m *MetricsFilter) GetNext() motan.EndPointFilter { } func (m *MetricsFilter) Filter(caller motan.Caller, request motan.Request) motan.Response { - start := time.Now() + var start time.Time + switch caller.(type) { + case motan.Provider: + start = request.GetRPCContext(true).RequestReceiveTime + case motan.EndPoint: + start = time.Now() + } response := m.GetNext().Filter(caller, request) proxy := false @@ -86,30 +91,30 @@ func (m *MetricsFilter) Filter(caller motan.Caller, request motan.Request) motan if provider { application = caller.GetURL().GetParam(motan.ApplicationKey, "") } - key := metrics.Escape(role) + - ":" + metrics.Escape(application) + - ":" + metrics.Escape(request.GetMethod()) - addMetric(metrics.Escape(request.GetAttachment(protocol.MGroup)), - metrics.Escape(request.GetAttachment(protocol.MPath)), - key, time.Since(start).Nanoseconds()/1e6, response) + //key := metrics.Escape(role) + + // ":" + metrics.Escape(application) + + // ":" + metrics.Escape(request.GetMethod()) + keys := []string{role, application, request.GetMethod()} + addMetricWithKeys(request.GetAttachment(protocol.MGroup), "", request.GetAttachment(protocol.MPath), + keys, time.Since(start).Nanoseconds()/1e6, response) return response } -func addMetric(group string, service string, key string, cost int64, response motan.Response) { - metrics.AddCounter(group, service, key+MetricsTotalCountSuffix, 1) //total_count - if response.GetException() != nil { //err_count +func addMetricWithKeys(group, groupSuffix string, service string, keys []string, cost int64, response motan.Response) { + metrics.AddCounterWithKeys(group, "", service, keys, MetricsTotalCountSuffix, 1) //total_count + if response.GetException() != nil { //err_count exception := response.GetException() if exception.ErrType == motan.BizException { - metrics.AddCounter(group, service, key+MetricsBizErrorCountSuffix, 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsBizErrorCountSuffix, 1) } else { - metrics.AddCounter(group, service, key+MetricsOtherErrorCountSuffix, 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsOtherErrorCountSuffix, 1) } } - metrics.AddCounter(group, service, key+metrics.ElapseTimeSuffix(cost), 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, metrics.ElapseTimeSuffix(cost), 1) if cost > 200 { - metrics.AddCounter(group, service, key+MetricsSlowCountSuffix, 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsSlowCountSuffix, 1) } - metrics.AddHistograms(group, service, key, cost) + metrics.AddHistogramsWithKeys(group, groupSuffix, service, keys, "", cost) } func (m *MetricsFilter) SetContext(context *motan.Context) { diff --git a/filter/metrics_test.go b/filter/metrics_test.go index f643882f..b0a9d18f 100644 --- a/filter/metrics_test.go +++ b/filter/metrics_test.go @@ -41,25 +41,28 @@ func TestMetricsFilter(t *testing.T) { name string caller motan.Caller request motan.Request - key string + keys []string }{ - {name: "proxyClient", caller: ep, request: request, key: "motan-client-agent:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, - {name: "proxyServer", caller: provider, request: request, key: "motan-server-agent:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, - {name: "Client", caller: ep, request: request2, key: "motan-client:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, - {name: "Server", caller: provider, request: request2, key: "motan-server:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, + {name: "proxyClient", caller: ep, request: request, keys: []string{"motan-client-agent", application, testMethod}}, + {name: "proxyServer", caller: provider, request: request, keys: []string{"motan-server-agent", application, testMethod}}, + {name: "Client", caller: ep, request: request2, keys: []string{"motan-client", application, testMethod}}, + {name: "Server", caller: provider, request: request2, keys: []string{"motan-server", application, testMethod}}, + } + var getKeysStr = func(keys []string) string { + return metrics.Escape(keys[0]) + ":" + metrics.Escape(keys[1]) + ":" + metrics.Escape(keys[2]) } for _, test := range tests { t.Run(test.name, func(t *testing.T) { mf.Filter(test.caller, test.request) time.Sleep(10 * time.Millisecond) // The metrics filter has do escape - assert.Equal(t, 1, int(metrics.GetStatItem(metrics.Escape(testGroup), metrics.Escape(testService)).SnapshotAndClear().Count(test.key+MetricsTotalCountSuffix)), "metric count") + assert.Equal(t, 1, int(metrics.GetStatItem(testGroup, testService).SnapshotAndClear().Count(getKeysStr(test.keys)+MetricsTotalCountSuffix)), "metric count") }) } } func TestAddMetric(t *testing.T) { - key := "motan-client-agent:testApplication:" + testMethod + keys := []string{"motan-client-agent", "testApplication", testMethod} factory := initFactory() mf := factory.GetFilter(Metrics).(motan.EndPointFilter) mf.(*MetricsFilter).SetContext(&motan.Context{Config: config.NewConfig()}) @@ -67,7 +70,9 @@ func TestAddMetric(t *testing.T) { response2 := &motan.MotanResponse{ProcessTime: 100, Exception: &motan.Exception{ErrType: motan.BizException}} response3 := &motan.MotanResponse{ProcessTime: 100, Exception: &motan.Exception{ErrType: motan.FrameworkException}} response4 := &motan.MotanResponse{ProcessTime: 1000} - + var getKeysStr = func(keys []string) string { + return metrics.Escape(keys[0]) + ":" + metrics.Escape(keys[1]) + ":" + metrics.Escape(keys[2]) + } tests := []struct { name string response motan.Response @@ -81,11 +86,11 @@ func TestAddMetric(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - addMetric(testGroup, testService, key, test.response.GetProcessTime(), test.response) + addMetricWithKeys(testGroup, "", testService, keys, test.response.GetProcessTime(), test.response) time.Sleep(10 * time.Millisecond) snap := metrics.GetStatItem(testGroup, testService).SnapshotAndClear() for _, k := range test.keys { - assert.True(t, snap.Count(key+k) > 0, fmt.Sprintf("key '%s'", k)) + assert.True(t, snap.Count(getKeysStr(keys)+k) > 0, fmt.Sprintf("key '%s'", k)) } }) } diff --git a/ha/backupRequestHA.go b/ha/backupRequestHA.go index d50c6e5b..892e3a5f 100644 --- a/ha/backupRequestHA.go +++ b/ha/backupRequestHA.go @@ -67,7 +67,7 @@ func (br *BackupRequestHA) Call(request motan.Request, loadBalance motan.LoadBal successCh := make(chan motan.Response, retries+1) if delay <= 0 { //no delay time configuration // TODO: we should use metrics of the cluster, with traffic control the group may changed - item := metrics.GetStatItem(metrics.Escape(request.GetAttachment(protocol.MGroup)), metrics.Escape(request.GetAttachment(protocol.MPath))) + item := metrics.GetStatItem(request.GetAttachment(protocol.MGroup), request.GetAttachment(protocol.MPath)) if item == nil || item.LastSnapshot() == nil { initDelay := int(br.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), "backupRequestInitDelayTime", 0)) if initDelay == 0 { diff --git a/log/bytes.go b/log/bytes.go new file mode 100644 index 00000000..2e6d6a7a --- /dev/null +++ b/log/bytes.go @@ -0,0 +1,82 @@ +package vlog + +import ( + "strconv" + "sync" +) + +var ( + initSize = 256 + innerBytesBufferPool = sync.Pool{New: func() interface{} { + return &innerBytesBuffer{buf: make([]byte, 0, initSize)} + }} +) + +// innerBytesBuffer is a variable-sized buffer of bytes with Write methods. +type innerBytesBuffer struct { + buf []byte // reuse +} + +// newInnerBytesBuffer create an empty innerBytesBuffer with initial size +func newInnerBytesBuffer() *innerBytesBuffer { + return acquireBytesBuffer() +} + +func createInnerBytesBuffer(data []byte) *innerBytesBuffer { + return &innerBytesBuffer{ + buf: data, + } +} + +// WriteString write a str string append the innerBytesBuffer +func (b *innerBytesBuffer) WriteString(str string) { + b.buf = append(b.buf, str...) +} + +// WriteBoolString append the string value of v(true/false) to innerBytesBuffer +func (b *innerBytesBuffer) WriteBoolString(v bool) { + if v { + b.WriteString("true") + } else { + b.WriteString("false") + } +} + +// WriteUint64String append the string value of u to innerBytesBuffer +func (b *innerBytesBuffer) WriteUint64String(u uint64) { + b.buf = strconv.AppendUint(b.buf, u, 10) +} + +// WriteInt64String append the string value of i to innerBytesBuffer +func (b *innerBytesBuffer) WriteInt64String(i int64) { + b.buf = strconv.AppendInt(b.buf, i, 10) +} + +func (b *innerBytesBuffer) Bytes() []byte { return b.buf } + +func (b *innerBytesBuffer) String() string { + return string(b.buf) +} + +func (b *innerBytesBuffer) Reset() { + b.buf = b.buf[:0] +} + +func (b *innerBytesBuffer) Len() int { return len(b.buf) } + +func (b *innerBytesBuffer) Cap() int { return cap(b.buf) } + +func acquireBytesBuffer() *innerBytesBuffer { + b := innerBytesBufferPool.Get() + if b == nil { + return &innerBytesBuffer{buf: make([]byte, 0, 256)} + } + return b.(*innerBytesBuffer) +} + +func releaseBytesBuffer(b *innerBytesBuffer) { + if b != nil { + b.Reset() + innerBytesBufferPool.Put(b) + } +} diff --git a/log/bytes_test.go b/log/bytes_test.go new file mode 100644 index 00000000..df2e598b --- /dev/null +++ b/log/bytes_test.go @@ -0,0 +1,104 @@ +package vlog + +import ( + "strconv" + "testing" +) + +func TestWrite(t *testing.T) { + // new BytesBuffer + buf := newInnerBytesBuffer() + if buf.Len() != 0 { + t.Errorf("new buf length not zero.") + } + if buf.Cap() != initSize { + t.Errorf("buf cap not correct.real:%d, expect:%d\n", buf.Cap(), initSize) + } + + // write string + buf.Reset() + buf.WriteString("string1") + buf.WriteString("string2") + tempbytes := buf.Bytes() + if "string1" != string(tempbytes[:7]) { + t.Errorf("write string not correct.buf:%+v\n", buf) + } + if "string2" != string(tempbytes[7:14]) { + t.Errorf("write string not correct.buf:%+v\n", buf) + } + + // write bool string + buf.Reset() + buf.WriteBoolString(true) + buf.WriteBoolString(false) + tempbytes = buf.Bytes() + if "true" != string(tempbytes[:4]) { + t.Errorf("write bool string not correct.buf:%+v\n", buf) + } + if "false" != string(tempbytes[4:9]) { + t.Errorf("write bool string not correct.buf:%+v\n", buf) + } + + // write uint64 string + buf.Reset() + var u1 uint64 = 11111111 + var u2 uint64 = 22222222 + buf.WriteUint64String(u1) + buf.WriteUint64String(u2) + tempbytes = buf.Bytes() + if "11111111" != string(tempbytes[:8]) { + t.Errorf("write unit64 string not correct.buf:%+v\n", buf) + } + if "22222222" != string(tempbytes[8:]) { + t.Errorf("write uint64 string not correct.buf:%+v\n", buf) + } + + // write int64 string + buf.Reset() + var i1 int64 = 11111111 + var i2 int64 = -22222222 + buf.WriteInt64String(i1) + buf.WriteInt64String(i2) + tempbytes = buf.Bytes() + if "11111111" != string(tempbytes[:8]) { + t.Errorf("write unit64 string not correct.buf:%+v\n", buf) + } + if "-22222222" != string(tempbytes[8:]) { + t.Errorf("write uint64 string not correct.buf:%+v\n", buf) + } +} + +func TestRead(t *testing.T) { + buf := newInnerBytesBuffer() + string1 := "aaaaaaaaaaaa" + buf.WriteString(string1) + buf.WriteUint64String(uint64(len(string1))) + buf.WriteBoolString(false) + buf.WriteInt64String(int64(-len(string1))) + + string2 := "bbbbbbbbbbbb" + buf.WriteString(string2) + buf.WriteUint64String(uint64(len(string2))) + buf.WriteBoolString(true) + buf.WriteInt64String(int64(-len(string2))) + + data := buf.Bytes() + buf2 := createInnerBytesBuffer(data) + rsize := len(string1) + 2 + 5 + 3 + len(string2) + 2 + 4 + 3 + if buf2.Len() != rsize { + t.Errorf("read buf len not correct. buf:%v\n", buf2) + } + + // read value + expectValue := string1 + + strconv.Itoa(len(string1)) + + "false" + + "-" + strconv.Itoa(len(string1)) + + string2 + + strconv.Itoa(len(string2)) + + "true" + + "-" + strconv.Itoa(len(string2)) + if expectValue != buf2.String() { + t.Errorf("read value not correct. buf:%v\n", buf2) + } +} diff --git a/log/log.go b/log/log.go index d1d484bb..c675691f 100644 --- a/log/log.go +++ b/log/log.go @@ -1,14 +1,12 @@ package vlog import ( - "bytes" "flag" "github.com/weibocom/motan-go/metrics/sampler" "log" "os" "path/filepath" "runtime/debug" - "strconv" "sync" "sync/atomic" "time" @@ -18,6 +16,9 @@ import ( ) var ( + accessLogEntityPool = sync.Pool{New: func() interface{} { + return new(AccessLogEntity) + }} loggerInstance Logger once sync.Once logDir = flag.String("log_dir", ".", "If non-empty, write log files in this directory") @@ -390,12 +391,13 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { zap.String("exception", logObject.Exception), zap.String("upstreamCode", logObject.UpstreamCode)) } else { - var buffer bytes.Buffer + buffer := newInnerBytesBuffer() + buffer.WriteString(logObject.FilterName) buffer.WriteString("|") buffer.WriteString(logObject.Role) buffer.WriteString("|") - buffer.WriteString(strconv.FormatUint(logObject.RequestID, 10)) + buffer.WriteUint64String(logObject.RequestID) buffer.WriteString("|") buffer.WriteString(logObject.Service) buffer.WriteString("|") @@ -405,15 +407,15 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { buffer.WriteString("|") buffer.WriteString(logObject.RemoteAddress) buffer.WriteString("|") - buffer.WriteString(strconv.Itoa(logObject.ReqSize)) + buffer.WriteInt64String(int64(logObject.ReqSize)) buffer.WriteString("|") - buffer.WriteString(strconv.Itoa(logObject.ResSize)) + buffer.WriteInt64String(int64(logObject.ResSize)) buffer.WriteString("|") - buffer.WriteString(strconv.FormatInt(logObject.BizTime, 10)) + buffer.WriteInt64String(logObject.BizTime) buffer.WriteString("|") - buffer.WriteString(strconv.FormatInt(logObject.TotalTime, 10)) + buffer.WriteInt64String(logObject.TotalTime) buffer.WriteString("|") - buffer.WriteString(strconv.FormatBool(logObject.Success)) + buffer.WriteBoolString(logObject.Success) buffer.WriteString("|") buffer.WriteString(logObject.ResponseCode) buffer.WriteString("|") @@ -421,7 +423,10 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { buffer.WriteString("|") buffer.WriteString(logObject.UpstreamCode) d.accessLogger.Info(buffer.String()) + + releaseBytesBuffer(buffer) } + ReleaseAccessLogEntity(logObject) } func (d *defaultLogger) MetricsLog(msg string) { @@ -490,3 +495,17 @@ func (d *defaultLogger) SetMetricsLogAvailable(status bool) { d.metricsLevel.SetLevel(zapcore.Level(defaultLogLevel + 1)) } } + +func AcquireAccessLogEntity() *AccessLogEntity { + v := accessLogEntityPool.Get() + if v == nil { + return &AccessLogEntity{} + } + return v.(*AccessLogEntity) +} + +func ReleaseAccessLogEntity(entity *AccessLogEntity) { + if entity != nil { + accessLogEntityPool.Put(entity) + } +} diff --git a/manageHandler.go b/manageHandler.go index 98fd4208..991877bb 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -158,7 +158,7 @@ func (s *StatusHandler) getStatus() []byte { exporter := v.(motan.Exporter) group := exporter.GetURL().Group service := exporter.GetURL().Path - statItem := metrics.GetStatItem(metrics.Escape(group), metrics.Escape(service)) + statItem := metrics.GetStatItem(group, service) if statItem == nil { return true } diff --git a/metrics/graphite.go b/metrics/graphite.go index 93f80cb5..07b4bbf3 100644 --- a/metrics/graphite.go +++ b/metrics/graphite.go @@ -102,19 +102,21 @@ func GenGraphiteMessages(localIP string, snapshots []Snapshot) []string { if len(pni) < minKeyLength { return } + escapedService := snap.GetEscapedService() + escapedGroup := snap.GetEscapedGroup() if snap.IsHistogram(k) { //histogram for slaK, slaV := range sla { segment += fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s.%s:%.2f|kv\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], slaK, snap.Percentile(k, slaV)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], slaK, snap.Percentile(k, slaV)) } segment += fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s.%s:%.2f|ms\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], "avg_time", snap.Mean(k)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], "avg_time", snap.Mean(k)) } else if snap.IsCounter(k) { //counter segment = fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s:%d|c\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], snap.Count(k)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], snap.Count(k)) } else { // gauge segment = fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s:%d|kv\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], snap.Value(k)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], snap.Value(k)) } if buf.Len() > 0 && buf.Len()+len(segment) > messageMaxLen { messages = append(messages, buf.String()) diff --git a/metrics/metrics.go b/metrics/metrics.go index 574917b1..6757bfc8 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -42,6 +42,7 @@ const ( ) var ( + metricsKeyBuilderBufferSize = 64 // NewStatItem is the factory func for StatItem NewStatItem = NewDefaultStatItem items = make(map[string]StatItem, 64) @@ -52,15 +53,20 @@ var ( processor: defaultEventProcessor, //sink processor size eventBus: make(chan *event, eventBufferSize), writers: make(map[string]StatWriter), - evtBuf: &sync.Pool{New: func() interface{} { return new(event) }}, + evtBuf: &sync.Pool{New: func() interface{} { + return &event{} + }}, } + escapeCache sync.Map ) type StatItem interface { SetService(service string) GetService() string + GetEscapedService() string SetGroup(group string) GetGroup() string + GetEscapedGroup() string AddCounter(key string, value int64) AddHistograms(key string, duration int64) AddGauge(key string, value int64) @@ -98,17 +104,18 @@ type StatWriter interface { } func GetOrRegisterStatItem(group string, service string) StatItem { + k := group + service itemsLock.RLock() - item := items[group+service] + item := items[k] itemsLock.RUnlock() if item != nil { return item } itemsLock.Lock() - item = items[group+service] + item = items[k] if item == nil { item = NewStatItem(group, service) - items[group+service] = item + items[k] = item } itemsLock.Unlock() return item @@ -165,14 +172,20 @@ func StatItemSize() int { return len(items) } +// Escape the string avoid invalid graphite key func Escape(s string) string { - return strings.Map(func(char rune) rune { + if v, ok := escapeCache.Load(s); ok { + return v.(string) + } + v := strings.Map(func(char rune) rune { if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || (char == '-') { return char } else { return '_' } }, s) + escapeCache.Store(s, v) + return v } func AddCounter(group string, service string, key string, value int64) { @@ -187,13 +200,27 @@ func AddGauge(group string, service string, key string, value int64) { sendEvent(eventGauge, group, service, key, value) } +func AddCounterWithKeys(group, groupSuffix string, service string, keys []string, keySuffix string, value int64) { + sendEventWithKeys(eventCounter, group, groupSuffix, service, keys, keySuffix, value) +} + +func AddHistogramsWithKeys(group, groupSuffix string, service string, keys []string, suffix string, duration int64) { + sendEventWithKeys(eventHistograms, group, groupSuffix, service, keys, suffix, duration) +} + func sendEvent(eventType int32, group string, service string, key string, value int64) { + sendEventWithKeys(eventType, group, "", service, []string{key}, "", value) +} + +func sendEventWithKeys(eventType int32, group, groupSuffix string, service string, keys []string, suffix string, value int64) { evt := rp.evtBuf.Get().(*event) evt.event = eventType - evt.key = key + evt.keys = keys evt.group = group evt.service = service evt.value = value + evt.keySuffix = suffix + evt.groupSuffix = groupSuffix select { case rp.eventBus <- evt: default: @@ -239,11 +266,46 @@ func startSampleStatus(application string) { } type event struct { - event int32 - key string - group string - service string - value int64 + event int32 + keys []string + keySuffix string + group string + groupSuffix string + service string + value int64 +} + +func (s *event) reset() { + s.event = 0 + s.keys = s.keys[:0] + s.keySuffix = "" + s.group = "" + s.service = "" + s.value = 0 + s.groupSuffix = "" +} + +func (s *event) getGroup() string { + if s.groupSuffix == "" { + return s.group + } + return s.group + s.groupSuffix +} + +func (s *event) getMetricKey() string { + keyBuilder := motan.NewBytesBuffer(metricsKeyBuilderBufferSize) + defer motan.ReleaseBytesBuffer(keyBuilder) + l := len(s.keys) + for idx, k := range s.keys { + keyBuilder.WriteString(Escape(k)) + if idx < l-1 { + keyBuilder.WriteString(":") + } + } + if s.keySuffix != "" { + keyBuilder.WriteString(s.keySuffix) + } + return string(keyBuilder.Bytes()) } type RegistryHolder struct { @@ -270,7 +332,9 @@ func (d *DefaultStatItem) SetService(service string) { func (d *DefaultStatItem) GetService() string { return d.service } - +func (d *DefaultStatItem) GetEscapedService() string { + return Escape(d.service) +} func (d *DefaultStatItem) SetGroup(group string) { d.group = group } @@ -279,6 +343,10 @@ func (d *DefaultStatItem) GetGroup() string { return d.group } +func (d *DefaultStatItem) GetEscapedGroup() string { + return Escape(d.group) +} + func (d *DefaultStatItem) AddCounter(key string, value int64) { c := d.getRegistry().Get(key) if c == nil { @@ -521,6 +589,10 @@ func (d *ReadonlyStatItem) GetService() string { return d.service } +func (d *ReadonlyStatItem) GetEscapedService() string { + return Escape(d.service) +} + func (d *ReadonlyStatItem) SetGroup(group string) { panic("action not supported") } @@ -529,6 +601,10 @@ func (d *ReadonlyStatItem) GetGroup() string { return d.group } +func (d *ReadonlyStatItem) GetEscapedGroup() string { + return Escape(d.group) +} + func (d *ReadonlyStatItem) AddCounter(key string, value int64) { panic("action not supported") } @@ -732,6 +808,9 @@ type reporter struct { func (r *reporter) eventLoop() { for evt := range r.eventBus { r.processEvent(evt) + // clean the event object before put it back + evt.reset() + r.evtBuf.Put(evt) } } @@ -746,14 +825,15 @@ func (r *reporter) addWriter(key string, sw StatWriter) { func (r *reporter) processEvent(evt *event) { defer motan.HandlePanic(nil) - item := GetOrRegisterStatItem(evt.group, evt.service) + item := GetOrRegisterStatItem(evt.getGroup(), evt.service) + key := evt.getMetricKey() switch evt.event { case eventCounter: - item.AddCounter(evt.key, evt.value) + item.AddCounter(key, evt.value) case eventHistograms: - item.AddHistograms(evt.key, evt.value) + item.AddHistograms(key, evt.value) case eventGauge: - item.AddGauge(evt.key, evt.value) + item.AddGauge(key, evt.value) } } diff --git a/protocol/motanProtocol.go b/protocol/motanProtocol.go index 0e1e413e..180fb021 100644 --- a/protocol/motanProtocol.go +++ b/protocol/motanProtocol.go @@ -22,7 +22,7 @@ const ( DefaultMaxContentLength = 10 * 1024 * 1024 ) -//message type +// message type const ( Req = iota Res @@ -60,6 +60,24 @@ type Header struct { RequestID uint64 } +func (h *Header) Reset() { + h.Magic = 0 + h.MsgType = 0 + h.VersionStatus = 0 + h.Serialize = 0 + h.RequestID = 0 +} + +func ResetHeader(h *Header) { + if h != nil { + h.Magic = 0 + h.MsgType = 0 + h.VersionStatus = 0 + h.Serialize = 0 + h.RequestID = 0 + } +} + func (h *Header) Clone() *Header { return &Header{ Magic: h.Magic, @@ -77,7 +95,7 @@ type Message struct { Type int } -//serialize +// serialize const ( Hessian = iota GrpcPb @@ -104,6 +122,9 @@ var ( writeBufPool = &sync.Pool{New: func() interface{} { // for gzip write buffer return &bytes.Buffer{} }} + messagePool = sync.Pool{New: func() interface{} { + return &Message{Metadata: motan.NewStringMap(DefaultMetaSize), Header: &Header{}} + }} ) // errors @@ -266,9 +287,9 @@ func (msg *Message) Encode() (buf *motan.BytesBuffer) { vlog.Errorf("metadata not correct.k:%s, v:%s", k, v) return true } - metabuf.Write([]byte(k)) + metabuf.WriteString(k) metabuf.WriteByte('\n') - metabuf.Write([]byte(v)) + metabuf.WriteString(v) metabuf.WriteByte('\n') return true }) @@ -291,6 +312,7 @@ func (msg *Message) Encode() (buf *motan.BytesBuffer) { if metasize > 0 { buf.Write(metabuf.Bytes()) } + motan.ReleaseBytesBuffer(metabuf) // encode body buf.WriteUint32(uint32(bodysize)) @@ -300,6 +322,13 @@ func (msg *Message) Encode() (buf *motan.BytesBuffer) { return buf } +func (msg *Message) Reset() { + msg.Type = 0 + msg.Body = msg.Body[:0] + msg.Header.Reset() + msg.Metadata.Reset() +} + func (msg *Message) Clone() interface{} { newMessage := &Message{ Header: msg.Header.Clone(), @@ -326,93 +355,105 @@ func CheckMotanVersion(buf *bufio.Reader) (version int, err error) { return int(b[3] >> 3 & 0x1f), nil } -func Decode(buf *bufio.Reader) (msg *Message, err error) { - msg, _, err = DecodeWithTime(buf, motan.DefaultMaxContentLength) +func Decode(buf *bufio.Reader, readSlice *[]byte) (msg *Message, err error) { + msg, _, err = DecodeWithTime(buf, readSlice, motan.DefaultMaxContentLength) return msg, err } -func DecodeWithTime(buf *bufio.Reader, maxContentLength int) (msg *Message, start time.Time, err error) { - temp := make([]byte, HeaderLength, HeaderLength) - +func DecodeWithTime(buf *bufio.Reader, rs *[]byte, maxContentLength int) (msg *Message, start time.Time, err error) { + readSlice := *rs // decode header - _, err = io.ReadAtLeast(buf, temp, HeaderLength) + _, err = io.ReadAtLeast(buf, readSlice[:HeaderLength], HeaderLength) start = time.Now() // record time when starting to read data if err != nil { return nil, start, err } - mn := binary.BigEndian.Uint16(temp[:2]) // TODO 不再验证 + mn := binary.BigEndian.Uint16(readSlice[:2]) // TODO 不再验证 if mn != MotanMagic { vlog.Errorf("wrong magic num:%d, err:%v", mn, err) return nil, start, ErrMagicNum } - - header := &Header{Magic: MotanMagic} - header.MsgType = temp[2] - header.VersionStatus = temp[3] - version := header.GetVersion() + msg = messagePool.Get().(*Message) + msg.Header.Magic = MotanMagic + msg.Header.MsgType = readSlice[2] + msg.Header.VersionStatus = readSlice[3] + version := msg.Header.GetVersion() if version != Version2 { // TODO 不再验证 vlog.Errorf("unsupported protocol version number: %d", version) return nil, start, ErrVersion } - header.Serialize = temp[4] - header.RequestID = binary.BigEndian.Uint64(temp[5:]) + msg.Header.Serialize = readSlice[4] + msg.Header.RequestID = binary.BigEndian.Uint64(readSlice[5:HeaderLength]) // decode meta - _, err = io.ReadAtLeast(buf, temp[:4], 4) + _, err = io.ReadAtLeast(buf, readSlice[:4], 4) if err != nil { + PutMessageBackToPool(msg) return nil, start, err } - metasize := int(binary.BigEndian.Uint32(temp[:4])) + metasize := int(binary.BigEndian.Uint32(readSlice[:4])) if metasize > maxContentLength { vlog.Errorf("meta over size. meta size:%d, max size:%d", metasize, maxContentLength) + PutMessageBackToPool(msg) return nil, start, ErrOverSize } - metamap := motan.NewStringMap(DefaultMetaSize) if metasize > 0 { - metadata, err := readBytes(buf, metasize) + if cap(readSlice) < metasize { + readSlice = make([]byte, metasize) + *rs = readSlice + } + err := readBytes(buf, readSlice, metasize) if err != nil { + PutMessageBackToPool(msg) return nil, start, err } s, e := 0, 0 var k string for i := 0; i <= metasize; i++ { - if i == metasize || metadata[i] == '\n' { + if i == metasize || readSlice[i] == '\n' { e = i if k == "" { - k = string(metadata[s:e]) + k = string(readSlice[s:e]) } else { - metamap.Store(k, string(metadata[s:e])) + msg.Metadata.Store(k, string(readSlice[s:e])) k = "" } s = i + 1 } } if k != "" { - vlog.Errorf("decode message fail, metadata not paired. header:%v, meta:%s", header, metadata) + vlog.Errorf("decode message fail, metadata not paired. header:%v, meta:%s", msg.Header, readSlice) + PutMessageBackToPool(msg) return nil, start, ErrMetadata } } //decode body - _, err = io.ReadAtLeast(buf, temp[:4], 4) + _, err = io.ReadAtLeast(buf, readSlice[:4], 4) if err != nil { + PutMessageBackToPool(msg) return nil, start, err } - bodysize := int(binary.BigEndian.Uint32(temp[:4])) + bodysize := int(binary.BigEndian.Uint32(readSlice[:4])) if bodysize > maxContentLength { vlog.Errorf("body over size. body size:%d, max size:%d", bodysize, maxContentLength) + PutMessageBackToPool(msg) return nil, start, ErrOverSize } - var body []byte + if bodysize > 0 { - body, err = readBytes(buf, bodysize) + if cap(msg.Body) < bodysize { + msg.Body = make([]byte, bodysize) + } + msg.Body = msg.Body[:bodysize] + err = readBytes(buf, msg.Body, bodysize) } else { - body = make([]byte, 0) + msg.Body = make([]byte, 0) } if err != nil { + PutMessageBackToPool(msg) return nil, start, err } - msg = &Message{header, metamap, body, Req} return msg, start, err } @@ -425,15 +466,14 @@ func DecodeGzipBody(body []byte) []byte { return ret } -func readBytes(buf *bufio.Reader, size int) ([]byte, error) { - tempbytes := make([]byte, size) +func readBytes(buf *bufio.Reader, readSlice []byte, size int) error { var s, n = 0, 0 var err error for s < size && err == nil { - n, err = buf.Read(tempbytes[s:]) + n, err = buf.Read(readSlice[s:size]) s += n } - return tempbytes, err + return err } func EncodeGzip(data []byte) ([]byte, error) { @@ -523,7 +563,7 @@ func DecodeGzip(data []byte) (ret []byte, err error) { // ConvertToRequest convert motan2 protocol request message to motan Request func ConvertToRequest(request *Message, serialize motan.Serialization) (motan.Request, error) { - motanRequest := &motan.MotanRequest{Arguments: make([]interface{}, 0)} + motanRequest := motan.GetMotanRequestFromPool() motanRequest.RequestID = request.Header.RequestID if idStr, ok := request.Metadata.Load(MRequestID); !ok { if request.Header.IsProxy() { @@ -549,12 +589,16 @@ func ConvertToRequest(request *Message, serialize motan.Serialization) (motan.Re request.Header.SetGzip(false) } if !rc.Proxy && serialize == nil { + motan.PutMotanRequestBackPool(motanRequest) return nil, ErrSerializeNil } - dv := &motan.DeserializableValue{Body: request.Body, Serialization: serialize} - motanRequest.Arguments = []interface{}{dv} + if len(motanRequest.Arguments) <= 0 { + motanRequest.Arguments = []interface{}{&motan.DeserializableValue{Body: request.Body, Serialization: serialize}} + } else { + motanRequest.Arguments[0].(*motan.DeserializableValue).Body = request.Body + motanRequest.Arguments[0].(*motan.DeserializableValue).Serialization = serialize + } } - return motanRequest, nil } @@ -634,7 +678,7 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) } } - res := &Message{} + res := messagePool.Get().(*Message) var msgType int if response.GetException() != nil { msgType = Exception @@ -660,11 +704,13 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) res.Body = b } else { vlog.Warningf("convert response value fail! serialized value not []byte. res:%+v", response) + PutMessageBackToPool(res) return nil, ErrSerializedData } } else { b, err := serialize.Serialize(response.GetValue()) if err != nil { + PutMessageBackToPool(res) return nil, err } res.Body = b @@ -683,7 +729,7 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) // ConvertToResponse convert protocol response to motan Response func ConvertToResponse(response *Message, serialize motan.Serialization) (motan.Response, error) { - mres := &motan.MotanResponse{} + mres := motan.GetMotanResponseFromPool() rc := mres.GetRPCContext(true) rc.Proxy = response.Header.IsProxy() mres.RequestID = response.Header.RequestID @@ -699,6 +745,7 @@ func ConvertToResponse(response *Message, serialize motan.Serialization) (motan. response.Header.SetGzip(false) } if !rc.Proxy && serialize == nil { + motan.PutMotanResponseBackPool(mres) return nil, ErrSerializeNil } dv := &motan.DeserializableValue{Body: response.Body, Serialization: serialize} @@ -710,6 +757,7 @@ func ConvertToResponse(response *Message, serialize motan.Serialization) (motan. var exception *motan.Exception err := json.Unmarshal([]byte(e), &exception) if err != nil { + motan.PutMotanResponseBackPool(mres) return nil, err } mres.Exception = exception @@ -732,3 +780,14 @@ func ExceptionToJSON(e *motan.Exception) string { errmsg, _ := json.Marshal(e) return string(errmsg) } + +func PutMessageBackToPool(msg *Message) { + if msg != nil { + //msg.Reset() + msg.Type = 0 + msg.Body = msg.Body[:0] + ResetHeader(msg.Header) + msg.Metadata.Reset() + messagePool.Put(msg) + } +} diff --git a/protocol/motanProtocol_test.go b/protocol/motanProtocol_test.go index 060ae0cb..acc41513 100644 --- a/protocol/motanProtocol_test.go +++ b/protocol/motanProtocol_test.go @@ -5,8 +5,10 @@ import ( "bytes" "compress/gzip" "fmt" + "github.com/stretchr/testify/assert" "github.com/weibocom/motan-go/serialize" "math/rand" + "strconv" "sync" "sync/atomic" "testing" @@ -145,7 +147,11 @@ func TestEncode(t *testing.T) { ebytes := msg.Encode() fmt.Println("len:", ebytes.Len()) - newMsg, err := Decode(bufio.NewReader(ebytes)) + readSlice := make([]byte, 100) + //for i := 0; i < len(readSlice); i++ { + // readSlice[i] = 't' + //} + newMsg, err := Decode(bufio.NewReader(ebytes), &readSlice) if newMsg == nil { t.Fatalf("encode message fail") } @@ -164,7 +170,7 @@ func TestEncode(t *testing.T) { msg.Header.SetGzip(true) msg.Body, _ = EncodeGzip([]byte("gzip encode")) b := msg.Encode() - newMsg, _ = Decode(bufio.NewReader(b)) + newMsg, _ = Decode(bufio.NewReader(b), &readSlice) // should not decode gzip if !newMsg.Header.IsGzip() { t.Fatalf("encode message fail") @@ -177,13 +183,106 @@ func TestEncode(t *testing.T) { assertTrue(string(nb) == "gzip encode", "body", t) } +func TestPool(t *testing.T) { + h := &Header{} + h.SetVersion(Version2) + h.SetStatus(6) + h.SetOneWay(true) + h.SetSerialize(5) + h.SetGzip(true) + h.SetHeartbeat(true) + h.SetProxy(true) + h.SetRequest(true) + h.Magic = MotanMagic + h.RequestID = 2349789 + meta := core.NewStringMap(0) + for mi := 0; mi < 10000; mi++ { + meta.Store(strconv.Itoa(mi), strconv.Itoa(mi)) + } + body := []byte("testbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbody") + msg := &Message{Header: h, Metadata: meta, Body: body} + ebytes := msg.Encode() + + fmt.Println("len:", ebytes.Len()) + readSlice := make([]byte, 100) + newMsg, err := Decode(bufio.NewReader(ebytes), &readSlice) + if newMsg == nil { + t.Fatalf("encode message fail") + } + assertTrue(newMsg.Header.IsOneWay(), "oneway", t) + assertTrue(newMsg.Header.IsGzip(), "gzip", t) + assertTrue(newMsg.Header.IsHeartbeat(), "heartbeat", t) + assertTrue(newMsg.Header.IsProxy(), "proxy", t) + assertTrue(newMsg.Header.isRequest(), "request", t) + assertTrue(newMsg.Header.GetVersion() == Version2, "version", t) + assertTrue(newMsg.Header.GetSerialize() == 5, "serialize", t) + assertTrue(newMsg.Header.GetStatus() == 6, "status", t) + assertTrue(newMsg.Metadata.LoadOrEmpty("1") == "1", "meta", t) + assertTrue(cap(readSlice) > 200, "readSlice", t) + assertTrue(len(newMsg.Body) == len(msg.Body), "body", t) + assert.Nil(t, err) + PutMessageBackToPool(newMsg) + body1 := []byte("testbody") + msg1 := &Message{Header: h, Metadata: meta, Body: body1} + ebytes1 := msg1.Encode() + newMsg, err = Decode(bufio.NewReader(ebytes1), &readSlice) + if newMsg == nil { + t.Fatalf("encode message fail") + } + assertTrue(newMsg.Header.IsOneWay(), "oneway", t) + assertTrue(newMsg.Header.IsGzip(), "gzip", t) + assertTrue(newMsg.Header.IsHeartbeat(), "heartbeat", t) + assertTrue(newMsg.Header.IsProxy(), "proxy", t) + assertTrue(newMsg.Header.isRequest(), "request", t) + assertTrue(newMsg.Header.GetVersion() == Version2, "version", t) + assertTrue(newMsg.Header.GetSerialize() == 5, "serialize", t) + assertTrue(newMsg.Header.GetStatus() == 6, "status", t) + assertTrue(newMsg.Metadata.LoadOrEmpty("1") == "1", "meta", t) + assertTrue(cap(readSlice) > 200, "readSlice", t) + assertTrue(len(newMsg.Body) == len(msg1.Body), "body", t) +} + func assertTrue(b bool, msg string, t *testing.T) { if !b { t.Fatalf("test fail, %s not correct.", msg) } } -//TODO convert +func TestConvertToResponse(t *testing.T) { + h := &Header{} + h.SetVersion(Version2) + h.SetStatus(6) + h.SetOneWay(true) + h.SetSerialize(6) + h.SetGzip(true) + h.SetHeartbeat(true) + h.SetProxy(true) + h.SetRequest(true) + h.Magic = MotanMagic + h.RequestID = 2349789 + meta := core.NewStringMap(0) + meta.Store("k1", "v1") + meta.Store(MGroup, "group") + meta.Store(MMethod, "method") + meta.Store(MPath, "path") + body := []byte("testbody") + msg := &Message{Header: h, Metadata: meta, Body: body} + + pMap := make(map[string]string) + for i := 0; i < 10000; i++ { + resp, err := ConvertToResponse(msg, &serialize.SimpleSerialization{}) + assertTrue(err == nil, "conver to request err", t) + assertTrue(resp.GetAttachment(MGroup) == "group", "response group", t) + assertTrue(resp.GetAttachment(MMethod) == "method", "response method", t) + assertTrue(resp.GetAttachment(MPath) == "path", "response path", t) + //assertTrue(resp.GetValue().(string) == "testbody", "response body", t) + pMap[fmt.Sprintf("%p", resp)] = "1" + core.PutMotanResponseBackPool(resp.(*core.MotanResponse)) + } + assert.True(t, len(pMap) < 10000) +} + +// TODO convert func TestConvertToRequest(t *testing.T) { h := &Header{} h.SetVersion(Version2) @@ -203,12 +302,19 @@ func TestConvertToRequest(t *testing.T) { meta.Store(MPath, "path") body := []byte("testbody") msg := &Message{Header: h, Metadata: meta, Body: body} - req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) - assertTrue(err == nil, "conver to request err", t) - assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) - assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) - assertTrue(req.GetAttachment(MPath) == "path", "request path", t) + pMap := make(map[string]string) + for i := 0; i < 10000; i++ { + req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) + assertTrue(err == nil, "conver to request err", t) + assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) + assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) + assertTrue(req.GetAttachment(MPath) == "path", "request path", t) + pMap[fmt.Sprintf("%p", req)] = "1" + core.PutMotanRequestBackPool(req.(*core.MotanRequest)) + } + assert.True(t, len(pMap) < 10000) + req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) // test request clone cloneReq := req.Clone().(core.Request) assertTrue(err == nil, "conver to request err", t) diff --git a/provider/httpProvider.go b/provider/httpProvider.go index f19cfe69..fdf34119 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -512,7 +512,5 @@ func fillException(resp *motan.MotanResponse, start int64, err error) { } func updateUpstreamStatusCode(resp *motan.MotanResponse, statusCode int) { - resCtx := resp.GetRPCContext(true) resp.SetAttachment(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) - resCtx.Meta.Store(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) } diff --git a/server/motanserver.go b/server/motanserver.go index 29fbfc04..5eecb3a2 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -151,7 +151,7 @@ func (m *MotanServer) handleConn(conn net.Conn) { } else { ip = getRemoteIP(conn.RemoteAddr().String()) } - + decodeBuf := make([]byte, motan.DefaultDecodeLength) for { v, err := mpro.CheckMotanVersion(buf) if err != nil { @@ -168,7 +168,7 @@ func (m *MotanServer) handleConn(conn net.Conn) { } go m.processV1(v1Msg, t, ip, conn) } else if v == mpro.Version2 { - msg, t, err := mpro.DecodeWithTime(buf, m.maxContextLength) + msg, t, err := mpro.DecodeWithTime(buf, &decodeBuf, m.maxContextLength) if err != nil { vlog.Warningf("decode motan v2 message fail! con:%s, err:%s.", conn.RemoteAddr().String(), err.Error()) break @@ -219,7 +219,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c tc.PutReqSpan(&motan.Span{Name: motan.Convert, Time: time.Now()}) req.GetRPCContext(true).Tc = tc } - callStart := time.Now() + mres = m.handler.Call(req) if tc != nil { // clusterFilter end @@ -229,7 +229,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c resCtx := mres.GetRPCContext(true) resCtx.Proxy = m.proxy if mres.GetAttachment(mpro.MProcessTime) == "" { - mres.SetAttachment(mpro.MProcessTime, strconv.FormatInt(int64(time.Now().Sub(callStart)/1e6), 10)) + mres.SetAttachment(mpro.MProcessTime, strconv.FormatInt(int64(time.Now().Sub(start)/1e6), 10)) } res, err = mpro.ConvertToResMessage(mres, serialization) if tc != nil { @@ -247,6 +247,8 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c // recover the communication identifier res.Header.RequestID = lastRequestID resBuf := res.Encode() + // reuse BytesBuffer + defer motan.ReleaseBytesBuffer(resBuf) if tc != nil { tc.PutResSpan(&motan.Span{Name: motan.Encode, Time: time.Now()}) } @@ -268,6 +270,16 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c if tc != nil { tc.PutResSpan(&motan.Span{Name: motan.Send, Time: resSendTime}) } + // 回收message + mpro.PutMessageBackToPool(msg) + mpro.PutMessageBackToPool(res) + // 回收request + if motanReq, ok := mreq.(*motan.MotanRequest); ok { + motan.PutMotanRequestBackPool(motanReq) + } + if motanResp, ok := mres.(*motan.MotanResponse); ok { + motan.PutMotanResponseBackPool(motanResp) + } } func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip string, conn net.Conn) { From 64558b2ba17d1e680fbf9e0e92ec71884cd016b7 Mon Sep 17 00:00:00 2001 From: snail007 Date: Wed, 6 Dec 2023 17:19:00 +0800 Subject: [PATCH 70/75] Revert "Profile base to dev (#350)" (#351) This reverts commit 3cf5d101d799da17f2e7cb50408ce58e582fd4f9. --- agent.go | 51 ++++-------- core/bytes.go | 59 +------------ core/constants.go | 4 - core/map.go | 22 ++--- core/motan.go | 124 ++++----------------------- endpoint/motanCommonEndpoint.go | 134 +++++++----------------------- endpoint/motanEndpoint.go | 119 +++++++------------------- endpoint/motanEndpoint_test.go | 5 +- filter/accessLog.go | 48 +++++------ filter/clusterMetrics.go | 13 +-- filter/metrics.go | 39 ++++----- filter/metrics_test.go | 25 +++--- ha/backupRequestHA.go | 2 +- log/bytes.go | 82 ------------------ log/bytes_test.go | 104 ----------------------- log/log.go | 37 ++------- manageHandler.go | 2 +- metrics/graphite.go | 10 +-- metrics/metrics.go | 112 ++++--------------------- protocol/motanProtocol.go | 143 ++++++++++---------------------- protocol/motanProtocol_test.go | 122 ++------------------------- provider/httpProvider.go | 2 + server/motanserver.go | 20 +---- 23 files changed, 252 insertions(+), 1027 deletions(-) delete mode 100644 log/bytes.go delete mode 100644 log/bytes_test.go diff --git a/agent.go b/agent.go index 541d8365..17884ad4 100644 --- a/agent.go +++ b/agent.go @@ -45,11 +45,6 @@ var ( setAgentLock sync.Mutex notFoundProviderCount int64 = 0 defaultInitClusterTimeout int64 = 10000 //ms - clusterSlicePool = sync.Pool{ - New: func() interface{} { - return make([]serviceMapItem, 0, 5) - }, - } ) type Agent struct { @@ -799,28 +794,32 @@ func (a *agentMessageHandler) Call(request motan.Request) (res motan.Response) { } return res } -func (a *agentMessageHandler) fillMatch(typ, cond, key string, data []serviceMapItem, f func(u *motan.URL) string, match *[]serviceMapItem) error { +func (a *agentMessageHandler) matchRule(typ, cond, key string, data []serviceMapItem, f func(u *motan.URL) string) (foundClusters []serviceMapItem, err error) { if cond == "" { - return fmt.Errorf("empty %s is not supported", typ) + err = fmt.Errorf("empty %s is not supported", typ) + return } for _, item := range data { if f(item.url) == cond { - *match = append(*match, item) + foundClusters = append(foundClusters, item) } } - if len(*match) == 0 { - return fmt.Errorf("cluster not found. cluster:%s", key) + if len(foundClusters) == 0 { + err = fmt.Errorf("cluster not found. cluster:%s", key) + return } - return nil + return } func (a *agentMessageHandler) findCluster(request motan.Request) (c *cluster.MotanCluster, key string, err error) { service := request.GetServiceName() group := request.GetAttachment(mpro.MGroup) version := request.GetAttachment(mpro.MVersion) protocol := request.GetAttachment(mpro.MProxyProtocol) + reqInfo := fmt.Sprintf("request information: {service: %s, group: %s, protocol: %s, version: %s}", + service, group, protocol, version) serviceItemArrI, exists := a.agent.serviceMap.Load(service) if !exists { - err = fmt.Errorf("cluster not found. cluster:%s, %s", service, getReqInfo(service, group, protocol, version)) + err = fmt.Errorf("cluster not found. cluster:%s, %s", service, reqInfo) return } search := []struct { @@ -833,31 +832,23 @@ func (a *agentMessageHandler) findCluster(request motan.Request) (c *cluster.Mot {"protocol", protocol, func(u *motan.URL) string { return u.Protocol }}, {"version", version, func(u *motan.URL) string { return u.GetParam(motan.VersionKey, "") }}, } - clusters := serviceItemArrI.([]serviceMapItem) - matched := clusterSlicePool.Get().([]serviceMapItem) - if cap(matched) < len(clusters) { - matched = make([]serviceMapItem, 0, len(clusters)) - } + foundClusters := serviceItemArrI.([]serviceMapItem) for i, rule := range search { if i == 0 { key = rule.cond } else { key += "_" + rule.cond } - err = a.fillMatch(rule.tip, rule.cond, key, clusters, rule.condFn, &matched) + foundClusters, err = a.matchRule(rule.tip, rule.cond, key, foundClusters, rule.condFn) if err != nil { - putBackClusterSlice(matched) return } - if len(matched) == 1 { - c = matched[0].cluster - putBackClusterSlice(matched) + if len(foundClusters) == 1 { + c = foundClusters[0].cluster return } - matched = matched[:0] } - err = fmt.Errorf("less condition to select cluster, maybe this service belongs to multiple group, protocol, version; cluster: %s, %s", key, getReqInfo(service, group, protocol, version)) - putBackClusterSlice(matched) + err = fmt.Errorf("less condition to select cluster, maybe this service belongs to multiple group, protocol, version; cluster: %s, %s", key, reqInfo) return } @@ -1231,11 +1222,6 @@ func urlExist(url *motan.URL, urls map[string]*motan.URL) bool { return false } -func getReqInfo(service, group, protocol, version string) string { - return fmt.Sprintf("request information: {service: %s, group: %s, protocol: %s, version: %s}", - service, group, protocol, version) -} - func (a *Agent) SubscribeService(url *motan.URL) error { if urlExist(url, a.Context.RefersURLs) { return fmt.Errorf("url exist, ignore subscribe, url: %s", url.GetIdentity()) @@ -1265,8 +1251,3 @@ func (a *Agent) UnexportService(url *motan.URL) error { } return nil } - -func putBackClusterSlice(s []serviceMapItem) { - s = s[:0] - clusterSlicePool.Put(s) -} diff --git a/core/bytes.go b/core/bytes.go index 457ff2d3..604c6d57 100644 --- a/core/bytes.go +++ b/core/bytes.go @@ -4,17 +4,6 @@ import ( "encoding/binary" "errors" "io" - "math/rand" - "sync" - "time" -) - -var ( - maxReuseBufSize = 204800 - discardRatio = 0.1 - bytesBufferPool = sync.Pool{New: func() interface{} { - return new(BytesBuffer) - }} ) // BytesBuffer is a variable-sized buffer of bytes with Read and Write methods. @@ -37,16 +26,10 @@ func NewBytesBuffer(initsize int) *BytesBuffer { // NewBytesBufferWithOrder create a empty BytesBuffer with initial size and byte order func NewBytesBufferWithOrder(initsize int, order binary.ByteOrder) *BytesBuffer { - bb := AcquireBytesBuffer() - if bb.buf == nil { - bb.buf = make([]byte, initsize) - } - if bb.temp == nil { - bb.temp = make([]byte, 8) + return &BytesBuffer{buf: make([]byte, initsize), + order: order, + temp: make([]byte, 8), } - bb.order = order - - return bb } // CreateBytesBuffer create a BytesBuffer from data bytes @@ -95,16 +78,6 @@ func (b *BytesBuffer) WriteByte(c byte) { b.wpos++ } -// WriteString write a str string append the BytesBuffer, and the wpos will increase len(str) -func (b *BytesBuffer) WriteString(str string) { - l := len(str) - if len(b.buf) < b.wpos+l { - b.grow(l) - } - copy(b.buf[b.wpos:], str) - b.wpos += l -} - // Write write a byte array append the BytesBuffer, and the wpos will increase len(bytes) func (b *BytesBuffer) Write(bytes []byte) { l := len(bytes) @@ -289,29 +262,3 @@ func (b *BytesBuffer) Remain() int { return b.wpos - b.rpos } func (b *BytesBuffer) Len() int { return b.wpos - 0 } func (b *BytesBuffer) Cap() int { return cap(b.buf) } - -func hitDiscard() bool { - r := rand.New(rand.NewSource(time.Now().UnixNano())).Intn(100) - if float64(r)/100 < discardRatio { - return true - } - return false -} - -func AcquireBytesBuffer() *BytesBuffer { - b := bytesBufferPool.Get() - if b == nil { - return &BytesBuffer{} - } - return b.(*BytesBuffer) -} - -func ReleaseBytesBuffer(b *BytesBuffer) { - if b != nil { - //if cap(b.buf) > maxReuseBufSize && hitDiscard() { - // return - //} - b.Reset() - bytesBufferPool.Put(b) - } -} diff --git a/core/constants.go b/core/constants.go index c873cb63..20ec7e76 100644 --- a/core/constants.go +++ b/core/constants.go @@ -129,7 +129,3 @@ const ( EUnkonwnMsg = 1003 EConvertMsg = 1004 ) - -const ( - DefaultDecodeLength = 100 -) diff --git a/core/map.go b/core/map.go index 7434eb89..4aa2bcf3 100644 --- a/core/map.go +++ b/core/map.go @@ -26,15 +26,6 @@ func (m *StringMap) Store(key, value string) { m.mu.Unlock() } -func (m *StringMap) Reset() { - //TODO: 这个地方是否应该加锁呢？ - m.mu.Lock() - for k := range m.innerMap { - delete(m.innerMap, k) - } - m.mu.Unlock() -} - func (m *StringMap) Delete(key string) { m.mu.Lock() delete(m.innerMap, key) @@ -57,8 +48,17 @@ func (m *StringMap) LoadOrEmpty(key string) string { // If f returns false, range stops the iteration func (m *StringMap) Range(f func(k, v string) bool) { m.mu.RLock() - defer m.mu.RUnlock() - for k, v := range m.innerMap { + keys := make([]string, 0, len(m.innerMap)) + for k := range m.innerMap { + keys = append(keys, k) + } + m.mu.RUnlock() + + for _, k := range keys { + v, ok := m.Load(k) + if !ok { + continue + } if !f(k, v) { break } diff --git a/core/motan.go b/core/motan.go index 8c0903aa..adfd9016 100644 --- a/core/motan.go +++ b/core/motan.go @@ -14,20 +14,7 @@ import ( type taskHandler func() -var ( - refreshTaskPool = make(chan taskHandler, 100) - requestPool = sync.Pool{New: func() interface{} { - return &MotanRequest{ - RPCContext: &RPCContext{}, - Arguments: []interface{}{}, - } - }} - responsePool = sync.Pool{New: func() interface{} { - return &MotanResponse{ - RPCContext: &RPCContext{}, - } - }} -) +var refreshTaskPool = make(chan taskHandler, 100) func init() { go func() { @@ -41,8 +28,10 @@ func init() { } const ( - DefaultAttachmentSize = 16 - ProtocolLocal = "local" + DefaultAttachmentSize = 16 + DefaultRPCContextMetaSize = 8 + + ProtocolLocal = "local" ) var ( @@ -384,6 +373,8 @@ type RPCContext struct { AsyncCall bool Result *AsyncResult Reply interface{} + + Meta *StringMap // various time, it's owned by motan request context RequestSendTime time.Time RequestReceiveTime time.Time @@ -400,44 +391,6 @@ type RPCContext struct { RemoteAddr string // remote address } -func ResetRPCContext(c *RPCContext) { - if c != nil { - c.ExtFactory = nil - c.OriginalMessage = nil - c.Oneway = false - c.Proxy = false - c.GzipSize = 0 - c.BodySize = 0 - c.SerializeNum = 0 - c.Serialized = false - c.AsyncCall = false - c.Result = nil - c.Reply = nil - c.FinishHandlers = c.FinishHandlers[:0] - c.Tc = nil - c.IsMotanV1 = false - c.RemoteAddr = "" - } -} - -func (c *RPCContext) Reset() { - c.ExtFactory = nil - c.OriginalMessage = nil - c.Oneway = false - c.Proxy = false - c.GzipSize = 0 - c.BodySize = 0 - c.SerializeNum = 0 - c.Serialized = false - c.AsyncCall = false - c.Result = nil - c.Reply = nil - c.FinishHandlers = c.FinishHandlers[:0] - c.Tc = nil - c.IsMotanV1 = false - c.RemoteAddr = "" -} - func (c *RPCContext) AddFinishHandler(handler FinishHandler) { c.FinishHandlers = append(c.FinishHandlers, handler) } @@ -490,34 +443,6 @@ type MotanRequest struct { mu sync.Mutex } -func GetMotanRequestFromPool() *MotanRequest { - return requestPool.Get().(*MotanRequest) -} - -func PutMotanRequestBackPool(req *MotanRequest) { - if req != nil { - req.Method = "" - req.RequestID = 0 - req.ServiceName = "" - req.MethodDesc = "" - ResetRPCContext(req.RPCContext) - req.Attachment = nil - req.Arguments = req.Arguments[:0] - requestPool.Put(req) - } -} - -// Reset: Reset -func (m *MotanRequest) Reset() { - m.Method = "" - m.RequestID = 0 - m.ServiceName = "" - m.MethodDesc = "" - m.RPCContext.Reset() - m.Attachment = nil - m.Arguments = m.Arguments[:0] -} - // GetAttachment GetAttachment func (m *MotanRequest) GetAttachment(key string) string { if m.Attachment == nil { @@ -575,7 +500,9 @@ func (m *MotanRequest) GetAttachments() *StringMap { func (m *MotanRequest) GetRPCContext(canCreate bool) *RPCContext { if m.RPCContext == nil && canCreate { - m.RPCContext = &RPCContext{} + m.RPCContext = &RPCContext{ + Meta: NewStringMap(DefaultRPCContextMetaSize), + } } return m.RPCContext } @@ -602,6 +529,7 @@ func (m *MotanRequest) Clone() interface{} { AsyncCall: m.RPCContext.AsyncCall, Result: m.RPCContext.Result, Reply: m.RPCContext.Reply, + Meta: m.RPCContext.Meta, RequestSendTime: m.RPCContext.RequestSendTime, RequestReceiveTime: m.RPCContext.RequestReceiveTime, ResponseSendTime: m.RPCContext.ResponseSendTime, @@ -645,32 +573,6 @@ type MotanResponse struct { mu sync.Mutex } -func GetMotanResponseFromPool() *MotanResponse { - return responsePool.Get().(*MotanResponse) -} - -func PutMotanResponseBackPool(resp *MotanResponse) { - if resp != nil { - //resp.Reset() - resp.RequestID = 0 - resp.Value = nil - resp.Exception = nil - resp.ProcessTime = 0 - resp.Attachment = nil - ResetRPCContext(resp.RPCContext) - responsePool.Put(resp) - } -} - -func (m *MotanResponse) Reset() { - m.RequestID = 0 - m.Value = nil - m.Exception = nil - m.ProcessTime = 0 - m.Attachment = nil - m.RPCContext.Reset() -} - func (m *MotanResponse) GetAttachment(key string) string { if m.Attachment == nil { return "" @@ -716,7 +618,9 @@ func (m *MotanResponse) GetAttachments() *StringMap { func (m *MotanResponse) GetRPCContext(canCreate bool) *RPCContext { if m.RPCContext == nil && canCreate { - m.RPCContext = &RPCContext{} + m.RPCContext = &RPCContext{ + Meta: NewStringMap(DefaultRPCContextMetaSize), + } } return m.RPCContext } diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 0c857bed..220587a3 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -14,12 +14,6 @@ import ( "time" ) -var ( - streamPool = sync.Pool{New: func() interface{} { - return new(Stream) - }} -) - // MotanCommonEndpoint supports motan v1, v2 protocols type MotanCommonEndpoint struct { url *motan.URL @@ -357,25 +351,11 @@ type Stream struct { isClose atomic.Value // bool isHeartbeat bool // for heartbeat heartbeatVersion int // for heartbeat - sendTimer *time.Timer - recvTimer *time.Timer - release bool // concurrency issues for stream timeout and channel recv msg -} - -func (s *Stream) Reset() { - s.channel = nil - s.req = nil - s.res = nil - s.rc = nil } func (s *Stream) Send() (err error) { - if s.sendTimer == nil { - s.sendTimer = time.NewTimer(s.deadline.Sub(time.Now())) - } else { - s.sendTimer.Reset(s.deadline.Sub(time.Now())) - } - defer s.sendTimer.Stop() + timer := time.NewTimer(s.deadline.Sub(time.Now())) + defer timer.Stop() var bytes []byte var msg *mpro.Message @@ -406,15 +386,13 @@ func (s *Stream) Send() (err error) { } } - ready := sendReady{} if msg != nil { // encode v2 message - ready.bytesBuffer = msg.Encode() - bytes = ready.bytesBuffer.Bytes() + bytes = msg.Encode().Bytes() } - ready.data = bytes if s.rc != nil && s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Encode, Addr: s.channel.address, Time: time.Now()}) } + ready := sendReady{data: bytes} select { case s.channel.sendCh <- ready: if s.rc != nil { @@ -425,7 +403,7 @@ func (s *Stream) Send() (err error) { } } return nil - case <-s.sendTimer.C: + case <-timer.C: return ErrSendRequestTimeout case <-s.channel.shutdownCh: return ErrChannelShutdown @@ -435,16 +413,10 @@ func (s *Stream) Send() (err error) { // Recv sync recv func (s *Stream) Recv() (motan.Response, error) { defer func() { - if s.Close() { - s.release = true - } + s.Close() }() - if s.recvTimer == nil { - s.recvTimer = time.NewTimer(s.deadline.Sub(time.Now())) - } else { - s.recvTimer.Reset(s.deadline.Sub(time.Now())) - } - defer s.recvTimer.Stop() + timer := time.NewTimer(s.deadline.Sub(time.Now())) + defer timer.Stop() select { case <-s.recvNotifyCh: msg := s.res @@ -452,16 +424,17 @@ func (s *Stream) Recv() (motan.Response, error) { return nil, errors.New("recv err: recvMsg is nil") } return msg, nil - case <-s.recvTimer.C: - s.release = false + case <-timer.C: return nil, ErrRecvRequestTimeout case <-s.channel.shutdownCh: - s.release = false return nil, ErrChannelShutdown } } func (s *Stream) notify(msg interface{}, t time.Time) { + defer func() { + s.Close() + }() decodeTime := time.Now() var res motan.Response var v2Msg *mpro.Message @@ -508,9 +481,6 @@ func (s *Stream) notify(msg interface{}, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) } if s.rc.AsyncCall { - defer func() { - s.Close() - }() result := s.rc.Result if err != nil { result.Error = err @@ -537,18 +507,15 @@ func (c *Channel) NewStream(req motan.Request, rc *motan.RPCContext) (*Stream, e if c.IsClosed() { return nil, ErrChannelShutdown } - s := AcquireStream() - s.streamId = GenerateRequestID() - s.channel = c - s.isHeartbeat = false - s.req = req - if s.recvNotifyCh == nil { - s.recvNotifyCh = make(chan struct{}, 1) + s := &Stream{ + streamId: GenerateRequestID(), + channel: c, + req: req, + recvNotifyCh: make(chan struct{}, 1), + deadline: time.Now().Add(defaultRequestTimeout), // default deadline + rc: rc, } - s.deadline = time.Now().Add(defaultRequestTimeout) - s.rc = rc s.isClose.Store(false) - s.release = true c.streamLock.Lock() c.streams[s.streamId] = s c.streamLock.Unlock() @@ -559,41 +526,34 @@ func (c *Channel) NewHeartbeatStream(heartbeatVersion int) (*Stream, error) { if c.IsClosed() { return nil, ErrChannelShutdown } - s := AcquireStream() - s.streamId = GenerateRequestID() - s.channel = c - s.isHeartbeat = true - s.heartbeatVersion = heartbeatVersion - if s.recvNotifyCh == nil { - s.recvNotifyCh = make(chan struct{}, 1) + s := &Stream{ + streamId: GenerateRequestID(), + channel: c, + isHeartbeat: true, + heartbeatVersion: heartbeatVersion, + recvNotifyCh: make(chan struct{}, 1), + deadline: time.Now().Add(defaultRequestTimeout), } - s.deadline = time.Now().Add(defaultRequestTimeout) s.isClose.Store(false) - s.release = true c.heartbeatLock.Lock() c.heartbeats[s.streamId] = s c.heartbeatLock.Unlock() return s, nil } -func (s *Stream) Close() bool { - var exist bool - if !s.isClose.Swap(true).(bool) { +func (s *Stream) Close() { + if !s.isClose.Load().(bool) { if s.isHeartbeat { s.channel.heartbeatLock.Lock() - if _, exist = s.channel.heartbeats[s.streamId]; exist { - delete(s.channel.heartbeats, s.streamId) - } + delete(s.channel.heartbeats, s.streamId) s.channel.heartbeatLock.Unlock() } else { s.channel.streamLock.Lock() - if _, exist = s.channel.heartbeats[s.streamId]; exist { - delete(s.channel.streams, s.streamId) - } + delete(s.channel.streams, s.streamId) s.channel.streamLock.Unlock() } + s.isClose.Store(true) } - return exist } // Call send request to the server. @@ -604,12 +564,6 @@ func (c *Channel) Call(req motan.Request, deadline time.Duration, rc *motan.RPCC if err != nil { return nil, err } - defer func() { - if rc == nil || !rc.AsyncCall { - ReleaseStream(stream) - } - }() - stream.SetDeadline(deadline) err = stream.Send() if err != nil { @@ -626,9 +580,6 @@ func (c *Channel) HeartBeat(heartbeatVersion int) (motan.Response, error) { if err != nil { return nil, err } - defer func() { - ReleaseStream(stream) - }() err = stream.Send() if err != nil { return nil, err @@ -650,7 +601,6 @@ func (c *Channel) recv() { } func (c *Channel) recvLoop() error { - decodeBuf := make([]byte, motan.DefaultDecodeLength) for { v, err := mpro.CheckMotanVersion(c.bufRead) if err != nil { @@ -661,7 +611,7 @@ func (c *Channel) recvLoop() error { if v == mpro.Version1 { msg, t, err = mpro.ReadV1Message(c.bufRead, c.config.MaxContentLength) } else if v == mpro.Version2 { - msg, t, err = mpro.DecodeWithTime(c.bufRead, &decodeBuf, c.config.MaxContentLength) + msg, t, err = mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) } else { vlog.Warningf("unsupported motan version! version:%d con:%s.", v, c.conn.RemoteAddr().String()) err = mpro.ErrVersion @@ -697,12 +647,10 @@ func (c *Channel) handleMsg(msg interface{}, t time.Time) { if isHeartbeat { c.heartbeatLock.Lock() stream = c.heartbeats[rid] - delete(c.heartbeats, rid) c.heartbeatLock.Unlock() } else { c.streamLock.Lock() stream = c.streams[rid] - delete(c.streams, rid) c.streamLock.Unlock() } if stream == nil { @@ -732,9 +680,6 @@ func (c *Channel) send() { sent += n } } - if ready.bytesBuffer != nil { - motan.ReleaseBytesBuffer(ready.bytesBuffer) - } case <-c.shutdownCh: return } @@ -900,18 +845,3 @@ func buildChannel(conn net.Conn, config *ChannelConfig, serialization motan.Seri return channel } - -func AcquireStream() *Stream { - v := streamPool.Get() - if v == nil { - return &Stream{} - } - return v.(*Stream) -} - -func ReleaseStream(stream *Stream) { - if stream != nil && stream.release { - stream.Reset() - streamPool.Put(stream) - } -} diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 90082b85..44c0ee50 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -32,10 +32,7 @@ var ( defaultAsyncResponse = &motan.MotanResponse{Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), RPCContext: &motan.RPCContext{AsyncCall: true}} - errPanic = errors.New("panic error") - v2StreamPool = sync.Pool{New: func() interface{} { - return new(V2Stream) - }} + errPanic = errors.New("panic error") ) type MotanEndpoint struct { @@ -145,8 +142,9 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { m.recordErrAndKeepalive() return m.defaultErrMotanResponse(request, "motanEndpoint error: channels is null") } + startTime := time.Now().UnixNano() if rc.AsyncCall { - rc.Result.StartTime = time.Now().UnixNano() + rc.Result.StartTime = startTime } // get a channel channel, err := m.channels.Get() @@ -382,9 +380,8 @@ type V2Channel struct { } type V2Stream struct { - channel *V2Channel - sendMsg *mpro.Message - streamId uint64 + channel *V2Channel + sendMsg *mpro.Message // recv msg recvMsg *mpro.Message recvNotifyCh chan struct{} @@ -394,31 +391,17 @@ type V2Stream struct { rc *motan.RPCContext isClose atomic.Value // bool isHeartBeat bool - sendTimer *time.Timer - recvTimer *time.Timer - release bool // concurrency issues for stream timeout and channel recv msg -} - -func (s *V2Stream) Reset() { - s.channel = nil - s.sendMsg = nil - s.recvMsg = nil - s.rc = nil } func (s *V2Stream) Send() error { - if s.sendTimer == nil { - s.sendTimer = time.NewTimer(s.deadline.Sub(time.Now())) - } else { - s.sendTimer.Reset(s.deadline.Sub(time.Now())) - } - defer s.sendTimer.Stop() + timer := time.NewTimer(s.deadline.Sub(time.Now())) + defer timer.Stop() buf := s.sendMsg.Encode() if s.rc != nil && s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Encode, Addr: s.channel.address, Time: time.Now()}) } - ready := sendReady{data: buf.Bytes(), bytesBuffer: buf} + ready := sendReady{data: buf.Bytes()} select { case s.channel.sendCh <- ready: if s.rc != nil { @@ -429,7 +412,7 @@ func (s *V2Stream) Send() error { } } return nil - case <-s.sendTimer.C: + case <-timer.C: return ErrSendRequestTimeout case <-s.channel.shutdownCh: return ErrChannelShutdown @@ -439,16 +422,10 @@ func (s *V2Stream) Send() error { // Recv sync recv func (s *V2Stream) Recv() (*mpro.Message, error) { defer func() { - if s.Close() { - s.release = true - } + s.Close() }() - if s.recvTimer == nil { - s.recvTimer = time.NewTimer(s.deadline.Sub(time.Now())) - } else { - s.recvTimer.Reset(s.deadline.Sub(time.Now())) - } - defer s.recvTimer.Stop() + timer := time.NewTimer(s.deadline.Sub(time.Now())) + defer timer.Stop() select { case <-s.recvNotifyCh: msg := s.recvMsg @@ -456,16 +433,17 @@ func (s *V2Stream) Recv() (*mpro.Message, error) { return nil, errors.New("recv err: recvMsg is nil") } return msg, nil - case <-s.recvTimer.C: - s.release = false + case <-timer.C: return nil, ErrRecvRequestTimeout case <-s.channel.shutdownCh: - s.release = false return nil, ErrChannelShutdown } } func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { + defer func() { + s.Close() + }() if s.rc != nil { s.rc.ResponseReceiveTime = t if s.rc.Tc != nil { @@ -473,9 +451,6 @@ func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Decode, Time: time.Now()}) } if s.rc.AsyncCall { - defer func() { - s.Close() - }() msg.Header.SetProxy(s.rc.Proxy) result := s.rc.Result response, err := mpro.ConvertToResponse(msg, s.channel.serialization) @@ -512,19 +487,16 @@ func (c *V2Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Strea if c.IsClosed() { return nil, ErrChannelShutdown } - - s := AcquireV2Stream() - s.channel = c - s.sendMsg = msg - if s.recvNotifyCh == nil { - s.recvNotifyCh = make(chan struct{}, 1) + s := &V2Stream{ + channel: c, + sendMsg: msg, + recvNotifyCh: make(chan struct{}, 1), + deadline: time.Now().Add(1 * time.Second), + rc: rc, } - s.deadline = time.Now().Add(1 * time.Second) - s.rc = rc s.isClose.Store(false) // RequestID is communication identifier, it is own by channel msg.Header.RequestID = GenerateRequestID() - s.streamId = msg.Header.RequestID if msg.Header.IsHeartbeat() { c.heartbeatLock.Lock() c.heartbeats[msg.Header.RequestID] = s @@ -534,35 +506,27 @@ func (c *V2Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Strea c.streamLock.Lock() c.streams[msg.Header.RequestID] = s c.streamLock.Unlock() - s.isHeartBeat = false } - s.release = true return s, nil } -func (s *V2Stream) Close() bool { - var exist bool - if !s.isClose.Swap(true).(bool) { +func (s *V2Stream) Close() { + if !s.isClose.Load().(bool) { if s.isHeartBeat { s.channel.heartbeatLock.Lock() - if _, exist = s.channel.heartbeats[s.streamId]; exist { - delete(s.channel.heartbeats, s.streamId) - } + delete(s.channel.heartbeats, s.sendMsg.Header.RequestID) s.channel.heartbeatLock.Unlock() } else { s.channel.streamLock.Lock() - if _, exist = s.channel.heartbeats[s.streamId]; exist { - delete(s.channel.streams, s.streamId) - } + delete(s.channel.streams, s.sendMsg.Header.RequestID) s.channel.streamLock.Unlock() } + s.isClose.Store(true) } - return exist } type sendReady struct { - data []byte - bytesBuffer *motan.BytesBuffer + data []byte } func (c *V2Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RPCContext) (*mpro.Message, error) { @@ -570,12 +534,6 @@ func (c *V2Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RP if err != nil { return nil, err } - defer func() { - if rc == nil || !rc.AsyncCall { - ReleaseV2Stream(stream) - } - }() - stream.SetDeadline(deadline) if err := stream.Send(); err != nil { return nil, err @@ -600,9 +558,8 @@ func (c *V2Channel) recv() { } func (c *V2Channel) recvLoop() error { - readSlice := make([]byte, motan.DefaultDecodeLength) for { - res, t, err := mpro.DecodeWithTime(c.bufRead, &readSlice, c.config.MaxContentLength) + res, t, err := mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) if err != nil { return err } @@ -640,7 +597,6 @@ func (c *V2Channel) send() { sent += n } } - motan.ReleaseBytesBuffer(ready.bytesBuffer) case <-c.shutdownCh: return } @@ -650,7 +606,6 @@ func (c *V2Channel) send() { func (c *V2Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { c.heartbeatLock.Lock() stream := c.heartbeats[msg.Header.RequestID] - delete(c.heartbeats, msg.Header.RequestID) c.heartbeatLock.Unlock() if stream == nil { vlog.Warningf("handle heartbeat message, missing stream: %d, ep:%s", msg.Header.RequestID, c.address) @@ -663,7 +618,6 @@ func (c *V2Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { func (c *V2Channel) handleMessage(msg *mpro.Message, t time.Time) error { c.streamLock.Lock() stream := c.streams[msg.Header.RequestID] - delete(c.streams, msg.Header.RequestID) c.streamLock.Unlock() if stream == nil { vlog.Warningf("handle recv message, missing stream: %d, ep:%s", msg.Header.RequestID, c.address) @@ -845,18 +799,3 @@ func GetDefaultMotanEPAsynInit() bool { } return res.(bool) } - -func AcquireV2Stream() *V2Stream { - v := v2StreamPool.Get() - if v == nil { - return &V2Stream{} - } - return v.(*V2Stream) -} - -func ReleaseV2Stream(stream *V2Stream) { - if stream != nil && stream.release { - stream.Reset() - v2StreamPool.Put(stream) - } -} diff --git a/endpoint/motanEndpoint_test.go b/endpoint/motanEndpoint_test.go index 1fd71939..9bb0efe6 100644 --- a/endpoint/motanEndpoint_test.go +++ b/endpoint/motanEndpoint_test.go @@ -21,7 +21,7 @@ func TestMain(m *testing.M) { m.Run() } -// TODO more UT +//TODO more UT func TestGetName(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") @@ -269,8 +269,7 @@ func handle(netListen net.Listener) { func handleConnection(conn net.Conn, timeout int) { buf := bufio.NewReader(conn) - readSlice := make([]byte, 100) - msg, _, err := protocol.DecodeWithTime(buf, &readSlice, 10*1024*1024) + msg, _, err := protocol.DecodeWithTime(buf, 10*1024*1024) if err != nil { time.Sleep(time.Millisecond * 1000) conn.Close() diff --git a/filter/accessLog.go b/filter/accessLog.go index ee980087..06c1e6e7 100644 --- a/filter/accessLog.go +++ b/filter/accessLog.go @@ -2,10 +2,11 @@ package filter import ( "encoding/json" - motan "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/log" "strconv" "time" + + motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/log" ) const ( @@ -33,17 +34,15 @@ func (t *AccessLogFilter) NewFilter(url *motan.URL) motan.Filter { func (t *AccessLogFilter) Filter(caller motan.Caller, request motan.Request) motan.Response { role := defaultRole var ip string - var start time.Time switch caller.(type) { case motan.Provider: role = serverAgentRole ip = request.GetAttachment(motan.HostKey) - start = request.GetRPCContext(true).RequestReceiveTime case motan.EndPoint: role = clientAgentRole ip = caller.GetURL().Host - start = time.Now() } + start := time.Now() response := t.GetNext().Filter(caller, request) address := ip + ":" + caller.GetURL().GetPortStr() if _, ok := caller.(motan.Provider); ok { @@ -82,6 +81,9 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 // response code should be same as upstream responseCode := "" metaUpstreamCode, _ := response.GetAttachments().Load(motan.MetaUpstreamCode) + if resCtx.Meta != nil { + responseCode = resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) + } var exceptionData []byte if exception != nil { exceptionData, _ = json.Marshal(exception) @@ -92,23 +94,21 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 responseCode = "200" } } - - logEntity := vlog.AcquireAccessLogEntity() - logEntity.FilterName = filterName - logEntity.Role = role - logEntity.RequestID = response.GetRequestID() - logEntity.Service = request.GetServiceName() - logEntity.Method = request.GetMethod() - logEntity.RemoteAddress = address - logEntity.Desc = request.GetMethodDesc() - logEntity.ReqSize = reqCtx.BodySize - logEntity.ResSize = resCtx.BodySize - logEntity.BizTime = response.GetProcessTime() //ms - logEntity.TotalTime = totalTime //ms - logEntity.ResponseCode = responseCode - logEntity.Success = exception == nil - logEntity.Exception = string(exceptionData) - logEntity.UpstreamCode = metaUpstreamCode - - vlog.AccessLog(logEntity) + vlog.AccessLog(&vlog.AccessLogEntity{ + FilterName: filterName, + Role: role, + RequestID: response.GetRequestID(), + Service: request.GetServiceName(), + Method: request.GetMethod(), + RemoteAddress: address, + Desc: request.GetMethodDesc(), + ReqSize: reqCtx.BodySize, + ResSize: resCtx.BodySize, + BizTime: response.GetProcessTime(), //ms + TotalTime: totalTime, //ms + ResponseCode: responseCode, + Success: exception == nil, + Exception: string(exceptionData), + UpstreamCode: metaUpstreamCode, + }) } diff --git a/filter/clusterMetrics.go b/filter/clusterMetrics.go index 75a6e435..bbbbcfbe 100644 --- a/filter/clusterMetrics.go +++ b/filter/clusterMetrics.go @@ -4,6 +4,7 @@ import ( "time" motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/metrics" "github.com/weibocom/motan-go/protocol" ) @@ -56,11 +57,11 @@ func (c *ClusterMetricsFilter) Filter(haStrategy motan.HaStrategy, loadBalance m if ctx != nil && ctx.Proxy { role = "motan-client-agent" } - //key := metrics.Escape(role) + - // ":" + metrics.Escape(request.GetAttachment(protocol.MSource)) + - // ":" + metrics.Escape(request.GetMethod()) - keys := []string{role, request.GetAttachment(protocol.MSource), request.GetMethod()} - addMetricWithKeys(request.GetAttachment(protocol.MGroup), ".cluster", - request.GetAttachment(protocol.MPath), keys, time.Since(start).Nanoseconds()/1e6, response) + key := metrics.Escape(role) + + ":" + metrics.Escape(request.GetAttachment(protocol.MSource)) + + ":" + metrics.Escape(request.GetMethod()) + addMetric(metrics.Escape(request.GetAttachment(protocol.MGroup))+".cluster", + metrics.Escape(request.GetAttachment(protocol.MPath)), + key, time.Since(start).Nanoseconds()/1e6, response) return response } diff --git a/filter/metrics.go b/filter/metrics.go index 1bb0fdb4..848eac88 100644 --- a/filter/metrics.go +++ b/filter/metrics.go @@ -1,10 +1,11 @@ package filter import ( + "time" + motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/metrics" "github.com/weibocom/motan-go/protocol" - "time" ) const ( @@ -56,13 +57,7 @@ func (m *MetricsFilter) GetNext() motan.EndPointFilter { } func (m *MetricsFilter) Filter(caller motan.Caller, request motan.Request) motan.Response { - var start time.Time - switch caller.(type) { - case motan.Provider: - start = request.GetRPCContext(true).RequestReceiveTime - case motan.EndPoint: - start = time.Now() - } + start := time.Now() response := m.GetNext().Filter(caller, request) proxy := false @@ -91,30 +86,30 @@ func (m *MetricsFilter) Filter(caller motan.Caller, request motan.Request) motan if provider { application = caller.GetURL().GetParam(motan.ApplicationKey, "") } - //key := metrics.Escape(role) + - // ":" + metrics.Escape(application) + - // ":" + metrics.Escape(request.GetMethod()) - keys := []string{role, application, request.GetMethod()} - addMetricWithKeys(request.GetAttachment(protocol.MGroup), "", request.GetAttachment(protocol.MPath), - keys, time.Since(start).Nanoseconds()/1e6, response) + key := metrics.Escape(role) + + ":" + metrics.Escape(application) + + ":" + metrics.Escape(request.GetMethod()) + addMetric(metrics.Escape(request.GetAttachment(protocol.MGroup)), + metrics.Escape(request.GetAttachment(protocol.MPath)), + key, time.Since(start).Nanoseconds()/1e6, response) return response } -func addMetricWithKeys(group, groupSuffix string, service string, keys []string, cost int64, response motan.Response) { - metrics.AddCounterWithKeys(group, "", service, keys, MetricsTotalCountSuffix, 1) //total_count - if response.GetException() != nil { //err_count +func addMetric(group string, service string, key string, cost int64, response motan.Response) { + metrics.AddCounter(group, service, key+MetricsTotalCountSuffix, 1) //total_count + if response.GetException() != nil { //err_count exception := response.GetException() if exception.ErrType == motan.BizException { - metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsBizErrorCountSuffix, 1) + metrics.AddCounter(group, service, key+MetricsBizErrorCountSuffix, 1) } else { - metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsOtherErrorCountSuffix, 1) + metrics.AddCounter(group, service, key+MetricsOtherErrorCountSuffix, 1) } } - metrics.AddCounterWithKeys(group, groupSuffix, service, keys, metrics.ElapseTimeSuffix(cost), 1) + metrics.AddCounter(group, service, key+metrics.ElapseTimeSuffix(cost), 1) if cost > 200 { - metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsSlowCountSuffix, 1) + metrics.AddCounter(group, service, key+MetricsSlowCountSuffix, 1) } - metrics.AddHistogramsWithKeys(group, groupSuffix, service, keys, "", cost) + metrics.AddHistograms(group, service, key, cost) } func (m *MetricsFilter) SetContext(context *motan.Context) { diff --git a/filter/metrics_test.go b/filter/metrics_test.go index b0a9d18f..f643882f 100644 --- a/filter/metrics_test.go +++ b/filter/metrics_test.go @@ -41,28 +41,25 @@ func TestMetricsFilter(t *testing.T) { name string caller motan.Caller request motan.Request - keys []string + key string }{ - {name: "proxyClient", caller: ep, request: request, keys: []string{"motan-client-agent", application, testMethod}}, - {name: "proxyServer", caller: provider, request: request, keys: []string{"motan-server-agent", application, testMethod}}, - {name: "Client", caller: ep, request: request2, keys: []string{"motan-client", application, testMethod}}, - {name: "Server", caller: provider, request: request2, keys: []string{"motan-server", application, testMethod}}, - } - var getKeysStr = func(keys []string) string { - return metrics.Escape(keys[0]) + ":" + metrics.Escape(keys[1]) + ":" + metrics.Escape(keys[2]) + {name: "proxyClient", caller: ep, request: request, key: "motan-client-agent:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, + {name: "proxyServer", caller: provider, request: request, key: "motan-server-agent:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, + {name: "Client", caller: ep, request: request2, key: "motan-client:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, + {name: "Server", caller: provider, request: request2, key: "motan-server:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { mf.Filter(test.caller, test.request) time.Sleep(10 * time.Millisecond) // The metrics filter has do escape - assert.Equal(t, 1, int(metrics.GetStatItem(testGroup, testService).SnapshotAndClear().Count(getKeysStr(test.keys)+MetricsTotalCountSuffix)), "metric count") + assert.Equal(t, 1, int(metrics.GetStatItem(metrics.Escape(testGroup), metrics.Escape(testService)).SnapshotAndClear().Count(test.key+MetricsTotalCountSuffix)), "metric count") }) } } func TestAddMetric(t *testing.T) { - keys := []string{"motan-client-agent", "testApplication", testMethod} + key := "motan-client-agent:testApplication:" + testMethod factory := initFactory() mf := factory.GetFilter(Metrics).(motan.EndPointFilter) mf.(*MetricsFilter).SetContext(&motan.Context{Config: config.NewConfig()}) @@ -70,9 +67,7 @@ func TestAddMetric(t *testing.T) { response2 := &motan.MotanResponse{ProcessTime: 100, Exception: &motan.Exception{ErrType: motan.BizException}} response3 := &motan.MotanResponse{ProcessTime: 100, Exception: &motan.Exception{ErrType: motan.FrameworkException}} response4 := &motan.MotanResponse{ProcessTime: 1000} - var getKeysStr = func(keys []string) string { - return metrics.Escape(keys[0]) + ":" + metrics.Escape(keys[1]) + ":" + metrics.Escape(keys[2]) - } + tests := []struct { name string response motan.Response @@ -86,11 +81,11 @@ func TestAddMetric(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - addMetricWithKeys(testGroup, "", testService, keys, test.response.GetProcessTime(), test.response) + addMetric(testGroup, testService, key, test.response.GetProcessTime(), test.response) time.Sleep(10 * time.Millisecond) snap := metrics.GetStatItem(testGroup, testService).SnapshotAndClear() for _, k := range test.keys { - assert.True(t, snap.Count(getKeysStr(keys)+k) > 0, fmt.Sprintf("key '%s'", k)) + assert.True(t, snap.Count(key+k) > 0, fmt.Sprintf("key '%s'", k)) } }) } diff --git a/ha/backupRequestHA.go b/ha/backupRequestHA.go index 892e3a5f..d50c6e5b 100644 --- a/ha/backupRequestHA.go +++ b/ha/backupRequestHA.go @@ -67,7 +67,7 @@ func (br *BackupRequestHA) Call(request motan.Request, loadBalance motan.LoadBal successCh := make(chan motan.Response, retries+1) if delay <= 0 { //no delay time configuration // TODO: we should use metrics of the cluster, with traffic control the group may changed - item := metrics.GetStatItem(request.GetAttachment(protocol.MGroup), request.GetAttachment(protocol.MPath)) + item := metrics.GetStatItem(metrics.Escape(request.GetAttachment(protocol.MGroup)), metrics.Escape(request.GetAttachment(protocol.MPath))) if item == nil || item.LastSnapshot() == nil { initDelay := int(br.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), "backupRequestInitDelayTime", 0)) if initDelay == 0 { diff --git a/log/bytes.go b/log/bytes.go deleted file mode 100644 index 2e6d6a7a..00000000 --- a/log/bytes.go +++ /dev/null @@ -1,82 +0,0 @@ -package vlog - -import ( - "strconv" - "sync" -) - -var ( - initSize = 256 - innerBytesBufferPool = sync.Pool{New: func() interface{} { - return &innerBytesBuffer{buf: make([]byte, 0, initSize)} - }} -) - -// innerBytesBuffer is a variable-sized buffer of bytes with Write methods. -type innerBytesBuffer struct { - buf []byte // reuse -} - -// newInnerBytesBuffer create an empty innerBytesBuffer with initial size -func newInnerBytesBuffer() *innerBytesBuffer { - return acquireBytesBuffer() -} - -func createInnerBytesBuffer(data []byte) *innerBytesBuffer { - return &innerBytesBuffer{ - buf: data, - } -} - -// WriteString write a str string append the innerBytesBuffer -func (b *innerBytesBuffer) WriteString(str string) { - b.buf = append(b.buf, str...) -} - -// WriteBoolString append the string value of v(true/false) to innerBytesBuffer -func (b *innerBytesBuffer) WriteBoolString(v bool) { - if v { - b.WriteString("true") - } else { - b.WriteString("false") - } -} - -// WriteUint64String append the string value of u to innerBytesBuffer -func (b *innerBytesBuffer) WriteUint64String(u uint64) { - b.buf = strconv.AppendUint(b.buf, u, 10) -} - -// WriteInt64String append the string value of i to innerBytesBuffer -func (b *innerBytesBuffer) WriteInt64String(i int64) { - b.buf = strconv.AppendInt(b.buf, i, 10) -} - -func (b *innerBytesBuffer) Bytes() []byte { return b.buf } - -func (b *innerBytesBuffer) String() string { - return string(b.buf) -} - -func (b *innerBytesBuffer) Reset() { - b.buf = b.buf[:0] -} - -func (b *innerBytesBuffer) Len() int { return len(b.buf) } - -func (b *innerBytesBuffer) Cap() int { return cap(b.buf) } - -func acquireBytesBuffer() *innerBytesBuffer { - b := innerBytesBufferPool.Get() - if b == nil { - return &innerBytesBuffer{buf: make([]byte, 0, 256)} - } - return b.(*innerBytesBuffer) -} - -func releaseBytesBuffer(b *innerBytesBuffer) { - if b != nil { - b.Reset() - innerBytesBufferPool.Put(b) - } -} diff --git a/log/bytes_test.go b/log/bytes_test.go deleted file mode 100644 index df2e598b..00000000 --- a/log/bytes_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package vlog - -import ( - "strconv" - "testing" -) - -func TestWrite(t *testing.T) { - // new BytesBuffer - buf := newInnerBytesBuffer() - if buf.Len() != 0 { - t.Errorf("new buf length not zero.") - } - if buf.Cap() != initSize { - t.Errorf("buf cap not correct.real:%d, expect:%d\n", buf.Cap(), initSize) - } - - // write string - buf.Reset() - buf.WriteString("string1") - buf.WriteString("string2") - tempbytes := buf.Bytes() - if "string1" != string(tempbytes[:7]) { - t.Errorf("write string not correct.buf:%+v\n", buf) - } - if "string2" != string(tempbytes[7:14]) { - t.Errorf("write string not correct.buf:%+v\n", buf) - } - - // write bool string - buf.Reset() - buf.WriteBoolString(true) - buf.WriteBoolString(false) - tempbytes = buf.Bytes() - if "true" != string(tempbytes[:4]) { - t.Errorf("write bool string not correct.buf:%+v\n", buf) - } - if "false" != string(tempbytes[4:9]) { - t.Errorf("write bool string not correct.buf:%+v\n", buf) - } - - // write uint64 string - buf.Reset() - var u1 uint64 = 11111111 - var u2 uint64 = 22222222 - buf.WriteUint64String(u1) - buf.WriteUint64String(u2) - tempbytes = buf.Bytes() - if "11111111" != string(tempbytes[:8]) { - t.Errorf("write unit64 string not correct.buf:%+v\n", buf) - } - if "22222222" != string(tempbytes[8:]) { - t.Errorf("write uint64 string not correct.buf:%+v\n", buf) - } - - // write int64 string - buf.Reset() - var i1 int64 = 11111111 - var i2 int64 = -22222222 - buf.WriteInt64String(i1) - buf.WriteInt64String(i2) - tempbytes = buf.Bytes() - if "11111111" != string(tempbytes[:8]) { - t.Errorf("write unit64 string not correct.buf:%+v\n", buf) - } - if "-22222222" != string(tempbytes[8:]) { - t.Errorf("write uint64 string not correct.buf:%+v\n", buf) - } -} - -func TestRead(t *testing.T) { - buf := newInnerBytesBuffer() - string1 := "aaaaaaaaaaaa" - buf.WriteString(string1) - buf.WriteUint64String(uint64(len(string1))) - buf.WriteBoolString(false) - buf.WriteInt64String(int64(-len(string1))) - - string2 := "bbbbbbbbbbbb" - buf.WriteString(string2) - buf.WriteUint64String(uint64(len(string2))) - buf.WriteBoolString(true) - buf.WriteInt64String(int64(-len(string2))) - - data := buf.Bytes() - buf2 := createInnerBytesBuffer(data) - rsize := len(string1) + 2 + 5 + 3 + len(string2) + 2 + 4 + 3 - if buf2.Len() != rsize { - t.Errorf("read buf len not correct. buf:%v\n", buf2) - } - - // read value - expectValue := string1 + - strconv.Itoa(len(string1)) + - "false" + - "-" + strconv.Itoa(len(string1)) + - string2 + - strconv.Itoa(len(string2)) + - "true" + - "-" + strconv.Itoa(len(string2)) - if expectValue != buf2.String() { - t.Errorf("read value not correct. buf:%v\n", buf2) - } -} diff --git a/log/log.go b/log/log.go index c675691f..d1d484bb 100644 --- a/log/log.go +++ b/log/log.go @@ -1,12 +1,14 @@ package vlog import ( + "bytes" "flag" "github.com/weibocom/motan-go/metrics/sampler" "log" "os" "path/filepath" "runtime/debug" + "strconv" "sync" "sync/atomic" "time" @@ -16,9 +18,6 @@ import ( ) var ( - accessLogEntityPool = sync.Pool{New: func() interface{} { - return new(AccessLogEntity) - }} loggerInstance Logger once sync.Once logDir = flag.String("log_dir", ".", "If non-empty, write log files in this directory") @@ -391,13 +390,12 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { zap.String("exception", logObject.Exception), zap.String("upstreamCode", logObject.UpstreamCode)) } else { - buffer := newInnerBytesBuffer() - + var buffer bytes.Buffer buffer.WriteString(logObject.FilterName) buffer.WriteString("|") buffer.WriteString(logObject.Role) buffer.WriteString("|") - buffer.WriteUint64String(logObject.RequestID) + buffer.WriteString(strconv.FormatUint(logObject.RequestID, 10)) buffer.WriteString("|") buffer.WriteString(logObject.Service) buffer.WriteString("|") @@ -407,15 +405,15 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { buffer.WriteString("|") buffer.WriteString(logObject.RemoteAddress) buffer.WriteString("|") - buffer.WriteInt64String(int64(logObject.ReqSize)) + buffer.WriteString(strconv.Itoa(logObject.ReqSize)) buffer.WriteString("|") - buffer.WriteInt64String(int64(logObject.ResSize)) + buffer.WriteString(strconv.Itoa(logObject.ResSize)) buffer.WriteString("|") - buffer.WriteInt64String(logObject.BizTime) + buffer.WriteString(strconv.FormatInt(logObject.BizTime, 10)) buffer.WriteString("|") - buffer.WriteInt64String(logObject.TotalTime) + buffer.WriteString(strconv.FormatInt(logObject.TotalTime, 10)) buffer.WriteString("|") - buffer.WriteBoolString(logObject.Success) + buffer.WriteString(strconv.FormatBool(logObject.Success)) buffer.WriteString("|") buffer.WriteString(logObject.ResponseCode) buffer.WriteString("|") @@ -423,10 +421,7 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { buffer.WriteString("|") buffer.WriteString(logObject.UpstreamCode) d.accessLogger.Info(buffer.String()) - - releaseBytesBuffer(buffer) } - ReleaseAccessLogEntity(logObject) } func (d *defaultLogger) MetricsLog(msg string) { @@ -495,17 +490,3 @@ func (d *defaultLogger) SetMetricsLogAvailable(status bool) { d.metricsLevel.SetLevel(zapcore.Level(defaultLogLevel + 1)) } } - -func AcquireAccessLogEntity() *AccessLogEntity { - v := accessLogEntityPool.Get() - if v == nil { - return &AccessLogEntity{} - } - return v.(*AccessLogEntity) -} - -func ReleaseAccessLogEntity(entity *AccessLogEntity) { - if entity != nil { - accessLogEntityPool.Put(entity) - } -} diff --git a/manageHandler.go b/manageHandler.go index 991877bb..98fd4208 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -158,7 +158,7 @@ func (s *StatusHandler) getStatus() []byte { exporter := v.(motan.Exporter) group := exporter.GetURL().Group service := exporter.GetURL().Path - statItem := metrics.GetStatItem(group, service) + statItem := metrics.GetStatItem(metrics.Escape(group), metrics.Escape(service)) if statItem == nil { return true } diff --git a/metrics/graphite.go b/metrics/graphite.go index 07b4bbf3..93f80cb5 100644 --- a/metrics/graphite.go +++ b/metrics/graphite.go @@ -102,21 +102,19 @@ func GenGraphiteMessages(localIP string, snapshots []Snapshot) []string { if len(pni) < minKeyLength { return } - escapedService := snap.GetEscapedService() - escapedGroup := snap.GetEscapedGroup() if snap.IsHistogram(k) { //histogram for slaK, slaV := range sla { segment += fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s.%s:%.2f|kv\n", - pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], slaK, snap.Percentile(k, slaV)) + pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], slaK, snap.Percentile(k, slaV)) } segment += fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s.%s:%.2f|ms\n", - pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], "avg_time", snap.Mean(k)) + pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], "avg_time", snap.Mean(k)) } else if snap.IsCounter(k) { //counter segment = fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s:%d|c\n", - pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], snap.Count(k)) + pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], snap.Count(k)) } else { // gauge segment = fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s:%d|kv\n", - pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], snap.Value(k)) + pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], snap.Value(k)) } if buf.Len() > 0 && buf.Len()+len(segment) > messageMaxLen { messages = append(messages, buf.String()) diff --git a/metrics/metrics.go b/metrics/metrics.go index 6757bfc8..574917b1 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -42,7 +42,6 @@ const ( ) var ( - metricsKeyBuilderBufferSize = 64 // NewStatItem is the factory func for StatItem NewStatItem = NewDefaultStatItem items = make(map[string]StatItem, 64) @@ -53,20 +52,15 @@ var ( processor: defaultEventProcessor, //sink processor size eventBus: make(chan *event, eventBufferSize), writers: make(map[string]StatWriter), - evtBuf: &sync.Pool{New: func() interface{} { - return &event{} - }}, + evtBuf: &sync.Pool{New: func() interface{} { return new(event) }}, } - escapeCache sync.Map ) type StatItem interface { SetService(service string) GetService() string - GetEscapedService() string SetGroup(group string) GetGroup() string - GetEscapedGroup() string AddCounter(key string, value int64) AddHistograms(key string, duration int64) AddGauge(key string, value int64) @@ -104,18 +98,17 @@ type StatWriter interface { } func GetOrRegisterStatItem(group string, service string) StatItem { - k := group + service itemsLock.RLock() - item := items[k] + item := items[group+service] itemsLock.RUnlock() if item != nil { return item } itemsLock.Lock() - item = items[k] + item = items[group+service] if item == nil { item = NewStatItem(group, service) - items[k] = item + items[group+service] = item } itemsLock.Unlock() return item @@ -172,20 +165,14 @@ func StatItemSize() int { return len(items) } -// Escape the string avoid invalid graphite key func Escape(s string) string { - if v, ok := escapeCache.Load(s); ok { - return v.(string) - } - v := strings.Map(func(char rune) rune { + return strings.Map(func(char rune) rune { if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || (char == '-') { return char } else { return '_' } }, s) - escapeCache.Store(s, v) - return v } func AddCounter(group string, service string, key string, value int64) { @@ -200,27 +187,13 @@ func AddGauge(group string, service string, key string, value int64) { sendEvent(eventGauge, group, service, key, value) } -func AddCounterWithKeys(group, groupSuffix string, service string, keys []string, keySuffix string, value int64) { - sendEventWithKeys(eventCounter, group, groupSuffix, service, keys, keySuffix, value) -} - -func AddHistogramsWithKeys(group, groupSuffix string, service string, keys []string, suffix string, duration int64) { - sendEventWithKeys(eventHistograms, group, groupSuffix, service, keys, suffix, duration) -} - func sendEvent(eventType int32, group string, service string, key string, value int64) { - sendEventWithKeys(eventType, group, "", service, []string{key}, "", value) -} - -func sendEventWithKeys(eventType int32, group, groupSuffix string, service string, keys []string, suffix string, value int64) { evt := rp.evtBuf.Get().(*event) evt.event = eventType - evt.keys = keys + evt.key = key evt.group = group evt.service = service evt.value = value - evt.keySuffix = suffix - evt.groupSuffix = groupSuffix select { case rp.eventBus <- evt: default: @@ -266,46 +239,11 @@ func startSampleStatus(application string) { } type event struct { - event int32 - keys []string - keySuffix string - group string - groupSuffix string - service string - value int64 -} - -func (s *event) reset() { - s.event = 0 - s.keys = s.keys[:0] - s.keySuffix = "" - s.group = "" - s.service = "" - s.value = 0 - s.groupSuffix = "" -} - -func (s *event) getGroup() string { - if s.groupSuffix == "" { - return s.group - } - return s.group + s.groupSuffix -} - -func (s *event) getMetricKey() string { - keyBuilder := motan.NewBytesBuffer(metricsKeyBuilderBufferSize) - defer motan.ReleaseBytesBuffer(keyBuilder) - l := len(s.keys) - for idx, k := range s.keys { - keyBuilder.WriteString(Escape(k)) - if idx < l-1 { - keyBuilder.WriteString(":") - } - } - if s.keySuffix != "" { - keyBuilder.WriteString(s.keySuffix) - } - return string(keyBuilder.Bytes()) + event int32 + key string + group string + service string + value int64 } type RegistryHolder struct { @@ -332,9 +270,7 @@ func (d *DefaultStatItem) SetService(service string) { func (d *DefaultStatItem) GetService() string { return d.service } -func (d *DefaultStatItem) GetEscapedService() string { - return Escape(d.service) -} + func (d *DefaultStatItem) SetGroup(group string) { d.group = group } @@ -343,10 +279,6 @@ func (d *DefaultStatItem) GetGroup() string { return d.group } -func (d *DefaultStatItem) GetEscapedGroup() string { - return Escape(d.group) -} - func (d *DefaultStatItem) AddCounter(key string, value int64) { c := d.getRegistry().Get(key) if c == nil { @@ -589,10 +521,6 @@ func (d *ReadonlyStatItem) GetService() string { return d.service } -func (d *ReadonlyStatItem) GetEscapedService() string { - return Escape(d.service) -} - func (d *ReadonlyStatItem) SetGroup(group string) { panic("action not supported") } @@ -601,10 +529,6 @@ func (d *ReadonlyStatItem) GetGroup() string { return d.group } -func (d *ReadonlyStatItem) GetEscapedGroup() string { - return Escape(d.group) -} - func (d *ReadonlyStatItem) AddCounter(key string, value int64) { panic("action not supported") } @@ -808,9 +732,6 @@ type reporter struct { func (r *reporter) eventLoop() { for evt := range r.eventBus { r.processEvent(evt) - // clean the event object before put it back - evt.reset() - r.evtBuf.Put(evt) } } @@ -825,15 +746,14 @@ func (r *reporter) addWriter(key string, sw StatWriter) { func (r *reporter) processEvent(evt *event) { defer motan.HandlePanic(nil) - item := GetOrRegisterStatItem(evt.getGroup(), evt.service) - key := evt.getMetricKey() + item := GetOrRegisterStatItem(evt.group, evt.service) switch evt.event { case eventCounter: - item.AddCounter(key, evt.value) + item.AddCounter(evt.key, evt.value) case eventHistograms: - item.AddHistograms(key, evt.value) + item.AddHistograms(evt.key, evt.value) case eventGauge: - item.AddGauge(key, evt.value) + item.AddGauge(evt.key, evt.value) } } diff --git a/protocol/motanProtocol.go b/protocol/motanProtocol.go index 180fb021..0e1e413e 100644 --- a/protocol/motanProtocol.go +++ b/protocol/motanProtocol.go @@ -22,7 +22,7 @@ const ( DefaultMaxContentLength = 10 * 1024 * 1024 ) -// message type +//message type const ( Req = iota Res @@ -60,24 +60,6 @@ type Header struct { RequestID uint64 } -func (h *Header) Reset() { - h.Magic = 0 - h.MsgType = 0 - h.VersionStatus = 0 - h.Serialize = 0 - h.RequestID = 0 -} - -func ResetHeader(h *Header) { - if h != nil { - h.Magic = 0 - h.MsgType = 0 - h.VersionStatus = 0 - h.Serialize = 0 - h.RequestID = 0 - } -} - func (h *Header) Clone() *Header { return &Header{ Magic: h.Magic, @@ -95,7 +77,7 @@ type Message struct { Type int } -// serialize +//serialize const ( Hessian = iota GrpcPb @@ -122,9 +104,6 @@ var ( writeBufPool = &sync.Pool{New: func() interface{} { // for gzip write buffer return &bytes.Buffer{} }} - messagePool = sync.Pool{New: func() interface{} { - return &Message{Metadata: motan.NewStringMap(DefaultMetaSize), Header: &Header{}} - }} ) // errors @@ -287,9 +266,9 @@ func (msg *Message) Encode() (buf *motan.BytesBuffer) { vlog.Errorf("metadata not correct.k:%s, v:%s", k, v) return true } - metabuf.WriteString(k) + metabuf.Write([]byte(k)) metabuf.WriteByte('\n') - metabuf.WriteString(v) + metabuf.Write([]byte(v)) metabuf.WriteByte('\n') return true }) @@ -312,7 +291,6 @@ func (msg *Message) Encode() (buf *motan.BytesBuffer) { if metasize > 0 { buf.Write(metabuf.Bytes()) } - motan.ReleaseBytesBuffer(metabuf) // encode body buf.WriteUint32(uint32(bodysize)) @@ -322,13 +300,6 @@ func (msg *Message) Encode() (buf *motan.BytesBuffer) { return buf } -func (msg *Message) Reset() { - msg.Type = 0 - msg.Body = msg.Body[:0] - msg.Header.Reset() - msg.Metadata.Reset() -} - func (msg *Message) Clone() interface{} { newMessage := &Message{ Header: msg.Header.Clone(), @@ -355,105 +326,93 @@ func CheckMotanVersion(buf *bufio.Reader) (version int, err error) { return int(b[3] >> 3 & 0x1f), nil } -func Decode(buf *bufio.Reader, readSlice *[]byte) (msg *Message, err error) { - msg, _, err = DecodeWithTime(buf, readSlice, motan.DefaultMaxContentLength) +func Decode(buf *bufio.Reader) (msg *Message, err error) { + msg, _, err = DecodeWithTime(buf, motan.DefaultMaxContentLength) return msg, err } -func DecodeWithTime(buf *bufio.Reader, rs *[]byte, maxContentLength int) (msg *Message, start time.Time, err error) { - readSlice := *rs +func DecodeWithTime(buf *bufio.Reader, maxContentLength int) (msg *Message, start time.Time, err error) { + temp := make([]byte, HeaderLength, HeaderLength) + // decode header - _, err = io.ReadAtLeast(buf, readSlice[:HeaderLength], HeaderLength) + _, err = io.ReadAtLeast(buf, temp, HeaderLength) start = time.Now() // record time when starting to read data if err != nil { return nil, start, err } - mn := binary.BigEndian.Uint16(readSlice[:2]) // TODO 不再验证 + mn := binary.BigEndian.Uint16(temp[:2]) // TODO 不再验证 if mn != MotanMagic { vlog.Errorf("wrong magic num:%d, err:%v", mn, err) return nil, start, ErrMagicNum } - msg = messagePool.Get().(*Message) - msg.Header.Magic = MotanMagic - msg.Header.MsgType = readSlice[2] - msg.Header.VersionStatus = readSlice[3] - version := msg.Header.GetVersion() + + header := &Header{Magic: MotanMagic} + header.MsgType = temp[2] + header.VersionStatus = temp[3] + version := header.GetVersion() if version != Version2 { // TODO 不再验证 vlog.Errorf("unsupported protocol version number: %d", version) return nil, start, ErrVersion } - msg.Header.Serialize = readSlice[4] - msg.Header.RequestID = binary.BigEndian.Uint64(readSlice[5:HeaderLength]) + header.Serialize = temp[4] + header.RequestID = binary.BigEndian.Uint64(temp[5:]) // decode meta - _, err = io.ReadAtLeast(buf, readSlice[:4], 4) + _, err = io.ReadAtLeast(buf, temp[:4], 4) if err != nil { - PutMessageBackToPool(msg) return nil, start, err } - metasize := int(binary.BigEndian.Uint32(readSlice[:4])) + metasize := int(binary.BigEndian.Uint32(temp[:4])) if metasize > maxContentLength { vlog.Errorf("meta over size. meta size:%d, max size:%d", metasize, maxContentLength) - PutMessageBackToPool(msg) return nil, start, ErrOverSize } + metamap := motan.NewStringMap(DefaultMetaSize) if metasize > 0 { - if cap(readSlice) < metasize { - readSlice = make([]byte, metasize) - *rs = readSlice - } - err := readBytes(buf, readSlice, metasize) + metadata, err := readBytes(buf, metasize) if err != nil { - PutMessageBackToPool(msg) return nil, start, err } s, e := 0, 0 var k string for i := 0; i <= metasize; i++ { - if i == metasize || readSlice[i] == '\n' { + if i == metasize || metadata[i] == '\n' { e = i if k == "" { - k = string(readSlice[s:e]) + k = string(metadata[s:e]) } else { - msg.Metadata.Store(k, string(readSlice[s:e])) + metamap.Store(k, string(metadata[s:e])) k = "" } s = i + 1 } } if k != "" { - vlog.Errorf("decode message fail, metadata not paired. header:%v, meta:%s", msg.Header, readSlice) - PutMessageBackToPool(msg) + vlog.Errorf("decode message fail, metadata not paired. header:%v, meta:%s", header, metadata) return nil, start, ErrMetadata } } //decode body - _, err = io.ReadAtLeast(buf, readSlice[:4], 4) + _, err = io.ReadAtLeast(buf, temp[:4], 4) if err != nil { - PutMessageBackToPool(msg) return nil, start, err } - bodysize := int(binary.BigEndian.Uint32(readSlice[:4])) + bodysize := int(binary.BigEndian.Uint32(temp[:4])) if bodysize > maxContentLength { vlog.Errorf("body over size. body size:%d, max size:%d", bodysize, maxContentLength) - PutMessageBackToPool(msg) return nil, start, ErrOverSize } - + var body []byte if bodysize > 0 { - if cap(msg.Body) < bodysize { - msg.Body = make([]byte, bodysize) - } - msg.Body = msg.Body[:bodysize] - err = readBytes(buf, msg.Body, bodysize) + body, err = readBytes(buf, bodysize) } else { - msg.Body = make([]byte, 0) + body = make([]byte, 0) } if err != nil { - PutMessageBackToPool(msg) return nil, start, err } + msg = &Message{header, metamap, body, Req} return msg, start, err } @@ -466,14 +425,15 @@ func DecodeGzipBody(body []byte) []byte { return ret } -func readBytes(buf *bufio.Reader, readSlice []byte, size int) error { +func readBytes(buf *bufio.Reader, size int) ([]byte, error) { + tempbytes := make([]byte, size) var s, n = 0, 0 var err error for s < size && err == nil { - n, err = buf.Read(readSlice[s:size]) + n, err = buf.Read(tempbytes[s:]) s += n } - return err + return tempbytes, err } func EncodeGzip(data []byte) ([]byte, error) { @@ -563,7 +523,7 @@ func DecodeGzip(data []byte) (ret []byte, err error) { // ConvertToRequest convert motan2 protocol request message to motan Request func ConvertToRequest(request *Message, serialize motan.Serialization) (motan.Request, error) { - motanRequest := motan.GetMotanRequestFromPool() + motanRequest := &motan.MotanRequest{Arguments: make([]interface{}, 0)} motanRequest.RequestID = request.Header.RequestID if idStr, ok := request.Metadata.Load(MRequestID); !ok { if request.Header.IsProxy() { @@ -589,16 +549,12 @@ func ConvertToRequest(request *Message, serialize motan.Serialization) (motan.Re request.Header.SetGzip(false) } if !rc.Proxy && serialize == nil { - motan.PutMotanRequestBackPool(motanRequest) return nil, ErrSerializeNil } - if len(motanRequest.Arguments) <= 0 { - motanRequest.Arguments = []interface{}{&motan.DeserializableValue{Body: request.Body, Serialization: serialize}} - } else { - motanRequest.Arguments[0].(*motan.DeserializableValue).Body = request.Body - motanRequest.Arguments[0].(*motan.DeserializableValue).Serialization = serialize - } + dv := &motan.DeserializableValue{Body: request.Body, Serialization: serialize} + motanRequest.Arguments = []interface{}{dv} } + return motanRequest, nil } @@ -678,7 +634,7 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) } } - res := messagePool.Get().(*Message) + res := &Message{} var msgType int if response.GetException() != nil { msgType = Exception @@ -704,13 +660,11 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) res.Body = b } else { vlog.Warningf("convert response value fail! serialized value not []byte. res:%+v", response) - PutMessageBackToPool(res) return nil, ErrSerializedData } } else { b, err := serialize.Serialize(response.GetValue()) if err != nil { - PutMessageBackToPool(res) return nil, err } res.Body = b @@ -729,7 +683,7 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) // ConvertToResponse convert protocol response to motan Response func ConvertToResponse(response *Message, serialize motan.Serialization) (motan.Response, error) { - mres := motan.GetMotanResponseFromPool() + mres := &motan.MotanResponse{} rc := mres.GetRPCContext(true) rc.Proxy = response.Header.IsProxy() mres.RequestID = response.Header.RequestID @@ -745,7 +699,6 @@ func ConvertToResponse(response *Message, serialize motan.Serialization) (motan. response.Header.SetGzip(false) } if !rc.Proxy && serialize == nil { - motan.PutMotanResponseBackPool(mres) return nil, ErrSerializeNil } dv := &motan.DeserializableValue{Body: response.Body, Serialization: serialize} @@ -757,7 +710,6 @@ func ConvertToResponse(response *Message, serialize motan.Serialization) (motan. var exception *motan.Exception err := json.Unmarshal([]byte(e), &exception) if err != nil { - motan.PutMotanResponseBackPool(mres) return nil, err } mres.Exception = exception @@ -780,14 +732,3 @@ func ExceptionToJSON(e *motan.Exception) string { errmsg, _ := json.Marshal(e) return string(errmsg) } - -func PutMessageBackToPool(msg *Message) { - if msg != nil { - //msg.Reset() - msg.Type = 0 - msg.Body = msg.Body[:0] - ResetHeader(msg.Header) - msg.Metadata.Reset() - messagePool.Put(msg) - } -} diff --git a/protocol/motanProtocol_test.go b/protocol/motanProtocol_test.go index acc41513..060ae0cb 100644 --- a/protocol/motanProtocol_test.go +++ b/protocol/motanProtocol_test.go @@ -5,10 +5,8 @@ import ( "bytes" "compress/gzip" "fmt" - "github.com/stretchr/testify/assert" "github.com/weibocom/motan-go/serialize" "math/rand" - "strconv" "sync" "sync/atomic" "testing" @@ -147,11 +145,7 @@ func TestEncode(t *testing.T) { ebytes := msg.Encode() fmt.Println("len:", ebytes.Len()) - readSlice := make([]byte, 100) - //for i := 0; i < len(readSlice); i++ { - // readSlice[i] = 't' - //} - newMsg, err := Decode(bufio.NewReader(ebytes), &readSlice) + newMsg, err := Decode(bufio.NewReader(ebytes)) if newMsg == nil { t.Fatalf("encode message fail") } @@ -170,7 +164,7 @@ func TestEncode(t *testing.T) { msg.Header.SetGzip(true) msg.Body, _ = EncodeGzip([]byte("gzip encode")) b := msg.Encode() - newMsg, _ = Decode(bufio.NewReader(b), &readSlice) + newMsg, _ = Decode(bufio.NewReader(b)) // should not decode gzip if !newMsg.Header.IsGzip() { t.Fatalf("encode message fail") @@ -183,106 +177,13 @@ func TestEncode(t *testing.T) { assertTrue(string(nb) == "gzip encode", "body", t) } -func TestPool(t *testing.T) { - h := &Header{} - h.SetVersion(Version2) - h.SetStatus(6) - h.SetOneWay(true) - h.SetSerialize(5) - h.SetGzip(true) - h.SetHeartbeat(true) - h.SetProxy(true) - h.SetRequest(true) - h.Magic = MotanMagic - h.RequestID = 2349789 - meta := core.NewStringMap(0) - for mi := 0; mi < 10000; mi++ { - meta.Store(strconv.Itoa(mi), strconv.Itoa(mi)) - } - body := []byte("testbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbody") - msg := &Message{Header: h, Metadata: meta, Body: body} - ebytes := msg.Encode() - - fmt.Println("len:", ebytes.Len()) - readSlice := make([]byte, 100) - newMsg, err := Decode(bufio.NewReader(ebytes), &readSlice) - if newMsg == nil { - t.Fatalf("encode message fail") - } - assertTrue(newMsg.Header.IsOneWay(), "oneway", t) - assertTrue(newMsg.Header.IsGzip(), "gzip", t) - assertTrue(newMsg.Header.IsHeartbeat(), "heartbeat", t) - assertTrue(newMsg.Header.IsProxy(), "proxy", t) - assertTrue(newMsg.Header.isRequest(), "request", t) - assertTrue(newMsg.Header.GetVersion() == Version2, "version", t) - assertTrue(newMsg.Header.GetSerialize() == 5, "serialize", t) - assertTrue(newMsg.Header.GetStatus() == 6, "status", t) - assertTrue(newMsg.Metadata.LoadOrEmpty("1") == "1", "meta", t) - assertTrue(cap(readSlice) > 200, "readSlice", t) - assertTrue(len(newMsg.Body) == len(msg.Body), "body", t) - assert.Nil(t, err) - PutMessageBackToPool(newMsg) - body1 := []byte("testbody") - msg1 := &Message{Header: h, Metadata: meta, Body: body1} - ebytes1 := msg1.Encode() - newMsg, err = Decode(bufio.NewReader(ebytes1), &readSlice) - if newMsg == nil { - t.Fatalf("encode message fail") - } - assertTrue(newMsg.Header.IsOneWay(), "oneway", t) - assertTrue(newMsg.Header.IsGzip(), "gzip", t) - assertTrue(newMsg.Header.IsHeartbeat(), "heartbeat", t) - assertTrue(newMsg.Header.IsProxy(), "proxy", t) - assertTrue(newMsg.Header.isRequest(), "request", t) - assertTrue(newMsg.Header.GetVersion() == Version2, "version", t) - assertTrue(newMsg.Header.GetSerialize() == 5, "serialize", t) - assertTrue(newMsg.Header.GetStatus() == 6, "status", t) - assertTrue(newMsg.Metadata.LoadOrEmpty("1") == "1", "meta", t) - assertTrue(cap(readSlice) > 200, "readSlice", t) - assertTrue(len(newMsg.Body) == len(msg1.Body), "body", t) -} - func assertTrue(b bool, msg string, t *testing.T) { if !b { t.Fatalf("test fail, %s not correct.", msg) } } -func TestConvertToResponse(t *testing.T) { - h := &Header{} - h.SetVersion(Version2) - h.SetStatus(6) - h.SetOneWay(true) - h.SetSerialize(6) - h.SetGzip(true) - h.SetHeartbeat(true) - h.SetProxy(true) - h.SetRequest(true) - h.Magic = MotanMagic - h.RequestID = 2349789 - meta := core.NewStringMap(0) - meta.Store("k1", "v1") - meta.Store(MGroup, "group") - meta.Store(MMethod, "method") - meta.Store(MPath, "path") - body := []byte("testbody") - msg := &Message{Header: h, Metadata: meta, Body: body} - - pMap := make(map[string]string) - for i := 0; i < 10000; i++ { - resp, err := ConvertToResponse(msg, &serialize.SimpleSerialization{}) - assertTrue(err == nil, "conver to request err", t) - assertTrue(resp.GetAttachment(MGroup) == "group", "response group", t) - assertTrue(resp.GetAttachment(MMethod) == "method", "response method", t) - assertTrue(resp.GetAttachment(MPath) == "path", "response path", t) - //assertTrue(resp.GetValue().(string) == "testbody", "response body", t) - pMap[fmt.Sprintf("%p", resp)] = "1" - core.PutMotanResponseBackPool(resp.(*core.MotanResponse)) - } - assert.True(t, len(pMap) < 10000) -} - -// TODO convert +//TODO convert func TestConvertToRequest(t *testing.T) { h := &Header{} h.SetVersion(Version2) @@ -302,19 +203,12 @@ func TestConvertToRequest(t *testing.T) { meta.Store(MPath, "path") body := []byte("testbody") msg := &Message{Header: h, Metadata: meta, Body: body} - pMap := make(map[string]string) - for i := 0; i < 10000; i++ { - req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) - assertTrue(err == nil, "conver to request err", t) - assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) - assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) - assertTrue(req.GetAttachment(MPath) == "path", "request path", t) - pMap[fmt.Sprintf("%p", req)] = "1" - core.PutMotanRequestBackPool(req.(*core.MotanRequest)) - } - assert.True(t, len(pMap) < 10000) - req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) + assertTrue(err == nil, "conver to request err", t) + assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) + assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) + assertTrue(req.GetAttachment(MPath) == "path", "request path", t) + // test request clone cloneReq := req.Clone().(core.Request) assertTrue(err == nil, "conver to request err", t) diff --git a/provider/httpProvider.go b/provider/httpProvider.go index fdf34119..f19cfe69 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -512,5 +512,7 @@ func fillException(resp *motan.MotanResponse, start int64, err error) { } func updateUpstreamStatusCode(resp *motan.MotanResponse, statusCode int) { + resCtx := resp.GetRPCContext(true) resp.SetAttachment(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) + resCtx.Meta.Store(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) } diff --git a/server/motanserver.go b/server/motanserver.go index 5eecb3a2..29fbfc04 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -151,7 +151,7 @@ func (m *MotanServer) handleConn(conn net.Conn) { } else { ip = getRemoteIP(conn.RemoteAddr().String()) } - decodeBuf := make([]byte, motan.DefaultDecodeLength) + for { v, err := mpro.CheckMotanVersion(buf) if err != nil { @@ -168,7 +168,7 @@ func (m *MotanServer) handleConn(conn net.Conn) { } go m.processV1(v1Msg, t, ip, conn) } else if v == mpro.Version2 { - msg, t, err := mpro.DecodeWithTime(buf, &decodeBuf, m.maxContextLength) + msg, t, err := mpro.DecodeWithTime(buf, m.maxContextLength) if err != nil { vlog.Warningf("decode motan v2 message fail! con:%s, err:%s.", conn.RemoteAddr().String(), err.Error()) break @@ -219,7 +219,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c tc.PutReqSpan(&motan.Span{Name: motan.Convert, Time: time.Now()}) req.GetRPCContext(true).Tc = tc } - + callStart := time.Now() mres = m.handler.Call(req) if tc != nil { // clusterFilter end @@ -229,7 +229,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c resCtx := mres.GetRPCContext(true) resCtx.Proxy = m.proxy if mres.GetAttachment(mpro.MProcessTime) == "" { - mres.SetAttachment(mpro.MProcessTime, strconv.FormatInt(int64(time.Now().Sub(start)/1e6), 10)) + mres.SetAttachment(mpro.MProcessTime, strconv.FormatInt(int64(time.Now().Sub(callStart)/1e6), 10)) } res, err = mpro.ConvertToResMessage(mres, serialization) if tc != nil { @@ -247,8 +247,6 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c // recover the communication identifier res.Header.RequestID = lastRequestID resBuf := res.Encode() - // reuse BytesBuffer - defer motan.ReleaseBytesBuffer(resBuf) if tc != nil { tc.PutResSpan(&motan.Span{Name: motan.Encode, Time: time.Now()}) } @@ -270,16 +268,6 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c if tc != nil { tc.PutResSpan(&motan.Span{Name: motan.Send, Time: resSendTime}) } - // 回收message - mpro.PutMessageBackToPool(msg) - mpro.PutMessageBackToPool(res) - // 回收request - if motanReq, ok := mreq.(*motan.MotanRequest); ok { - motan.PutMotanRequestBackPool(motanReq) - } - if motanResp, ok := mres.(*motan.MotanResponse); ok { - motan.PutMotanResponseBackPool(motanResp) - } } func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip string, conn net.Conn) { From ecfa6a0bb4d7a59d1ff58a81b9180f66e5e99f8a Mon Sep 17 00:00:00 2001 From: snail007 Date: Tue, 26 Dec 2023 14:19:18 +0800 Subject: [PATCH 71/75] Profile base to dev (#352) profile done --- .github/workflows/test.yml | 2 +- agent.go | 93 ++++---- agent_test.go | 25 ++- cluster/command.go | 4 +- cluster/motanCluster.go | 9 + core/bytes.go | 52 ++++- core/bytes_test.go | 53 +++++ core/constants.go | 4 + core/map.go | 25 ++- core/map_test.go | 13 ++ core/motan.go | 292 ++++++++++++++++--------- core/test.go | 31 +++ core/url.go | 151 ++++++++++--- core/url_test.go | 107 +++++++-- dynamicConfig.go | 8 +- endpoint/motanCommonEndpoint.go | 237 +++++++++++++------- endpoint/motanCommonEndpoint_test.go | 109 ++++++++++ endpoint/motanEndpoint.go | 200 +++++++++++------ endpoint/motanEndpoint_test.go | 123 ++++++++++- filter/accessLog.go | 48 ++-- filter/clusterMetrics.go | 10 +- filter/filter.go | 12 + filter/metrics.go | 32 ++- filter/metrics_test.go | 29 ++- filter/rateLimit.go | 20 +- filter/tracing.go | 50 ++--- go.mod | 3 +- ha/backupRequestHA.go | 2 +- ha/failoverHA.go | 3 + http/httpProxy.go | 88 ++++++-- lb/randomLb.go | 2 + log/bytes.go | 67 ++++++ log/bytes_test.go | 104 +++++++++ log/log.go | 33 ++- log/log_test.go | 6 +- log/rotate.go | 6 +- manageHandler.go | 2 +- manageHandler_test.go | 2 +- metrics/graphite.go | 14 +- metrics/graphite_test.go | 8 +- metrics/metrics.go | 155 ++++++++++--- metrics/metrics_test.go | 30 +-- protocol/motan1Protocol.go | 5 + protocol/motanProtocol.go | 266 ++++++++++++++++------- protocol/motanProtocol_test.go | 307 +++++++++++++++++++++++++- provider/httpProvider.go | 314 ++++++++++++++------------- provider/motanProvider.go | 2 +- registry/consulRegistry.go | 10 + registry/directRegistry.go | 11 + registry/localRegistry.go | 4 + registry/zkRegistry_test.go | 2 +- server.go | 1 + server/motanserver.go | 52 ++++- tools/nginx/parser.go | 10 +- 54 files changed, 2408 insertions(+), 840 deletions(-) create mode 100644 log/bytes.go create mode 100644 log/bytes_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df431f1a..7f511c5f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: testing: strategy: matrix: - go-version: [1.12.x,1.13.x,1.14.x,1.15.x,1.16.x,1.17.x,1.18.x,1.19.x] + go-version: [1.12.x,1.13.x,1.14.x,1.15.x,1.16.x,1.17.x,1.18.x,1.19.x,1.20.x,1.21.x] platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} steps: diff --git a/agent.go b/agent.go index 17884ad4..6428ff09 100644 --- a/agent.go +++ b/agent.go @@ -133,7 +133,7 @@ func (a *Agent) RegisterCommandHandler(f CommandHandler) { a.commandHandlers = append(a.commandHandlers, f) } -func (a *Agent) GetDynamicRegistryInfo() *registrySnapInfoStorage { +func (a *Agent) GetDynamicRegistryInfo() *RegistrySnapInfoStorage { return a.configurer.getRegistryInfo() } @@ -150,7 +150,7 @@ func (a *Agent) RuntimeDir() string { return a.runtimedir } -// get Agent server +// GetAgentServer get Agent server func (a *Agent) GetAgentServer() motan.Server { return a.agentServer } @@ -300,7 +300,7 @@ func (a *Agent) initStatus() { func (a *Agent) saveStatus() { statSnapFile := a.runtimedir + string(filepath.Separator) + defaultStatusSnap - err := ioutil.WriteFile(statSnapFile, []byte(strconv.Itoa(int(http.StatusOK))), 0644) + err := ioutil.WriteFile(statSnapFile, []byte(strconv.Itoa(http.StatusOK)), 0644) if err != nil { vlog.Errorln("Save status error: " + err.Error()) return @@ -351,6 +351,14 @@ func (a *Agent) initParam() { initLog(logDir, section) registerSwitchers(a.Context) + processPoolSize := 0 + if section != nil && section["processPoolSize"] != nil { + processPoolSize = section["processPoolSize"].(int) + } + if processPoolSize > 0 { + mserver.SetProcessPoolSize(processPoolSize) + } + port := *motan.Port if port == 0 && section != nil && section["port"] != nil { port = section["port"].(int) @@ -474,7 +482,7 @@ func (a *Agent) reloadClusters(ctx *motan.Context) { serviceItemKeep := make(map[string]bool) clusterMap := make(map[interface{}]interface{}) serviceMap := make(map[interface{}]interface{}) - var allRefersURLs = []*motan.URL{} + var allRefersURLs []*motan.URL if a.configurer != nil { //keep all dynamic refers for _, url := range a.configurer.subscribeNodes { @@ -490,7 +498,7 @@ func (a *Agent) reloadClusters(ctx *motan.Context) { } service := url.Path - mapKey := getClusterKey(url.Group, url.GetStringParamsWithDefault(motan.VersionKey, "0.1"), url.Protocol, url.Path) + mapKey := getClusterKey(url.Group, url.GetStringParamsWithDefault(motan.VersionKey, motan.DefaultReferVersion), url.Protocol, url.Path) // find exists old serviceMap var serviceMapValue serviceMapItem @@ -589,7 +597,7 @@ func (a *Agent) initCluster(url *motan.URL) { } a.serviceMap.UnsafeStore(url.Path, serviceMapItemArr) }) - mapKey := getClusterKey(url.Group, url.GetStringParamsWithDefault(motan.VersionKey, "0.1"), url.Protocol, url.Path) + mapKey := getClusterKey(url.Group, url.GetStringParamsWithDefault(motan.VersionKey, motan.DefaultReferVersion), url.Protocol, url.Path) a.clsLock.Lock() // Mutually exclusive with the reloadClusters method defer a.clsLock.Unlock() a.clusterMap.Store(mapKey, c) @@ -748,7 +756,9 @@ func (a *agentMessageHandler) httpCall(request motan.Request, ck string, httpClu if err != nil { return getDefaultResponse(request.GetRequestID(), "do http request failed : "+err.Error()) } - res = &motan.MotanResponse{RequestID: request.GetRequestID()} + httpMotanResp := mhttp.AcquireHttpMotanResponse() + httpMotanResp.RequestID = request.GetRequestID() + res = httpMotanResp mhttp.FasthttpResponseToMotanResponse(res, httpResponse) return res } @@ -794,61 +804,38 @@ func (a *agentMessageHandler) Call(request motan.Request) (res motan.Response) { } return res } -func (a *agentMessageHandler) matchRule(typ, cond, key string, data []serviceMapItem, f func(u *motan.URL) string) (foundClusters []serviceMapItem, err error) { - if cond == "" { - err = fmt.Errorf("empty %s is not supported", typ) + +func (a *agentMessageHandler) findCluster(request motan.Request) (c *cluster.MotanCluster, key string, err error) { + service := request.GetServiceName() + if service == "" { + err = fmt.Errorf("empty service is not supported. service: %s", service) return } - for _, item := range data { - if f(item.url) == cond { - foundClusters = append(foundClusters, item) - } + serviceItemArrI, exists := a.agent.serviceMap.Load(service) + if !exists { + err = fmt.Errorf("cluster not found. service: %s", service) + return } - if len(foundClusters) == 0 { - err = fmt.Errorf("cluster not found. cluster:%s", key) + clusters := serviceItemArrI.([]serviceMapItem) + if len(clusters) == 1 { + //TODO: add strict mode to avoid incorrect group call + c = clusters[0].cluster return } - return -} -func (a *agentMessageHandler) findCluster(request motan.Request) (c *cluster.MotanCluster, key string, err error) { - service := request.GetServiceName() group := request.GetAttachment(mpro.MGroup) - version := request.GetAttachment(mpro.MVersion) - protocol := request.GetAttachment(mpro.MProxyProtocol) - reqInfo := fmt.Sprintf("request information: {service: %s, group: %s, protocol: %s, version: %s}", - service, group, protocol, version) - serviceItemArrI, exists := a.agent.serviceMap.Load(service) - if !exists { - err = fmt.Errorf("cluster not found. cluster:%s, %s", service, reqInfo) + if group == "" { + err = fmt.Errorf("multiple clusters are matched with service: %s, but the group is empty", service) return } - search := []struct { - tip string - cond string - condFn func(u *motan.URL) string - }{ - {"service", service, func(u *motan.URL) string { return u.Path }}, - {"group", group, func(u *motan.URL) string { return u.Group }}, - {"protocol", protocol, func(u *motan.URL) string { return u.Protocol }}, - {"version", version, func(u *motan.URL) string { return u.GetParam(motan.VersionKey, "") }}, - } - foundClusters := serviceItemArrI.([]serviceMapItem) - for i, rule := range search { - if i == 0 { - key = rule.cond - } else { - key += "_" + rule.cond - } - foundClusters, err = a.matchRule(rule.tip, rule.cond, key, foundClusters, rule.condFn) - if err != nil { - return - } - if len(foundClusters) == 1 { - c = foundClusters[0].cluster + version := request.GetAttachment(mpro.MVersion) + protocol := request.GetAttachment(mpro.MProxyProtocol) + for _, j := range clusters { + if j.url.IsMatch(service, group, protocol, version) { + c = j.cluster return } } - err = fmt.Errorf("less condition to select cluster, maybe this service belongs to multiple group, protocol, version; cluster: %s, %s", key, reqInfo) + err = fmt.Errorf("no cluster matches the request; info: {service: %s, group: %s, protocol: %s, version: %s}", service, group, protocol, version) return } @@ -1145,7 +1132,7 @@ func (a *Agent) startMServer() { continue } a.mport = port - managementListener = motan.TCPKeepAliveListener{listener.(*net.TCPListener)} + managementListener = motan.TCPKeepAliveListener{TCPListener: listener.(*net.TCPListener)} break } if managementListener == nil { @@ -1158,7 +1145,7 @@ func (a *Agent) startMServer() { vlog.Infof("listen manage port %d failed:%s", a.mport, err.Error()) return } - managementListener = motan.TCPKeepAliveListener{listener.(*net.TCPListener)} + managementListener = motan.TCPKeepAliveListener{TCPListener: listener.(*net.TCPListener)} } vlog.Infof("start listen manage for address: %s", managementListener.Addr().String()) diff --git a/agent_test.go b/agent_test.go index 55b8929d..3942d5ad 100644 --- a/agent_test.go +++ b/agent_test.go @@ -11,6 +11,7 @@ import ( vlog "github.com/weibocom/motan-go/log" "github.com/weibocom/motan-go/registry" "github.com/weibocom/motan-go/serialize" + "github.com/weibocom/motan-go/server" _ "github.com/weibocom/motan-go/server" _ "golang.org/x/net/context" "io/ioutil" @@ -63,6 +64,7 @@ motan-agent: log_dir: "stdout" snapshot_dir: "./snapshot" application: "testing" + processPoolSize: 100 motan-registry: direct: @@ -87,6 +89,7 @@ motan-refer: resp := c1.BaseCall(req, nil) assert.Nil(t, resp.GetException()) assert.Equal(t, "Hello jack from motan server", resp.GetValue()) + assert.Equal(t, 100, server.GetProcessPoolSize()) } func Test_unixClientCall2(t *testing.T) { t.Parallel() @@ -387,7 +390,7 @@ func TestAgent_InitCall(t *testing.T) { } //test init cluster with one path and one groups in clusterMap - temp := agent.clusterMap.LoadOrNil(getClusterKey("test1", "0.1", "", "")) + temp := agent.clusterMap.LoadOrNil(getClusterKey("test1", "1.0", "", "")) assert.NotNil(t, temp, "init cluster with one path and two groups in clusterMap fail") //test agentHandler call with group @@ -413,15 +416,18 @@ func TestAgent_InitCall(t *testing.T) { version string except string }{ + // only input service，and there is only one cluster，findCluster would return successfully {"test0", "", "", "", "No refers for request"}, - {"test-1", "111", "222", "333", "cluster not found. cluster:test-1"}, - {"test3", "", "", "", "empty group is not supported"}, + {"test0", "g0", "", "", "No refers for request"}, + {"test0", "g0", "http", "", "No refers for request"}, + {"test0", "g0", "", "1.3", "No refers for request"}, + {"test-1", "111", "222", "333", "cluster not found"}, {"test", "g2", "", "", "No refers for request"}, - {"test", "g1", "", "", "empty protocol is not supported"}, {"test", "g1", "motan2", "", "No refers for request"}, - {"test", "g1", "motan", "", "empty version is not supported"}, {"test", "g1", "http", "1.3", "No refers for request"}, - {"test", "g1", "http", "1.2", "less condition to select cluster"}, + {"test", "b", "c", "d", "no cluster matches the request"}, + // one service matches multiple clusters, without passing group + {"test", "", "c", "d", "multiple clusters are matched with service"}, } { request.ServiceName = v.service request.SetAttachment(mpro.MGroup, v.group) @@ -479,10 +485,9 @@ func TestAgent_InitCall(t *testing.T) { version string except string }{ - {"test3", "111", "222", "333", "cluster not found. cluster:test3"}, - {"test4", "", "", "", "empty group is not supported"}, + {"test3", "111", "222", "333", "cluster not found. service: test3"}, {"test5", "", "", "", "No refers for request"}, - {"helloService2", "", "", "", "cluster not found. cluster:helloService2"}, + {"helloService2", "", "", "", "cluster not found. service: helloService2"}, } { request = newRequest(v.service, "") request.SetAttachment(mpro.MGroup, v.group) @@ -633,7 +638,7 @@ motan-service: c1.Initialize() var reply []byte req := c1.BuildRequestWithGroup("helloService", "/unixclient", []interface{}{}, "hello") - req.SetAttachment("HTTP_HOST", "test.com") + req.SetAttachment("http_Host", "test.com") resp := c1.BaseCall(req, &reply) assert.Nil(t, resp.GetException()) assert.Equal(t, "okay", string(reply)) diff --git a/cluster/command.go b/cluster/command.go index f49ad7af..f80405f7 100644 --- a/cluster/command.go +++ b/cluster/command.go @@ -108,9 +108,11 @@ type CmdList []ClientCommand func (c CmdList) Len() int { return len(c) } + func (c CmdList) Swap(i, j int) { c[i], c[j] = c[j], c[i] } + func (c CmdList) Less(i, j int) bool { return c[i].Index < c[j].Index } @@ -149,7 +151,7 @@ func GetCommandRegistryWrapper(cluster *MotanCluster, registry motan.Registry) m mixGroups := cluster.GetURL().GetParam(motan.MixGroups, "") if mixGroups != "" { groups := strings.Split(mixGroups, ",") - command := &ClientCommand{CommandType: CMDTrafficControl, Index: 0, Version: "1.0", MergeGroups: make([]string, 0, len(groups)+1)} + command := &ClientCommand{CommandType: CMDTrafficControl, Index: 0, Version: motan.DefaultReferVersion, MergeGroups: make([]string, 0, len(groups)+1)} ownGroup := cluster.GetURL().Group command.MergeGroups = append(command.MergeGroups, ownGroup) for _, group := range groups { diff --git a/cluster/motanCluster.go b/cluster/motanCluster.go index b0180ac4..4bae0690 100644 --- a/cluster/motanCluster.go +++ b/cluster/motanCluster.go @@ -60,6 +60,7 @@ func (m *MotanCluster) GetURL() *motan.URL { func (m *MotanCluster) SetURL(url *motan.URL) { m.url = url } + func (m *MotanCluster) Call(request motan.Request) (res motan.Response) { defer motan.HandlePanic(func() { res = motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "cluster call panic", ErrType: motan.ServiceException}) @@ -71,6 +72,7 @@ func (m *MotanCluster) Call(request motan.Request) (res motan.Response) { vlog.Infoln("cluster:" + m.GetIdentity() + "is not available!") return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "cluster not available, maybe caused by degrade", ErrType: motan.ServiceException}) } + func (m *MotanCluster) initCluster() bool { m.registryRefers = make(map[string][]motan.EndPoint) //ha @@ -99,15 +101,19 @@ func (m *MotanCluster) initCluster() bool { vlog.Infof("init MotanCluster %s", m.GetIdentity()) return true } + func (m *MotanCluster) SetLoadBalance(loadBalance motan.LoadBalance) { m.LoadBalance = loadBalance } + func (m *MotanCluster) SetHaStrategy(haStrategy motan.HaStrategy) { m.HaStrategy = haStrategy } + func (m *MotanCluster) GetRefers() []motan.EndPoint { return m.Refers } + func (m *MotanCluster) refresh() { newRefers := make([]motan.EndPoint, 0, 32) for _, v := range m.registryRefers { @@ -120,14 +126,17 @@ func (m *MotanCluster) refresh() { m.Refers = newRefers m.LoadBalance.OnRefresh(newRefers) } + func (m *MotanCluster) ShuffleEndpoints(endpoints []motan.EndPoint) []motan.EndPoint { rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) return endpoints } + func (m *MotanCluster) AddRegistry(registry motan.Registry) { m.Registries = append(m.Registries, registry) } + func (m *MotanCluster) Notify(registryURL *motan.URL, urls []*motan.URL) { vlog.Infof("cluster %s receive notify size %d. ", m.GetIdentity(), len(urls)) m.notifyLock.Lock() diff --git a/core/bytes.go b/core/bytes.go index 604c6d57..26dedc27 100644 --- a/core/bytes.go +++ b/core/bytes.go @@ -4,6 +4,16 @@ import ( "encoding/binary" "errors" "io" + "sync" +) + +var ( + bytesBufferPool = sync.Pool{New: func() interface{} { + return &BytesBuffer{ + temp: make([]byte, 8), + order: binary.BigEndian, + } + }} ) // BytesBuffer is a variable-sized buffer of bytes with Read and Write methods. @@ -19,14 +29,15 @@ type BytesBuffer struct { var ErrNotEnough = errors.New("BytesBuffer: not enough bytes") var ErrOverflow = errors.New("BytesBuffer: integer overflow") -// NewBytesBuffer create a empty BytesBuffer with initial size +// NewBytesBuffer create an empty BytesBuffer with initial size func NewBytesBuffer(initsize int) *BytesBuffer { return NewBytesBufferWithOrder(initsize, binary.BigEndian) } -// NewBytesBufferWithOrder create a empty BytesBuffer with initial size and byte order +// NewBytesBufferWithOrder create an empty BytesBuffer with initial size and byte order func NewBytesBufferWithOrder(initsize int, order binary.ByteOrder) *BytesBuffer { - return &BytesBuffer{buf: make([]byte, initsize), + return &BytesBuffer{ + buf: make([]byte, initsize), order: order, temp: make([]byte, 8), } @@ -78,6 +89,16 @@ func (b *BytesBuffer) WriteByte(c byte) { b.wpos++ } +// WriteString write a str string append the BytesBuffer, and the wpos will increase len(str) +func (b *BytesBuffer) WriteString(str string) { + l := len(str) + if len(b.buf) < b.wpos+l { + b.grow(l) + } + copy(b.buf[b.wpos:], str) + b.wpos += l +} + // Write write a byte array append the BytesBuffer, and the wpos will increase len(bytes) func (b *BytesBuffer) Write(bytes []byte) { l := len(bytes) @@ -117,11 +138,11 @@ func (b *BytesBuffer) WriteUint64(u uint64) { } func (b *BytesBuffer) WriteZigzag32(u uint32) int { - return b.WriteVarint(uint64((u << 1) ^ uint32((int32(u) >> 31)))) + return b.WriteVarint(uint64((u << 1) ^ uint32(int32(u)>>31))) } func (b *BytesBuffer) WriteZigzag64(u uint64) int { - return b.WriteVarint(uint64((u << 1) ^ uint64((int64(u) >> 63)))) + return b.WriteVarint((u << 1) ^ uint64(int64(u)>>63)) } func (b *BytesBuffer) WriteVarint(u uint64) int { @@ -225,10 +246,10 @@ func (b *BytesBuffer) ReadVarint() (x uint64, err error) { return 0, err } if (temp & 0x80) != 0x80 { - x |= (uint64(temp) << offset) + x |= uint64(temp) << offset return x, nil } - x |= (uint64(temp&0x7f) << offset) + x |= uint64(temp&0x7f) << offset } return 0, ErrOverflow } @@ -262,3 +283,20 @@ func (b *BytesBuffer) Remain() int { return b.wpos - b.rpos } func (b *BytesBuffer) Len() int { return b.wpos - 0 } func (b *BytesBuffer) Cap() int { return cap(b.buf) } + +// AcquireBytesBuffer create an empty BytesBuffer with initial size and byte order from bytesBufferPool +func AcquireBytesBuffer(initSize int) *BytesBuffer { + bb := bytesBufferPool.Get().(*BytesBuffer) + if bb.buf == nil { + bb.buf = make([]byte, initSize) + } + return bb +} + +// ReleaseBytesBuffer put the BytesBuffer to bytesBufferPool +func ReleaseBytesBuffer(b *BytesBuffer) { + if b != nil { + b.Reset() + bytesBufferPool.Put(b) + } +} diff --git a/core/bytes_test.go b/core/bytes_test.go index f842cf76..07b329cb 100644 --- a/core/bytes_test.go +++ b/core/bytes_test.go @@ -3,6 +3,7 @@ package core import ( "encoding/binary" "fmt" + "github.com/stretchr/testify/assert" "testing" ) @@ -238,3 +239,55 @@ func TestZigzag(t *testing.T) { } } } + +func TestBytesBuffer_WriteString_Grow(t *testing.T) { + a := BytesBuffer{} + a.WriteString("") + assert.Equal(t, 0, len(a.buf)) + a.WriteString("abc") + assert.Equal(t, "abc", string(a.Bytes())) + assert.Equal(t, 3, len(a.buf)) + a.WriteString("abc") + assert.Equal(t, "abcabc", string(a.Bytes())) + assert.Equal(t, 9, len(a.buf)) +} + +func TestBytesBuffer_WriteString_NoGrow(t *testing.T) { + a := BytesBuffer{buf: make([]byte, 4)} + a.WriteString("abc") + assert.Equal(t, "abc", string(a.Bytes())) + assert.Equal(t, 4, len(a.buf)) +} + +func TestDefaultBytesBufferPool(t *testing.T) { + // consume pool + for { + bb := bytesBufferPool.Get().(*BytesBuffer) + if bb.buf == nil { + break + } + } + // test new BytesBuffer + bb := AcquireBytesBuffer(10) + assert.Equal(t, 0, bb.Len()) + assert.Equal(t, 10, len(bb.buf)) + assert.Equal(t, 10, bb.Cap()) + + // test release and acquire + ReleaseBytesBuffer(bb) + newBb := AcquireBytesBuffer(10) + assert.NotEqual(t, nil, newBb) + assert.Equal(t, 0, bb.Len()) + assert.Equal(t, 10, len(bb.buf)) + assert.Equal(t, 10, bb.Cap()) + + // test put nil + var nilByteBuffer *BytesBuffer + // can not put nil to pool + ReleaseBytesBuffer(nilByteBuffer) + nilBb := bytesBufferPool.Get().(*BytesBuffer) + assert.NotEqual(t, nil, nilBb) + ReleaseBytesBuffer(nilBb) + notNilBb := AcquireBytesBuffer(10) + assert.NotEqual(t, nil, notNilBb) +} diff --git a/core/constants.go b/core/constants.go index 20ec7e76..a3108104 100644 --- a/core/constants.go +++ b/core/constants.go @@ -129,3 +129,7 @@ const ( EUnkonwnMsg = 1003 EConvertMsg = 1004 ) + +const ( + DefaultReferVersion = "1.0" +) diff --git a/core/map.go b/core/map.go index 4aa2bcf3..5d5502e8 100644 --- a/core/map.go +++ b/core/map.go @@ -26,6 +26,14 @@ func (m *StringMap) Store(key, value string) { m.mu.Unlock() } +func (m *StringMap) Reset() { + m.mu.Lock() + for k := range m.innerMap { + delete(m.innerMap, k) + } + m.mu.Unlock() +} + func (m *StringMap) Delete(key string) { m.mu.Lock() delete(m.innerMap, key) @@ -45,20 +53,13 @@ func (m *StringMap) LoadOrEmpty(key string) string { } // Range calls f sequentially for each key and value present in the map -// If f returns false, range stops the iteration +// If f returns false, range stops the iteration. +// +// Notice: do not delete elements in range function,because of Range loop the inner map directly. func (m *StringMap) Range(f func(k, v string) bool) { m.mu.RLock() - keys := make([]string, 0, len(m.innerMap)) - for k := range m.innerMap { - keys = append(keys, k) - } - m.mu.RUnlock() - - for _, k := range keys { - v, ok := m.Load(k) - if !ok { - continue - } + defer m.mu.RUnlock() + for k, v := range m.innerMap { if !f(k, v) { break } diff --git a/core/map_test.go b/core/map_test.go index ec1c95a5..aeddce9f 100644 --- a/core/map_test.go +++ b/core/map_test.go @@ -220,3 +220,16 @@ func BenchmarkCopyOnWriteMap_Load(b *testing.B) { } }) } + +func TestStringMap_Range(t *testing.T) { + a := NewStringMap(10) + a.Store("a", "a") + a.Store("b", "a") + a.Store("c", "a") + s := "" + a.Range(func(k, v string) bool { + s += v + return true + }) + assert.Equal(t, "aaa", s) +} diff --git a/core/motan.go b/core/motan.go index adfd9016..7b420cb5 100644 --- a/core/motan.go +++ b/core/motan.go @@ -14,7 +14,20 @@ import ( type taskHandler func() -var refreshTaskPool = make(chan taskHandler, 100) +var ( + refreshTaskPool = make(chan taskHandler, 100) + requestPool = sync.Pool{New: func() interface{} { + return &MotanRequest{ + RPCContext: &RPCContext{}, + Arguments: []interface{}{}, + } + }} + responsePool = sync.Pool{New: func() interface{} { + return &MotanResponse{ + RPCContext: &RPCContext{}, + } + }} +) func init() { go func() { @@ -28,10 +41,8 @@ func init() { } const ( - DefaultAttachmentSize = 16 - DefaultRPCContextMetaSize = 8 - - ProtocolLocal = "local" + DefaultAttachmentSize = 16 + ProtocolLocal = "local" ) var ( @@ -373,8 +384,6 @@ type RPCContext struct { AsyncCall bool Result *AsyncResult Reply interface{} - - Meta *StringMap // various time, it's owned by motan request context RequestSendTime time.Time RequestReceiveTime time.Time @@ -391,6 +400,27 @@ type RPCContext struct { RemoteAddr string // remote address } +func (c *RPCContext) Reset() { + // because there is a binding between RPCContext and request/response, + // some attributes such as RequestSendTime、RequestReceiveTime will be reset by request/response + // therefore, these attributes do not need to be reset here. + c.ExtFactory = nil + c.OriginalMessage = nil + c.Oneway = false + c.Proxy = false + c.GzipSize = 0 + c.BodySize = 0 + c.SerializeNum = 0 + c.Serialized = false + c.AsyncCall = false + c.Result = nil + c.Reply = nil + c.FinishHandlers = c.FinishHandlers[:0] + c.Tc = nil + c.IsMotanV1 = false + c.RemoteAddr = "" +} + func (c *RPCContext) AddFinishHandler(handler FinishHandler) { c.FinishHandlers = append(c.FinishHandlers, handler) } @@ -443,105 +473,125 @@ type MotanRequest struct { mu sync.Mutex } +func AcquireMotanRequest() *MotanRequest { + return requestPool.Get().(*MotanRequest) +} + +func ReleaseMotanRequest(req *MotanRequest) { + if req != nil { + req.Reset() + requestPool.Put(req) + } +} + +// Reset reset motan request +func (req *MotanRequest) Reset() { + req.Method = "" + req.RequestID = 0 + req.ServiceName = "" + req.MethodDesc = "" + req.RPCContext.Reset() + req.Attachment = nil + req.Arguments = req.Arguments[:0] +} + // GetAttachment GetAttachment -func (m *MotanRequest) GetAttachment(key string) string { - if m.Attachment == nil { +func (req *MotanRequest) GetAttachment(key string) string { + if req.Attachment == nil { return "" } - return m.Attachment.LoadOrEmpty(key) + return req.Attachment.LoadOrEmpty(key) } // SetAttachment : SetAttachment -func (m *MotanRequest) SetAttachment(key string, value string) { - m.GetAttachments().Store(key, value) +func (req *MotanRequest) SetAttachment(key string, value string) { + req.GetAttachments().Store(key, value) } // GetServiceName GetServiceName -func (m *MotanRequest) GetServiceName() string { - return m.ServiceName +func (req *MotanRequest) GetServiceName() string { + return req.ServiceName } // GetMethod GetMethod -func (m *MotanRequest) GetMethod() string { - return m.Method +func (req *MotanRequest) GetMethod() string { + return req.Method } // GetMethodDesc GetMethodDesc -func (m *MotanRequest) GetMethodDesc() string { - return m.MethodDesc +func (req *MotanRequest) GetMethodDesc() string { + return req.MethodDesc } -func (m *MotanRequest) GetArguments() []interface{} { - return m.Arguments +func (req *MotanRequest) GetArguments() []interface{} { + return req.Arguments } -func (m *MotanRequest) GetRequestID() uint64 { - return m.RequestID + +func (req *MotanRequest) GetRequestID() uint64 { + return req.RequestID } -func (m *MotanRequest) SetArguments(arguments []interface{}) { - m.Arguments = arguments +func (req *MotanRequest) SetArguments(arguments []interface{}) { + req.Arguments = arguments } -func (m *MotanRequest) GetAttachments() *StringMap { - attachment := (*StringMap)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&m.Attachment)))) +func (req *MotanRequest) GetAttachments() *StringMap { + attachment := (*StringMap)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&req.Attachment)))) if attachment != nil { return attachment } - m.mu.Lock() - defer m.mu.Unlock() - if m.Attachment == nil { + req.mu.Lock() + defer req.mu.Unlock() + if req.Attachment == nil { attachment = NewStringMap(DefaultAttachmentSize) - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&m.Attachment)), unsafe.Pointer(attachment)) + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&req.Attachment)), unsafe.Pointer(attachment)) } else { - attachment = m.Attachment + attachment = req.Attachment } return attachment } -func (m *MotanRequest) GetRPCContext(canCreate bool) *RPCContext { - if m.RPCContext == nil && canCreate { - m.RPCContext = &RPCContext{ - Meta: NewStringMap(DefaultRPCContextMetaSize), - } +func (req *MotanRequest) GetRPCContext(canCreate bool) *RPCContext { + if req.RPCContext == nil && canCreate { + req.RPCContext = &RPCContext{} } - return m.RPCContext + return req.RPCContext } -func (m *MotanRequest) Clone() interface{} { +func (req *MotanRequest) Clone() interface{} { newRequest := &MotanRequest{ - RequestID: m.RequestID, - ServiceName: m.ServiceName, - Method: m.Method, - MethodDesc: m.MethodDesc, - Arguments: m.Arguments, + RequestID: req.RequestID, + ServiceName: req.ServiceName, + Method: req.Method, + MethodDesc: req.MethodDesc, + Arguments: req.Arguments, } - if m.Attachment != nil { - newRequest.Attachment = m.Attachment.Copy() + if req.Attachment != nil { + newRequest.Attachment = req.Attachment.Copy() } - if m.RPCContext != nil { + if req.RPCContext != nil { newRequest.RPCContext = &RPCContext{ - ExtFactory: m.RPCContext.ExtFactory, - Oneway: m.RPCContext.Oneway, - Proxy: m.RPCContext.Proxy, - GzipSize: m.RPCContext.GzipSize, - SerializeNum: m.RPCContext.SerializeNum, - Serialized: m.RPCContext.Serialized, - AsyncCall: m.RPCContext.AsyncCall, - Result: m.RPCContext.Result, - Reply: m.RPCContext.Reply, - Meta: m.RPCContext.Meta, - RequestSendTime: m.RPCContext.RequestSendTime, - RequestReceiveTime: m.RPCContext.RequestReceiveTime, - ResponseSendTime: m.RPCContext.ResponseSendTime, - ResponseReceiveTime: m.RPCContext.ResponseReceiveTime, - FinishHandlers: m.RPCContext.FinishHandlers, - Tc: m.RPCContext.Tc, + ExtFactory: req.RPCContext.ExtFactory, + Oneway: req.RPCContext.Oneway, + Proxy: req.RPCContext.Proxy, + GzipSize: req.RPCContext.GzipSize, + SerializeNum: req.RPCContext.SerializeNum, + Serialized: req.RPCContext.Serialized, + AsyncCall: req.RPCContext.AsyncCall, + Result: req.RPCContext.Result, + Reply: req.RPCContext.Reply, + RequestSendTime: req.RPCContext.RequestSendTime, + RequestReceiveTime: req.RPCContext.RequestReceiveTime, + ResponseSendTime: req.RPCContext.ResponseSendTime, + ResponseReceiveTime: req.RPCContext.ResponseReceiveTime, + FinishHandlers: req.RPCContext.FinishHandlers, + Tc: req.RPCContext.Tc, } - if m.RPCContext.OriginalMessage != nil { - if oldMessage, ok := m.RPCContext.OriginalMessage.(Cloneable); ok { + if req.RPCContext.OriginalMessage != nil { + if oldMessage, ok := req.RPCContext.OriginalMessage.(Cloneable); ok { newRequest.RPCContext.OriginalMessage = oldMessage.Clone() } else { - newRequest.RPCContext.OriginalMessage = m.RPCContext.OriginalMessage + newRequest.RPCContext.OriginalMessage = req.RPCContext.OriginalMessage } } } @@ -550,14 +600,14 @@ func (m *MotanRequest) Clone() interface{} { // ProcessDeserializable : DeserializableValue to real params according toType // some serialization can deserialize without toType, so nil toType can be accepted in these serializations -func (m *MotanRequest) ProcessDeserializable(toTypes []interface{}) error { - if m.GetArguments() != nil && len(m.GetArguments()) == 1 { - if d, ok := m.GetArguments()[0].(*DeserializableValue); ok { +func (req *MotanRequest) ProcessDeserializable(toTypes []interface{}) error { + if req.GetArguments() != nil && len(req.GetArguments()) == 1 { + if d, ok := req.GetArguments()[0].(*DeserializableValue); ok { v, err := d.DeserializeMulti(toTypes) if err != nil { return err } - m.SetArguments(v) + req.SetArguments(v) } } return nil @@ -573,78 +623,99 @@ type MotanResponse struct { mu sync.Mutex } -func (m *MotanResponse) GetAttachment(key string) string { - if m.Attachment == nil { +func AcquireMotanResponse() *MotanResponse { + return responsePool.Get().(*MotanResponse) +} + +func ReleaseMotanResponse(m *MotanResponse) { + if m != nil { + m.Reset() + responsePool.Put(m) + } +} + +func (res *MotanResponse) Reset() { + res.RequestID = 0 + res.Value = nil + res.Exception = nil + res.ProcessTime = 0 + res.Attachment = nil + res.RPCContext.Reset() +} + +func (res *MotanResponse) GetAttachment(key string) string { + if res.Attachment == nil { return "" } - return m.Attachment.LoadOrEmpty(key) + return res.Attachment.LoadOrEmpty(key) } -func (m *MotanResponse) SetAttachment(key string, value string) { - m.GetAttachments().Store(key, value) +func (res *MotanResponse) SetAttachment(key string, value string) { + res.GetAttachments().Store(key, value) } -func (m *MotanResponse) GetValue() interface{} { - return m.Value +func (res *MotanResponse) GetValue() interface{} { + return res.Value } -func (m *MotanResponse) GetException() *Exception { - return m.Exception +func (res *MotanResponse) GetException() *Exception { + return res.Exception } -func (m *MotanResponse) GetRequestID() uint64 { - return m.RequestID +func (res *MotanResponse) GetRequestID() uint64 { + return res.RequestID } -func (m *MotanResponse) GetProcessTime() int64 { - return m.ProcessTime +func (res *MotanResponse) GetProcessTime() int64 { + return res.ProcessTime } -func (m *MotanResponse) GetAttachments() *StringMap { - attachment := (*StringMap)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&m.Attachment)))) +func (res *MotanResponse) GetAttachments() *StringMap { + attachment := (*StringMap)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&res.Attachment)))) if attachment != nil { return attachment } - m.mu.Lock() - defer m.mu.Unlock() - if m.Attachment == nil { + res.mu.Lock() + defer res.mu.Unlock() + if res.Attachment == nil { attachment = NewStringMap(DefaultAttachmentSize) - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&m.Attachment)), unsafe.Pointer(attachment)) + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&res.Attachment)), unsafe.Pointer(attachment)) } else { - attachment = m.Attachment + attachment = res.Attachment } return attachment } -func (m *MotanResponse) GetRPCContext(canCreate bool) *RPCContext { - if m.RPCContext == nil && canCreate { - m.RPCContext = &RPCContext{ - Meta: NewStringMap(DefaultRPCContextMetaSize), - } +func (res *MotanResponse) GetRPCContext(canCreate bool) *RPCContext { + if res.RPCContext == nil && canCreate { + res.RPCContext = &RPCContext{} } - return m.RPCContext + return res.RPCContext } -func (m *MotanResponse) SetProcessTime(time int64) { - m.ProcessTime = time +func (res *MotanResponse) SetProcessTime(time int64) { + res.ProcessTime = time } // ProcessDeserializable : same with MotanRequest -func (m *MotanResponse) ProcessDeserializable(toType interface{}) error { - if m.GetValue() != nil { - if d, ok := m.GetValue().(*DeserializableValue); ok { +func (res *MotanResponse) ProcessDeserializable(toType interface{}) error { + if res.GetValue() != nil { + if d, ok := res.GetValue().(*DeserializableValue); ok { v, err := d.Deserialize(toType) if err != nil { return err } - m.Value = v + res.Value = v } } return nil } func BuildExceptionResponse(requestid uint64, e *Exception) *MotanResponse { - return &MotanResponse{RequestID: requestid, Exception: e} + resp := AcquireMotanResponse() + resp.RequestID = requestid + resp.Exception = e + return resp } // extensions factory-func @@ -695,8 +766,8 @@ func (d *DefaultExtensionFactory) GetLB(url *URL) LoadBalance { } func (d *DefaultExtensionFactory) GetFilter(name string) Filter { - if newDefualt, ok := d.filterFactories[strings.TrimSpace(name)]; ok { - return newDefualt() + if newDefault, ok := d.filterFactories[strings.TrimSpace(name)]; ok { + return newDefault() } vlog.Errorf("filter name %s is not found in DefaultExtensionFactory!", name) return nil @@ -870,12 +941,15 @@ func (l *lastEndPointFilter) HasNext() bool { func (l *lastEndPointFilter) SetNext(nextFilter EndPointFilter) { vlog.Errorf("should not set next in lastEndPointFilter! filer:%s", nextFilter.GetName()) } + func (l *lastEndPointFilter) GetNext() EndPointFilter { return nil } + func (l *lastEndPointFilter) GetIndex() int { return 100 } + func (l *lastEndPointFilter) GetType() int32 { return EndPointFilterType } @@ -885,6 +959,7 @@ type lastClusterFilter struct{} func (l *lastClusterFilter) GetName() string { return "lastClusterFilter" } + func (l *lastClusterFilter) NewFilter(url *URL) Filter { return GetLastClusterFilter() } @@ -905,15 +980,19 @@ func (l *lastClusterFilter) Filter(haStrategy HaStrategy, loadBalance LoadBalanc func (l *lastClusterFilter) HasNext() bool { return false } + func (l *lastClusterFilter) SetNext(nextFilter ClusterFilter) { vlog.Errorf("should not set next in lastClusterFilter! filer:%s", nextFilter.GetName()) } + func (l *lastClusterFilter) GetNext() ClusterFilter { return nil } + func (l *lastClusterFilter) GetIndex() int { return 100 } + func (l *lastClusterFilter) GetType() int32 { return ClusterFilterType } @@ -931,12 +1010,15 @@ func (f *FilterEndPoint) Call(request Request) Response { } return f.Filter.Filter(f.Caller, request) } + func (f *FilterEndPoint) GetURL() *URL { return f.URL } + func (f *FilterEndPoint) SetURL(url *URL) { f.URL = url } + func (f *FilterEndPoint) GetName() string { return "FilterEndPoint" } @@ -1034,7 +1116,7 @@ func newRegistryGroupServiceCacheInfo(sr ServiceDiscoverableRegistry, group stri func (c *registryGroupServiceCacheInfo) getServices() ([]string, map[string]string) { if time.Now().Sub(c.lastUpdTime.Load().(time.Time)) >= registryGroupServiceInfoMaxCacheTime { select { - case refreshTaskPool <- taskHandler(func() { c.refreshServices() }): + case refreshTaskPool <- func() { c.refreshServices() }: default: vlog.Warningf("Task pool is full, refresh service of group [%s] delay", c.group) } diff --git a/core/test.go b/core/test.go index af3d857d..047683a8 100644 --- a/core/test.go +++ b/core/test.go @@ -18,6 +18,7 @@ type TestFilter struct { func (t *TestFilter) GetName() string { return "TestFilter" } + func (t *TestFilter) NewFilter(url *URL) Filter { //init with url in here return &TestFilter{URL: url} @@ -29,18 +30,23 @@ func (t *TestFilter) Filter(haStrategy HaStrategy, loadBalance LoadBalance, requ return t.GetNext().Filter(haStrategy, loadBalance, request) } + func (t *TestFilter) HasNext() bool { return t.next != nil } + func (t *TestFilter) SetNext(nextFilter ClusterFilter) { t.next = nextFilter } + func (t *TestFilter) GetNext() ClusterFilter { return t.next } + func (t *TestFilter) GetIndex() int { return t.Index } + func (t *TestFilter) GetType() int32 { return ClusterFilterType } @@ -54,6 +60,7 @@ type TestEndPointFilter struct { func (t *TestEndPointFilter) GetName() string { return "TestEndPointFilter" } + func (t *TestEndPointFilter) NewFilter(url *URL) Filter { //init with url in here return &TestEndPointFilter{URL: url} @@ -71,15 +78,19 @@ func (t *TestEndPointFilter) Filter(caller Caller, request Request) Response { func (t *TestEndPointFilter) HasNext() bool { return t.next != nil } + func (t *TestEndPointFilter) SetNext(nextFilter EndPointFilter) { t.next = nextFilter } + func (t *TestEndPointFilter) GetNext() EndPointFilter { return t.next } + func (t *TestEndPointFilter) GetIndex() int { return t.Index } + func (t *TestEndPointFilter) GetType() int32 { return EndPointFilterType } @@ -123,12 +134,15 @@ type TestEndPoint struct { func (t *TestEndPoint) GetURL() *URL { return t.URL } + func (t *TestEndPoint) SetURL(url *URL) { t.URL = url } + func (t *TestEndPoint) GetName() string { return "testEndPoint" } + func (t *TestEndPoint) Call(request Request) Response { fmt.Println("mock rpc request..") if t.ProcessTime != 0 { @@ -173,9 +187,11 @@ func (t *TestHaStrategy) GetName() string { func (t *TestHaStrategy) GetURL() *URL { return t.URL } + func (t *TestHaStrategy) SetURL(url *URL) { t.URL = url } + func (t *TestHaStrategy) Call(request Request, loadBalance LoadBalance) Response { fmt.Println("in testHaStrategy call") refer := loadBalance.Select(request) @@ -189,6 +205,7 @@ type TestLoadBalance struct { func (t *TestLoadBalance) OnRefresh(endpoints []EndPoint) { t.Endpoints = endpoints } + func (t *TestLoadBalance) Select(request Request) EndPoint { fmt.Println("in testLoadBalance select") endpoint := &TestEndPoint{} @@ -201,9 +218,11 @@ func (t *TestLoadBalance) Select(request Request) EndPoint { filterEndPoint.Filter = efilter1 return filterEndPoint } + func (t *TestLoadBalance) SelectArray(request Request) []EndPoint { return []EndPoint{&TestEndPoint{}} } + func (t *TestLoadBalance) SetWeight(weight string) { } @@ -217,42 +236,54 @@ type TestRegistry struct { func (t *TestRegistry) GetName() string { return "testRegistry" } + func (t *TestRegistry) Subscribe(url *URL, listener NotifyListener) { } + func (t *TestRegistry) Unsubscribe(url *URL, listener NotifyListener) { } + func (t *TestRegistry) Discover(url *URL) []*URL { return make([]*URL, 0) } + func (t *TestRegistry) Register(serverURL *URL) { } + func (t *TestRegistry) UnRegister(serverURL *URL) { } + func (t *TestRegistry) Available(serverURL *URL) { } + func (t *TestRegistry) Unavailable(serverURL *URL) { } + func (t *TestRegistry) GetRegisteredServices() []*URL { return make([]*URL, 0) } + func (t *TestRegistry) GetURL() *URL { if t.URL == nil { t.URL = &URL{} } return t.URL } + func (t *TestRegistry) SetURL(url *URL) { t.URL = url } + func (t *TestRegistry) InitRegistry() { } + func (t *TestRegistry) StartSnapshot(conf *SnapshotConf) { } diff --git a/core/url.go b/core/url.go index ec94faa6..11ac3be6 100644 --- a/core/url.go +++ b/core/url.go @@ -2,13 +2,13 @@ package core import ( "bytes" + "github.com/weibocom/motan-go/log" "sort" "strconv" "strings" + "sync" "sync/atomic" "time" - - "github.com/weibocom/motan-go/log" ) type URL struct { @@ -20,18 +20,26 @@ type URL struct { Parameters map[string]string `json:"parameters"` // cached info - address atomic.Value - identity atomic.Value + address atomic.Value + portStr atomic.Value + identity atomic.Value + hasMethodParamsCache atomic.Value // Whether it has method parameters + intParamCache sync.Map +} + +type int64Cache struct { + value int64 + isMiss bool // miss cache if true } var ( - defaultSerialize = "simple" + defaultSerialize = "simple" + defaultMethodParamsSubStr = ")." + defaultMissCache = &int64Cache{value: 0, isMiss: true} // Uniform miss cache ) -//TODO int param cache - // GetIdentity return the identity of url. identity info includes protocol, host, port, path, group -// the identity will cached, so must clear cached info after update above info by calling ClearCachedInfo() +// the identity will be cached, so must clear cached info after update above info by calling ClearCachedInfo() func (u *URL) GetIdentity() string { temp := u.identity.Load() if temp != nil && temp != "" { @@ -45,6 +53,33 @@ func (u *URL) GetIdentity() string { return idt } +// IsMatch is a tool function for comparing parameters: service, group, protocol and version +// with URL. When 'protocol' or 'version' is empty, it will be ignored +func (u *URL) IsMatch(service, group, protocol, version string) bool { + if u.Path != service { + return false + } + if group != "" && u.Group != group { + return false + } + // for motan v1 request, parameter protocol should be empty + if protocol != "" { + if u.Protocol == "motanV1Compatible" { + if protocol != "motan2" && protocol != "motan" { + return false + } + } else { + if u.Protocol != protocol { + return false + } + } + } + if version != "" && u.GetParam(VersionKey, "") != "" { + return version == u.GetParam(VersionKey, "") + } + return true +} + func (u *URL) GetIdentityWithRegistry() string { id := u.GetIdentity() registryId := u.GetParam(RegistryKey, "") @@ -54,14 +89,20 @@ func (u *URL) GetIdentityWithRegistry() string { func (u *URL) ClearCachedInfo() { u.address.Store("") u.identity.Store("") + u.portStr.Store("") + u.hasMethodParamsCache.Store("") + u.intParamCache.Range(func(key interface{}, value interface{}) bool { + u.intParamCache.Delete(key) + return true + }) } -func (u *URL) GetPositiveIntValue(key string, defaultvalue int64) int64 { - intvalue := u.GetIntValue(key, defaultvalue) - if intvalue < 1 { - return defaultvalue +func (u *URL) GetPositiveIntValue(key string, defaultValue int64) int64 { + intValue := u.GetIntValue(key, defaultValue) + if intValue < 1 { + return defaultValue } - return intvalue + return intValue } func (u *URL) GetBoolValue(key string, defaultValue bool) bool { @@ -83,39 +124,70 @@ func (u *URL) GetIntValue(key string, defaultValue int64) int64 { } func (u *URL) GetInt(key string) (int64, bool) { + if cache, ok := u.intParamCache.Load(key); ok { + if c, ok := cache.(*int64Cache); ok { // from cache + if c.isMiss { + return 0, false + } + return c.value, true + } + } + if v, ok := u.Parameters[key]; ok { intValue, err := strconv.ParseInt(v, 10, 64) if err == nil { + u.intParamCache.Store(key, &int64Cache{value: intValue, isMiss: false}) return intValue, true } } + u.intParamCache.Store(key, defaultMissCache) // set miss cache return 0, false } -func (u *URL) GetStringParamsWithDefault(key string, defaultvalue string) string { +func (u *URL) GetStringParamsWithDefault(key string, defaultValue string) string { var ret string if u.Parameters != nil { ret = u.Parameters[key] } if ret == "" { - ret = defaultvalue + ret = defaultValue } return ret } func (u *URL) GetMethodIntValue(method string, methodDesc string, key string, defaultValue int64) int64 { - mkey := method + "(" + methodDesc + ")." + key - result, b := u.GetInt(mkey) - if b { - return result + if u.hasMethodParams() { + mk := method + "(" + methodDesc + ")." + key + result, b := u.GetInt(mk) + if b { + return result + } } - result, b = u.GetInt(key) + result, b := u.GetInt(key) if b { return result } return defaultValue } +func (u *URL) hasMethodParams() bool { + v := u.hasMethodParamsCache.Load() + if v == nil || v == "" { // Check if method parameters exist + if u.Parameters != nil { + for k := range u.Parameters { + if strings.Contains(k, defaultMethodParamsSubStr) { + v = "t" + u.hasMethodParamsCache.Store("t") + return true + } + } + } + v = "f" + u.hasMethodParamsCache.Store("f") + } + return v == "t" +} + func (u *URL) GetMethodPositiveIntValue(method string, methodDesc string, key string, defaultValue int64) int64 { result := u.GetMethodIntValue(method, methodDesc, key, defaultValue) if result > 0 { @@ -146,6 +218,10 @@ func (u *URL) GetTimeDuration(key string, unit time.Duration, defaultDuration ti } func (u *URL) PutParam(key string, value string) { + u.intParamCache.Delete(key) // remove cache + if strings.Contains(key, defaultMethodParamsSubStr) { // Check if method parameter + u.hasMethodParamsCache.Store("t") + } if u.Parameters == nil { u.Parameters = make(map[string]string) } @@ -181,20 +257,20 @@ func (u *URL) ToExtInfo() string { } -func FromExtInfo(extinfo string) *URL { - defer func() { // if extinfo format not correct, just return nil URL +func FromExtInfo(extInfo string) *URL { + defer func() { // if extInfo format not correct, just return nil URL if err := recover(); err != nil { - vlog.Warningf("from ext to url fail. extinfo:%s, err:%v", extinfo, err) + vlog.Warningf("from ext to url fail. extInfo:%s, err:%v", extInfo, err) } }() - arr := strings.Split(extinfo, "?") - nodeinfos := strings.Split(arr[0], "://") - protocol := nodeinfos[0] - nodeinfos = strings.Split(nodeinfos[1], "/") - path := nodeinfos[1] - nodeinfos = strings.Split(nodeinfos[0], ":") - host := nodeinfos[0] - port, _ := strconv.ParseInt(nodeinfos[1], 10, 64) + arr := strings.Split(extInfo, "?") + nodeInfos := strings.Split(arr[0], "://") + protocol := nodeInfos[0] + nodeInfos = strings.Split(nodeInfos[1], "/") + path := nodeInfos[1] + nodeInfos = strings.Split(nodeInfos[0], ":") + host := nodeInfos[0] + port, _ := strconv.ParseInt(nodeInfos[1], 10, 64) paramsMap := make(map[string]string) params := strings.Split(arr[1], "&") @@ -212,7 +288,13 @@ func FromExtInfo(extinfo string) *URL { } func (u *URL) GetPortStr() string { - return strconv.FormatInt(int64(u.Port), 10) + temp := u.portStr.Load() + if temp != nil && temp != "" { + return temp.(string) + } + p := strconv.FormatInt(int64(u.Port), 10) + u.portStr.Store(p) + return p } func (u *URL) GetAddressStr() string { @@ -258,7 +340,8 @@ func (u *URL) CanServe(other *URL) bool { vlog.Errorf("can not serve serialization, err : s1:%s, s2:%s", u.Parameters[SerializationKey], other.Parameters[SerializationKey]) return false } - if !IsSame(u.Parameters, other.Parameters, VersionKey, "0.1") { + // compatible with old version: 0.1 + if !(IsSame(u.Parameters, other.Parameters, VersionKey, "0.1") || IsSame(u.Parameters, other.Parameters, VersionKey, DefaultReferVersion)) { vlog.Errorf("can not serve version, err : v1:%s, v2:%s", u.Parameters[VersionKey], other.Parameters[VersionKey]) return false } @@ -328,9 +411,11 @@ type filterSlice []Filter func (f filterSlice) Len() int { return len(f) } + func (f filterSlice) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + func (f filterSlice) Less(i, j int) bool { // desc return f[i].GetIndex() > f[j].GetIndex() diff --git a/core/url_test.go b/core/url_test.go index 59baa269..73ca3a73 100644 --- a/core/url_test.go +++ b/core/url_test.go @@ -2,16 +2,17 @@ package core import ( "fmt" + "github.com/stretchr/testify/assert" "strconv" "strings" "testing" ) -func TestFromExtinfo(t *testing.T) { - extinfo := "triggerMemcache://10.73.32.175:22222/com.weibo.trigger.common.bean.TriggerMemcacheClient?nodeType=service&version=1.0&group=status2-core" - url := FromExtInfo(extinfo) +func TestFromExtInfo(t *testing.T) { + extInfo := "triggerMemcache://10.73.32.175:22222/com.weibo.trigger.common.bean.TriggerMemcacheClient?nodeType=service&version=1.0&group=status2-core" + url := FromExtInfo(extInfo) if url == nil { - t.Fatal("parse form extinfo fail") + t.Fatal("parse form extInfo fail") } fmt.Printf("url:%+v", url) if url.Host != "10.73.32.175" || url.Port != 22222 || url.Protocol != "triggerMemcache" || @@ -27,14 +28,14 @@ func TestFromExtinfo(t *testing.T) { !strings.Contains(ext2, "group=status2-core") || !strings.Contains(ext2, "nodeType=service") || !strings.Contains(ext2, "version=1.0") { - t.Fatalf("convert url to extinfo not correct. ext2: %s", ext2) + t.Fatalf("convert url to extInfo not correct. ext2: %s", ext2) } fmt.Println("ext2:", ext2) //invalid invalidInfo := "motan://123.23.33.32" url = FromExtInfo(invalidInfo) if url != nil { - t.Fatal("url should be nil when parse invalid extinfo") + t.Fatal("url should be nil when parse invalid extInfo") } } @@ -42,35 +43,36 @@ func TestGetInt(t *testing.T) { url := &URL{} params := make(map[string]string) url.Parameters = params - key := "keyt" + key := "keyT" method := "method1" methodDesc := "string,string" params[key] = "12" v, _ := url.GetInt(key) - intequals(12, v, t) + intEquals(12, v, t) - params[key] = "-20" + url.PutParam(key, "-20") // use PutParam set value will update the cache v, _ = url.GetInt(key) - intequals(-20, v, t) + intEquals(-20, v, t) v = url.GetPositiveIntValue(key, 8) - intequals(8, v, t) + intEquals(8, v, t) delete(params, key) + url.ClearCachedInfo() // clear cache v = url.GetMethodIntValue(method, methodDesc, key, 6) - intequals(6, v, t) + intEquals(6, v, t) - url.Parameters[method+"("+methodDesc+")."+key] = "-17" + url.PutParam(method+"("+methodDesc+")."+key, "-17") v = url.GetMethodIntValue(method, methodDesc, key, 6) - intequals(-17, v, t) + intEquals(-17, v, t) v = url.GetMethodPositiveIntValue(method, methodDesc, key, 9) - intequals(9, v, t) + intEquals(9, v, t) } -func intequals(expect int64, realvalue int64, t *testing.T) { - if realvalue != expect { - t.Fatalf("getint test fail, expect :%d, real :%d", expect, realvalue) +func intEquals(expect int64, realValue int64, t *testing.T) { + if realValue != expect { + t.Fatalf("getint test fail, expect :%d, real :%d", expect, realValue) } } @@ -99,7 +101,6 @@ func TestCopyAndMerge(t *testing.T) { if url.Parameters["key1"] != "xxx" { t.Fatalf("url merge not correct. expect v :%s, real v: %s", "xxx", url.Parameters["key1"]) } - } func TestCanServer(t *testing.T) { @@ -140,7 +141,7 @@ func TestCanServer(t *testing.T) { url1.Protocol = "" url2.Protocol = "" url1.Path = "test/path" - url2.Path = "xxxx" + url2.Path = "whatever" if url1.CanServe(url2) { t.Fatalf("url CanServe testFail url1: %+v, url2: %+v\n", url1, url2) } @@ -156,3 +157,69 @@ func TestGetPositiveIntValue(t *testing.T) { t.Errorf("get positive int fail. v:%d", v) } } + +func TestIntParamCache(t *testing.T) { + url := &URL{} + // test normal + url.PutParam(SessionTimeOutKey, "20000000") + _, ok := url.intParamCache.Load(SessionTimeOutKey) // cache will remove after new value set + assert.False(t, ok) + checkIntCache(t, url, SessionTimeOutKey, 20000000) + + url.PutParam(SessionTimeOutKey, "15") + _, ok = url.intParamCache.Load(SessionTimeOutKey) // cache will remove after new value set + assert.False(t, ok) + checkIntCache(t, url, SessionTimeOutKey, 15) + + // clear cache + url.ClearCachedInfo() + _, ok = url.intParamCache.Load(SessionTimeOutKey) // cache will remove after new value set + assert.False(t, ok) + checkIntCache(t, url, SessionTimeOutKey, 15) + + // test miss cache + _, ok = url.GetInt("notExist") + assert.False(t, ok) + nv, ok := url.intParamCache.Load("notExist") + assert.True(t, ok) + assert.NotNil(t, nv) + if ic, ok := nv.(*int64Cache); ok { + assert.True(t, ic.isMiss) + assert.Equal(t, defaultMissCache, ic) + } + assert.True(t, ok) + + // test hasMethod cache + v := url.GetMethodIntValue("method", "desc", "testKey", 10) + assert.Equal(t, int64(10), v) // default value + assert.False(t, url.hasMethodParams()) + url.PutParam("method(desc).testKey", "100") + assert.True(t, url.hasMethodParams()) + v = url.GetMethodIntValue("method", "desc", "testKey", 10) + assert.Equal(t, int64(100), v) + url.ClearCachedInfo() + // test init with method params + assert.True(t, url.hasMethodParams()) + // test init without method params + delete(url.Parameters, "method(desc).testKey") + url.ClearCachedInfo() + assert.False(t, url.hasMethodParams()) + + // test GetPortStr + url.Port = 8080 + assert.Equal(t, "", url.portStr.Load()) + assert.Equal(t, "8080", url.GetPortStr()) + assert.Equal(t, "8080", url.portStr.Load().(string)) +} + +func checkIntCache(t *testing.T, url *URL, k string, v int64) { + dv := url.GetIntValue(k, 20) + assert.Equal(t, v, dv) + cv, ok := url.intParamCache.Load(k) + assert.True(t, ok) + assert.NotNil(t, cv) + if i, ok := cv.(*int64Cache); ok { + assert.Equal(t, v, i.value) + } + assert.True(t, ok) +} diff --git a/dynamicConfig.go b/dynamicConfig.go index 98ca0538..afc10cc5 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -32,7 +32,7 @@ type DynamicConfigurer struct { agent *Agent } -type registrySnapInfoStorage struct { +type RegistrySnapInfoStorage struct { RegisterNodes []*core.URL `json:"register_nodes"` SubscribeNodes []*core.URL `json:"subscribe_nodes"` } @@ -60,7 +60,7 @@ func (c *DynamicConfigurer) doRecover() error { vlog.Warningln("Read configuration snapshot file error: " + err.Error()) return err } - registerSnapInfo := new(registrySnapInfoStorage) + registerSnapInfo := new(RegistrySnapInfoStorage) err = json.Unmarshal(bytes, registerSnapInfo) if err != nil { vlog.Errorln("Parse snapshot string error: " + err.Error()) @@ -162,8 +162,8 @@ func (c *DynamicConfigurer) saveSnapshot() { } } -func (c *DynamicConfigurer) getRegistryInfo() *registrySnapInfoStorage { - registrySnapInfo := registrySnapInfoStorage{} +func (c *DynamicConfigurer) getRegistryInfo() *RegistrySnapInfoStorage { + registrySnapInfo := RegistrySnapInfoStorage{} c.regLock.Lock() defer c.regLock.Unlock() diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 220587a3..91bee948 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -3,6 +3,7 @@ package endpoint import ( "bufio" "errors" + "github.com/panjf2000/ants/v2" motan "github.com/weibocom/motan-go/core" vlog "github.com/weibocom/motan-go/log" mpro "github.com/weibocom/motan-go/protocol" @@ -14,6 +15,15 @@ import ( "time" ) +var ( + streamPool = sync.Pool{New: func() interface{} { + return &Stream{ + recvNotifyCh: make(chan struct{}, 1), + } + }} + handleMsgPool, _ = ants.NewPool(10000) +) + // MotanCommonEndpoint supports motan v1, v2 protocols type MotanCommonEndpoint struct { url *motan.URL @@ -33,6 +43,7 @@ type MotanCommonEndpoint struct { lazyInit bool maxContentLength int heartbeatVersion int + gzipSize int keepaliveRunning bool serialization motan.Serialization @@ -67,6 +78,7 @@ func (m *MotanCommonEndpoint) Initialize() { asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, GetDefaultMotanEPAsynInit()) m.heartbeatVersion = -1 m.DefaultVersion = mpro.Version2 + m.gzipSize = int(m.url.GetIntValue(motan.GzipSizeKey, 0)) factory := func() (net.Conn, error) { address := m.url.GetAddressStr() if strings.HasPrefix(address, motan.UnixSockProtocolFlag) { @@ -107,9 +119,12 @@ func (m *MotanCommonEndpoint) GetRequestTimeout(request motan.Request) time.Dura if maxTimeout == 0 { maxTimeout = timeout * 2 } - reqTimeout, _ := strconv.ParseInt(request.GetAttachment(mpro.MTimeout), 10, 64) - if reqTimeout >= minTimeout && reqTimeout <= maxTimeout { - timeout = reqTimeout + rt := request.GetAttachment(mpro.MTimeout) + if rt != "" { + reqTimeout, _ := strconv.ParseInt(rt, 10, 64) + if reqTimeout >= minTimeout && reqTimeout <= maxTimeout { + timeout = reqTimeout + } } return time.Duration(timeout) * time.Millisecond } @@ -117,7 +132,7 @@ func (m *MotanCommonEndpoint) GetRequestTimeout(request motan.Request) time.Dura func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { rc := request.GetRPCContext(true) rc.Proxy = m.proxy - rc.GzipSize = int(m.url.GetIntValue(motan.GzipSizeKey, 0)) + rc.GzipSize = m.gzipSize if m.channels == nil { vlog.Errorf("motanEndpoint %s error: channels is null", m.url.GetAddressStr()) @@ -285,16 +300,11 @@ func (m *MotanCommonEndpoint) keepalive() { } func (m *MotanCommonEndpoint) defaultErrMotanResponse(request motan.Request, errMsg string) motan.Response { - response := &motan.MotanResponse{ - RequestID: request.GetRequestID(), - Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), - Exception: &motan.Exception{ - ErrCode: 400, - ErrMsg: errMsg, - ErrType: motan.ServiceException, - }, - } - return response + return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ + ErrCode: 400, + ErrMsg: errMsg, + ErrType: motan.ServiceException, + }) } func (m *MotanCommonEndpoint) GetName() string { @@ -348,14 +358,31 @@ type Stream struct { recvNotifyCh chan struct{} deadline time.Time // for timeout rc *motan.RPCContext - isClose atomic.Value // bool - isHeartbeat bool // for heartbeat - heartbeatVersion int // for heartbeat + isHeartbeat bool // for heartbeat + heartbeatVersion int // for heartbeat + timer *time.Timer + canRelease atomic.Value // state indicates whether the stream needs to be recycled by the channel +} + +func (s *Stream) Reset() { + // try consume + select { + case <-s.recvNotifyCh: + default: + } + s.channel = nil + s.req = nil + s.res = nil + s.rc = nil } func (s *Stream) Send() (err error) { - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.timer == nil { + s.timer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.timer.Reset(s.deadline.Sub(time.Now())) + } + defer s.timer.Stop() var bytes []byte var msg *mpro.Message @@ -386,13 +413,16 @@ func (s *Stream) Send() (err error) { } } + ready := sendReady{} if msg != nil { // encode v2 message - bytes = msg.Encode().Bytes() + msg.Encode0() + ready.message = msg + } else { + ready.v1Message = bytes } if s.rc != nil && s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Encode, Addr: s.channel.address, Time: time.Now()}) } - ready := sendReady{data: bytes} select { case s.channel.sendCh <- ready: if s.rc != nil { @@ -401,9 +431,13 @@ func (s *Stream) Send() (err error) { if s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Send, Addr: s.channel.address, Time: sendTime}) } + if s.rc.AsyncCall { + // only return send success, it can release in Stream.notify + s.canRelease.Store(true) + } } return nil - case <-timer.C: + case <-s.timer.C: return ErrSendRequestTimeout case <-s.channel.shutdownCh: return ErrChannelShutdown @@ -413,10 +447,18 @@ func (s *Stream) Send() (err error) { // Recv sync recv func (s *Stream) Recv() (motan.Response, error) { defer func() { - s.Close() + // only timeout or shutdown before channel.handleMsg, call RemoveFromChannel will be true, + // which means stream can release + if s.RemoveFromChannel() { + s.canRelease.Store(true) + } }() - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.timer == nil { + s.timer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.timer.Reset(s.deadline.Sub(time.Now())) + } + defer s.timer.Stop() select { case <-s.recvNotifyCh: msg := s.res @@ -424,17 +466,18 @@ func (s *Stream) Recv() (motan.Response, error) { return nil, errors.New("recv err: recvMsg is nil") } return msg, nil - case <-timer.C: + case <-s.timer.C: + // stream may be referenced by V2Stream.notify, can`t release + s.canRelease.Store(false) return nil, ErrRecvRequestTimeout case <-s.channel.shutdownCh: + // stream may be referenced by V2Stream.notify, can`t release + s.canRelease.Store(false) return nil, ErrChannelShutdown } } func (s *Stream) notify(msg interface{}, t time.Time) { - defer func() { - s.Close() - }() decodeTime := time.Now() var res motan.Response var v2Msg *mpro.Message @@ -481,6 +524,10 @@ func (s *Stream) notify(msg interface{}, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) } if s.rc.AsyncCall { + defer func() { + s.RemoveFromChannel() + releaseStream(s) + }() result := s.rc.Result if err != nil { result.Error = err @@ -503,70 +550,78 @@ func (s *Stream) SetDeadline(deadline time.Duration) { s.deadline = time.Now().Add(deadline) } -func (c *Channel) NewStream(req motan.Request, rc *motan.RPCContext) (*Stream, error) { +func (c *Channel) newStream(req motan.Request, rc *motan.RPCContext, deadline time.Duration) (*Stream, error) { if c.IsClosed() { return nil, ErrChannelShutdown } - s := &Stream{ - streamId: GenerateRequestID(), - channel: c, - req: req, - recvNotifyCh: make(chan struct{}, 1), - deadline: time.Now().Add(defaultRequestTimeout), // default deadline - rc: rc, + s := acquireStream() + s.streamId = GenerateRequestID() + s.channel = c + s.isHeartbeat = false + s.req = req + s.deadline = time.Now().Add(deadline) + s.rc = rc + if rc != nil && rc.AsyncCall { // release by Stream.Notify + s.canRelease.Store(false) + } else { // release by Channel self + s.canRelease.Store(true) } - s.isClose.Store(false) c.streamLock.Lock() c.streams[s.streamId] = s c.streamLock.Unlock() return s, nil } -func (c *Channel) NewHeartbeatStream(heartbeatVersion int) (*Stream, error) { +func (c *Channel) newHeartbeatStream(heartbeatVersion int) (*Stream, error) { if c.IsClosed() { return nil, ErrChannelShutdown } - s := &Stream{ - streamId: GenerateRequestID(), - channel: c, - isHeartbeat: true, - heartbeatVersion: heartbeatVersion, - recvNotifyCh: make(chan struct{}, 1), - deadline: time.Now().Add(defaultRequestTimeout), - } - s.isClose.Store(false) + s := acquireStream() + s.streamId = GenerateRequestID() + s.channel = c + s.isHeartbeat = true + s.heartbeatVersion = heartbeatVersion + s.deadline = time.Now().Add(defaultRequestTimeout) + s.canRelease.Store(true) c.heartbeatLock.Lock() c.heartbeats[s.streamId] = s c.heartbeatLock.Unlock() return s, nil } -func (s *Stream) Close() { - if !s.isClose.Load().(bool) { - if s.isHeartbeat { - s.channel.heartbeatLock.Lock() +func (s *Stream) RemoveFromChannel() bool { + var exist bool + if s.isHeartbeat { + s.channel.heartbeatLock.Lock() + if _, exist = s.channel.heartbeats[s.streamId]; exist { delete(s.channel.heartbeats, s.streamId) - s.channel.heartbeatLock.Unlock() - } else { - s.channel.streamLock.Lock() + } + s.channel.heartbeatLock.Unlock() + } else { + s.channel.streamLock.Lock() + if _, exist = s.channel.streams[s.streamId]; exist { delete(s.channel.streams, s.streamId) - s.channel.streamLock.Unlock() } - s.isClose.Store(true) + s.channel.streamLock.Unlock() } + return exist } // Call send request to the server. // // about return: exception in response will record error count, err will not. func (c *Channel) Call(req motan.Request, deadline time.Duration, rc *motan.RPCContext) (motan.Response, error) { - stream, err := c.NewStream(req, rc) + stream, err := c.newStream(req, rc, deadline) if err != nil { return nil, err } - stream.SetDeadline(deadline) - err = stream.Send() - if err != nil { + defer func() { + if rc == nil || !rc.AsyncCall { + releaseStream(stream) + } + }() + + if err = stream.Send(); err != nil { return nil, err } if rc != nil && rc.AsyncCall { @@ -576,12 +631,13 @@ func (c *Channel) Call(req motan.Request, deadline time.Duration, rc *motan.RPCC } func (c *Channel) HeartBeat(heartbeatVersion int) (motan.Response, error) { - stream, err := c.NewHeartbeatStream(heartbeatVersion) + stream, err := c.newHeartbeatStream(heartbeatVersion) if err != nil { return nil, err } - err = stream.Send() - if err != nil { + defer releaseStream(stream) + + if err = stream.Send(); err != nil { return nil, err } return stream.Recv() @@ -601,6 +657,7 @@ func (c *Channel) recv() { } func (c *Channel) recvLoop() error { + decodeBuf := make([]byte, mpro.DefaultBufferSize) for { v, err := mpro.CheckMotanVersion(c.bufRead) if err != nil { @@ -611,7 +668,7 @@ func (c *Channel) recvLoop() error { if v == mpro.Version1 { msg, t, err = mpro.ReadV1Message(c.bufRead, c.config.MaxContentLength) } else if v == mpro.Version2 { - msg, t, err = mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) + msg, t, err = mpro.DecodeWithTime(c.bufRead, &decodeBuf, c.config.MaxContentLength) } else { vlog.Warningf("unsupported motan version! version:%d con:%s.", v, c.conn.RemoteAddr().String()) err = mpro.ErrVersion @@ -619,7 +676,10 @@ func (c *Channel) recvLoop() error { if err != nil { return err } - go c.handleMsg(msg, t) + + handleMsgPool.Submit(func() { + c.handleMsg(msg, t) + }) } } @@ -647,10 +707,12 @@ func (c *Channel) handleMsg(msg interface{}, t time.Time) { if isHeartbeat { c.heartbeatLock.Lock() stream = c.heartbeats[rid] + delete(c.heartbeats, rid) c.heartbeatLock.Unlock() } else { c.streamLock.Lock() stream = c.streams[rid] + delete(c.streams, rid) c.streamLock.Unlock() } if stream == nil { @@ -667,18 +729,24 @@ func (c *Channel) send() { for { select { case ready := <-c.sendCh: - if ready.data != nil { - c.conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) - sent := 0 - for sent < len(ready.data) { - n, err := c.conn.Write(ready.data[sent:]) - if err != nil { - vlog.Errorf("Failed to write channel. ep: %s, err: %s", c.address, err.Error()) - c.closeOnErr(err) - return - } - sent += n - } + // can`t reuse net.Buffers + // len and cap will be 0 after writev, 'out of range panic' will happen when reuse + var sendBuf net.Buffers + if ready.message != nil { // motan2 + sendBuf = ready.message.GetEncodedBytes() + } else { + sendBuf = [][]byte{ready.v1Message} + } + c.conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) + _, err := sendBuf.WriteTo(c.conn) + if ready.message != nil { + // message canRelease condition before reset + ready.message.SetCanRelease() + } + if err != nil { + vlog.Errorf("Failed to write channel. ep: %s, err: %s", c.address, err.Error()) + c.closeOnErr(err) + return } case <-c.shutdownCh: return @@ -845,3 +913,16 @@ func buildChannel(conn net.Conn, config *ChannelConfig, serialization motan.Seri return channel } + +func acquireStream() *Stream { + return streamPool.Get().(*Stream) +} + +func releaseStream(stream *Stream) { + if stream != nil { + if v, ok := stream.canRelease.Load().(bool); ok && v { + stream.Reset() + streamPool.Put(stream) + } + } +} diff --git a/endpoint/motanCommonEndpoint_test.go b/endpoint/motanCommonEndpoint_test.go index 3cd2715f..75c460c5 100644 --- a/endpoint/motanCommonEndpoint_test.go +++ b/endpoint/motanCommonEndpoint_test.go @@ -48,6 +48,8 @@ func TestV1RecordErrEmptyThreshold(t *testing.T) { ep.Call(request) assert.True(t, ep.IsAvailable()) } + + assertChanelStreamEmpty(ep, t) ep.Destroy() } @@ -79,6 +81,8 @@ func TestV1RecordErrWithErrThreshold(t *testing.T) { _ = conn.(*net.TCPConn).SetNoDelay(true) ep.channels.channels <- buildChannel(conn, ep.channels.config, ep.channels.serialization) time.Sleep(time.Second * 2) + + assertChanelStreamEmpty(ep, t) //assert.True(t, ep.IsAvailable()) ep.Destroy() } @@ -102,6 +106,8 @@ func TestMotanCommonEndpoint_SuccessCall(t *testing.T) { s, ok := v.(string) assert.True(t, ok) assert.Equal(t, s, "hello") + + assertChanelStreamEmpty(ep, t) } func TestMotanCommonEndpoint_AsyncCall(t *testing.T) { @@ -123,6 +129,8 @@ func TestMotanCommonEndpoint_AsyncCall(t *testing.T) { resp := <-request.GetRPCContext(false).Result.Done assert.Nil(t, resp.Error) assert.Equal(t, resStr, "hello") + + assertChanelStreamEmpty(ep, t) } func TestMotanCommonEndpoint_ErrorCall(t *testing.T) { @@ -147,6 +155,8 @@ func TestMotanCommonEndpoint_ErrorCall(t *testing.T) { ep.Call(request) time.Sleep(1 * time.Millisecond) assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + + assertChanelStreamEmpty(ep, t) ep.Destroy() } @@ -173,6 +183,8 @@ func TestMotanCommonEndpoint_RequestTimeout(t *testing.T) { ep.Call(request) time.Sleep(1 * time.Millisecond) assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + + assertChanelStreamEmpty(ep, t) ep.Destroy() } @@ -215,3 +227,100 @@ func TestV1AsyncInit(t *testing.T) { ep.Initialize() time.Sleep(time.Second * 5) } + +// TestMotanCommonEndpoint_AsyncCallNoResponse verify V2Channel streams memory leak when server not reply response +// TODO:: bugs to be fixed +func TestMotanCommonEndpoint_AsyncCallNoResponse(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url.PutParam(motan.TimeOutKey, "2000") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + var resStr string + request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} + request.Attachment = motan.NewStringMap(0) + // server not reply + request.SetAttachment("no_response", "true") + + res := ep.Call(request) + assert.Nil(t, res.GetException()) + timeoutTimer := time.NewTimer(time.Second * 3) + defer timeoutTimer.Stop() + select { + case <-request.GetRPCContext(false).Result.Done: + t.Errorf("unexpect condition, recv response singnal") + case <-timeoutTimer.C: + t.Logf("expect condition, not recv response singnal") + } + + // Channel.streams can`t release stream + c := <-ep.channels.getChannels() + // it will be zero if server not reply response, bug to be fixed + assert.Equal(t, 1, len(c.streams)) +} + +func TestStreamPool(t *testing.T) { + var oldStream *Stream + // consume stream poll until call New func + for { + oldStream = acquireStream() + if v, ok := oldStream.canRelease.Load().(bool); !ok || !v { + break + } + } + // test new Stream + assert.NotNil(t, oldStream) + assert.NotNil(t, oldStream.recvNotifyCh) + oldStream.streamId = GenerateRequestID() + // verify reset + oldStream.recvNotifyCh <- struct{}{} + + // test canRelease + // oldStream.canRelease is not ture，release fail + // test reset recvNotifyCh + assert.Equal(t, 1, len(oldStream.recvNotifyCh)) + releaseStream(oldStream) + assert.Equal(t, 1, len(oldStream.recvNotifyCh)) + // release success + oldStream.canRelease.Store(true) + releaseStream(oldStream) + assert.Equal(t, 0, len(oldStream.recvNotifyCh)) + + // test put nil + var nilStream *Stream + // can not put nil to pool + releaseStream(nilStream) + newStream3 := acquireStream() + assert.NotEqual(t, nil, newStream3) +} + +func assertChanelStreamEmpty(ep *MotanCommonEndpoint, t *testing.T) { + if ep == nil { + return + } + channels := ep.channels.getChannels() + for { + select { + case c, ok := <-channels: + if !ok || c == nil { + return + } else { + c.streamLock.Lock() + // it should be zero + assert.Equal(t, 0, len(c.streams)) + c.streamLock.Unlock() + + c.heartbeatLock.Lock() + assert.Equal(t, 0, len(c.heartbeats)) + c.heartbeatLock.Unlock() + } + default: + return + } + } +} diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 44c0ee50..3ea7fdb4 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -4,16 +4,15 @@ import ( "bufio" "errors" "fmt" + motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/log" + mpro "github.com/weibocom/motan-go/protocol" "net" "strconv" "strings" "sync" "sync/atomic" "time" - - motan "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/log" - mpro "github.com/weibocom/motan-go/protocol" ) var ( @@ -32,7 +31,12 @@ var ( defaultAsyncResponse = &motan.MotanResponse{Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), RPCContext: &motan.RPCContext{AsyncCall: true}} - errPanic = errors.New("panic error") + errPanic = errors.New("panic error") + v2StreamPool = sync.Pool{New: func() interface{} { + return &V2Stream{ + recvNotifyCh: make(chan struct{}, 1), + } + }} ) type MotanEndpoint struct { @@ -142,9 +146,8 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { m.recordErrAndKeepalive() return m.defaultErrMotanResponse(request, "motanEndpoint error: channels is null") } - startTime := time.Now().UnixNano() if rc.AsyncCall { - rc.Result.StartTime = startTime + rc.Result.StartTime = time.Now().UnixNano() } // get a channel channel, err := m.channels.Get() @@ -306,16 +309,11 @@ func (m *MotanEndpoint) keepalive() { } func (m *MotanEndpoint) defaultErrMotanResponse(request motan.Request, errMsg string) motan.Response { - response := &motan.MotanResponse{ - RequestID: request.GetRequestID(), - Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), - Exception: &motan.Exception{ - ErrCode: 400, - ErrMsg: errMsg, - ErrType: motan.ServiceException, - }, - } - return response + return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ + ErrCode: 400, + ErrMsg: errMsg, + ErrType: motan.ServiceException, + }) } func (m *MotanEndpoint) GetName() string { @@ -380,8 +378,9 @@ type V2Channel struct { } type V2Stream struct { - channel *V2Channel - sendMsg *mpro.Message + channel *V2Channel + sendMsg *mpro.Message + streamId uint64 // recv msg recvMsg *mpro.Message recvNotifyCh chan struct{} @@ -389,30 +388,52 @@ type V2Stream struct { deadline time.Time rc *motan.RPCContext - isClose atomic.Value // bool isHeartBeat bool + timer *time.Timer + canRelease atomic.Value // state indicates whether the stream needs to be recycled by the V2Channel +} + +func (s *V2Stream) Reset() { + // try consume + select { + case <-s.recvNotifyCh: + default: + } + s.channel = nil + s.sendMsg = nil + s.recvMsg = nil + s.rc = nil } func (s *V2Stream) Send() error { - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.timer == nil { + s.timer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.timer.Reset(s.deadline.Sub(time.Now())) + } + defer s.timer.Stop() - buf := s.sendMsg.Encode() + s.sendMsg.Encode0() if s.rc != nil && s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Encode, Addr: s.channel.address, Time: time.Now()}) } - ready := sendReady{data: buf.Bytes()} + ready := sendReady{message: s.sendMsg} select { case s.channel.sendCh <- ready: + // read/write data race if s.rc != nil { sendTime := time.Now() s.rc.RequestSendTime = sendTime if s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Send, Addr: s.channel.address, Time: sendTime}) } + if s.rc.AsyncCall { + // only return send success, it can release in V2Stream.notify + s.canRelease.Store(true) + } } return nil - case <-timer.C: + case <-s.timer.C: return ErrSendRequestTimeout case <-s.channel.shutdownCh: return ErrChannelShutdown @@ -422,10 +443,18 @@ func (s *V2Stream) Send() error { // Recv sync recv func (s *V2Stream) Recv() (*mpro.Message, error) { defer func() { - s.Close() + // only timeout or shutdown before channel.handleMsg, call RemoveFromChannel will be true, + // which means stream can release + if s.RemoveFromChannel() { + s.canRelease.Store(true) + } }() - timer := time.NewTimer(s.deadline.Sub(time.Now())) - defer timer.Stop() + if s.timer == nil { + s.timer = time.NewTimer(s.deadline.Sub(time.Now())) + } else { + s.timer.Reset(s.deadline.Sub(time.Now())) + } + defer s.timer.Stop() select { case <-s.recvNotifyCh: msg := s.recvMsg @@ -433,17 +462,18 @@ func (s *V2Stream) Recv() (*mpro.Message, error) { return nil, errors.New("recv err: recvMsg is nil") } return msg, nil - case <-timer.C: + case <-s.timer.C: + // stream may be referenced by V2Stream.notify, can`t release + s.canRelease.Store(false) return nil, ErrRecvRequestTimeout case <-s.channel.shutdownCh: + // stream may be referenced by V2Stream.notify, can`t release + s.canRelease.Store(false) return nil, ErrChannelShutdown } } func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { - defer func() { - s.Close() - }() if s.rc != nil { s.rc.ResponseReceiveTime = t if s.rc.Tc != nil { @@ -451,6 +481,10 @@ func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Decode, Time: time.Now()}) } if s.rc.AsyncCall { + defer func() { + s.RemoveFromChannel() + releaseV2Stream(s) + }() msg.Header.SetProxy(s.rc.Proxy) result := s.rc.Result response, err := mpro.ConvertToResponse(msg, s.channel.serialization) @@ -463,7 +497,7 @@ func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { if err = response.ProcessDeserializable(result.Reply); err != nil { result.Error = err } - response.SetProcessTime(int64((time.Now().UnixNano() - result.StartTime) / 1000000)) + response.SetProcessTime((time.Now().UnixNano() - result.StartTime) / 1000000) if s.rc.Tc != nil { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) } @@ -480,23 +514,25 @@ func (s *V2Stream) SetDeadline(deadline time.Duration) { s.deadline = time.Now().Add(deadline) } -func (c *V2Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Stream, error) { +func (c *V2Channel) newStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Stream, error) { if msg == nil || msg.Header == nil { return nil, errors.New("msg is invalid") } if c.IsClosed() { return nil, ErrChannelShutdown } - s := &V2Stream{ - channel: c, - sendMsg: msg, - recvNotifyCh: make(chan struct{}, 1), - deadline: time.Now().Add(1 * time.Second), - rc: rc, + + s := acquireV2Stream() + s.channel = c + s.sendMsg = msg + if s.recvNotifyCh == nil { + s.recvNotifyCh = make(chan struct{}, 1) } - s.isClose.Store(false) + s.deadline = time.Now().Add(1 * time.Second) + s.rc = rc // RequestID is communication identifier, it is own by channel msg.Header.RequestID = GenerateRequestID() + s.streamId = msg.Header.RequestID if msg.Header.IsHeartbeat() { c.heartbeatLock.Lock() c.heartbeats[msg.Header.RequestID] = s @@ -506,36 +542,52 @@ func (c *V2Channel) NewStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Strea c.streamLock.Lock() c.streams[msg.Header.RequestID] = s c.streamLock.Unlock() + s.isHeartBeat = false + } + if rc != nil && rc.AsyncCall { // release by Stream.Notify + s.canRelease.Store(false) + } else { // release by Channel self + s.canRelease.Store(true) } return s, nil } -func (s *V2Stream) Close() { - if !s.isClose.Load().(bool) { - if s.isHeartBeat { - s.channel.heartbeatLock.Lock() - delete(s.channel.heartbeats, s.sendMsg.Header.RequestID) - s.channel.heartbeatLock.Unlock() - } else { - s.channel.streamLock.Lock() - delete(s.channel.streams, s.sendMsg.Header.RequestID) - s.channel.streamLock.Unlock() +func (s *V2Stream) RemoveFromChannel() bool { + var exist bool + if s.isHeartBeat { + s.channel.heartbeatLock.Lock() + if _, exist = s.channel.heartbeats[s.streamId]; exist { + delete(s.channel.heartbeats, s.streamId) } - s.isClose.Store(true) + s.channel.heartbeatLock.Unlock() + } else { + s.channel.streamLock.Lock() + if _, exist = s.channel.streams[s.streamId]; exist { + delete(s.channel.streams, s.streamId) + } + s.channel.streamLock.Unlock() } + return exist } type sendReady struct { - data []byte + message *mpro.Message // motan2 protocol message + v1Message []byte //motan1 protocol, synchronize subsequent if motan1 protocol message optimization } func (c *V2Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RPCContext) (*mpro.Message, error) { - stream, err := c.NewStream(msg, rc) + stream, err := c.newStream(msg, rc) if err != nil { return nil, err } + defer func() { + if rc == nil || !rc.AsyncCall { + releaseV2Stream(stream) + } + }() + stream.SetDeadline(deadline) - if err := stream.Send(); err != nil { + if err = stream.Send(); err != nil { return nil, err } if rc != nil && rc.AsyncCall { @@ -558,8 +610,9 @@ func (c *V2Channel) recv() { } func (c *V2Channel) recvLoop() error { + decodeBuf := make([]byte, mpro.DefaultBufferSize) for { - res, t, err := mpro.DecodeWithTime(c.bufRead, c.config.MaxContentLength) + res, t, err := mpro.DecodeWithTime(c.bufRead, &decodeBuf, c.config.MaxContentLength) if err != nil { return err } @@ -583,18 +636,18 @@ func (c *V2Channel) send() { for { select { case ready := <-c.sendCh: - if ready.data != nil { + if ready.message != nil { + // can`t reuse net.Buffers + // len and cap will be 0 after writev consume, 'out of range panic' will happen when reuse + var sendBuf net.Buffers = ready.message.GetEncodedBytes() // TODO need async? c.conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) - sent := 0 - for sent < len(ready.data) { - n, err := c.conn.Write(ready.data[sent:]) - if err != nil { - vlog.Errorf("Failed to write channel. ep: %s, err: %s", c.address, err.Error()) - c.closeOnErr(err) - return - } - sent += n + _, err := sendBuf.WriteTo(c.conn) + ready.message.SetCanRelease() + if err != nil { + vlog.Errorf("Failed to write channel. ep: %s, err: %s", c.address, err.Error()) + c.closeOnErr(err) + return } } case <-c.shutdownCh: @@ -606,6 +659,7 @@ func (c *V2Channel) send() { func (c *V2Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { c.heartbeatLock.Lock() stream := c.heartbeats[msg.Header.RequestID] + delete(c.heartbeats, msg.Header.RequestID) c.heartbeatLock.Unlock() if stream == nil { vlog.Warningf("handle heartbeat message, missing stream: %d, ep:%s", msg.Header.RequestID, c.address) @@ -618,6 +672,7 @@ func (c *V2Channel) handleHeartbeat(msg *mpro.Message, t time.Time) error { func (c *V2Channel) handleMessage(msg *mpro.Message, t time.Time) error { c.streamLock.Lock() stream := c.streams[msg.Header.RequestID] + delete(c.streams, msg.Header.RequestID) c.streamLock.Unlock() if stream == nil { vlog.Warningf("handle recv message, missing stream: %d, ep:%s", msg.Header.RequestID, c.address) @@ -799,3 +854,16 @@ func GetDefaultMotanEPAsynInit() bool { } return res.(bool) } + +func acquireV2Stream() *V2Stream { + return v2StreamPool.Get().(*V2Stream) +} + +func releaseV2Stream(stream *V2Stream) { + if stream != nil { + if v, ok := stream.canRelease.Load().(bool); ok && v { + stream.Reset() + v2StreamPool.Put(stream) + } + } +} diff --git a/endpoint/motanEndpoint_test.go b/endpoint/motanEndpoint_test.go index 9bb0efe6..acb98912 100644 --- a/endpoint/motanEndpoint_test.go +++ b/endpoint/motanEndpoint_test.go @@ -21,7 +21,7 @@ func TestMain(m *testing.M) { m.Run() } -//TODO more UT +// TODO more UT func TestGetName(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") @@ -57,6 +57,8 @@ func TestRecordErrEmptyThreshold(t *testing.T) { ep.Call(request) assert.True(t, ep.IsAvailable()) } + + assertV2ChanelStreamEmpty(ep, t) ep.Destroy() } @@ -89,6 +91,8 @@ func TestRecordErrWithErrThreshold(t *testing.T) { ep.channels.channels <- buildV2Channel(conn, ep.channels.config, ep.channels.serialization) time.Sleep(time.Second * 2) //assert.True(t, ep.IsAvailable()) + + assertV2ChanelStreamEmpty(ep, t) ep.Destroy() } @@ -111,6 +115,8 @@ func TestMotanEndpoint_SuccessCall(t *testing.T) { s, ok := v.(string) assert.True(t, ok) assert.Equal(t, s, "hello") + + assertV2ChanelStreamEmpty(ep, t) ep.Destroy() } @@ -133,6 +139,8 @@ func TestMotanEndpoint_AsyncCall(t *testing.T) { resp := <-request.GetRPCContext(false).Result.Done assert.Nil(t, resp.Error) assert.Equal(t, resStr, "hello") + + assertV2ChanelStreamEmpty(ep, t) ep.Destroy() } @@ -158,6 +166,8 @@ func TestMotanEndpoint_ErrorCall(t *testing.T) { ep.Call(request) time.Sleep(1 * time.Millisecond) assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + + assertV2ChanelStreamEmpty(ep, t) ep.Destroy() } @@ -184,6 +194,8 @@ func TestMotanEndpoint_RequestTimeout(t *testing.T) { ep.Call(request) time.Sleep(1 * time.Millisecond) assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + + assertV2ChanelStreamEmpty(ep, t) ep.Destroy() } @@ -211,6 +223,8 @@ func TestLazyInit(t *testing.T) { ep.Call(request) time.Sleep(1 * time.Millisecond) assert.Equal(t, beforeNGoroutine, runtime.NumGoroutine()) + + assertV2ChanelStreamEmpty(ep, t) ep.Destroy() } @@ -227,6 +241,104 @@ func TestAsyncInit(t *testing.T) { time.Sleep(time.Second * 5) } +// TestMotanEndpoint_AsyncCallNoResponse verify V2Channel streams memory leak when server not reply response +// bugs to be fixed +func TestMotanEndpoint_AsyncCallNoResponse(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan2"} + url.PutParam(motan.TimeOutKey, "2000") + url.PutParam(motan.ErrorCountThresholdKey, "1") + url.PutParam(motan.ClientConnectionKey, "1") + url.PutParam(motan.AsyncInitConnection, "false") + ep := &MotanEndpoint{} + ep.SetURL(url) + ep.SetSerialization(&serialize.SimpleSerialization{}) + ep.Initialize() + assert.Equal(t, 1, ep.clientConnection) + var resStr string + request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} + request.Attachment = motan.NewStringMap(0) + + // server not reply + request.SetAttachment("no_response", "true") + + res := ep.Call(request) + assert.Nil(t, res.GetException()) + timeoutTimer := time.NewTimer(time.Second * 3) + defer timeoutTimer.Stop() + select { + case <-request.GetRPCContext(false).Result.Done: + t.Errorf("unexpect condition, recv response singnal") + case <-timeoutTimer.C: + t.Logf("expect condition, not recv response singnal") + } + + // Channel.streams cant`t release stream + c := <-ep.channels.getChannels() + // it will be zero if server not reply response, bug to be fixed + assert.Equal(t, 1, len(c.streams)) +} + +func TestV2StreamPool(t *testing.T) { + var oldStream *V2Stream + // consume v2stream poll until call New func + for { + oldStream = acquireV2Stream() + if v, ok := oldStream.canRelease.Load().(bool); !ok || !v { + break + } + } + // test new Stream + assert.NotNil(t, oldStream) + assert.NotNil(t, oldStream.recvNotifyCh) + oldStream.streamId = GenerateRequestID() + // verify reset + oldStream.recvNotifyCh <- struct{}{} + + // test canRelease + // oldStream.canRelease is not ture，release fail + // test reset recvNotifyCh + assert.Equal(t, 1, len(oldStream.recvNotifyCh)) + releaseV2Stream(oldStream) + assert.Equal(t, 1, len(oldStream.recvNotifyCh)) + // canRelease success + oldStream.canRelease.Store(true) + releaseV2Stream(oldStream) + assert.Equal(t, 0, len(oldStream.recvNotifyCh)) + + // test put nil + var nilStream *V2Stream + // can not put nil to pool + releaseV2Stream(nilStream) + newStream3 := acquireV2Stream() + assert.NotEqual(t, nil, newStream3) +} + +func assertV2ChanelStreamEmpty(ep *MotanEndpoint, t *testing.T) { + if ep == nil { + return + } + channels := ep.channels.getChannels() + for { + select { + case c, ok := <-channels: + if !ok || c == nil { + return + } else { + c.streamLock.Lock() + // it should be zero + assert.Equal(t, 0, len(c.streams)) + c.streamLock.Unlock() + + c.heartbeatLock.Lock() + assert.Equal(t, 0, len(c.heartbeats)) + c.heartbeatLock.Unlock() + } + default: + return + } + } +} + func StartTestServer(port int) *MockServer { m := &MockServer{Port: port} m.Start() @@ -268,8 +380,9 @@ func handle(netListen net.Listener) { } func handleConnection(conn net.Conn, timeout int) { - buf := bufio.NewReader(conn) - msg, _, err := protocol.DecodeWithTime(buf, 10*1024*1024) + reader := bufio.NewReader(conn) + decodeBuf := make([]byte, 100) + msg, err := protocol.Decode(reader, &decodeBuf) if err != nil { time.Sleep(time.Millisecond * 1000) conn.Close() @@ -279,6 +392,10 @@ func handleConnection(conn net.Conn, timeout int) { } func processMsg(msg *protocol.Message, conn net.Conn) { + // mock async call, but server not reply response + if _, ok := msg.Metadata.Load("no_response"); ok { + return + } var res *protocol.Message var tc *motan.TraceContext var err error diff --git a/filter/accessLog.go b/filter/accessLog.go index 06c1e6e7..ee980087 100644 --- a/filter/accessLog.go +++ b/filter/accessLog.go @@ -2,11 +2,10 @@ package filter import ( "encoding/json" - "strconv" - "time" - motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/log" + "strconv" + "time" ) const ( @@ -34,15 +33,17 @@ func (t *AccessLogFilter) NewFilter(url *motan.URL) motan.Filter { func (t *AccessLogFilter) Filter(caller motan.Caller, request motan.Request) motan.Response { role := defaultRole var ip string + var start time.Time switch caller.(type) { case motan.Provider: role = serverAgentRole ip = request.GetAttachment(motan.HostKey) + start = request.GetRPCContext(true).RequestReceiveTime case motan.EndPoint: role = clientAgentRole ip = caller.GetURL().Host + start = time.Now() } - start := time.Now() response := t.GetNext().Filter(caller, request) address := ip + ":" + caller.GetURL().GetPortStr() if _, ok := caller.(motan.Provider); ok { @@ -81,9 +82,6 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 // response code should be same as upstream responseCode := "" metaUpstreamCode, _ := response.GetAttachments().Load(motan.MetaUpstreamCode) - if resCtx.Meta != nil { - responseCode = resCtx.Meta.LoadOrEmpty(motan.MetaUpstreamCode) - } var exceptionData []byte if exception != nil { exceptionData, _ = json.Marshal(exception) @@ -94,21 +92,23 @@ func doAccessLog(filterName string, role string, address string, totalTime int64 responseCode = "200" } } - vlog.AccessLog(&vlog.AccessLogEntity{ - FilterName: filterName, - Role: role, - RequestID: response.GetRequestID(), - Service: request.GetServiceName(), - Method: request.GetMethod(), - RemoteAddress: address, - Desc: request.GetMethodDesc(), - ReqSize: reqCtx.BodySize, - ResSize: resCtx.BodySize, - BizTime: response.GetProcessTime(), //ms - TotalTime: totalTime, //ms - ResponseCode: responseCode, - Success: exception == nil, - Exception: string(exceptionData), - UpstreamCode: metaUpstreamCode, - }) + + logEntity := vlog.AcquireAccessLogEntity() + logEntity.FilterName = filterName + logEntity.Role = role + logEntity.RequestID = response.GetRequestID() + logEntity.Service = request.GetServiceName() + logEntity.Method = request.GetMethod() + logEntity.RemoteAddress = address + logEntity.Desc = request.GetMethodDesc() + logEntity.ReqSize = reqCtx.BodySize + logEntity.ResSize = resCtx.BodySize + logEntity.BizTime = response.GetProcessTime() //ms + logEntity.TotalTime = totalTime //ms + logEntity.ResponseCode = responseCode + logEntity.Success = exception == nil + logEntity.Exception = string(exceptionData) + logEntity.UpstreamCode = metaUpstreamCode + + vlog.AccessLog(logEntity) } diff --git a/filter/clusterMetrics.go b/filter/clusterMetrics.go index bbbbcfbe..8019f3ce 100644 --- a/filter/clusterMetrics.go +++ b/filter/clusterMetrics.go @@ -4,7 +4,6 @@ import ( "time" motan "github.com/weibocom/motan-go/core" - "github.com/weibocom/motan-go/metrics" "github.com/weibocom/motan-go/protocol" ) @@ -57,11 +56,8 @@ func (c *ClusterMetricsFilter) Filter(haStrategy motan.HaStrategy, loadBalance m if ctx != nil && ctx.Proxy { role = "motan-client-agent" } - key := metrics.Escape(role) + - ":" + metrics.Escape(request.GetAttachment(protocol.MSource)) + - ":" + metrics.Escape(request.GetMethod()) - addMetric(metrics.Escape(request.GetAttachment(protocol.MGroup))+".cluster", - metrics.Escape(request.GetAttachment(protocol.MPath)), - key, time.Since(start).Nanoseconds()/1e6, response) + keys := []string{role, request.GetAttachment(protocol.MSource), request.GetMethod()} + addMetricWithKeys(request.GetAttachment(protocol.MGroup), ".cluster", + request.GetAttachment(protocol.MPath), keys, time.Since(start).Nanoseconds()/1e6, response) return response } diff --git a/filter/filter.go b/filter/filter.go index 0126f291..38552ac8 100644 --- a/filter/filter.go +++ b/filter/filter.go @@ -2,6 +2,7 @@ package filter import ( motan "github.com/weibocom/motan-go/core" + "time" ) // ext name @@ -59,3 +60,14 @@ func RegistDefaultFilters(extFactory motan.ExtensionFactory) { return &ClusterCircuitBreakerFilter{} }) } + +func getFilterStartTime(caller motan.Caller, request motan.Request) time.Time { + switch caller.(type) { + case motan.Provider: + return request.GetRPCContext(true).RequestReceiveTime + case motan.EndPoint: + return time.Now() + default: + return time.Now() + } +} diff --git a/filter/metrics.go b/filter/metrics.go index 848eac88..c69b41fd 100644 --- a/filter/metrics.go +++ b/filter/metrics.go @@ -1,11 +1,10 @@ package filter import ( - "time" - motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/metrics" "github.com/weibocom/motan-go/protocol" + "time" ) const ( @@ -57,9 +56,8 @@ func (m *MetricsFilter) GetNext() motan.EndPointFilter { } func (m *MetricsFilter) Filter(caller motan.Caller, request motan.Request) motan.Response { - start := time.Now() + start := getFilterStartTime(caller, request) response := m.GetNext().Filter(caller, request) - proxy := false provider := false ctx := request.GetRPCContext(false) @@ -86,30 +84,28 @@ func (m *MetricsFilter) Filter(caller motan.Caller, request motan.Request) motan if provider { application = caller.GetURL().GetParam(motan.ApplicationKey, "") } - key := metrics.Escape(role) + - ":" + metrics.Escape(application) + - ":" + metrics.Escape(request.GetMethod()) - addMetric(metrics.Escape(request.GetAttachment(protocol.MGroup)), - metrics.Escape(request.GetAttachment(protocol.MPath)), - key, time.Since(start).Nanoseconds()/1e6, response) + keys := []string{role, application, request.GetMethod()} + addMetricWithKeys(request.GetAttachment(protocol.MGroup), "", request.GetAttachment(protocol.MPath), + keys, time.Since(start).Nanoseconds()/1e6, response) return response } -func addMetric(group string, service string, key string, cost int64, response motan.Response) { - metrics.AddCounter(group, service, key+MetricsTotalCountSuffix, 1) //total_count - if response.GetException() != nil { //err_count +// addMetricWithKeys arguments: group & groupSuffix & service & keys elements is text without escaped +func addMetricWithKeys(group, groupSuffix string, service string, keys []string, cost int64, response motan.Response) { + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsTotalCountSuffix, 1) //total_count + if response.GetException() != nil { //err_count exception := response.GetException() if exception.ErrType == motan.BizException { - metrics.AddCounter(group, service, key+MetricsBizErrorCountSuffix, 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsBizErrorCountSuffix, 1) } else { - metrics.AddCounter(group, service, key+MetricsOtherErrorCountSuffix, 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsOtherErrorCountSuffix, 1) } } - metrics.AddCounter(group, service, key+metrics.ElapseTimeSuffix(cost), 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, metrics.ElapseTimeSuffix(cost), 1) if cost > 200 { - metrics.AddCounter(group, service, key+MetricsSlowCountSuffix, 1) + metrics.AddCounterWithKeys(group, groupSuffix, service, keys, MetricsSlowCountSuffix, 1) } - metrics.AddHistograms(group, service, key, cost) + metrics.AddHistogramsWithKeys(group, groupSuffix, service, keys, "", cost) } func (m *MetricsFilter) SetContext(context *motan.Context) { diff --git a/filter/metrics_test.go b/filter/metrics_test.go index f643882f..63cc5cc9 100644 --- a/filter/metrics_test.go +++ b/filter/metrics_test.go @@ -31,7 +31,7 @@ func TestMetricsFilter(t *testing.T) { request.GetRPCContext(true).Proxy = true request.SetAttachment(protocol.MSource, application) request.SetAttachment(protocol.MPath, testService) - assert.Nil(t, metrics.GetStatItem(testGroup, testService), "metric stat") + assert.Nil(t, metrics.GetStatItem(testGroup, "", testService), "metric stat") ep := factory.GetEndPoint(url) provider := factory.GetProvider(url) @@ -41,25 +41,28 @@ func TestMetricsFilter(t *testing.T) { name string caller motan.Caller request motan.Request - key string + keys []string }{ - {name: "proxyClient", caller: ep, request: request, key: "motan-client-agent:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, - {name: "proxyServer", caller: provider, request: request, key: "motan-server-agent:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, - {name: "Client", caller: ep, request: request2, key: "motan-client:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, - {name: "Server", caller: provider, request: request2, key: "motan-server:" + metrics.Escape(application) + ":" + metrics.Escape(testMethod)}, + {name: "proxyClient", caller: ep, request: request, keys: []string{"motan-client-agent", application, testMethod}}, + {name: "proxyServer", caller: provider, request: request, keys: []string{"motan-server-agent", application, testMethod}}, + {name: "Client", caller: ep, request: request2, keys: []string{"motan-client", application, testMethod}}, + {name: "Server", caller: provider, request: request2, keys: []string{"motan-server", application, testMethod}}, + } + var getKeysStr = func(keys []string) string { + return metrics.Escape(keys[0]) + ":" + metrics.Escape(keys[1]) + ":" + metrics.Escape(keys[2]) } for _, test := range tests { t.Run(test.name, func(t *testing.T) { mf.Filter(test.caller, test.request) time.Sleep(10 * time.Millisecond) // The metrics filter has do escape - assert.Equal(t, 1, int(metrics.GetStatItem(metrics.Escape(testGroup), metrics.Escape(testService)).SnapshotAndClear().Count(test.key+MetricsTotalCountSuffix)), "metric count") + assert.Equal(t, 1, int(metrics.GetStatItem(testGroup, "", testService).SnapshotAndClear().Count(getKeysStr(test.keys)+MetricsTotalCountSuffix)), "metric count") }) } } func TestAddMetric(t *testing.T) { - key := "motan-client-agent:testApplication:" + testMethod + keys := []string{"motan-client-agent", "testApplication", testMethod} factory := initFactory() mf := factory.GetFilter(Metrics).(motan.EndPointFilter) mf.(*MetricsFilter).SetContext(&motan.Context{Config: config.NewConfig()}) @@ -67,7 +70,9 @@ func TestAddMetric(t *testing.T) { response2 := &motan.MotanResponse{ProcessTime: 100, Exception: &motan.Exception{ErrType: motan.BizException}} response3 := &motan.MotanResponse{ProcessTime: 100, Exception: &motan.Exception{ErrType: motan.FrameworkException}} response4 := &motan.MotanResponse{ProcessTime: 1000} - + var getKeysStr = func(keys []string) string { + return metrics.Escape(keys[0]) + ":" + metrics.Escape(keys[1]) + ":" + metrics.Escape(keys[2]) + } tests := []struct { name string response motan.Response @@ -81,11 +86,11 @@ func TestAddMetric(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - addMetric(testGroup, testService, key, test.response.GetProcessTime(), test.response) + addMetricWithKeys(testGroup, "", testService, keys, test.response.GetProcessTime(), test.response) time.Sleep(10 * time.Millisecond) - snap := metrics.GetStatItem(testGroup, testService).SnapshotAndClear() + snap := metrics.GetStatItem(testGroup, "", testService).SnapshotAndClear() for _, k := range test.keys { - assert.True(t, snap.Count(key+k) > 0, fmt.Sprintf("key '%s'", k)) + assert.True(t, snap.Count(getKeysStr(keys)+k) > 0, fmt.Sprintf("key '%s'", k)) } }) } diff --git a/filter/rateLimit.go b/filter/rateLimit.go index 85d5ac22..e98692b1 100644 --- a/filter/rateLimit.go +++ b/filter/rateLimit.go @@ -137,20 +137,20 @@ func (r *RateLimitFilter) GetType() int32 { func getKeyValue(key, value, prefix string) (string, float64, bool) { if strings.HasPrefix(key, prefix) { if temp := strings.Split(key, prefix); len(temp) == 2 { - if r, err := strconv.ParseFloat(value, 64); err == nil && temp[1] != "" && r > 0 { + r, err := strconv.ParseFloat(value, 64) + if err == nil && temp[1] != "" && r > 0 { return temp[1], r, true + } + if err != nil { + vlog.Warningf("[rateLimit] parse %s config error:%s", key, err.Error()) } else { - if err != nil { - vlog.Warningf("[rateLimit] parse %s config error:%s", key, err.Error()) - } else { - if r <= 0 { - vlog.Warningf("[rateLimit] parse %s config error: value is 0 or negative", key) - } - } - if temp[1] == "" { - vlog.Warningf("[rateLimit] parse %s config error: key is empty", key) + if r <= 0 { + vlog.Warningf("[rateLimit] parse %s config error: value is 0 or negative", key) } } + if temp[1] == "" { + vlog.Warningf("[rateLimit] parse %s config error: key is empty", key) + } } } return "", 0, false diff --git a/filter/tracing.go b/filter/tracing.go index ac602c85..aa855087 100644 --- a/filter/tracing.go +++ b/filter/tracing.go @@ -65,48 +65,48 @@ func tracingFunc() func(span ot.Span, data *CallData) { // As described by OpenTracing, for a single call from client to server, both sides will start a span, // with the server side span to be child of the client side. Described as following // -// call -// caller ---------------> callee -// [span1] [span2] +// call +// caller ---------------> callee +// [span1] [span2] // // here [span1] is parent of [span2]. // // When this filter is applied, it will filter both the incoming and // outgoing requests to record trace information. The following diagram is a demonstration. // -// filter -// +---------+ -// | | [span1] -// [span2] *-----+-- in <--+--------------------- | user | -// | | | -// V | | -// | ------- | | -// pass-thru | service | | -// [span2] V ------- | | -// | | | -// | | | [span3] -// [span2] *-----+-> out --+--------------------> | dep | -// | | -// +---------+ +// filter +// +---------+ +// | | [span1] +// [span2] *-----+-- in <--+--------------------- | user | +// | | | +// V | | +// | ------- | | +// pass-thru | service | | +// [span2] V ------- | | +// | | | +// | | | [span3] +// [span2] *-----+-> out --+--------------------> | dep | +// | | +// +---------+ // // When the filter receives an incoming request, it will: // -// 1. extract span context from request (will get [span1]) -// 2. start a child span of the extracted span ([span2], child of [span1]) -// 3. forward the request with [span2] to the service +// 1. extract span context from request (will get [span1]) +// 2. start a child span of the extracted span ([span2], child of [span1]) +// 3. forward the request with [span2] to the service // // Then the service may make an outgoing request to some dependent services, // it should pass-through the span information ([span2]). // The filter will receive the outgoing request with [span2], then it will. // -// 1. extract span context from the outgoing request (it should the [span2]) -// 2. start a child span of the extracted span ([span3], child of [span2]) -// 3. forward the request with [span3] to the dependent service +// 1. extract span context from the outgoing request (it should the [span2]) +// 2. start a child span of the extracted span ([span3], child of [span2]) +// 3. forward the request with [span3] to the dependent service // // So here // -// (parent) (parent) -// [span1] <------ [span2] <------ [span3] +// (parent) (parent) +// [span1] <------ [span2] <------ [span3] // // NOTE: // diff --git a/go.mod b/go.mod index 989f8b55..e7623234 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,13 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/mitchellh/mapstructure v1.1.2 github.com/opentracing/opentracing-go v1.0.2 + github.com/panjf2000/ants/v2 v2.9.0 github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec github.com/shirou/gopsutil/v3 v3.21.9 github.com/smartystreets/goconvey v1.6.4 // indirect - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.2 github.com/valyala/fasthttp v1.2.0 github.com/weibreeze/breeze-go v0.1.1 go.uber.org/atomic v1.4.0 // indirect diff --git a/ha/backupRequestHA.go b/ha/backupRequestHA.go index d50c6e5b..8ee949d9 100644 --- a/ha/backupRequestHA.go +++ b/ha/backupRequestHA.go @@ -67,7 +67,7 @@ func (br *BackupRequestHA) Call(request motan.Request, loadBalance motan.LoadBal successCh := make(chan motan.Response, retries+1) if delay <= 0 { //no delay time configuration // TODO: we should use metrics of the cluster, with traffic control the group may changed - item := metrics.GetStatItem(metrics.Escape(request.GetAttachment(protocol.MGroup)), metrics.Escape(request.GetAttachment(protocol.MPath))) + item := metrics.GetStatItem(request.GetAttachment(protocol.MGroup), "", request.GetAttachment(protocol.MPath)) if item == nil || item.LastSnapshot() == nil { initDelay := int(br.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), "backupRequestInitDelayTime", 0)) if initDelay == 0 { diff --git a/ha/failoverHA.go b/ha/failoverHA.go index 1701cf50..e5d20bab 100644 --- a/ha/failoverHA.go +++ b/ha/failoverHA.go @@ -18,12 +18,15 @@ type FailOverHA struct { func (f *FailOverHA) GetName() string { return FailOver } + func (f *FailOverHA) GetURL() *motan.URL { return f.url } + func (f *FailOverHA) SetURL(url *motan.URL) { f.url = url } + func (f *FailOverHA) Call(request motan.Request, loadBalance motan.LoadBalance) motan.Response { retries := f.url.GetMethodPositiveIntValue(request.GetMethod(), request.GetMethodDesc(), motan.RetriesKey, defaultRetries) var lastErr *motan.Exception diff --git a/http/httpProxy.go b/http/httpProxy.go index dc048722..33738633 100644 --- a/http/httpProxy.go +++ b/http/httpProxy.go @@ -4,10 +4,12 @@ import ( "bytes" "errors" "fmt" + mpro "github.com/weibocom/motan-go/protocol" "net/url" "reflect" "regexp" "strings" + "sync" "github.com/valyala/fasthttp" "github.com/weibocom/motan-go/core" @@ -21,8 +23,10 @@ const ( ) const ( - HeaderPrefix = "http_" - HeaderContentType = "Content-Type" + HeaderPrefix = "http_" + HeaderContentType = "Content-Type" + HeaderContentTypeLowerCase = "content-type" + HeaderHost = "Host" ) const ( @@ -60,8 +64,20 @@ const ( var ( WhitespaceSplitPattern = regexp.MustCompile(`\s+`) findRewriteVarPattern = regexp.MustCompile(`\{[0-9a-zA-Z_-]+\}`) - httpProxySpecifiedAttachments = []string{Proxy, Method, QueryString, core.HostKey} - rewriteVarFunc = func(condType ProxyRewriteType, uri string, queryBytes []byte) string { + httpProxySpecifiedAttachments = []string{Proxy, Method, QueryString, core.HostKey, HeaderHost} + InnerAttachmentsConvertMap = map[string]string{ + mpro.MPath: "MOTAN-p", + mpro.MMethod: "MOTAN-m", + mpro.MMethodDesc: "MOTAN-md", + mpro.MGroup: "MOTAN-g", + mpro.MProxyProtocol: "MOTAN-pp", + mpro.MVersion: "MOTAN-v", + mpro.MModule: "MOTAN-mdu", + mpro.MSource: "MOTAN-s", + mpro.MRequestID: "MOTAN-rid", + mpro.MTimeout: "MOTAN-tmo", + } + rewriteVarFunc = func(condType ProxyRewriteType, uri string, queryBytes []byte) string { if condType != proxyRewriteTypeRegexpVar || len(queryBytes) == 0 { return uri } @@ -73,8 +89,43 @@ var ( } return uri } + httpResponsePool = sync.Pool{New: func() interface{} { + res := &HttpMotanResponse{} + res.RPCContext = &core.RPCContext{} + return res + }} ) +// HttpMotanResponse Resposne used in http provider to deal with http response +type HttpMotanResponse struct { + core.MotanResponse + HttpResponse *fasthttp.Response +} + +func (m *HttpMotanResponse) Reset() { + m.RequestID = 0 + m.Value = nil + m.Exception = nil + m.ProcessTime = 0 + m.Attachment = nil + m.RPCContext.Reset() + m.HttpResponse = nil +} + +func AcquireHttpMotanResponse() *HttpMotanResponse { + return httpResponsePool.Get().(*HttpMotanResponse) +} + +func ReleaseHttpMotanResponse(m *HttpMotanResponse) { + if m != nil { + if m.HttpResponse != nil { + fasthttp.ReleaseResponse(m.HttpResponse) + } + m.Reset() + httpResponsePool.Put(m) + } +} + func PatternSplit(s string, pattern *regexp.Regexp) []string { matches := pattern.FindAllStringIndex(s, -1) strings := make([]string, 0, len(matches)) @@ -431,25 +482,31 @@ func MotanRequestToFasthttpRequest(motanRequest core.Request, fasthttpRequest *f fasthttpRequest.URI().SetQueryString(queryString) } motanRequest.GetAttachments().Range(func(k, v string) bool { + if convertK, ok := InnerAttachmentsConvertMap[k]; ok { + k = convertK + fasthttpRequest.Header.Set(k, v) + return true + } // ignore some specified key for _, attachmentKey := range httpProxySpecifiedAttachments { - if strings.EqualFold(k, attachmentKey) { + if k == attachmentKey { return true } } // fasthttp will use a special field to store this header - if strings.EqualFold(k, HeaderPrefix+core.HostKey) { + if k == HeaderPrefix+core.HostKey || k == HeaderPrefix+HeaderHost { fasthttpRequest.Header.SetHost(v) return true } - if strings.EqualFold(k, HeaderContentType) { + if k == HeaderContentType || k == HeaderContentTypeLowerCase { fasthttpRequest.Header.SetContentType(v) return true } - k = strings.Replace(k, "M_", "MOTAN-", -1) // http header private prefix - k = strings.Replace(k, HeaderPrefix, "", -1) - fasthttpRequest.Header.Add(k, v) + if strings.HasPrefix(k, HeaderPrefix) { + k = strings.Replace(k, HeaderPrefix, "", 1) + } + fasthttpRequest.Header.Set(k, v) return true }) fasthttpRequest.Header.Del("Connection") @@ -501,12 +558,11 @@ func FasthttpResponseToMotanResponse(motanResponse core.Response, fasthttpRespon fasthttpResponse.Header.VisitAll(func(k, v []byte) { motanResponse.SetAttachment(string(k), string(v)) }) - if resp, ok := motanResponse.(*core.MotanResponse); ok { + if resp, ok := motanResponse.(*HttpMotanResponse); ok { httpResponseBody := fasthttpResponse.Body() - if httpResponseBody != nil { - motanResponseBody := make([]byte, len(httpResponseBody)) - copy(motanResponseBody, httpResponseBody) - resp.Value = motanResponseBody - } + // direct assign to resp.Value + resp.Value = httpResponseBody + // record this http response, release it when releasing HttpMotanResponse + resp.HttpResponse = fasthttpResponse } } diff --git a/lb/randomLb.go b/lb/randomLb.go index a8869526..48ef206b 100644 --- a/lb/randomLb.go +++ b/lb/randomLb.go @@ -13,11 +13,13 @@ type RandomLB struct { func (r *RandomLB) OnRefresh(endpoints []motan.EndPoint) { r.endpoints = endpoints } + func (r *RandomLB) Select(request motan.Request) motan.EndPoint { eps := r.endpoints _, endpoint := SelectOneAtRandom(eps) return endpoint } + func (r *RandomLB) SelectArray(request motan.Request) []motan.EndPoint { eps := r.endpoints index, endpoint := SelectOneAtRandom(eps) diff --git a/log/bytes.go b/log/bytes.go new file mode 100644 index 00000000..f9582fc0 --- /dev/null +++ b/log/bytes.go @@ -0,0 +1,67 @@ +package vlog + +import ( + "strconv" + "sync" +) + +var ( + initSize = 256 + innerBytesBufferPool = sync.Pool{New: func() interface{} { + return &innerBytesBuffer{buf: make([]byte, 0, initSize)} + }} +) + +// innerBytesBuffer is a variable-sized buffer of bytes with Write methods. +type innerBytesBuffer struct { + buf []byte // reuse +} + +// WriteString write a str string append the innerBytesBuffer +func (b *innerBytesBuffer) WriteString(str string) { + b.buf = append(b.buf, str...) +} + +// WriteBoolString append the string value of v(true/false) to innerBytesBuffer +func (b *innerBytesBuffer) WriteBoolString(v bool) { + if v { + b.WriteString("true") + } else { + b.WriteString("false") + } +} + +// WriteUint64String append the string value of u to innerBytesBuffer +func (b *innerBytesBuffer) WriteUint64String(u uint64) { + b.buf = strconv.AppendUint(b.buf, u, 10) +} + +// WriteInt64String append the string value of i to innerBytesBuffer +func (b *innerBytesBuffer) WriteInt64String(i int64) { + b.buf = strconv.AppendInt(b.buf, i, 10) +} + +func (b *innerBytesBuffer) Bytes() []byte { return b.buf } + +func (b *innerBytesBuffer) String() string { + return string(b.buf) +} + +func (b *innerBytesBuffer) Reset() { + b.buf = b.buf[:0] +} + +func (b *innerBytesBuffer) Len() int { return len(b.buf) } + +func (b *innerBytesBuffer) Cap() int { return cap(b.buf) } + +func acquireBytesBuffer() *innerBytesBuffer { + return innerBytesBufferPool.Get().(*innerBytesBuffer) +} + +func releaseBytesBuffer(b *innerBytesBuffer) { + if b != nil { + b.Reset() + innerBytesBufferPool.Put(b) + } +} diff --git a/log/bytes_test.go b/log/bytes_test.go new file mode 100644 index 00000000..333c0ece --- /dev/null +++ b/log/bytes_test.go @@ -0,0 +1,104 @@ +package vlog + +import ( + "strconv" + "testing" +) + +func TestWrite(t *testing.T) { + // new BytesBuffer + buf := acquireBytesBuffer() + if buf.Len() != 0 { + t.Errorf("new buf length not zero.") + } + if buf.Cap() != initSize { + t.Errorf("buf cap not correct.real:%d, expect:%d\n", buf.Cap(), initSize) + } + + // write string + buf.Reset() + buf.WriteString("string1") + buf.WriteString("string2") + tempbytes := buf.Bytes() + if "string1" != string(tempbytes[:7]) { + t.Errorf("write string not correct.buf:%+v\n", buf) + } + if "string2" != string(tempbytes[7:14]) { + t.Errorf("write string not correct.buf:%+v\n", buf) + } + + // write bool string + buf.Reset() + buf.WriteBoolString(true) + buf.WriteBoolString(false) + tempbytes = buf.Bytes() + if "true" != string(tempbytes[:4]) { + t.Errorf("write bool string not correct.buf:%+v\n", buf) + } + if "false" != string(tempbytes[4:9]) { + t.Errorf("write bool string not correct.buf:%+v\n", buf) + } + + // write uint64 string + buf.Reset() + var u1 uint64 = 11111111 + var u2 uint64 = 22222222 + buf.WriteUint64String(u1) + buf.WriteUint64String(u2) + tempbytes = buf.Bytes() + if "11111111" != string(tempbytes[:8]) { + t.Errorf("write unit64 string not correct.buf:%+v\n", buf) + } + if "22222222" != string(tempbytes[8:]) { + t.Errorf("write uint64 string not correct.buf:%+v\n", buf) + } + + // write int64 string + buf.Reset() + var i1 int64 = 11111111 + var i2 int64 = -22222222 + buf.WriteInt64String(i1) + buf.WriteInt64String(i2) + tempbytes = buf.Bytes() + if "11111111" != string(tempbytes[:8]) { + t.Errorf("write unit64 string not correct.buf:%+v\n", buf) + } + if "-22222222" != string(tempbytes[8:]) { + t.Errorf("write uint64 string not correct.buf:%+v\n", buf) + } +} + +func TestRead(t *testing.T) { + buf := acquireBytesBuffer() + string1 := "aaaaaaaaaaaa" + buf.WriteString(string1) + buf.WriteUint64String(uint64(len(string1))) + buf.WriteBoolString(false) + buf.WriteInt64String(int64(-len(string1))) + + string2 := "bbbbbbbbbbbb" + buf.WriteString(string2) + buf.WriteUint64String(uint64(len(string2))) + buf.WriteBoolString(true) + buf.WriteInt64String(int64(-len(string2))) + + data := buf.Bytes() + buf2 := &innerBytesBuffer{data} + rsize := len(string1) + 2 + 5 + 3 + len(string2) + 2 + 4 + 3 + if buf2.Len() != rsize { + t.Errorf("read buf len not correct. buf:%v\n", buf2) + } + + // read value + expectValue := string1 + + strconv.Itoa(len(string1)) + + "false" + + "-" + strconv.Itoa(len(string1)) + + string2 + + strconv.Itoa(len(string2)) + + "true" + + "-" + strconv.Itoa(len(string2)) + if expectValue != buf2.String() { + t.Errorf("read value not correct. buf:%v\n", buf2) + } +} diff --git a/log/log.go b/log/log.go index d1d484bb..a3b71c2e 100644 --- a/log/log.go +++ b/log/log.go @@ -1,14 +1,12 @@ package vlog import ( - "bytes" "flag" "github.com/weibocom/motan-go/metrics/sampler" "log" "os" "path/filepath" "runtime/debug" - "strconv" "sync" "sync/atomic" "time" @@ -18,6 +16,9 @@ import ( ) var ( + accessLogEntityPool = sync.Pool{New: func() interface{} { + return new(AccessLogEntity) + }} loggerInstance Logger once sync.Once logDir = flag.String("log_dir", ".", "If non-empty, write log files in this directory") @@ -390,12 +391,13 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { zap.String("exception", logObject.Exception), zap.String("upstreamCode", logObject.UpstreamCode)) } else { - var buffer bytes.Buffer + buffer := acquireBytesBuffer() + buffer.WriteString(logObject.FilterName) buffer.WriteString("|") buffer.WriteString(logObject.Role) buffer.WriteString("|") - buffer.WriteString(strconv.FormatUint(logObject.RequestID, 10)) + buffer.WriteUint64String(logObject.RequestID) buffer.WriteString("|") buffer.WriteString(logObject.Service) buffer.WriteString("|") @@ -405,15 +407,15 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { buffer.WriteString("|") buffer.WriteString(logObject.RemoteAddress) buffer.WriteString("|") - buffer.WriteString(strconv.Itoa(logObject.ReqSize)) + buffer.WriteInt64String(int64(logObject.ReqSize)) buffer.WriteString("|") - buffer.WriteString(strconv.Itoa(logObject.ResSize)) + buffer.WriteInt64String(int64(logObject.ResSize)) buffer.WriteString("|") - buffer.WriteString(strconv.FormatInt(logObject.BizTime, 10)) + buffer.WriteInt64String(logObject.BizTime) buffer.WriteString("|") - buffer.WriteString(strconv.FormatInt(logObject.TotalTime, 10)) + buffer.WriteInt64String(logObject.TotalTime) buffer.WriteString("|") - buffer.WriteString(strconv.FormatBool(logObject.Success)) + buffer.WriteBoolString(logObject.Success) buffer.WriteString("|") buffer.WriteString(logObject.ResponseCode) buffer.WriteString("|") @@ -421,7 +423,10 @@ func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { buffer.WriteString("|") buffer.WriteString(logObject.UpstreamCode) d.accessLogger.Info(buffer.String()) + + releaseBytesBuffer(buffer) } + ReleaseAccessLogEntity(logObject) } func (d *defaultLogger) MetricsLog(msg string) { @@ -490,3 +495,13 @@ func (d *defaultLogger) SetMetricsLogAvailable(status bool) { d.metricsLevel.SetLevel(zapcore.Level(defaultLogLevel + 1)) } } + +func AcquireAccessLogEntity() *AccessLogEntity { + return accessLogEntityPool.Get().(*AccessLogEntity) +} + +func ReleaseAccessLogEntity(entity *AccessLogEntity) { + if entity != nil { + accessLogEntityPool.Put(entity) + } +} diff --git a/log/log_test.go b/log/log_test.go index fe025fbc..b3552bec 100644 --- a/log/log_test.go +++ b/log/log_test.go @@ -31,7 +31,7 @@ func init() { Exception: "Exception"} } -//BenchmarkLogSprintf: 736 ns/op +// BenchmarkLogSprintf: 736 ns/op func BenchmarkLogSprintf(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { @@ -52,7 +52,7 @@ func BenchmarkLogSprintf(b *testing.B) { } } -//BenchmarkLogBufferWritePlus: 438 ns/op +// BenchmarkLogBufferWritePlus: 438 ns/op func BenchmarkLogBufferWritePlus(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { @@ -74,7 +74,7 @@ func BenchmarkLogBufferWritePlus(b *testing.B) { } } -//BenchmarkLogBufferWrite: 406 ns/op +// BenchmarkLogBufferWrite: 406 ns/op func BenchmarkLogBufferWrite(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/log/rotate.go b/log/rotate.go index 18d7ca7b..652027bb 100644 --- a/log/rotate.go +++ b/log/rotate.go @@ -1,10 +1,10 @@ -//********** this file is modified based on "gopkg.in/natefinch/lumberjack.v2" ********** +// Package vlog ********** this file is modified based on "gopkg.in/natefinch/lumberjack.v2" ********** // Package lumberjack provides a rolling logger. // // Note that this is v2.0 of lumberjack, and should be imported using gopkg.in // thusly: // -// import "gopkg.in/natefinch/lumberjack.v2" +// import "gopkg.in/natefinch/lumberjack.v2" // // The package name remains simply lumberjack, and the code resides at // https://github.com/natefinch/lumberjack under the v2.0 branch. @@ -69,7 +69,7 @@ var _ io.WriteCloser = (*RotateWriter)(nil) // `/var/log/foo/server.log`, a backup created at 6:30pm on Nov 11 2016 would // use the filename `/var/log/foo/server-2016-11-04T18-30-00.000.log` // -// Cleaning Up Old Log Files +// # Cleaning Up Old Log Files // // Whenever a new logfile gets created, old log files may be deleted. The most // recent files according to the encoded timestamp will be retained, up to a diff --git a/manageHandler.go b/manageHandler.go index 98fd4208..f8e79085 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -158,7 +158,7 @@ func (s *StatusHandler) getStatus() []byte { exporter := v.(motan.Exporter) group := exporter.GetURL().Group service := exporter.GetURL().Path - statItem := metrics.GetStatItem(metrics.Escape(group), metrics.Escape(service)) + statItem := metrics.GetStatItem(group, "", service) if statItem == nil { return true } diff --git a/manageHandler_test.go b/manageHandler_test.go index 55bafec9..10655465 100644 --- a/manageHandler_test.go +++ b/manageHandler_test.go @@ -20,7 +20,7 @@ func TestGetAllService(t *testing.T) { i := InfoHandler{} ctx := &motan.Context{ ServiceURLs: map[string]*motan.URL{ - "test": &motan.URL{ + "test": { Group: "testgroup", Path: "testpath", }, diff --git a/metrics/graphite.go b/metrics/graphite.go index 93f80cb5..45324687 100644 --- a/metrics/graphite.go +++ b/metrics/graphite.go @@ -32,7 +32,7 @@ type graphite struct { Port int Name string localIP string - lock sync.Mutex + lock *sync.Mutex //using pointer avoid shadow copy, cause lock issue conn net.Conn } @@ -50,7 +50,7 @@ func newGraphite(ip, pool string, port int) *graphite { Host: ip, Port: port, Name: pool, - lock: sync.Mutex{}, + lock: &sync.Mutex{}, conn: getUDPConn(ip, port), } } @@ -102,19 +102,21 @@ func GenGraphiteMessages(localIP string, snapshots []Snapshot) []string { if len(pni) < minKeyLength { return } + escapedService := snap.GetEscapedService() + escapedGroup := snap.GetEscapedGroup() + snap.GetGroupSuffix() if snap.IsHistogram(k) { //histogram for slaK, slaV := range sla { segment += fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s.%s:%.2f|kv\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], slaK, snap.Percentile(k, slaV)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], slaK, snap.Percentile(k, slaV)) } segment += fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s.%s:%.2f|ms\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], "avg_time", snap.Mean(k)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], "avg_time", snap.Mean(k)) } else if snap.IsCounter(k) { //counter segment = fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s:%d|c\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], snap.Count(k)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], snap.Count(k)) } else { // gauge segment = fmt.Sprintf("%s.%s.%s.byhost.%s.%s.%s:%d|kv\n", - pni[0], pni[1], snap.GetGroup(), localIP, snap.GetService(), pni[2], snap.Value(k)) + pni[0], pni[1], escapedGroup, localIP, escapedService, pni[2], snap.Value(k)) } if buf.Len() > 0 && buf.Len()+len(segment) > messageMaxLen { messages = append(messages, buf.String()) diff --git a/metrics/graphite_test.go b/metrics/graphite_test.go index fceb856d..dbc3c068 100644 --- a/metrics/graphite_test.go +++ b/metrics/graphite_test.go @@ -25,7 +25,7 @@ func Test_graphite_Write(t *testing.T) { go server.start() time.Sleep(100 * time.Millisecond) g := newGraphite("127.0.0.1", "test pool", 3456) - item := NewStatItem(group, service) + item := NewStatItem(group, "", service) item.AddCounter(keyPrefix+"c1", 1) item.AddHistograms(keyPrefix+" h1", 100) err := g.Write([]Snapshot{item.SnapshotAndClear()}) @@ -48,7 +48,7 @@ func Test_graphite_Write(t *testing.T) { } func TestGenGraphiteMessages(t *testing.T) { - item1 := NewDefaultStatItem(group, service) + item1 := NewDefaultStatItem(group, "", service) // counter message item1.AddCounter(keyPrefix+"c1", 1) @@ -69,8 +69,8 @@ func TestGenGraphiteMessages(t *testing.T) { role, application, group, localhost, service, methodPrefix+"h1", "avg_time", float32(100))), "histogram message") // multi items - item2 := NewDefaultStatItem(group+"2", service+"2") - item3 := NewDefaultStatItem(group+"3", service+"3") + item2 := NewDefaultStatItem(group+"2", "", service+"2") + item3 := NewDefaultStatItem(group+"3", "", service+"3") item3.SetReport(false) // item3 not report length := 10 diff --git a/metrics/metrics.go b/metrics/metrics.go index 574917b1..614e9dc1 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -42,6 +42,7 @@ const ( ) var ( + metricsKeyBuilderBufferSize = 64 // NewStatItem is the factory func for StatItem NewStatItem = NewDefaultStatItem items = make(map[string]StatItem, 64) @@ -52,15 +53,21 @@ var ( processor: defaultEventProcessor, //sink processor size eventBus: make(chan *event, eventBufferSize), writers: make(map[string]StatWriter), - evtBuf: &sync.Pool{New: func() interface{} { return new(event) }}, + eventPool: &sync.Pool{New: func() interface{} { + return &event{} + }}, } + escapeCache sync.Map ) type StatItem interface { SetService(service string) GetService() string + GetEscapedService() string SetGroup(group string) GetGroup() string + GetEscapedGroup() string + GetGroupSuffix() string AddCounter(key string, value int64) AddHistograms(key string, duration int64) AddGauge(key string, value int64) @@ -97,42 +104,43 @@ type StatWriter interface { Write(snapshots []Snapshot) error } -func GetOrRegisterStatItem(group string, service string) StatItem { +func GetOrRegisterStatItem(group, groupSuffix string, service string) StatItem { + k := group + groupSuffix + service itemsLock.RLock() - item := items[group+service] + item := items[k] itemsLock.RUnlock() if item != nil { return item } itemsLock.Lock() - item = items[group+service] + item = items[k] if item == nil { - item = NewStatItem(group, service) - items[group+service] = item + item = NewStatItem(group, groupSuffix, service) + items[k] = item } itemsLock.Unlock() return item } -func GetStatItem(group string, service string) StatItem { +func GetStatItem(group, groupSuffix string, service string) StatItem { itemsLock.RLock() defer itemsLock.RUnlock() - return items[group+service] + return items[group+groupSuffix+service] } // NewDefaultStatItem create a new statistic item, you should escape input parameter before call this function -func NewDefaultStatItem(group string, service string) StatItem { - return &DefaultStatItem{group: group, service: service, holder: &RegistryHolder{registry: metrics.NewRegistry()}, isReport: true} +func NewDefaultStatItem(group, groupSuffix string, service string) StatItem { + return &DefaultStatItem{group: group, groupSuffix: groupSuffix, service: service, holder: &RegistryHolder{registry: metrics.NewRegistry()}, isReport: true} } -func RMStatItem(group string, service string) { +func RMStatItem(group, groupSuffix string, service string) { itemsLock.RLock() - i := items[group+service] + i := items[group+groupSuffix+service] itemsLock.RUnlock() if i != nil { i.Clear() itemsLock.Lock() - delete(items, group+service) + delete(items, group+groupSuffix+service) itemsLock.Unlock() } } @@ -165,14 +173,19 @@ func StatItemSize() int { return len(items) } +// Escape the string avoid invalid graphite key func Escape(s string) string { - return strings.Map(func(char rune) rune { + if v, ok := escapeCache.Load(s); ok { + return v.(string) + } + v := strings.Map(func(char rune) rune { if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || (char == '-') { return char - } else { - return '_' } + return '_' }, s) + escapeCache.Store(s, v) + return v } func AddCounter(group string, service string, key string, value int64) { @@ -187,13 +200,29 @@ func AddGauge(group string, service string, key string, value int64) { sendEvent(eventGauge, group, service, key, value) } +// AddCounterWithKeys arguments: group & groupSuffix & service & keys elements & keySuffix is text without escaped +func AddCounterWithKeys(group, groupSuffix string, service string, keys []string, keySuffix string, value int64) { + sendEventWithKeys(eventCounter, group, groupSuffix, service, keys, keySuffix, value) +} + +// AddHistogramsWithKeys arguments: group & groupSuffix & service & keys elements & keySuffix is text without escaped +func AddHistogramsWithKeys(group, groupSuffix string, service string, keys []string, suffix string, duration int64) { + sendEventWithKeys(eventHistograms, group, groupSuffix, service, keys, suffix, duration) +} + func sendEvent(eventType int32, group string, service string, key string, value int64) { - evt := rp.evtBuf.Get().(*event) + sendEventWithKeys(eventType, group, "", service, []string{key}, "", value) +} + +func sendEventWithKeys(eventType int32, group, groupSuffix string, service string, keys []string, suffix string, value int64) { + evt := rp.eventPool.Get().(*event) evt.event = eventType - evt.key = key + evt.keys = keys evt.group = group evt.service = service evt.value = value + evt.keySuffix = suffix + evt.groupSuffix = groupSuffix select { case rp.eventBus <- evt: default: @@ -239,11 +268,43 @@ func startSampleStatus(application string) { } type event struct { - event int32 - key string - group string - service string - value int64 + event int32 + keys []string + keySuffix string + group string + groupSuffix string + service string + value int64 +} + +// reset used to reset the event object before put it back +// to event objects pool +func (s *event) reset() { + s.event = 0 + s.keys = s.keys[:0] + s.keySuffix = "" + s.group = "" + s.service = "" + s.value = 0 + s.groupSuffix = "" +} + +// getMetricKey get the metrics key when add metrics data into metrics object, +// the key split by : used to when send data to graphite +func (s *event) getMetricKey() string { + keyBuilder := motan.AcquireBytesBuffer(metricsKeyBuilderBufferSize) + defer motan.ReleaseBytesBuffer(keyBuilder) + l := len(s.keys) + for idx, k := range s.keys { + keyBuilder.WriteString(Escape(k)) + if idx < l-1 { + keyBuilder.WriteString(":") + } + } + if s.keySuffix != "" { + keyBuilder.WriteString(s.keySuffix) + } + return string(keyBuilder.Bytes()) } type RegistryHolder struct { @@ -252,6 +313,7 @@ type RegistryHolder struct { type DefaultStatItem struct { group string + groupSuffix string service string holder *RegistryHolder isReport bool @@ -271,6 +333,11 @@ func (d *DefaultStatItem) GetService() string { return d.service } +// GetEscapedService return the escaped service used as graphite key +func (d *DefaultStatItem) GetEscapedService() string { + return Escape(d.service) +} + func (d *DefaultStatItem) SetGroup(group string) { d.group = group } @@ -279,6 +346,15 @@ func (d *DefaultStatItem) GetGroup() string { return d.group } +func (d *DefaultStatItem) GetGroupSuffix() string { + return d.groupSuffix +} + +// GetEscapedGroup return the escaped group used as graphite key +func (d *DefaultStatItem) GetEscapedGroup() string { + return Escape(d.group) +} + func (d *DefaultStatItem) AddCounter(key string, value int64) { c := d.getRegistry().Get(key) if c == nil { @@ -315,6 +391,7 @@ func (d *DefaultStatItem) SnapshotAndClear() Snapshot { old := atomic.SwapPointer((*unsafe.Pointer)(unsafe.Pointer(&d.holder)), unsafe.Pointer(&RegistryHolder{registry: metrics.NewRegistry()})) d.lastSnapshot = &ReadonlyStatItem{ group: d.group, + groupSuffix: d.groupSuffix, service: d.service, holder: (*RegistryHolder)(old), isReport: d.isReport, @@ -324,8 +401,8 @@ func (d *DefaultStatItem) SnapshotAndClear() Snapshot { return d.lastSnapshot } +// SnapshotAndClearV0 Using SnapshotAndClear instead. // Deprecated. -// Using SnapshotAndClear instead. // Because of Snapshot(DefaultStatItem) calculates metrics will call locker to do that, // cause low performance func (d *DefaultStatItem) SnapshotAndClearV0() Snapshot { @@ -467,6 +544,7 @@ func (d *DefaultStatItem) IsGauge(key string) bool { type ReadonlyStatItem struct { group string + groupSuffix string service string holder *RegistryHolder isReport bool @@ -477,6 +555,7 @@ type ReadonlyStatItem struct { func (d *ReadonlyStatItem) getRegistry() metrics.Registry { return (*RegistryHolder)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&d.holder)))).registry } + func (d *ReadonlyStatItem) getCache(key string) interface{} { d.buildCacheLock.RLock() if v, ok := d.cache[key]; ok { @@ -521,6 +600,11 @@ func (d *ReadonlyStatItem) GetService() string { return d.service } +// GetEscapedService return the escaped service used as graphite key +func (d *ReadonlyStatItem) GetEscapedService() string { + return Escape(d.service) +} + func (d *ReadonlyStatItem) SetGroup(group string) { panic("action not supported") } @@ -529,6 +613,15 @@ func (d *ReadonlyStatItem) GetGroup() string { return d.group } +func (d *ReadonlyStatItem) GetGroupSuffix() string { + return d.groupSuffix +} + +// GetEscapedGroup return the escaped group used as graphite key +func (d *ReadonlyStatItem) GetEscapedGroup() string { + return Escape(d.group) +} + func (d *ReadonlyStatItem) AddCounter(key string, value int64) { panic("action not supported") } @@ -725,13 +818,16 @@ type reporter struct { interval time.Duration processor int writers map[string]StatWriter - evtBuf *sync.Pool + eventPool *sync.Pool writersLock sync.RWMutex } func (r *reporter) eventLoop() { for evt := range r.eventBus { r.processEvent(evt) + // clean the event object before put it back + evt.reset() + r.eventPool.Put(evt) } } @@ -746,14 +842,15 @@ func (r *reporter) addWriter(key string, sw StatWriter) { func (r *reporter) processEvent(evt *event) { defer motan.HandlePanic(nil) - item := GetOrRegisterStatItem(evt.group, evt.service) + item := GetOrRegisterStatItem(evt.group, evt.groupSuffix, evt.service) + key := evt.getMetricKey() switch evt.event { case eventCounter: - item.AddCounter(evt.key, evt.value) + item.AddCounter(key, evt.value) case eventHistograms: - item.AddHistograms(evt.key, evt.value) + item.AddHistograms(key, evt.value) case eventGauge: - item.AddGauge(evt.key, evt.value) + item.AddGauge(key, evt.value) } } diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go index 604f4359..612640b6 100644 --- a/metrics/metrics_test.go +++ b/metrics/metrics_test.go @@ -19,19 +19,19 @@ var ( func TestGetStatItem(t *testing.T) { ClearStatItems() // get - si := GetStatItem(group, service) + si := GetStatItem(group, "", service) assert.Nil(t, si, "GetStatItem") assert.Equal(t, 0, StatItemSize(), "item size") // register - si = GetOrRegisterStatItem(group, service) + si = GetOrRegisterStatItem(group, "", service) assert.NotNil(t, si, "GetOrRegisterStatItem") assert.Equal(t, group, si.GetGroup(), "StatItem group") assert.Equal(t, service, si.GetService(), "StatItem service") assert.Equal(t, 1, StatItemSize(), "item size") - again := GetStatItem(group, service) + again := GetStatItem(group, "", service) assert.True(t, si == again, "get StatItem not the same one") - si2 := GetOrRegisterStatItem(group+"2", service+"2") + si2 := GetOrRegisterStatItem(group+"2", "", service+"2") assert.NotNil(t, si2, "GetOrRegisterStatItem") assert.Equal(t, group+"2", si2.GetGroup(), "StatItem group") assert.Equal(t, service+"2", si2.GetService(), "StatItem service") @@ -39,17 +39,17 @@ func TestGetStatItem(t *testing.T) { assert.Equal(t, 2, StatItemSize(), "item size") // rm - RMStatItem(group, service) - si = GetStatItem(group, service) + RMStatItem(group, "", service) + si = GetStatItem(group, "", service) assert.Nil(t, si, "GetStatItem") - si2 = GetStatItem(group+"2", service+"2") + si2 = GetStatItem(group+"2", "", service+"2") assert.NotNil(t, si2, "GetOrRegisterStatItem") // clear ClearStatItems() - si = GetStatItem(group, service) + si = GetStatItem(group, "", service) assert.Nil(t, si, "clear not work") - si2 = GetStatItem(group+"2", service+"2") + si2 = GetStatItem(group+"2", "", service+"2") assert.Nil(t, si2, "clear not work") // multi thread @@ -59,7 +59,7 @@ func TestGetStatItem(t *testing.T) { for i := 0; i < size; i++ { j := i go func() { - s := GetOrRegisterStatItem(group, service) + s := GetOrRegisterStatItem(group, "", service) lock.Lock() sia[j] = s lock.Unlock() @@ -75,14 +75,14 @@ func TestGetStatItem(t *testing.T) { } func TestNewDefaultStatItem(t *testing.T) { - si := NewStatItem(group, service) + si := NewStatItem(group, "", service) assert.NotNil(t, si, "NewStatItem") _, ok := si.(StatItem) assert.True(t, ok, "type not StatItem") } func TestDefaultStatItem(t *testing.T) { - item := NewDefaultStatItem(group, service) + item := NewDefaultStatItem(group, "", service) assert.NotNil(t, item, "GetOrRegisterStatItem") assert.Equal(t, group, item.GetGroup(), "StatItem group") assert.Equal(t, service, item.GetService(), "StatItem service") @@ -161,7 +161,7 @@ func TestStat(t *testing.T) { AddGauge(group, service, "g2", 200) } time.Sleep(100 * time.Millisecond) - item := GetStatItem(group, service) + item := GetStatItem(group, "", service) assert.NotNil(t, item, "item not exist") snap := item.SnapshotAndClear() // test counters @@ -236,7 +236,7 @@ func (m *mockWriter) GetSnapshot() []Snapshot { func TestReadonlyStatItem(t *testing.T) { assert := assert.New(t) - item := NewDefaultStatItem(group, service) + item := NewDefaultStatItem(group, "", service) item.SetService(service) item.SetGroup(group) roItem := item.SnapshotAndClear() @@ -281,7 +281,7 @@ func TestReadonlyStatItem(t *testing.T) { } func TestDefaultStatItem_1(t *testing.T) { - item := NewDefaultStatItem(group, service) + item := NewDefaultStatItem(group, "", service) assert.Equal(t, "str", Escape("str")) assert.NotNil(t, item, "GetOrRegisterStatItem") assert.Equal(t, group, item.GetGroup(), "StatItem group") diff --git a/protocol/motan1Protocol.go b/protocol/motan1Protocol.go index 136c525f..19d9004c 100644 --- a/protocol/motan1Protocol.go +++ b/protocol/motan1Protocol.go @@ -60,6 +60,11 @@ const ( HEARTBEAT_RESPONSE_STRING = HEARTBEAT_METHOD_NAME ) +const ( + V1Group = "group" + V1Version = "version" +) + const MAX_BLOCK_SIZE = 1024 // base binary arrays diff --git a/protocol/motanProtocol.go b/protocol/motanProtocol.go index 0e1e413e..8b102c16 100644 --- a/protocol/motanProtocol.go +++ b/protocol/motanProtocol.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" motan "github.com/weibocom/motan-go/core" @@ -20,9 +21,10 @@ import ( const ( DefaultMetaSize = 16 DefaultMaxContentLength = 10 * 1024 * 1024 + DefaultBufferSize = 256 ) -//message type +// message type const ( Req = iota Res @@ -32,6 +34,7 @@ const ( const ( MotanMagic = 0xf1f1 HeaderLength = 13 + MetaStartOffset = HeaderLength + 4 Version1 = 0 Version2 = 1 defaultProtocol = "motan2" @@ -40,7 +43,7 @@ const ( const ( MPath = "M_p" MMethod = "M_m" - MExceptionn = "M_e" + MException = "M_e" MProcessTime = "M_pt" MMethodDesc = "M_md" MGroup = "M_g" @@ -60,6 +63,14 @@ type Header struct { RequestID uint64 } +func (h *Header) Reset() { + h.Magic = 0 + h.MsgType = 0 + h.VersionStatus = 0 + h.Serialize = 0 + h.RequestID = 0 +} + func (h *Header) Clone() *Header { return &Header{ Magic: h.Magic, @@ -71,13 +82,16 @@ func (h *Header) Clone() *Header { } type Message struct { - Header *Header - Metadata *motan.StringMap - Body []byte - Type int + Header *Header + Metadata *motan.StringMap + Body []byte + Type int + canRelease atomic.Value + // lazy init or reset when call Message.Encode or Message.Encode0 + bytesBuffer *motan.BytesBuffer } -//serialize +// serialize const ( Hessian = iota GrpcPb @@ -104,6 +118,9 @@ var ( writeBufPool = &sync.Pool{New: func() interface{} { // for gzip write buffer return &bytes.Buffer{} }} + messagePool = sync.Pool{New: func() interface{} { + return &Message{Metadata: motan.NewStringMap(DefaultMetaSize), Header: &Header{}} + }} ) // errors @@ -248,16 +265,34 @@ func BuildHeader(msgType int, proxy bool, serialize int, requestID uint64, msgSt } //status - status := uint8(0x08 | (uint8(msgStatus) & 0x07)) + status := 0x08 | (uint8(msgStatus) & 0x07) - serial := uint8(0x00 | (uint8(serialize) << 3)) + serial := 0x00 | (uint8(serialize) << 3) header := &Header{MotanMagic, mtype, status, serial, requestID} return header } -func (msg *Message) Encode() (buf *motan.BytesBuffer) { - metabuf := motan.NewBytesBuffer(256) +// Encode0 encode header、meta、body size +// used with GetEncodedBytes method +// unexpected if call Message.GetEncodedBytes after rewrite result(*motan.BytesBuffer) +func (msg *Message) Encode0() { + if msg.bytesBuffer == nil { + msg.bytesBuffer = motan.NewBytesBuffer(DefaultBufferSize) + } else { + msg.bytesBuffer.Reset() + } + + // encode header. + msg.bytesBuffer.WriteUint16(MotanMagic) + msg.bytesBuffer.WriteByte(msg.Header.MsgType) + msg.bytesBuffer.WriteByte(msg.Header.VersionStatus) + msg.bytesBuffer.WriteByte(msg.Header.Serialize) + msg.bytesBuffer.WriteUint64(msg.Header.RequestID) + + // 4 byte for meta size + msg.bytesBuffer.SetWPos(MetaStartOffset) + // encode meta msg.Metadata.Range(func(k, v string) bool { if k == "" || v == "" { return true @@ -266,38 +301,60 @@ func (msg *Message) Encode() (buf *motan.BytesBuffer) { vlog.Errorf("metadata not correct.k:%s, v:%s", k, v) return true } - metabuf.Write([]byte(k)) - metabuf.WriteByte('\n') - metabuf.Write([]byte(v)) - metabuf.WriteByte('\n') + msg.bytesBuffer.WriteString(k) + msg.bytesBuffer.WriteByte('\n') + msg.bytesBuffer.WriteString(v) + msg.bytesBuffer.WriteByte('\n') return true }) + metaWpos := msg.bytesBuffer.GetWPos() + metaSize := 0 + if metaWpos != MetaStartOffset { + // rewrite meta last char '\n' + metaWpos -= 1 + metaSize = metaWpos - MetaStartOffset - if metabuf.Len() > 0 { - metabuf.SetWPos(metabuf.GetWPos() - 1) } - metasize := metabuf.Len() - bodysize := len(msg.Body) - buf = motan.NewBytesBuffer(int(HeaderLength + bodysize + metasize + 8)) - // encode header. - buf.WriteUint16(MotanMagic) - buf.WriteByte(msg.Header.MsgType) - buf.WriteByte(msg.Header.VersionStatus) - buf.WriteByte(msg.Header.Serialize) - buf.WriteUint64(msg.Header.RequestID) + msg.bytesBuffer.SetWPos(HeaderLength) + msg.bytesBuffer.WriteUint32(uint32(metaSize)) - // encode meta - buf.WriteUint32(uint32(metasize)) - if metasize > 0 { - buf.Write(metabuf.Bytes()) + msg.bytesBuffer.SetWPos(metaWpos) + // encode body size + bodySize := len(msg.Body) + msg.bytesBuffer.WriteUint32(uint32(bodySize)) +} + +// GetEncodedBytes get encoded headers Message.Body +// used with Encode0 method +// unexpected if repeat call Message.GetEncodedBytes after rewrite result([][]byte) +func (msg *Message) GetEncodedBytes() [][]byte { + if msg.bytesBuffer == nil { // maybe not call Message.Encode or Message.Encode0 + msg.Encode0() } + return [][]byte{msg.bytesBuffer.Bytes(), msg.Body} +} + +// Encode result encoded result +// unexpected if call Message.GetEncodedBytes after rewrite result(*motan.BytesBuffer) +func (msg *Message) Encode() *motan.BytesBuffer { + msg.Encode0() + bodySize := len(msg.Body) + if bodySize > 0 { + msg.bytesBuffer.Write(msg.Body) + } + return msg.bytesBuffer +} - // encode body - buf.WriteUint32(uint32(bodysize)) - if bodysize > 0 { - buf.Write(msg.Body) +func (msg *Message) Reset() bool { + if ok, v := msg.canRelease.Load().(bool); ok && v { + msg.Type = 0 + msg.Body = msg.Body[:0] + msg.Header.Reset() + msg.Metadata.Reset() + msg.canRelease.Store(false) + return true } - return buf + return false } func (msg *Message) Clone() interface{} { @@ -312,6 +369,10 @@ func (msg *Message) Clone() interface{} { return newMessage } +func (msg *Message) SetCanRelease() { + msg.canRelease.Store(true) +} + func CheckMotanVersion(buf *bufio.Reader) (version int, err error) { var b []byte b, err = buf.Peek(4) @@ -326,93 +387,111 @@ func CheckMotanVersion(buf *bufio.Reader) (version int, err error) { return int(b[3] >> 3 & 0x1f), nil } -func Decode(buf *bufio.Reader) (msg *Message, err error) { - msg, _, err = DecodeWithTime(buf, motan.DefaultMaxContentLength) +func Decode(reader *bufio.Reader, buf *[]byte) (msg *Message, err error) { + msg, _, err = DecodeWithTime(reader, buf, motan.DefaultMaxContentLength) return msg, err } -func DecodeWithTime(buf *bufio.Reader, maxContentLength int) (msg *Message, start time.Time, err error) { - temp := make([]byte, HeaderLength, HeaderLength) - +// DecodeWithTime the parameter buf is a slice pointer, so the +// extension of *buf will affect the slice outside in order to +// reuse the buf which is used by reading header info from reader +func DecodeWithTime(reader *bufio.Reader, buf *[]byte, maxContentLength int) (msg *Message, start time.Time, err error) { + start = time.Now() // record time when starting to reader data + readSlice := *buf // decode header - _, err = io.ReadAtLeast(buf, temp, HeaderLength) - start = time.Now() // record time when starting to read data + _, err = io.ReadAtLeast(reader, readSlice[:HeaderLength], HeaderLength) if err != nil { return nil, start, err } - mn := binary.BigEndian.Uint16(temp[:2]) // TODO 不再验证 + mn := binary.BigEndian.Uint16(readSlice[:2]) // TODO 不再验证 if mn != MotanMagic { vlog.Errorf("wrong magic num:%d, err:%v", mn, err) return nil, start, ErrMagicNum } - - header := &Header{Magic: MotanMagic} - header.MsgType = temp[2] - header.VersionStatus = temp[3] - version := header.GetVersion() + // get a message from pool + msg = AcquireMessage() + defer func() { + if err != nil { + msg.SetCanRelease() + ReleaseMessage(msg) + } + }() + msg.Header.Magic = MotanMagic + msg.Header.MsgType = readSlice[2] + msg.Header.VersionStatus = readSlice[3] + version := msg.Header.GetVersion() if version != Version2 { // TODO 不再验证 + err = ErrVersion vlog.Errorf("unsupported protocol version number: %d", version) return nil, start, ErrVersion } - header.Serialize = temp[4] - header.RequestID = binary.BigEndian.Uint64(temp[5:]) + msg.Header.Serialize = readSlice[4] + msg.Header.RequestID = binary.BigEndian.Uint64(readSlice[5:HeaderLength]) // decode meta - _, err = io.ReadAtLeast(buf, temp[:4], 4) + _, err = io.ReadAtLeast(reader, readSlice[:4], 4) if err != nil { return nil, start, err } - metasize := int(binary.BigEndian.Uint32(temp[:4])) + metasize := int(binary.BigEndian.Uint32(readSlice[:4])) if metasize > maxContentLength { + err = ErrOverSize vlog.Errorf("meta over size. meta size:%d, max size:%d", metasize, maxContentLength) return nil, start, ErrOverSize } - metamap := motan.NewStringMap(DefaultMetaSize) if metasize > 0 { - metadata, err := readBytes(buf, metasize) + if cap(readSlice) < metasize { + readSlice = make([]byte, metasize) + *buf = readSlice + } + err = readBytes(reader, readSlice, metasize) if err != nil { return nil, start, err } s, e := 0, 0 var k string for i := 0; i <= metasize; i++ { - if i == metasize || metadata[i] == '\n' { + if i == metasize || readSlice[i] == '\n' { e = i if k == "" { - k = string(metadata[s:e]) + k = string(readSlice[s:e]) } else { - metamap.Store(k, string(metadata[s:e])) + msg.Metadata.Store(k, string(readSlice[s:e])) k = "" } s = i + 1 } } if k != "" { - vlog.Errorf("decode message fail, metadata not paired. header:%v, meta:%s", header, metadata) + err = ErrMetadata + vlog.Errorf("decode message fail, metadata not paired. header:%v, meta:%s", msg.Header, readSlice) return nil, start, ErrMetadata } } //decode body - _, err = io.ReadAtLeast(buf, temp[:4], 4) + _, err = io.ReadAtLeast(reader, readSlice[:4], 4) if err != nil { return nil, start, err } - bodysize := int(binary.BigEndian.Uint32(temp[:4])) - if bodysize > maxContentLength { - vlog.Errorf("body over size. body size:%d, max size:%d", bodysize, maxContentLength) + bodySize := int(binary.BigEndian.Uint32(readSlice[:4])) + if bodySize > maxContentLength { + err = ErrOverSize + vlog.Errorf("body over size. body size:%d, max size:%d", bodySize, maxContentLength) return nil, start, ErrOverSize } - var body []byte - if bodysize > 0 { - body, err = readBytes(buf, bodysize) + if bodySize > 0 { + if cap(msg.Body) < bodySize { + msg.Body = make([]byte, bodySize) + } + msg.Body = msg.Body[:bodySize] + err = readBytes(reader, msg.Body, bodySize) } else { - body = make([]byte, 0) + msg.Body = make([]byte, 0) } if err != nil { return nil, start, err } - msg = &Message{header, metamap, body, Req} return msg, start, err } @@ -425,15 +504,14 @@ func DecodeGzipBody(body []byte) []byte { return ret } -func readBytes(buf *bufio.Reader, size int) ([]byte, error) { - tempbytes := make([]byte, size) +func readBytes(buf *bufio.Reader, readSlice []byte, size int) error { var s, n = 0, 0 var err error for s < size && err == nil { - n, err = buf.Read(tempbytes[s:]) + n, err = buf.Read(readSlice[s:size]) s += n } - return tempbytes, err + return err } func EncodeGzip(data []byte) ([]byte, error) { @@ -523,7 +601,7 @@ func DecodeGzip(data []byte) (ret []byte, err error) { // ConvertToRequest convert motan2 protocol request message to motan Request func ConvertToRequest(request *Message, serialize motan.Serialization) (motan.Request, error) { - motanRequest := &motan.MotanRequest{Arguments: make([]interface{}, 0)} + motanRequest := motan.AcquireMotanRequest() motanRequest.RequestID = request.Header.RequestID if idStr, ok := request.Metadata.Load(MRequestID); !ok { if request.Header.IsProxy() { @@ -549,12 +627,11 @@ func ConvertToRequest(request *Message, serialize motan.Serialization) (motan.Re request.Header.SetGzip(false) } if !rc.Proxy && serialize == nil { + motan.ReleaseMotanRequest(motanRequest) return nil, ErrSerializeNil } - dv := &motan.DeserializableValue{Body: request.Body, Serialization: serialize} - motanRequest.Arguments = []interface{}{dv} + motanRequest.Arguments = append(motanRequest.Arguments, &motan.DeserializableValue{Body: request.Body, Serialization: serialize}) } - return motanRequest, nil } @@ -633,12 +710,11 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) return msg, nil } } - - res := &Message{} + res := AcquireMessage() var msgType int if response.GetException() != nil { msgType = Exception - response.SetAttachment(MExceptionn, ExceptionToJSON(response.GetException())) + response.SetAttachment(MException, ExceptionToJSON(response.GetException())) if rc.Proxy { rc.Serialized = true } @@ -660,11 +736,15 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) res.Body = b } else { vlog.Warningf("convert response value fail! serialized value not []byte. res:%+v", response) + res.SetCanRelease() + ReleaseMessage(res) return nil, ErrSerializedData } } else { b, err := serialize.Serialize(response.GetValue()) if err != nil { + res.SetCanRelease() + ReleaseMessage(res) return nil, err } res.Body = b @@ -683,7 +763,7 @@ func ConvertToResMessage(response motan.Response, serialize motan.Serialization) // ConvertToResponse convert protocol response to motan Response func ConvertToResponse(response *Message, serialize motan.Serialization) (motan.Response, error) { - mres := &motan.MotanResponse{} + mres := motan.AcquireMotanResponse() rc := mres.GetRPCContext(true) rc.Proxy = response.Header.IsProxy() mres.RequestID = response.Header.RequestID @@ -699,17 +779,19 @@ func ConvertToResponse(response *Message, serialize motan.Serialization) (motan. response.Header.SetGzip(false) } if !rc.Proxy && serialize == nil { + motan.ReleaseMotanResponse(mres) return nil, ErrSerializeNil } dv := &motan.DeserializableValue{Body: response.Body, Serialization: serialize} mres.Value = dv } if response.Header.GetStatus() == Exception { - e := response.Metadata.LoadOrEmpty(MExceptionn) + e := response.Metadata.LoadOrEmpty(MException) if e != "" { var exception *motan.Exception err := json.Unmarshal([]byte(e), &exception) if err != nil { + motan.ReleaseMotanResponse(mres) return nil, err } mres.Exception = exception @@ -722,13 +804,29 @@ func ConvertToResponse(response *Message, serialize motan.Serialization) (motan. } func BuildExceptionResponse(requestID uint64, errmsg string) *Message { - header := BuildHeader(Res, false, defaultSerialize, requestID, Exception) - msg := &Message{Header: header, Metadata: motan.NewStringMap(DefaultMetaSize)} - msg.Metadata.Store(MExceptionn, errmsg) - return msg + message := AcquireMessage() + message.Header.RequestID = requestID + message.Header.SetRequest(false) + message.Header.SetProxy(false) + message.Header.SetVersion(Version2) + message.Header.SetStatus(Exception) + message.Header.SetSerialize(defaultSerialize) + message.Metadata.Store(MException, errmsg) + + return message } func ExceptionToJSON(e *motan.Exception) string { errmsg, _ := json.Marshal(e) return string(errmsg) } + +func AcquireMessage() *Message { + return messagePool.Get().(*Message) +} + +func ReleaseMessage(msg *Message) { + if msg != nil && msg.Reset() { + messagePool.Put(msg) + } +} diff --git a/protocol/motanProtocol_test.go b/protocol/motanProtocol_test.go index 060ae0cb..81563f16 100644 --- a/protocol/motanProtocol_test.go +++ b/protocol/motanProtocol_test.go @@ -5,8 +5,10 @@ import ( "bytes" "compress/gzip" "fmt" + "github.com/stretchr/testify/assert" "github.com/weibocom/motan-go/serialize" "math/rand" + "strconv" "sync" "sync/atomic" "testing" @@ -145,7 +147,8 @@ func TestEncode(t *testing.T) { ebytes := msg.Encode() fmt.Println("len:", ebytes.Len()) - newMsg, err := Decode(bufio.NewReader(ebytes)) + readSlice := make([]byte, 100) + newMsg, err := Decode(bufio.NewReader(ebytes), &readSlice) if newMsg == nil { t.Fatalf("encode message fail") } @@ -164,12 +167,11 @@ func TestEncode(t *testing.T) { msg.Header.SetGzip(true) msg.Body, _ = EncodeGzip([]byte("gzip encode")) b := msg.Encode() - newMsg, _ = Decode(bufio.NewReader(b)) + newMsg, _ = Decode(bufio.NewReader(b), &readSlice) // should not decode gzip if !newMsg.Header.IsGzip() { t.Fatalf("encode message fail") } - nb, err := DecodeGzip(newMsg.Body) if err != nil { t.Errorf("decode gzip fail. err:%v", err) @@ -177,20 +179,127 @@ func TestEncode(t *testing.T) { assertTrue(string(nb) == "gzip encode", "body", t) } +func TestMessage_GetEncodedBytes(t *testing.T) { + h := &Header{} + h.SetVersion(Version2) + h.SetStatus(6) + h.SetOneWay(true) + h.SetSerialize(5) + h.SetGzip(true) + h.SetHeartbeat(true) + h.SetProxy(true) + h.SetRequest(true) + h.Magic = MotanMagic + h.RequestID = 2349789 + meta := core.NewStringMap(0) + meta.Store("k1", "v1") + body := []byte("testbody") + msg := &Message{Header: h, Metadata: meta, Body: body} + msg.Encode0() + encodedBytes := msg.GetEncodedBytes() + buf := core.CreateBytesBuffer(encodedBytes[0]) + + assert.Equal(t, "testbody", string(encodedBytes[1])) + + // append body to buf + buf.Write(msg.Body) + // verify decode + readSlice := make([]byte, 100) + newMsg, err := Decode(bufio.NewReader(buf), &readSlice) + if err != nil || newMsg == nil { + t.Fatalf("encode message fail") + } + + // verify header + assertTrue(newMsg.Header.IsOneWay(), "oneway", t) + assertTrue(newMsg.Header.IsGzip(), "gzip", t) + assertTrue(newMsg.Header.IsHeartbeat(), "heartbeat", t) + assertTrue(newMsg.Header.IsProxy(), "proxy", t) + assertTrue(newMsg.Header.isRequest(), "request", t) + assertTrue(newMsg.Header.GetVersion() == Version2, "version", t) + assertTrue(newMsg.Header.GetSerialize() == 5, "serialize", t) + assertTrue(newMsg.Header.GetStatus() == 6, "status", t) + + // verify meta + assertTrue(newMsg.Metadata.LoadOrEmpty("k1") == "v1", "meta", t) + + // verify body nil + assertTrue(len(newMsg.Body) == len(msg.Body), "body", t) +} + +func TestPool(t *testing.T) { + h := &Header{} + h.SetVersion(Version2) + h.SetStatus(6) + h.SetOneWay(true) + h.SetSerialize(5) + h.SetGzip(true) + h.SetHeartbeat(true) + h.SetProxy(true) + h.SetRequest(true) + h.Magic = MotanMagic + h.RequestID = 2349789 + meta := core.NewStringMap(0) + for mi := 0; mi < 10000; mi++ { + meta.Store(strconv.Itoa(mi), strconv.Itoa(mi)) + } + body := []byte("testbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbodytestbody") + msg := &Message{Header: h, Metadata: meta, Body: body} + ebytes := msg.Encode() + + fmt.Println("len:", ebytes.Len()) + readSlice := make([]byte, 100) + newMsg, err := Decode(bufio.NewReader(ebytes), &readSlice) + if newMsg == nil { + t.Fatalf("encode message fail") + } + assertTrue(newMsg.Header.IsOneWay(), "oneway", t) + assertTrue(newMsg.Header.IsGzip(), "gzip", t) + assertTrue(newMsg.Header.IsHeartbeat(), "heartbeat", t) + assertTrue(newMsg.Header.IsProxy(), "proxy", t) + assertTrue(newMsg.Header.isRequest(), "request", t) + assertTrue(newMsg.Header.GetVersion() == Version2, "version", t) + assertTrue(newMsg.Header.GetSerialize() == 5, "serialize", t) + assertTrue(newMsg.Header.GetStatus() == 6, "status", t) + assertTrue(newMsg.Metadata.LoadOrEmpty("1") == "1", "meta", t) + assertTrue(cap(readSlice) > 200, "readSlice", t) + assertTrue(len(newMsg.Body) == len(msg.Body), "body", t) + assert.Nil(t, err) + ReleaseMessage(newMsg) + body1 := []byte("testbody") + msg1 := &Message{Header: h, Metadata: meta, Body: body1} + ebytes1 := msg1.Encode() + newMsg, err = Decode(bufio.NewReader(ebytes1), &readSlice) + if newMsg == nil { + t.Fatalf("encode message fail") + } + assertTrue(newMsg.Header.IsOneWay(), "oneway", t) + assertTrue(newMsg.Header.IsGzip(), "gzip", t) + assertTrue(newMsg.Header.IsHeartbeat(), "heartbeat", t) + assertTrue(newMsg.Header.IsProxy(), "proxy", t) + assertTrue(newMsg.Header.isRequest(), "request", t) + assertTrue(newMsg.Header.GetVersion() == Version2, "version", t) + assertTrue(newMsg.Header.GetSerialize() == 5, "serialize", t) + assertTrue(newMsg.Header.GetStatus() == 6, "status", t) + assertTrue(newMsg.Metadata.LoadOrEmpty("1") == "1", "meta", t) + assertTrue(cap(readSlice) > 200, "readSlice", t) + assertTrue(len(newMsg.Body) == len(msg1.Body), "body", t) +} + func assertTrue(b bool, msg string, t *testing.T) { if !b { t.Fatalf("test fail, %s not correct.", msg) } } -//TODO convert -func TestConvertToRequest(t *testing.T) { +func TestConvertToResponse(t *testing.T) { h := &Header{} h.SetVersion(Version2) h.SetStatus(6) h.SetOneWay(true) h.SetSerialize(6) - h.SetGzip(true) + h.SetStatus(0) + h.SetGzip(false) h.SetHeartbeat(true) h.SetProxy(true) h.SetRequest(true) @@ -203,13 +312,160 @@ func TestConvertToRequest(t *testing.T) { meta.Store(MPath, "path") body := []byte("testbody") msg := &Message{Header: h, Metadata: meta, Body: body} - req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) - assertTrue(err == nil, "conver to request err", t) - assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) - assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) - assertTrue(req.GetAttachment(MPath) == "path", "request path", t) + // To test convert when the method use pool + pMap := make(map[string]string) + for i := 0; i < 10000; i++ { + resp, err := ConvertToResponse(msg, &serialize.SimpleSerialization{}) + assertTrue(err == nil, "conver to request err", t) + assertTrue(resp.GetAttachment(MGroup) == "group", "response group", t) + assertTrue(resp.GetAttachment(MMethod) == "method", "response method", t) + assertTrue(resp.GetAttachment(MPath) == "path", "response path", t) + assertTrue(string(resp.GetValue().(*core.DeserializableValue).Body) == "testbody", "response body", t) + pMap[fmt.Sprintf("%p", resp)] = "1" + core.ReleaseMotanResponse(resp.(*core.MotanResponse)) + } + // check if responses are reused + assert.True(t, len(pMap) < 10000) + // To test if convert is correct when the method is called in concurrent situation + h1 := &Header{} + h1.SetVersion(Version2) + h1.SetStatus(1) + h1.SetOneWay(true) + h1.SetSerialize(6) + h1.SetGzip(false) + h1.SetHeartbeat(true) + h1.SetProxy(true) + h1.SetRequest(true) + h1.Magic = MotanMagic + h1.RequestID = 1234456 + meta1 := core.NewStringMap(0) + meta1.Store("k2", "v2") + meta1.Store(MGroup, "group1") + meta1.Store(MMethod, "method1") + meta1.Store(MPath, "path1") + meta1.Store(MException, `{"errcode": 0, "errmsg": "test exception", "errtype": 1}`) + msg1 := &Message{Header: h1, Metadata: meta1, Body: nil} + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + time.Sleep(time.Millisecond * 500) + for i := 0; i < 10000; i++ { + resp, err := ConvertToResponse(msg, &serialize.SimpleSerialization{}) + assertTrue(resp.GetRequestID() == 2349789, "request id is incorrect", t) + assertTrue(err == nil, "convert to response err", t) + assertTrue(resp.GetAttachment(MGroup) == "group", "response group", t) + assertTrue(resp.GetAttachment(MMethod) == "method", "response method", t) + assertTrue(resp.GetAttachment(MPath) == "path", "response path", t) + assertTrue(string(resp.GetValue().(*core.DeserializableValue).Body) == "testbody", "response body", t) + core.ReleaseMotanResponse(resp.(*core.MotanResponse)) + } + wg.Done() + }() + go func() { + time.Sleep(time.Millisecond * 500) + for i := 0; i < 10000; i++ { + resp, err := ConvertToResponse(msg1, &serialize.SimpleSerialization{}) + assertTrue(resp.GetRequestID() == 1234456, "request id is incorrect", t) + assertTrue(err == nil, "convert to response err", t) + assertTrue(resp.GetAttachment(MGroup) == "group1", "response group", t) + assertTrue(resp.GetAttachment(MMethod) == "method1", "response method", t) + assertTrue(resp.GetAttachment(MPath) == "path1", "response path", t) + assertTrue(resp.GetValue() == nil, "response body", t) + assertTrue(resp.GetException().ErrMsg == "test exception", "response exception error message", t) + assertTrue(resp.GetException().ErrCode == 0, "response exception error code", t) + assertTrue(resp.GetException().ErrType == 1, "response exception error type", t) + core.ReleaseMotanResponse(resp.(*core.MotanResponse)) + } + wg.Done() + }() + wg.Wait() +} +func TestConvertToRequest(t *testing.T) { + h := &Header{} + h.SetVersion(Version2) + h.SetStatus(6) + h.SetOneWay(true) + h.SetSerialize(6) + h.SetGzip(false) + h.SetHeartbeat(true) + h.SetProxy(true) + h.SetRequest(true) + h.Magic = MotanMagic + h.RequestID = 2349789 + meta := core.NewStringMap(0) + meta.Store("k1", "v1") + meta.Store(MGroup, "group") + meta.Store(MMethod, "method") + meta.Store(MPath, "path") + body := []byte("testbody") + msg := &Message{Header: h, Metadata: meta, Body: body} + pMap := make(map[string]string) + // To test convert when the method use pool + for i := 0; i < 10000; i++ { + req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) + assertTrue(req.GetRequestID() == 2349789, "request id", t) + assertTrue(err == nil, "conver to request err", t) + assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) + assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) + assertTrue(req.GetAttachment(MPath) == "path", "request path", t) + assertTrue(len(req.GetArguments()) == 1, "request argument", t) + pMap[fmt.Sprintf("%p", req)] = "1" + core.ReleaseMotanRequest(req.(*core.MotanRequest)) + } + // check if requests are reused + assert.True(t, len(pMap) < 10000) + // To test if convert is correct when the method is called in concurrent situation + h1 := &Header{} + h1.SetVersion(Version2) + h1.SetStatus(6) + h1.SetOneWay(true) + h1.SetSerialize(6) + h1.SetGzip(false) + h1.SetHeartbeat(true) + h1.SetProxy(true) + h1.SetRequest(true) + h1.Magic = MotanMagic + h1.RequestID = 1234456 + meta1 := core.NewStringMap(0) + meta1.Store("k2", "v2") + meta1.Store(MGroup, "group1") + meta1.Store(MMethod, "method1") + meta1.Store(MPath, "path1") + msg1 := &Message{Header: h1, Metadata: meta1, Body: nil} + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + time.Sleep(time.Millisecond * 500) + for i := 0; i < 10000; i++ { + req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) + assertTrue(req.GetRequestID() == 2349789, "request id is incorrect", t) + assertTrue(err == nil, "conver to request err", t) + assertTrue(req.GetAttachment(MGroup) == "group", "request group", t) + assertTrue(req.GetAttachment(MMethod) == "method", "request method", t) + assertTrue(req.GetAttachment(MPath) == "path", "request path", t) + assertTrue(len(req.GetArguments()) == 1, "request argument", t) + core.ReleaseMotanRequest(req.(*core.MotanRequest)) + } + wg.Done() + }() + go func() { + time.Sleep(time.Millisecond * 500) + for i := 0; i < 10000; i++ { + req, err := ConvertToRequest(msg1, &serialize.SimpleSerialization{}) + assertTrue(req.GetRequestID() == 1234456, "request id is incorrect", t) + assertTrue(err == nil, "conver to request err", t) + assertTrue(req.GetAttachment(MGroup) == "group1", "request group", t) + assertTrue(req.GetAttachment(MMethod) == "method1", "request method", t) + assertTrue(req.GetAttachment(MPath) == "path1", "request path", t) + assertTrue(len(req.GetArguments()) == 0, "request argument", t) + core.ReleaseMotanRequest(req.(*core.MotanRequest)) + } + wg.Done() + }() + wg.Wait() // test request clone + req, err := ConvertToRequest(msg, &serialize.SimpleSerialization{}) cloneReq := req.Clone().(core.Request) assertTrue(err == nil, "conver to request err", t) assertTrue(cloneReq.GetAttachment(MGroup) == "group", "clone request group", t) @@ -337,6 +593,35 @@ func TestConcurrentGzip(t *testing.T) { fmt.Printf("count:%v, errCount: %v\n", count, errCount) } +func TestBuildExceptionResponse(t *testing.T) { + // BuildExceptionResponse + var requestId uint64 = 1234 + err := fmt.Errorf("test error") + exception := &core.Exception{ErrCode: 500, ErrMsg: err.Error(), ErrType: core.ServiceException} + msg := ExceptionToJSON(exception) + + // verify exception message + message := BuildExceptionResponse(requestId, msg) + assert.Equal(t, requestId, message.Header.RequestID) + assert.Equal(t, false, message.Header.isRequest()) + assert.Equal(t, Res, int(message.Header.MsgType)) + assert.Equal(t, false, message.Header.IsProxy()) + assert.Equal(t, Exception, message.Header.GetStatus()) + assert.Equal(t, msg, message.Metadata.LoadOrEmpty(MException)) + + buf := message.Encode() + readSlice := make([]byte, 100) + newMessage, err := Decode(bufio.NewReader(buf), &readSlice) + + // verify encode and decode exception message + assert.Equal(t, message.Header.RequestID, newMessage.Header.RequestID) + assert.Equal(t, message.Header.IsProxy(), newMessage.Header.IsProxy()) + assert.Equal(t, Res, int(message.Header.MsgType)) + assert.Equal(t, message.Header.isRequest(), newMessage.Header.isRequest()) + assert.Equal(t, Exception, newMessage.Header.GetStatus()) + assert.Equal(t, msg, message.Metadata.LoadOrEmpty(MException)) +} + func buildBytes(size int) []byte { baseBytes := []byte("0123456789abcdefghijklmnopqrstuvwxyz") result := bytes.NewBuffer(make([]byte, 0, size)) diff --git a/provider/httpProvider.go b/provider/httpProvider.go index f19cfe69..a5b47ec7 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -140,6 +140,23 @@ func (h *HTTPProvider) SetContext(context *motan.Context) { h.gctx = context } +// rewrite do rewrite +func (h *HTTPProvider) rewrite(httpReq *fasthttp.Request, request motan.Request) (string, error) { + if h.enableRewrite { + var query []byte + // init query string bytes if needed. + if h.locationMatcher.NeedURLQueryString() { + query = httpReq.URI().QueryString() + } + _, path, ok := h.locationMatcher.Pick(request.GetMethod(), query, true) + if !ok { + return "", errors.New("service not found") + } + return path, nil + } + return request.GetMethod(), nil +} + func buildReqURL(request motan.Request, h *HTTPProvider) (string, string, error) { method := request.GetMethod() httpReqURLFmt := h.url.Parameters["URL_FORMAT"] @@ -219,162 +236,135 @@ func buildQueryStr(request motan.Request, url *motan.URL, mixVars []string) (res return res, err } -// Call for do a motan call through this provider -func (h *HTTPProvider) Call(request motan.Request) motan.Response { - t := time.Now().UnixNano() - resp := &motan.MotanResponse{Attachment: motan.NewStringMap(motan.DefaultAttachmentSize)} +func (h *HTTPProvider) DoTransparentProxy(request motan.Request, t int64, ip string) motan.Response { + resp := mhttp.AcquireHttpMotanResponse() + resp.RequestID = request.GetRequestID() var headerBytes []byte var bodyBytes []byte - doTransparentProxy, _ := strconv.ParseBool(request.GetAttachment(mhttp.Proxy)) - var toType []interface{} - if doTransparentProxy { - // Header and body with []byte - toType = []interface{}{&headerBytes, &bodyBytes} - } else if h.proxyAddr != "" { - toType = nil - } else { - toType = make([]interface{}, 1) - } + toType := []interface{}{&headerBytes, &bodyBytes} if err := request.ProcessDeserializable(toType); err != nil { - fillExceptionWithCode(resp, http.StatusBadRequest, t, err) + fillHttpException(resp, http.StatusBadRequest, t, err.Error()) return resp } - resp.RequestID = request.GetRequestID() - ip := "" - if remoteIP, exist := request.GetAttachments().Load(motan.RemoteIPKey); exist { - ip = remoteIP - } else { - ip = request.GetAttachment(motan.HostKey) + // acquires new fasthttp Request and Response object + httpReq := fasthttp.AcquireRequest() + httpRes := fasthttp.AcquireResponse() + // only release fast http request. The response will be released when Response is released + defer fasthttp.ReleaseRequest(httpReq) + // read http header into Request + httpReq.Header.Read(bufio.NewReader(bytes.NewReader(headerBytes))) + //do rewrite + rewritePath := request.GetMethod() + var err error + rewritePath, err = h.rewrite(httpReq, request) + if err != nil { + fillHttpException(resp, http.StatusNotFound, t, err.Error()) + return resp } - // Ok here we do transparent http proxy and return - if doTransparentProxy { - // acquires new fasthttp Request and Response object - httpReq := fasthttp.AcquireRequest() - httpRes := fasthttp.AcquireResponse() - defer fasthttp.ReleaseRequest(httpReq) - defer fasthttp.ReleaseResponse(httpRes) - // read http header into Request - httpReq.Header.Read(bufio.NewReader(bytes.NewReader(headerBytes))) - - //do rewrite - rewritePath := request.GetMethod() - if h.enableRewrite { - // Do not check upstream for compatibility - - var query []byte - // init query string bytes if needed. - if h.locationMatcher.NeedURLQueryString() { - query = httpReq.URI().QueryString() - } - _, path, ok := h.locationMatcher.Pick(request.GetMethod(), query, true) - if !ok { - fillExceptionWithCode(resp, http.StatusNotFound, t, errors.New("service not found")) - return resp - } - rewritePath = path - } - // sets rewrite - httpReq.URI().SetScheme(h.proxySchema) - httpReq.URI().SetPath(rewritePath) - request.GetAttachments().Range(func(k, v string) bool { - if strings.HasPrefix(k, "M_") { - httpReq.Header.Add(strings.Replace(k, "M_", "MOTAN-", -1), v) - } - return true - }) - httpReq.Header.Del("Connection") - if httpReq.Header.Peek(motan.XForwardedFor) == nil { - httpReq.Header.Set(motan.XForwardedFor, ip) - } - if len(bodyBytes) != 0 { - httpReq.BodyWriter().Write(bodyBytes) - } - err := h.fastClient.Do(httpReq, httpRes) - if err != nil { - fillExceptionWithCode(resp, http.StatusServiceUnavailable, t, err) - return resp - } - if h.enableHttpException { - if httpRes.StatusCode() >= 400 { - fillHttpException(resp, httpRes.StatusCode(), t, httpRes.Body()) - return resp - } + // sets rewrite + httpReq.URI().SetScheme(h.proxySchema) + httpReq.URI().SetPath(rewritePath) + request.GetAttachments().Range(func(k, v string) bool { + if kk, ok := mhttp.InnerAttachmentsConvertMap[k]; ok { + httpReq.Header.Set(kk, v) } - headerBuffer := &bytes.Buffer{} - httpRes.Header.Del("Connection") - httpRes.Header.WriteTo(headerBuffer) - body := httpRes.Body() - resp.ProcessTime = (time.Now().UnixNano() - t) / 1e6 - // copy response body is needed - responseBodyBytes := make([]byte, len(body)) - copy(responseBodyBytes, body) - resp.Value = []interface{}{headerBuffer.Bytes(), responseBodyBytes} - updateUpstreamStatusCode(resp, httpRes.StatusCode()) + return true + }) + httpReq.Header.Del("Connection") + if httpReq.Header.Peek(motan.XForwardedFor) == nil { + httpReq.Header.Set(motan.XForwardedFor, ip) + } + if len(bodyBytes) != 0 { + httpReq.BodyWriter().Write(bodyBytes) + } + err = h.fastClient.Do(httpReq, httpRes) + if err != nil { + fillHttpException(resp, http.StatusServiceUnavailable, t, err.Error()) return resp } + if h.enableHttpException && httpRes.StatusCode() >= 400 { + fillHttpException(resp, httpRes.StatusCode(), t, string(httpRes.Body())) + return resp + } + headerBuffer := &bytes.Buffer{} + httpRes.Header.Del("Connection") + httpRes.Header.WriteTo(headerBuffer) + body := httpRes.Body() + resp.ProcessTime = (time.Now().UnixNano() - t) / 1e6 + // record the response and release later + resp.HttpResponse = httpRes + resp.Value = []interface{}{headerBuffer.Bytes(), body} + updateUpstreamStatusCode(resp, httpRes.StatusCode()) + return resp +} - if h.proxyAddr != "" { - // rpc client call to this server - - // acquires new fasthttp Request and Response object - httpReq := fasthttp.AcquireRequest() - httpRes := fasthttp.AcquireResponse() - defer fasthttp.ReleaseRequest(httpReq) - defer fasthttp.ReleaseResponse(httpRes) - // convert motan request to fasthttp request - err := mhttp.MotanRequestToFasthttpRequest(request, httpReq, h.defaultHTTPMethod) - if err != nil { - fillExceptionWithCode(resp, http.StatusBadRequest, t, err) - return resp - } - rewritePath := request.GetMethod() - if h.enableRewrite { - var query []byte - // init query string bytes if needed. - if h.locationMatcher.NeedURLQueryString() { - query = httpReq.URI().QueryString() - } - _, path, ok := h.locationMatcher.Pick(request.GetMethod(), query, true) - if !ok { - fillExceptionWithCode(resp, http.StatusNotFound, t, errors.New("service not found")) - return resp - } - rewritePath = path - } - - httpReq.URI().SetScheme(h.proxySchema) - httpReq.URI().SetPath(rewritePath) - if len(httpReq.Header.Host()) == 0 { - httpReq.Header.SetHost(h.domain) - } - if httpReq.Header.Peek(motan.XForwardedFor) == nil { - httpReq.Header.Set(motan.XForwardedFor, ip) - } - err = h.fastClient.Do(httpReq, httpRes) - if err != nil { - fillExceptionWithCode(resp, http.StatusServiceUnavailable, t, err) - return resp - } - if h.enableHttpException { - if httpRes.StatusCode() >= 400 { - fillHttpException(resp, httpRes.StatusCode(), t, httpRes.Body()) - return resp - } - } - mhttp.FasthttpResponseToMotanResponse(resp, httpRes) - resp.ProcessTime = (time.Now().UnixNano() - t) / 1e6 - updateUpstreamStatusCode(resp, httpRes.StatusCode()) +// DoProxy deal with Request start from a rpc client +func (h *HTTPProvider) DoProxy(request motan.Request, t int64, ip string) motan.Response { + resp := mhttp.AcquireHttpMotanResponse() + resp.RequestID = request.GetRequestID() + if err := request.ProcessDeserializable(nil); err != nil { + fillHttpException(resp, http.StatusBadRequest, t, err.Error()) + return resp + } + // rpc client call to this server + + // acquires new fasthttp Request and Response object + httpReq := fasthttp.AcquireRequest() + httpRes := fasthttp.AcquireResponse() + // do not release http response + defer fasthttp.ReleaseRequest(httpReq) + // convert motan request to fasthttp request + err := mhttp.MotanRequestToFasthttpRequest(request, httpReq, h.defaultHTTPMethod) + if err != nil { + fillHttpException(resp, http.StatusBadRequest, t, err.Error()) + return resp + } + rewritePath := request.GetMethod() + rewritePath, err = h.rewrite(httpReq, request) + if err != nil { + fillHttpException(resp, http.StatusNotFound, t, err.Error()) + return resp + } + httpReq.URI().SetScheme(h.proxySchema) + httpReq.URI().SetPath(rewritePath) + if len(httpReq.Header.Host()) == 0 { + httpReq.Header.SetHost(h.domain) + } + if httpReq.Header.Peek(motan.XForwardedFor) == nil { + httpReq.Header.Set(motan.XForwardedFor, ip) + } + err = h.fastClient.Do(httpReq, httpRes) + if err != nil { + fillHttpException(resp, http.StatusServiceUnavailable, t, err.Error()) + return resp + } + if h.enableHttpException && httpRes.StatusCode() >= 400 { + fillHttpException(resp, httpRes.StatusCode(), t, string(httpRes.Body())) return resp } + mhttp.FasthttpResponseToMotanResponse(resp, httpRes) + resp.ProcessTime = (time.Now().UnixNano() - t) / 1e6 + updateUpstreamStatusCode(resp, httpRes.StatusCode()) + return resp +} +// DoFormatURLQuery use ordinary client and parse the format url +func (h *HTTPProvider) DoFormatURLQuery(request motan.Request, t int64, ip string) motan.Response { + resp := mhttp.AcquireHttpMotanResponse() + resp.RequestID = request.GetRequestID() + toType := make([]interface{}, 1) + if err := request.ProcessDeserializable(toType); err != nil { + fillHttpException(resp, http.StatusBadRequest, t, err.Error()) + return resp + } httpReqURL, httpReqMethod, err := buildReqURL(request, h) if err != nil { - fillException(resp, t, err) + fillHttpException(resp, http.StatusServiceUnavailable, t, err.Error()) return resp } queryStr, err := buildQueryStr(request, h.url, h.mixVars) if err != nil { - fillException(resp, t, err) + fillHttpException(resp, http.StatusServiceUnavailable, t, err.Error()) return resp } var reqBody io.Reader @@ -390,7 +380,7 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { req, err := http.NewRequest(httpReqMethod, httpReqURL, reqBody) if err != nil { vlog.Errorf("new HTTP Provider NewRequest err: %v", err) - fillException(resp, t, err) + fillHttpException(resp, http.StatusServiceUnavailable, t, err.Error()) return resp } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") //设置后，post参数才可正常传递 @@ -421,7 +411,7 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { httpResp, err := c.Do(req) if err != nil { vlog.Errorf("new HTTP Provider Do HTTP Call err: %v", err) - fillException(resp, t, err) + fillHttpException(resp, http.StatusServiceUnavailable, t, err.Error()) return resp } defer httpResp.Body.Close() @@ -432,18 +422,16 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { if l == 0 { vlog.Warningf("server_agent result is empty :%d,%d,%s", statusCode, request.GetRequestID(), httpReqURL) } - resp.ProcessTime = int64((time.Now().UnixNano() - t) / 1e6) + resp.ProcessTime = (time.Now().UnixNano() - t) / 1e6 if err != nil { vlog.Errorf("new HTTP Provider Read body err: %v", err) resp.Exception = &motan.Exception{ErrCode: statusCode, ErrMsg: fmt.Sprintf("%s", err), ErrType: http.StatusServiceUnavailable} return resp } - if h.enableHttpException { - if statusCode >= 400 { - fillHttpException(resp, statusCode, t, body) - return resp - } + if h.enableHttpException && statusCode >= 400 { + fillHttpException(resp, statusCode, t, string(body)) + return resp } request.GetAttachments().Range(func(k, v string) bool { resp.SetAttachment(k, v) @@ -457,6 +445,26 @@ func (h *HTTPProvider) Call(request motan.Request) motan.Response { return resp } +// Call for do a motan call through this provider +func (h *HTTPProvider) Call(request motan.Request) motan.Response { + t := time.Now().UnixNano() + doTransparentProxy, _ := strconv.ParseBool(request.GetAttachment(mhttp.Proxy)) + ip := "" + if remoteIP, exist := request.GetAttachments().Load(motan.RemoteIPKey); exist { + ip = remoteIP + } else { + ip = request.GetAttachment(motan.HostKey) + } + // Ok here we do transparent http proxy and return + if doTransparentProxy { + return h.DoTransparentProxy(request, t, ip) + } + if h.proxyAddr != "" { + return h.DoProxy(request, t, ip) + } + return h.DoFormatURLQuery(request, t, ip) +} + // GetName return this provider name func (h *HTTPProvider) GetName() string { return "HTTPProvider" @@ -498,21 +506,23 @@ func (h *HTTPProvider) GetPath() string { } func fillExceptionWithCode(resp *motan.MotanResponse, code int, start int64, err error) { - resp.ProcessTime = int64((time.Now().UnixNano() - start) / 1e6) + resp.ProcessTime = (time.Now().UnixNano() - start) / 1e6 resp.Exception = &motan.Exception{ErrCode: code, ErrMsg: fmt.Sprintf("%s", err), ErrType: code} } -func fillHttpException(resp *motan.MotanResponse, statusCode int, start int64, body []byte) { - resp.ProcessTime = int64((time.Now().UnixNano() - start) / 1e6) - resp.Exception = &motan.Exception{ErrCode: statusCode, ErrMsg: string(body), ErrType: motan.BizException} +func fillHttpExceptionWithCode(resp *mhttp.HttpMotanResponse, statusCode int, errType int, start int64, msg string) { + resp.ProcessTime = (time.Now().UnixNano() - start) / 1e6 + resp.Exception = &motan.Exception{ErrCode: statusCode, ErrMsg: msg, ErrType: errType} +} + +func fillHttpException(resp *mhttp.HttpMotanResponse, statusCode int, start int64, msg string) { + fillHttpExceptionWithCode(resp, statusCode, motan.BizException, start, msg) } func fillException(resp *motan.MotanResponse, start int64, err error) { fillExceptionWithCode(resp, http.StatusServiceUnavailable, start, err) } -func updateUpstreamStatusCode(resp *motan.MotanResponse, statusCode int) { - resCtx := resp.GetRPCContext(true) +func updateUpstreamStatusCode(resp *mhttp.HttpMotanResponse, statusCode int) { resp.SetAttachment(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) - resCtx.Meta.Store(motan.MetaUpstreamCode, strconv.Itoa(statusCode)) } diff --git a/provider/motanProvider.go b/provider/motanProvider.go index c885d216..b2c5f860 100644 --- a/provider/motanProvider.go +++ b/provider/motanProvider.go @@ -65,7 +65,7 @@ func (m *MotanProvider) Call(request motan.Request) motan.Response { return m.ep.Call(request) } t := time.Now().UnixNano() - res := &motan.MotanResponse{Attachment: motan.NewStringMap(motan.DefaultAttachmentSize)} + res := motan.AcquireMotanResponse() fillException(res, t, errors.New("reverse proxy call err: motanProvider is unavailable")) return res } diff --git a/registry/consulRegistry.go b/registry/consulRegistry.go index 943583e7..ae1c1168 100644 --- a/registry/consulRegistry.go +++ b/registry/consulRegistry.go @@ -16,9 +16,11 @@ type ConsulRegistry struct { func (v *ConsulRegistry) GetURL() *motan.URL { return v.url } + func (v *ConsulRegistry) SetURL(url *motan.URL) { v.url = url } + func (v *ConsulRegistry) GetName() string { return "consulRegistry" } @@ -26,6 +28,7 @@ func (v *ConsulRegistry) GetName() string { func (v *ConsulRegistry) Initialize() { } + func (v *ConsulRegistry) Subscribe(url *motan.URL, listener motan.NotifyListener) { } @@ -33,22 +36,29 @@ func (v *ConsulRegistry) Subscribe(url *motan.URL, listener motan.NotifyListener func (v *ConsulRegistry) Unsubscribe(url *motan.URL, listener motan.NotifyListener) { } + func (v *ConsulRegistry) Discover(url *motan.URL) []*motan.URL { return nil } + func (v *ConsulRegistry) Register(serverURL *motan.URL) { } + func (v *ConsulRegistry) UnRegister(serverURL *motan.URL) { } + func (v *ConsulRegistry) Available(serverURL *motan.URL) { } + func (v *ConsulRegistry) Unavailable(serverURL *motan.URL) { } + func (v *ConsulRegistry) GetRegisteredServices() []*motan.URL { return nil } + func (v *ConsulRegistry) StartSnapshot(conf *motan.SnapshotConf) {} diff --git a/registry/directRegistry.go b/registry/directRegistry.go index fcdc6876..9bc46783 100644 --- a/registry/directRegistry.go +++ b/registry/directRegistry.go @@ -16,21 +16,25 @@ type DirectRegistry struct { func (d *DirectRegistry) GetURL() *motan.URL { return d.url } + func (d *DirectRegistry) SetURL(url *motan.URL) { d.url = url d.urls = parseURLs(url) } + func (d *DirectRegistry) GetName() string { return "direct" } func (d *DirectRegistry) InitRegistry() { } + func (d *DirectRegistry) Subscribe(url *motan.URL, listener motan.NotifyListener) { } func (d *DirectRegistry) Unsubscribe(url *motan.URL, listener motan.NotifyListener) { } + func (d *DirectRegistry) Discover(url *motan.URL) []*motan.URL { if d.urls == nil { d.urls = parseURLs(d.url) @@ -47,22 +51,29 @@ func (d *DirectRegistry) Discover(url *motan.URL) []*motan.URL { } return result } + func (d *DirectRegistry) Register(serverURL *motan.URL) { vlog.Infof("direct registry:register url :%+v", serverURL) } + func (d *DirectRegistry) UnRegister(serverURL *motan.URL) { } + func (d *DirectRegistry) Available(serverURL *motan.URL) { } + func (d *DirectRegistry) Unavailable(serverURL *motan.URL) { } + func (d *DirectRegistry) GetRegisteredServices() []*motan.URL { return nil } + func (d *DirectRegistry) StartSnapshot(conf *motan.SnapshotConf) {} + func parseURLs(url *motan.URL) []*motan.URL { urls := make([]*motan.URL, 0) if len(url.Host) > 0 && url.Port > 0 { diff --git a/registry/localRegistry.go b/registry/localRegistry.go index 7cff8316..cda744b9 100644 --- a/registry/localRegistry.go +++ b/registry/localRegistry.go @@ -15,6 +15,7 @@ func (d *LocalRegistry) GetURL() *motan.URL { func (d *LocalRegistry) SetURL(url *motan.URL) { d.url = url } + func (d *LocalRegistry) GetName() string { return "local" } @@ -40,12 +41,15 @@ func (d *LocalRegistry) Register(serverURL *motan.URL) { func (d *LocalRegistry) UnRegister(serverURL *motan.URL) { } + func (d *LocalRegistry) Available(serverURL *motan.URL) { } + func (d *LocalRegistry) Unavailable(serverURL *motan.URL) { } + func (d *LocalRegistry) GetRegisteredServices() []*motan.URL { return nil } diff --git a/registry/zkRegistry_test.go b/registry/zkRegistry_test.go index 534c8d28..4af4a75b 100644 --- a/registry/zkRegistry_test.go +++ b/registry/zkRegistry_test.go @@ -35,7 +35,7 @@ var ( z = &ZkRegistry{} ) -//Test path generation methods. +// Test path generation methods. func TestZkRegistryToPath(t *testing.T) { //Test path create methods. if p := toNodePath(testURL, zkNodeTypeServer); p != serverPath { diff --git a/server.go b/server.go index 808b8e82..63672ac2 100644 --- a/server.go +++ b/server.go @@ -113,6 +113,7 @@ func (m *MSContext) hashInt(s string) int { h.Write([]byte(s)) return int(h.Sum32()) } + func (m *MSContext) export(url *motan.URL) { defer motan.HandlePanic(nil) service := m.serviceImpls[url.Parameters[motan.RefKey]] diff --git a/server/motanserver.go b/server/motanserver.go index 29fbfc04..23e0059b 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -3,6 +3,8 @@ package server import ( "bufio" "errors" + "github.com/panjf2000/ants/v2" + mhttp "github.com/weibocom/motan-go/http" "net" "strconv" "strings" @@ -19,6 +21,15 @@ import ( var currentConnections int64 var motanServerOnce sync.Once +var processPool, _ = ants.NewPool(50000, ants.WithMaxBlockingTasks(1024)) + +func SetProcessPoolSize(size int) { + processPool.Tune(size) +} + +func GetProcessPoolSize() int { + return processPool.Cap() +} func incrConnections() { atomic.AddInt64(¤tConnections, 1) @@ -61,7 +72,7 @@ func (m *MotanServer) Open(block bool, proxy bool, handler motan.MessageHandler, } lis = listener } else { - addr := ":" + strconv.Itoa(int(m.URL.Port)) + addr := ":" + strconv.Itoa(m.URL.Port) if registry.IsAgent(m.URL) { addr = m.URL.Host + addr } @@ -151,7 +162,7 @@ func (m *MotanServer) handleConn(conn net.Conn) { } else { ip = getRemoteIP(conn.RemoteAddr().String()) } - + decodeBuf := make([]byte, mpro.DefaultBufferSize) for { v, err := mpro.CheckMotanVersion(buf) if err != nil { @@ -168,13 +179,15 @@ func (m *MotanServer) handleConn(conn net.Conn) { } go m.processV1(v1Msg, t, ip, conn) } else if v == mpro.Version2 { - msg, t, err := mpro.DecodeWithTime(buf, m.maxContextLength) + msg, t, err := mpro.DecodeWithTime(buf, &decodeBuf, m.maxContextLength) if err != nil { vlog.Warningf("decode motan v2 message fail! con:%s, err:%s.", conn.RemoteAddr().String(), err.Error()) break } - go m.processV2(msg, t, ip, conn) + processPool.Submit(func() { + m.processV2(msg, t, ip, conn) + }) } else { vlog.Warningf("unsupported motan version! version:%d con:%s", v, conn.RemoteAddr().String()) break @@ -219,7 +232,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c tc.PutReqSpan(&motan.Span{Name: motan.Convert, Time: time.Now()}) req.GetRPCContext(true).Tc = tc } - callStart := time.Now() + mres = m.handler.Call(req) if tc != nil { // clusterFilter end @@ -229,7 +242,7 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c resCtx := mres.GetRPCContext(true) resCtx.Proxy = m.proxy if mres.GetAttachment(mpro.MProcessTime) == "" { - mres.SetAttachment(mpro.MProcessTime, strconv.FormatInt(int64(time.Now().Sub(callStart)/1e6), 10)) + mres.SetAttachment(mpro.MProcessTime, strconv.FormatInt(int64(time.Now().Sub(start)/1e6), 10)) } res, err = mpro.ConvertToResMessage(mres, serialization) if tc != nil { @@ -246,12 +259,14 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c } // recover the communication identifier res.Header.RequestID = lastRequestID - resBuf := res.Encode() + res.Encode0() if tc != nil { tc.PutResSpan(&motan.Span{Name: motan.Encode, Time: time.Now()}) } + var sendBuf net.Buffers = res.GetEncodedBytes() conn.SetWriteDeadline(time.Now().Add(motan.DefaultWriteTimeout)) - _, err := conn.Write(resBuf.Bytes()) + _, err := sendBuf.WriteTo(conn) + res.SetCanRelease() if err != nil { vlog.Errorf("connection will close. conn: %s, err:%s", conn.RemoteAddr().String(), err.Error()) conn.Close() @@ -268,6 +283,18 @@ func (m *MotanServer) processV2(msg *mpro.Message, start time.Time, ip string, c if tc != nil { tc.PutResSpan(&motan.Span{Name: motan.Send, Time: resSendTime}) } + // 回收message + mpro.ReleaseMessage(msg) + mpro.ReleaseMessage(res) + // 回收request + if motanReq, ok := mreq.(*motan.MotanRequest); ok { + motan.ReleaseMotanRequest(motanReq) + } + if motanResp, ok := mres.(*motan.MotanResponse); ok { + motan.ReleaseMotanResponse(motanResp) + } else if motanHttpRes, ok := mres.(*mhttp.HttpMotanResponse); ok { + mhttp.ReleaseHttpMotanResponse(motanHttpRes) + } } func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip string, conn net.Conn) { @@ -276,6 +303,13 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st var result []byte var reqCtx *motan.RPCContext req, err := mpro.DecodeMotanV1Request(msg) + // fill v2 attachment + if req.GetAttachment(mpro.MGroup) == "" { + req.SetAttachment(mpro.MGroup, req.GetAttachment(mpro.V1Group)) + } + if req.GetAttachment(mpro.MVersion) == "" { + req.SetAttachment(mpro.MVersion, req.GetAttachment(mpro.V1Version)) + } if err != nil { vlog.Errorf("decode v1 request fail. conn: %s, err:%s", conn.RemoteAddr().String(), err.Error()) result = mpro.BuildV1ExceptionResponse(msg.Rid, err.Error()) @@ -337,7 +371,7 @@ func getRemoteIP(address string) string { var ip string index := strings.Index(address, ":") if index > 0 { - ip = string(address[:index]) + ip = address[:index] } else { ip = address } diff --git a/tools/nginx/parser.go b/tools/nginx/parser.go index f8c28f20..118beca7 100644 --- a/tools/nginx/parser.go +++ b/tools/nginx/parser.go @@ -29,10 +29,11 @@ const ( Error ) -// location / { # directive location -// if ($request_uri ~= '/*') { # directive if -// } -// } +// Directive location / { # directive location +// +// if ($request_uri ~= '/*') { # directive if +// } +// } type Directive struct { name string args []string @@ -69,6 +70,7 @@ func init() { func NewParser(reader io.Reader) *Parser { return &Parser{reader: bufio.NewReader(reader)} } + func (p *Parser) readToken() token { buf := bytes.Buffer{} sharpComment := false From ac18034302d46f715b759d1ac85c584f294f7604 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Wed, 3 Jan 2024 16:42:11 +0800 Subject: [PATCH 72/75] feat motanv1 metrics attachments (#372) * add filter env * feat motan v1 metrics attachments --- core/constants.go | 1 + core/globalContext.go | 17 ++++++++++++++++- dynamicConfig.go | 1 + filter/clusterMetrics.go | 2 +- filter/metrics.go | 2 +- protocol/motan1Protocol.go | 5 +++-- server/motanserver.go | 21 ++++++++++++++------- 7 files changed, 37 insertions(+), 12 deletions(-) diff --git a/core/constants.go b/core/constants.go index a3108104..ff271ddd 100644 --- a/core/constants.go +++ b/core/constants.go @@ -115,6 +115,7 @@ const ( const ( GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" DirectRPCEnvironmentName = "MESH_DIRECT_RPC" + FilterEnvironmentName = "MESH_FILTERS" ) // meta keys diff --git a/core/globalContext.go b/core/globalContext.go index c910199c..6efd7b7f 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -397,10 +397,11 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { newURL = url } - //final filters: defaultFilter + globalFilter + filters + //final filters: defaultFilter + globalFilter + filters + envFilter finalFilters := c.MergeFilterSet( c.GetDefaultFilterSet(newURL), c.GetGlobalFilterSet(newURL), + c.GetEnvGlobalFilterSet(), c.GetFilterSet(newURL.GetStringParamsWithDefault(FilterKey, ""), ""), ) if len(finalFilters) > 0 { @@ -474,6 +475,20 @@ func (c *Context) GetGlobalFilterSet(newURL *URL) map[string]bool { newURL.GetStringParamsWithDefault(DisableGlobalFilter, "")) } +func (c *Context) GetEnvGlobalFilterSet() map[string]bool { + res := make(map[string]bool) + if filters := os.Getenv(FilterEnvironmentName); filters != "" { + for _, k := range strings.Split(filters, ",") { + k = strings.TrimSpace(k) + if k == "" { + continue + } + res[k] = true + } + } + return res +} + // parseMultipleServiceGroup add motan-service group support of multiple comma split group name func (c *Context) parseMultipleServiceGroup(motanServiceMap map[string]*URL) { addMotanServiceMap := map[string]*URL{} diff --git a/dynamicConfig.go b/dynamicConfig.go index afc10cc5..10ffe417 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -327,6 +327,7 @@ func (h *DynamicConfigurerHandler) parseURL(url *core.URL) (*core.URL, error) { finalFilters := h.agent.Context.MergeFilterSet( h.agent.Context.GetDefaultFilterSet(url), h.agent.Context.GetGlobalFilterSet(url), + h.agent.Context.GetEnvGlobalFilterSet(), h.agent.Context.GetFilterSet(url.GetStringParamsWithDefault(core.FilterKey, ""), ""), ) if len(finalFilters) > 0 { diff --git a/filter/clusterMetrics.go b/filter/clusterMetrics.go index 8019f3ce..46b2678b 100644 --- a/filter/clusterMetrics.go +++ b/filter/clusterMetrics.go @@ -58,6 +58,6 @@ func (c *ClusterMetricsFilter) Filter(haStrategy motan.HaStrategy, loadBalance m } keys := []string{role, request.GetAttachment(protocol.MSource), request.GetMethod()} addMetricWithKeys(request.GetAttachment(protocol.MGroup), ".cluster", - request.GetAttachment(protocol.MPath), keys, time.Since(start).Nanoseconds()/1e6, response) + request.GetServiceName(), keys, time.Since(start).Nanoseconds()/1e6, response) return response } diff --git a/filter/metrics.go b/filter/metrics.go index c69b41fd..49cfd17a 100644 --- a/filter/metrics.go +++ b/filter/metrics.go @@ -85,7 +85,7 @@ func (m *MetricsFilter) Filter(caller motan.Caller, request motan.Request) motan application = caller.GetURL().GetParam(motan.ApplicationKey, "") } keys := []string{role, application, request.GetMethod()} - addMetricWithKeys(request.GetAttachment(protocol.MGroup), "", request.GetAttachment(protocol.MPath), + addMetricWithKeys(request.GetAttachment(protocol.MGroup), "", request.GetServiceName(), keys, time.Since(start).Nanoseconds()/1e6, response) return response } diff --git a/protocol/motan1Protocol.go b/protocol/motan1Protocol.go index 19d9004c..65a37099 100644 --- a/protocol/motan1Protocol.go +++ b/protocol/motan1Protocol.go @@ -61,8 +61,9 @@ const ( ) const ( - V1Group = "group" - V1Version = "version" + V1Group = "group" + V1Version = "version" + V1Application = "application" ) const MAX_BLOCK_SIZE = 1024 diff --git a/server/motanserver.go b/server/motanserver.go index 23e0059b..7fdd21fb 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -303,13 +303,7 @@ func (m *MotanServer) processV1(msg *mpro.MotanV1Message, start time.Time, ip st var result []byte var reqCtx *motan.RPCContext req, err := mpro.DecodeMotanV1Request(msg) - // fill v2 attachment - if req.GetAttachment(mpro.MGroup) == "" { - req.SetAttachment(mpro.MGroup, req.GetAttachment(mpro.V1Group)) - } - if req.GetAttachment(mpro.MVersion) == "" { - req.SetAttachment(mpro.MVersion, req.GetAttachment(mpro.V1Version)) - } + setV1Attachments(req) if err != nil { vlog.Errorf("decode v1 request fail. conn: %s, err:%s", conn.RemoteAddr().String(), err.Error()) result = mpro.BuildV1ExceptionResponse(msg.Rid, err.Error()) @@ -377,3 +371,16 @@ func getRemoteIP(address string) string { } return ip } + +func setV1Attachments(req motan.Request) { + // fill v2 attachment + if req.GetAttachment(mpro.MGroup) == "" { + req.SetAttachment(mpro.MGroup, req.GetAttachment(mpro.V1Group)) + } + if req.GetAttachment(mpro.MVersion) == "" { + req.SetAttachment(mpro.MVersion, req.GetAttachment(mpro.V1Version)) + } + if req.GetAttachment(mpro.MSource) == "" { + req.SetAttachment(mpro.MSource, req.GetAttachment(mpro.V1Application)) + } +} From 0057ffce27f1f127aaf83e6ab892b86a0bddd5a9 Mon Sep 17 00:00:00 2001 From: Hoofffman <55658814+Hoofffman@users.noreply.github.com> Date: Thu, 4 Jan 2024 17:03:30 +0800 Subject: [PATCH 73/75] add admin handler environment feature and add relevant filters feature (#373) * add filter env --- agent.go | 29 +++++++++++++++++++++--- agent_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++ core/constants.go | 1 + core/globalContext.go | 17 ++++++++++++-- dynamicConfig.go | 1 + 5 files changed, 95 insertions(+), 5 deletions(-) diff --git a/agent.go b/agent.go index 6428ff09..93c53f13 100644 --- a/agent.go +++ b/agent.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -75,6 +76,7 @@ type Agent struct { httpProxyServer *mserver.HTTPProxyServer manageHandlers map[string]http.Handler + envHandlers map[string]map[string]http.Handler svcLock sync.Mutex clsLock sync.Mutex @@ -109,6 +111,7 @@ func NewAgent(extfactory motan.ExtensionFactory) *Agent { agent.agentPortServer = make(map[int]motan.Server) agent.serviceRegistries = motan.NewCopyOnWriteMap() agent.manageHandlers = make(map[string]http.Handler) + agent.envHandlers = make(map[string]map[string]http.Handler) agent.serviceMap = motan.NewCopyOnWriteMap() return agent } @@ -777,9 +780,13 @@ func fillDefaultReqInfo(r motan.Request, url *motan.URL) { } } else { if r.GetAttachment(mpro.MSource) == "" { - application := url.GetParam(motan.ApplicationKey, "") - if application != "" { - r.SetAttachment(mpro.MSource, application) + if app := r.GetAttachment(motan.ApplicationKey); app != "" { + r.SetAttachment(mpro.MSource, app) + } else { + application := url.GetParam(motan.ApplicationKey, "") + if application != "" { + r.SetAttachment(mpro.MSource, application) + } } } if r.GetAttachment(mpro.MGroup) == "" { @@ -1090,6 +1097,12 @@ func (a *Agent) RegisterManageHandler(path string, handler http.Handler) { } } +func (a *Agent) RegisterEnvHandlers(envStr string, handlers map[string]http.Handler) { + if envStr != "" && handlers != nil { + a.envHandlers[envStr] = handlers // override + } +} + func (a *Agent) startMServer() { handlers := make(map[string]http.Handler, 16) for k, v := range GetDefaultManageHandlers() { @@ -1098,6 +1111,16 @@ func (a *Agent) startMServer() { for k, v := range a.manageHandlers { handlers[k] = v } + // register env handlers + extHandelrs := os.Getenv(motan.HandlerEnvironmentName) + for _, k := range strings.Split(extHandelrs, ",") { + if v, ok := a.envHandlers[strings.TrimSpace(k)]; ok { + for kk, vv := range v { + handlers[kk] = vv + } + + } + } for k, v := range handlers { a.mhandle(k, v) } diff --git a/agent_test.go b/agent_test.go index 3942d5ad..d21166e7 100644 --- a/agent_test.go +++ b/agent_test.go @@ -91,6 +91,58 @@ motan-refer: assert.Equal(t, "Hello jack from motan server", resp.GetValue()) assert.Equal(t, 100, server.GetProcessPoolSize()) } + +func Test_envHandler(t *testing.T) { + t.Parallel() + time.Sleep(time.Second * 3) + // start client mesh + ext := GetDefaultExtFactory() + os.Remove("agent.sock") + config, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` +motan-agent: + mport: 13500 + port: 14821 + eport: 14281 + htport: 25282 + +motan-registry: + direct: + protocol: direct + address: 127.0.0.1:22991 + +motan-refer: + recom-engine-refer: + group: hello + path: helloService + protocol: motan2 + registry: direct + asyncInitConnection: false + serialization: breeze`))) + agent := NewAgent(ext) + agent.RegisterEnvHandlers("testHandler", map[string]http.Handler{ + "/test/test": testHandler(), + }) + os.Setenv(core.HandlerEnvironmentName, "testHandler") + go agent.StartMotanAgentFromConfig(config) + time.Sleep(time.Second * 3) + client := http.Client{ + Timeout: time.Second, + } + resp, err := client.Get("http://127.0.0.1:13500/test/test") + assert.Nil(t, err) + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, string(b), "OK") + os.Unsetenv(core.HandlerEnvironmentName) +} + +func testHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("OK")) + } +} + func Test_unixClientCall2(t *testing.T) { t.Parallel() startServer(t, "helloService", 22992) diff --git a/core/constants.go b/core/constants.go index ff271ddd..63799607 100644 --- a/core/constants.go +++ b/core/constants.go @@ -116,6 +116,7 @@ const ( GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" DirectRPCEnvironmentName = "MESH_DIRECT_RPC" FilterEnvironmentName = "MESH_FILTERS" + HandlerEnvironmentName = "MESH_ADMIN_EXT_HANDLERS" ) // meta keys diff --git a/core/globalContext.go b/core/globalContext.go index 6efd7b7f..81ef149f 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -69,7 +69,8 @@ var ( defaultConfigPath = "./" defaultFileSuffix = ".yaml" - urlFields = map[string]bool{"protocol": true, "host": true, "port": true, "path": true, "group": true} + urlFields = map[string]bool{"protocol": true, "host": true, "port": true, "path": true, "group": true} + extFilters = make(map[string]bool) ) // all env flag in motan-go @@ -87,6 +88,17 @@ var ( Recover = flag.Bool("recover", false, "recover from accidental exit") ) +func AddRelevantFilter(filterStr string) { + k := strings.TrimSpace(filterStr) + if k != "" { + extFilters[k] = true + } +} + +func GetRelevantFilters() map[string]bool { + return extFilters +} + func (c *Context) confToURLs(section string) map[string]*URL { urls := map[string]*URL{} sectionConf, _ := c.Config.GetSection(section) @@ -397,11 +409,12 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { newURL = url } - //final filters: defaultFilter + globalFilter + filters + envFilter + //final filters: defaultFilter + globalFilter + filters + envFilter + relevantFilters finalFilters := c.MergeFilterSet( c.GetDefaultFilterSet(newURL), c.GetGlobalFilterSet(newURL), c.GetEnvGlobalFilterSet(), + GetRelevantFilters(), c.GetFilterSet(newURL.GetStringParamsWithDefault(FilterKey, ""), ""), ) if len(finalFilters) > 0 { diff --git a/dynamicConfig.go b/dynamicConfig.go index 10ffe417..442fd019 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -328,6 +328,7 @@ func (h *DynamicConfigurerHandler) parseURL(url *core.URL) (*core.URL, error) { h.agent.Context.GetDefaultFilterSet(url), h.agent.Context.GetGlobalFilterSet(url), h.agent.Context.GetEnvGlobalFilterSet(), + core.GetRelevantFilters(), h.agent.Context.GetFilterSet(url.GetStringParamsWithDefault(core.FilterKey, ""), ""), ) if len(finalFilters) > 0 { From 35e2bd1a27acf40c7c918441934b4373991a102f Mon Sep 17 00:00:00 2001 From: snail007 Date: Thu, 4 Jan 2024 17:09:00 +0800 Subject: [PATCH 74/75] Revert "add admin handler environment feature and add relevant filters feature (#373)" (#374) This reverts commit 0057ffce27f1f127aaf83e6ab892b86a0bddd5a9. --- agent.go | 29 +++--------------------- agent_test.go | 52 ------------------------------------------- core/constants.go | 1 - core/globalContext.go | 17 ++------------ dynamicConfig.go | 1 - 5 files changed, 5 insertions(+), 95 deletions(-) diff --git a/agent.go b/agent.go index 93c53f13..6428ff09 100644 --- a/agent.go +++ b/agent.go @@ -13,7 +13,6 @@ import ( "path/filepath" "runtime" "strconv" - "strings" "sync" "sync/atomic" "time" @@ -76,7 +75,6 @@ type Agent struct { httpProxyServer *mserver.HTTPProxyServer manageHandlers map[string]http.Handler - envHandlers map[string]map[string]http.Handler svcLock sync.Mutex clsLock sync.Mutex @@ -111,7 +109,6 @@ func NewAgent(extfactory motan.ExtensionFactory) *Agent { agent.agentPortServer = make(map[int]motan.Server) agent.serviceRegistries = motan.NewCopyOnWriteMap() agent.manageHandlers = make(map[string]http.Handler) - agent.envHandlers = make(map[string]map[string]http.Handler) agent.serviceMap = motan.NewCopyOnWriteMap() return agent } @@ -780,13 +777,9 @@ func fillDefaultReqInfo(r motan.Request, url *motan.URL) { } } else { if r.GetAttachment(mpro.MSource) == "" { - if app := r.GetAttachment(motan.ApplicationKey); app != "" { - r.SetAttachment(mpro.MSource, app) - } else { - application := url.GetParam(motan.ApplicationKey, "") - if application != "" { - r.SetAttachment(mpro.MSource, application) - } + application := url.GetParam(motan.ApplicationKey, "") + if application != "" { + r.SetAttachment(mpro.MSource, application) } } if r.GetAttachment(mpro.MGroup) == "" { @@ -1097,12 +1090,6 @@ func (a *Agent) RegisterManageHandler(path string, handler http.Handler) { } } -func (a *Agent) RegisterEnvHandlers(envStr string, handlers map[string]http.Handler) { - if envStr != "" && handlers != nil { - a.envHandlers[envStr] = handlers // override - } -} - func (a *Agent) startMServer() { handlers := make(map[string]http.Handler, 16) for k, v := range GetDefaultManageHandlers() { @@ -1111,16 +1098,6 @@ func (a *Agent) startMServer() { for k, v := range a.manageHandlers { handlers[k] = v } - // register env handlers - extHandelrs := os.Getenv(motan.HandlerEnvironmentName) - for _, k := range strings.Split(extHandelrs, ",") { - if v, ok := a.envHandlers[strings.TrimSpace(k)]; ok { - for kk, vv := range v { - handlers[kk] = vv - } - - } - } for k, v := range handlers { a.mhandle(k, v) } diff --git a/agent_test.go b/agent_test.go index d21166e7..3942d5ad 100644 --- a/agent_test.go +++ b/agent_test.go @@ -91,58 +91,6 @@ motan-refer: assert.Equal(t, "Hello jack from motan server", resp.GetValue()) assert.Equal(t, 100, server.GetProcessPoolSize()) } - -func Test_envHandler(t *testing.T) { - t.Parallel() - time.Sleep(time.Second * 3) - // start client mesh - ext := GetDefaultExtFactory() - os.Remove("agent.sock") - config, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` -motan-agent: - mport: 13500 - port: 14821 - eport: 14281 - htport: 25282 - -motan-registry: - direct: - protocol: direct - address: 127.0.0.1:22991 - -motan-refer: - recom-engine-refer: - group: hello - path: helloService - protocol: motan2 - registry: direct - asyncInitConnection: false - serialization: breeze`))) - agent := NewAgent(ext) - agent.RegisterEnvHandlers("testHandler", map[string]http.Handler{ - "/test/test": testHandler(), - }) - os.Setenv(core.HandlerEnvironmentName, "testHandler") - go agent.StartMotanAgentFromConfig(config) - time.Sleep(time.Second * 3) - client := http.Client{ - Timeout: time.Second, - } - resp, err := client.Get("http://127.0.0.1:13500/test/test") - assert.Nil(t, err) - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - assert.Nil(t, err) - assert.Equal(t, string(b), "OK") - os.Unsetenv(core.HandlerEnvironmentName) -} - -func testHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) - } -} - func Test_unixClientCall2(t *testing.T) { t.Parallel() startServer(t, "helloService", 22992) diff --git a/core/constants.go b/core/constants.go index 63799607..ff271ddd 100644 --- a/core/constants.go +++ b/core/constants.go @@ -116,7 +116,6 @@ const ( GroupEnvironmentName = "MESH_SERVICE_ADDITIONAL_GROUP" DirectRPCEnvironmentName = "MESH_DIRECT_RPC" FilterEnvironmentName = "MESH_FILTERS" - HandlerEnvironmentName = "MESH_ADMIN_EXT_HANDLERS" ) // meta keys diff --git a/core/globalContext.go b/core/globalContext.go index 81ef149f..6efd7b7f 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -69,8 +69,7 @@ var ( defaultConfigPath = "./" defaultFileSuffix = ".yaml" - urlFields = map[string]bool{"protocol": true, "host": true, "port": true, "path": true, "group": true} - extFilters = make(map[string]bool) + urlFields = map[string]bool{"protocol": true, "host": true, "port": true, "path": true, "group": true} ) // all env flag in motan-go @@ -88,17 +87,6 @@ var ( Recover = flag.Bool("recover", false, "recover from accidental exit") ) -func AddRelevantFilter(filterStr string) { - k := strings.TrimSpace(filterStr) - if k != "" { - extFilters[k] = true - } -} - -func GetRelevantFilters() map[string]bool { - return extFilters -} - func (c *Context) confToURLs(section string) map[string]*URL { urls := map[string]*URL{} sectionConf, _ := c.Config.GetSection(section) @@ -409,12 +397,11 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { newURL = url } - //final filters: defaultFilter + globalFilter + filters + envFilter + relevantFilters + //final filters: defaultFilter + globalFilter + filters + envFilter finalFilters := c.MergeFilterSet( c.GetDefaultFilterSet(newURL), c.GetGlobalFilterSet(newURL), c.GetEnvGlobalFilterSet(), - GetRelevantFilters(), c.GetFilterSet(newURL.GetStringParamsWithDefault(FilterKey, ""), ""), ) if len(finalFilters) > 0 { diff --git a/dynamicConfig.go b/dynamicConfig.go index 442fd019..10ffe417 100644 --- a/dynamicConfig.go +++ b/dynamicConfig.go @@ -328,7 +328,6 @@ func (h *DynamicConfigurerHandler) parseURL(url *core.URL) (*core.URL, error) { h.agent.Context.GetDefaultFilterSet(url), h.agent.Context.GetGlobalFilterSet(url), h.agent.Context.GetEnvGlobalFilterSet(), - core.GetRelevantFilters(), h.agent.Context.GetFilterSet(url.GetStringParamsWithDefault(core.FilterKey, ""), ""), ) if len(finalFilters) > 0 { From c671e0d132646d65b1bf9e49b2088e1d4b1c4935 Mon Sep 17 00:00:00 2001 From: snail007 Date: Thu, 11 Jan 2024 10:21:37 +0800 Subject: [PATCH 75/75] Fix report (#378) * fix report cpu etc. --- agent.go | 5 +++-- metrics/metrics.go | 12 +++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/agent.go b/agent.go index 6428ff09..903e09b3 100644 --- a/agent.go +++ b/agent.go @@ -291,8 +291,9 @@ func (a *Agent) initStatus() { a.recoverStatus() // here we add the metrics for recover application := a.agentURL.GetParam(motan.ApplicationKey, metrics.DefaultStatApplication) - key := metrics.DefaultStatRole + metrics.KeyDelimiter + application + metrics.KeyDelimiter + "abnormal_exit.total_count" - metrics.AddCounter(metrics.DefaultStatGroup, metrics.DefaultStatService, key, 1) + keys := []string{metrics.DefaultStatRole, application, "abnormal_exit"} + metrics.AddCounterWithKeys(metrics.DefaultStatGroup, "", metrics.DefaultStatService, + keys, ".total_count", 1) } else { atomic.StoreInt64(&a.status, http.StatusServiceUnavailable) } diff --git a/metrics/metrics.go b/metrics/metrics.go index 614e9dc1..1739f600 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -200,6 +200,10 @@ func AddGauge(group string, service string, key string, value int64) { sendEvent(eventGauge, group, service, key, value) } +func AddGaugeWithKeys(group, groupSuffix string, service string, keys []string, keySuffix string, value int64) { + sendEventWithKeys(eventGauge, group, groupSuffix, service, keys, keySuffix, value) +} + // AddCounterWithKeys arguments: group & groupSuffix & service & keys elements & keySuffix is text without escaped func AddCounterWithKeys(group, groupSuffix string, service string, keys []string, keySuffix string, value int64) { sendEventWithKeys(eventCounter, group, groupSuffix, service, keys, keySuffix, value) @@ -252,7 +256,8 @@ func RegisterStatusSampleFunc(key string, sf func() int64) { func sampleStatus(application string) { defer motan.HandlePanic(nil) sampler.RangeDo(func(key string, value sampler.StatusSampler) bool { - AddGauge(DefaultStatGroup, DefaultStatService, DefaultStatRole+KeyDelimiter+application+KeyDelimiter+key, value.Sample()) + AddGaugeWithKeys(DefaultStatGroup, "", DefaultStatService, + []string{DefaultStatRole, application, key}, "", value.Sample()) return true }) } @@ -801,8 +806,9 @@ func StartReporter(ctx *motan.Context) { if ctx.AgentURL != nil { application := ctx.AgentURL.GetParam(motan.ApplicationKey, DefaultStatApplication) motan.PanicStatFunc = func() { - key := DefaultStatRole + KeyDelimiter + application + KeyDelimiter + "panic.total_count" - AddCounter(DefaultStatGroup, DefaultStatService, key, 1) + keys := []string{DefaultStatRole, application, "panic"} + AddCounterWithKeys(DefaultStatGroup, "", DefaultStatService, + keys, ".total_count", 1) } startSampleStatus(application) }