diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f511c5f..ba52da95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: testing: strategy: matrix: - go-version: [1.12.x,1.13.x,1.14.x,1.15.x,1.16.x,1.17.x,1.18.x,1.19.x,1.20.x,1.21.x] + go-version: [1.16.x,1.17.x,1.18.x,1.19.x,1.20.x,1.21.x] platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} steps: @@ -45,10 +45,10 @@ jobs: name: codecov runs-on: ubuntu-latest steps: - - name: Set up Go 1.15 + - name: Set up Go 1.16 uses: actions/setup-go@v3 with: - go-version: 1.15.x + go-version: 1.16.x id: go - name: Checkout code diff --git a/.gitignore b/.gitignore index b3c37487..f203523c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ zj* # gdb *.gdb_history __* +*.log # motan-go *.pid @@ -40,4 +41,4 @@ main/magent* log/log.test* go.sum agent_runtime -test/ +test/ \ No newline at end of file diff --git a/agent.go b/agent.go index 9c9052ac..013f08f6 100644 --- a/agent.go +++ b/agent.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/weibocom/motan-go/endpoint" vlog "github.com/weibocom/motan-go/log" + "github.com/weibocom/motan-go/meta" + "github.com/weibocom/motan-go/provider" "gopkg.in/yaml.v2" "io/ioutil" "net" @@ -193,6 +195,8 @@ func (a *Agent) StartMotanAgentFromConfig(config *cfg.Config) { return } fmt.Println("init agent context success.") + // initialize meta package + meta.Initialize(a.Context) a.initParam() a.SetSanpshotConf() a.initAgentURL() @@ -225,7 +229,7 @@ func (a *Agent) StartMotanAgentFromConfig(config *cfg.Config) { func (a *Agent) startRegistryFailback() { vlog.Infoln("start agent failback") - ticker := time.NewTicker(registry.DefaultFailbackInterval * time.Millisecond) + ticker := time.NewTicker(time.Duration(registry.GetFailbackInterval()) * time.Millisecond) defer ticker.Stop() for range ticker.C { a.registryLock.Lock() @@ -265,22 +269,8 @@ func (a *Agent) GetRegistryStatus() []map[string]*motan.RegistryStatus { } func (a *Agent) registerStatusSampler() { - metrics.RegisterStatusSampleFunc("memory", func() int64 { - p, _ := process.NewProcess(int32(os.Getpid())) - memInfo, err := p.MemoryInfo() - if err != nil { - return 0 - } - return int64(memInfo.RSS >> 20) - }) - metrics.RegisterStatusSampleFunc("cpu", func() int64 { - p, _ := process.NewProcess(int32(os.Getpid())) - cpuPercent, err := p.CPUPercent() - if err != nil { - return 0 - } - return int64(cpuPercent) - }) + metrics.RegisterStatusSampleFunc("memory", GetRssMemory) + metrics.RegisterStatusSampleFunc("cpu", GetCpuPercent) metrics.RegisterStatusSampleFunc("goroutine_count", func() int64 { return int64(runtime.NumGoroutine()) }) @@ -289,6 +279,24 @@ func (a *Agent) registerStatusSampler() { }) } +func GetRssMemory() int64 { + p, _ := process.NewProcess(int32(os.Getpid())) + memInfo, err := p.MemoryInfo() + if err != nil { + return 0 + } + return int64(memInfo.RSS >> 20) +} + +func GetCpuPercent() int64 { + p, _ := process.NewProcess(int32(os.Getpid())) + cpuPercent, err := p.CPUPercent() + if err != nil { + return 0 + } + return int64(cpuPercent) +} + func (a *Agent) initStatus() { if a.recover { a.recoverStatus() @@ -371,7 +379,7 @@ func (a *Agent) initParam() { port = defaultPort } - mPort := *motan.Mport + mPort := motan.GetMport() if mPort == 0 { if envMPort, ok := os.LookupEnv("mport"); ok { if envMPortInt, err := strconv.Atoi(envMPort); err == nil { @@ -629,6 +637,7 @@ func (a *Agent) initAgentURL() { } else { agentURL.Parameters[motan.ApplicationKey] = agentURL.Group } + motan.SetApplication(agentURL.Parameters[motan.ApplicationKey]) if agentURL.Group == "" { agentURL.Group = defaultAgentGroup agentURL.Parameters[motan.ApplicationKey] = defaultAgentGroup @@ -652,8 +661,9 @@ func (a *Agent) initAgentURL() { func (a *Agent) startAgent() { url := a.agentURL.Copy() url.Port = a.port + url.Protocol = mserver.Motan2 handler := &agentMessageHandler{agent: a} - server := &mserver.MotanServer{URL: url} + server := defaultExtFactory.GetServer(url) server.SetMessageHandler(handler) vlog.Infof("Motan agent is started. port:%d", a.port) fmt.Println("Motan agent start.") @@ -673,7 +683,7 @@ func (a *Agent) registerAgent() { if agentURL.Host == "" { agentURL.Host = motan.GetLocalIP() } - if registryURL, regexit := a.Context.RegistryURLs[reg]; regexit { + if registryURL, regExist := a.Context.RegistryURLs[reg]; regExist { registry := a.extFactory.GetRegistry(registryURL) if registry != nil { vlog.Infof("agent register in registry:%s, agent url:%s", registry.GetURL().GetIdentity(), agentURL.GetIdentity()) @@ -697,6 +707,16 @@ type agentMessageHandler struct { agent *Agent } +func (a *agentMessageHandler) GetName() string { + return "agentMessageHandler" +} + +func (a *agentMessageHandler) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + info[motan.RuntimeMessageHandlerTypeKey] = a.GetName() + return info +} + func (a *agentMessageHandler) clusterCall(request motan.Request, ck string, motanCluster *cluster.MotanCluster) (res motan.Response) { // fill default request info fillDefaultReqInfo(request, motanCluster.GetURL()) @@ -747,8 +767,8 @@ func (a *agentMessageHandler) httpCall(request motan.Request, ck string, httpClu } httpRequest := fasthttp.AcquireRequest() httpResponse := fasthttp.AcquireResponse() + // do not release http response defer fasthttp.ReleaseRequest(httpRequest) - defer fasthttp.ReleaseResponse(httpResponse) httpRequest.Header.Del("Host") httpRequest.SetHost(originalService) httpRequest.URI().SetPath(request.GetMethod()) @@ -886,6 +906,9 @@ func (a *Agent) unavailableAllServices() { func (a *Agent) doExportService(url *motan.URL) { a.svcLock.Lock() defer a.svcLock.Unlock() + if _, ok := a.serviceExporters.Load(url.GetIdentityWithRegistry()); ok { + return + } globalContext := a.Context exporter := &mserver.DefaultExporter{} @@ -932,11 +955,38 @@ func (a *Agent) doExportService(url *motan.URL) { } type serverAgentMessageHandler struct { - providers *motan.CopyOnWriteMap + providers *motan.CopyOnWriteMap + frameworkProviders *motan.CopyOnWriteMap +} + +func (sa *serverAgentMessageHandler) GetName() string { + return "serverAgentMessageHandler" +} + +func (sa *serverAgentMessageHandler) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + info[motan.RuntimeMessageHandlerTypeKey] = sa.GetName() + providersInfo := map[string]interface{}{} + sa.providers.Range(func(k, v interface{}) bool { + provider, ok := v.(motan.Provider) + if !ok { + return true + } + providersInfo[k.(string)] = provider.GetRuntimeInfo() + return true + }) + info[motan.RuntimeProvidersKey] = providersInfo + return info } func (sa *serverAgentMessageHandler) Initialize() { sa.providers = motan.NewCopyOnWriteMap() + sa.frameworkProviders = motan.NewCopyOnWriteMap() + sa.initFrameworkServiceProvider() +} + +func (sa *serverAgentMessageHandler) initFrameworkServiceProvider() { + sa.frameworkProviders.Store(meta.MetaServiceName, &provider.MetaProvider{}) } func getServiceKey(group, path string) string { @@ -954,15 +1004,22 @@ func (sa *serverAgentMessageHandler) Call(request motan.Request) (res motan.Resp group = request.GetAttachment(motan.GroupKey) } serviceKey := getServiceKey(group, request.GetServiceName()) + if mfs := request.GetAttachment(mpro.MFrameworkService); mfs != "" { + if fp, ok := sa.frameworkProviders.Load(request.GetServiceName()); ok { + return fp.(motan.Provider).Call(request) + } + //throw specific exception to avoid triggering forced fusing on the client side。 + return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 501, ErrMsg: motan.ServiceNotSupport, ErrType: motan.ServiceException}) + } if p := sa.providers.LoadOrNil(serviceKey); p != nil { p := p.(motan.Provider) res = p.Call(request) res.GetRPCContext(true).GzipSize = int(p.GetURL().GetIntValue(motan.GzipSizeKey, 0)) return res } - vlog.Errorf("not found provider for %s", motan.GetReqInfo(request)) + vlog.Errorf("%s%s, %s", motan.ProviderNotExistPrefix, serviceKey, motan.GetReqInfo(request)) atomic.AddInt64(¬FoundProviderCount, 1) - return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "not found provider for " + serviceKey, ErrType: motan.ServiceException}) + return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: motan.EProviderNotExist, ErrMsg: motan.ProviderNotExistPrefix + serviceKey, ErrType: motan.ServiceException}) } func (sa *serverAgentMessageHandler) AddProvider(p motan.Provider) error { @@ -1119,13 +1176,12 @@ func (a *Agent) startMServer() { for kk, vv := range v { handlers[kk] = vv } - } } for k, v := range handlers { a.mhandle(k, v) } - + var mPort int var managementListener net.Listener if managementUnixSockAddr := a.agentURL.GetParam(motan.ManagementUnixSockKey, ""); managementUnixSockAddr != "" { listener, err := motan.ListenUnixSock(managementUnixSockAddr) @@ -1159,19 +1215,21 @@ func (a *Agent) startMServer() { managementListener = motan.TCPKeepAliveListener{TCPListener: listener.(*net.TCPListener)} break } + mPort = a.mport if managementListener == nil { vlog.Warningf("start management server failed for port range %s", startAndPortStr) return } } else { listener, err := net.Listen("tcp", ":"+strconv.Itoa(a.mport)) + mPort = a.mport if err != nil { vlog.Infof("listen manage port %d failed:%s", a.mport, err.Error()) return } managementListener = motan.TCPKeepAliveListener{TCPListener: listener.(*net.TCPListener)} } - + motan.SetMport(mPort) vlog.Infof("start listen manage for address: %s", managementListener.Addr().String()) err := http.Serve(managementListener, nil) if err != nil { @@ -1191,8 +1249,10 @@ func (a *Agent) mhandle(k string, h http.Handler) { setAgentLock.Unlock() } http.HandleFunc(k, func(w http.ResponseWriter, r *http.Request) { + vlog.Infof("mport request: %s, address: %s", r.URL.Path, r.RemoteAddr) if !PermissionCheck(r) { w.Write([]byte("need permission!")) + vlog.Warningf("mport request no permission: %s, address: %s", r.URL.Path, r.RemoteAddr) return } defer func() { diff --git a/agent_test.go b/agent_test.go index d21166e7..1546a367 100644 --- a/agent_test.go +++ b/agent_test.go @@ -48,13 +48,13 @@ var ( ) func Test_unixClientCall1(t *testing.T) { - t.Parallel() + //t.Parallel() startServer(t, "helloService", 22991) time.Sleep(time.Second * 3) // start client mesh ext := GetDefaultExtFactory() os.Remove("agent.sock") - config, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` + clientAgentConfig, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: mport: 12500 port: 13821 @@ -79,9 +79,10 @@ motan-refer: registry: direct asyncInitConnection: false serialization: breeze`))) - agent := NewAgent(ext) - go agent.StartMotanAgentFromConfig(config) + targetAgent := NewAgent(ext) + go targetAgent.StartMotanAgentFromConfig(clientAgentConfig) time.Sleep(time.Second * 3) + core.SetMport(0) c1 := NewMeshClient() c1.SetAddress("unix://./agent.sock") c1.Initialize() @@ -93,7 +94,7 @@ motan-refer: } func Test_envHandler(t *testing.T) { - t.Parallel() + //t.Parallel() time.Sleep(time.Second * 3) // start client mesh ext := GetDefaultExtFactory() @@ -118,13 +119,14 @@ motan-refer: registry: direct asyncInitConnection: false serialization: breeze`))) - agent := NewAgent(ext) - agent.RegisterEnvHandlers("testHandler", map[string]http.Handler{ + clientAgent := NewAgent(ext) + clientAgent.RegisterEnvHandlers("testHandler", map[string]http.Handler{ "/test/test": testHandler(), }) os.Setenv(core.HandlerEnvironmentName, "testHandler") - go agent.StartMotanAgentFromConfig(config) + go clientAgent.StartMotanAgentFromConfig(config) time.Sleep(time.Second * 3) + core.SetMport(0) client := http.Client{ Timeout: time.Second, } @@ -144,13 +146,13 @@ func testHandler() http.HandlerFunc { } func Test_unixClientCall2(t *testing.T) { - t.Parallel() + //t.Parallel() startServer(t, "helloService", 22992) time.Sleep(time.Second * 3) // start client mesh ext := GetDefaultExtFactory() os.Remove("agent2.sock") - config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` + clientAgentconfig1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: mport: 12501 port: 12921 @@ -171,17 +173,18 @@ motan-refer: test-refer: group: hello path: helloService - protocol: motanV1Compatible + protocol: motan2 registry: direct serialization: breeze asyncInitConnection: false `))) - agent := NewAgent(ext) - go agent.StartMotanAgentFromConfig(config1) + clientAgent := NewAgent(ext) + go clientAgent.StartMotanAgentFromConfig(clientAgentconfig1) time.Sleep(time.Second * 3) + core.SetMport(0) ext1 := GetDefaultExtFactory() - cfg, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` + clientConfig, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-client: log_dir: stdout application: client-test @@ -193,13 +196,13 @@ motan-refer: test-refer: registry: local serialization: breeze - protocol: motanV1Compatible + protocol: motan2 group: hello path: helloService requestTimeout: 3000 asyncInitConnection: false `))) - mccontext := NewClientContextFromConfig(cfg) + mccontext := NewClientContextFromConfig(clientConfig) mccontext.Start(ext1) mclient := mccontext.GetClient("test-refer") var reply string @@ -207,6 +210,7 @@ motan-refer: assert.Nil(t, err) assert.Equal(t, "Hello jack from motan server", reply) } + func TestMain(m *testing.M) { core.RegistLocalProvider("LocalTestService", &LocalTestServiceProvider{}) cfgFile := filepath.Join("testdata", "agent.yaml") @@ -224,6 +228,8 @@ func TestMain(m *testing.M) { agent.ConfigFile = cfgFile agent.StartMotanAgent() }() + time.Sleep(time.Second * 3) + core.SetMport(0) proxyURL, _ := url.Parse("http://localhost:9983") proxyClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} time.Sleep(1 * time.Second) @@ -248,8 +254,8 @@ func Test_initParam(t *testing.T) { assert := assert.New(t) conf, err := config.NewConfigFromFile(filepath.Join("testdata", "agent.yaml")) assert.Nil(err) - a := NewAgent(nil) - a.Context = &core.Context{Config: conf} + testAgent := NewAgent(nil) + testAgent.Context = &core.Context{Config: conf} logFilterCallerFalseConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: log_filter_caller: false @@ -258,7 +264,7 @@ motan-agent: section, err := conf.GetSection("motan-agent") assert.Nil(err) assert.Equal(false, section["log_filter_caller"].(bool)) - a.initParam() + testAgent.initParam() logFilterCallerTrueConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: @@ -269,7 +275,7 @@ motan-agent: section, err = conf.GetSection("motan-agent") assert.Nil(err) assert.Equal(true, section["log_filter_caller"].(bool)) - a.initParam() + testAgent.initParam() logDirConfig, err := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: @@ -279,12 +285,12 @@ motan-agent: conf.Merge(logDirConfig) section, err = conf.GetSection("motan-agent") assert.Nil(err) - a.initParam() - assert.Equal(a.logdir, "./test/abcd") + testAgent.initParam() + assert.Equal(testAgent.logdir, "./test/abcd") os.Args = append(os.Args, "-log_dir", "./test/cdef") _ = flag.Set("log_dir", "./test/cdef") - a.initParam() - assert.Equal(a.logdir, "./test/cdef") + testAgent.initParam() + assert.Equal(testAgent.logdir, "./test/cdef") // test export log dir assert.Equal(vlog.GetLogDir(), "./test/cdef") @@ -296,8 +302,8 @@ motan-agent: conf.Merge(mportConfig) section, err = conf.GetSection("motan-agent") assert.Nil(err) - a.initParam() - assert.Equal(a.mport, 8003) + testAgent.initParam() + assert.Equal(testAgent.mport, 8003) mportConfigENV, err := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: @@ -308,8 +314,8 @@ motan-agent: section, err = conf.GetSection("motan-agent") assert.Nil(err) err = os.Setenv("mport", "8006") - a.initParam() - assert.Equal(a.mport, 8006) + testAgent.initParam() + assert.Equal(testAgent.mport, 8006) mportConfigENVParam, err := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: @@ -321,8 +327,8 @@ motan-agent: assert.Nil(err) //err = os.Setenv("mport", "8006") //_ = flag.Set("mport", "8007") - //a.initParam() - //assert.Equal(a.mport, 8007) + //testAgent.initParam() + //assert.Equal(testAgent.mport, 8007) os.Unsetenv("mport") } @@ -403,7 +409,6 @@ func TestRpcToHTTPProxy(t *testing.T) { } close(requests) wg.Wait() - } func TestLocalEndpoint(t *testing.T) { @@ -418,13 +423,13 @@ func TestAgent_RuntimeDir(t *testing.T) { func TestAgent_InitCall(t *testing.T) { //init - agent := NewAgent(nil) - agent.agentURL = &core.URL{Parameters: make(map[string]string)} + targetAgent := NewAgent(nil) + targetAgent.agentURL = &core.URL{Parameters: make(map[string]string)} urlTest := &core.URL{Parameters: make(map[string]string)} urlTest.Group = "test1" urlTest.Parameters[core.AsyncInitConnection] = "false" - agent.initCluster(urlTest) - agentHandler := &agentMessageHandler{agent: agent} + targetAgent.initCluster(urlTest) + agentHandler := &agentMessageHandler{agent: targetAgent} for _, v := range []*core.URL{ {Parameters: map[string]string{core.VersionKey: ""}, Path: "test3", Group: "", Protocol: ""}, @@ -438,11 +443,11 @@ func TestAgent_InitCall(t *testing.T) { {Parameters: map[string]string{core.VersionKey: "1.3"}, Path: "test", Group: "g1", Protocol: "http"}, {Parameters: map[string]string{core.VersionKey: "1.3"}, Path: "test0", Group: "g0", Protocol: "http"}, } { - agent.initCluster(v) + targetAgent.initCluster(v) } //test init cluster with one path and one groups in clusterMap - temp := agent.clusterMap.LoadOrNil(getClusterKey("test1", "1.0", "", "")) + temp := targetAgent.clusterMap.LoadOrNil(getClusterKey("test1", "1.0", "", "")) assert.NotNil(t, temp, "init cluster with one path and two groups in clusterMap fail") //test agentHandler call with group @@ -502,8 +507,8 @@ func TestAgent_InitCall(t *testing.T) { // wait ha time.Sleep(time.Second * 1) - agent.reloadClusters(ctx) - assert.Equal(t, agent.serviceMap.Len(), 1, "hot-load serviceMap helloService2 length error") + targetAgent.reloadClusters(ctx) + assert.Equal(t, targetAgent.serviceMap.Len(), 1, "hot-load serviceMap helloService2 length error") request = newRequest("helloService2", "hello", "Ray") motanResponse := agentHandler.Call(request) @@ -521,14 +526,14 @@ func TestAgent_InitCall(t *testing.T) { dynamicURLs := map[string]*core.URL{ "test6": {Parameters: map[string]string{core.VersionKey: ""}, Path: "test6", Group: "g1", Protocol: ""}, } - agent.serviceMap.Store("test6", []serviceMapItem{ + targetAgent.serviceMap.Store("test6", []serviceMapItem{ {url: dynamicURLs["test6"], cluster: nil}, }) - agent.configurer = NewDynamicConfigurer(agent) - agent.configurer.subscribeNodes = dynamicURLs + targetAgent.configurer = NewDynamicConfigurer(targetAgent) + targetAgent.configurer.subscribeNodes = dynamicURLs ctx.RefersURLs = reloadUrls - agent.reloadClusters(ctx) - assert.Equal(t, agent.serviceMap.Len(), 3, "hot-load serviceMap except length error") + targetAgent.reloadClusters(ctx) + assert.Equal(t, targetAgent.serviceMap.Len(), 3, "hot-load serviceMap except length error") for _, v := range []struct { service string @@ -554,10 +559,10 @@ func TestNotFoundProvider(t *testing.T) { notFoundService := "notFoundService" request := meshClient.BuildRequest(notFoundService, "test", []interface{}{}) epUrl := &core.URL{ - Protocol: "motanV1Compatible", + Protocol: endpoint.Motan2, Host: "127.0.0.1", Port: 9982, - Path: "notFoundService", + Path: notFoundService, Group: "test", Parameters: map[string]string{ core.TimeOutKey: "3000", @@ -575,7 +580,7 @@ func TestNotFoundProvider(t *testing.T) { ep.SetProxy(true) resp := ep.Call(request) assert.NotNil(t, resp.GetException()) - assert.Contains(t, "not found provider for test_notFoundService", resp.GetException().ErrMsg) + assert.Contains(t, core.ProviderNotExistPrefix+"test_notFoundService", resp.GetException().ErrMsg) assert.Equal(t, int64(1), atomic.LoadInt64(¬FoundProviderCount)) ep.Destroy() } @@ -595,6 +600,10 @@ type LocalTestServiceProvider struct { url *core.URL } +func (l *LocalTestServiceProvider) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (l *LocalTestServiceProvider) SetService(s interface{}) { } @@ -638,7 +647,7 @@ func (l *LocalTestServiceProvider) GetPath() string { } func Test_unixHTTPClientCall(t *testing.T) { - t.Parallel() + //t.Parallel() go func() { http.HandleFunc("/unixclient", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("okay")) @@ -656,7 +665,7 @@ func Test_unixHTTPClientCall(t *testing.T) { }() // start unix server mesh ext := GetDefaultExtFactory() - config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` + serverAgentConfig1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: port: 12822 mport: 12504 @@ -682,9 +691,11 @@ motan-service: enableRewrite: false export: motan2:23282 `))) - agent := NewAgent(ext) - go agent.StartMotanAgentFromConfig(config1) + serverAgent := NewAgent(ext) + go serverAgent.StartMotanAgentFromConfig(serverAgentConfig1) time.Sleep(time.Second * 3) + core.SetMport(0) + c1 := NewMeshClient() c1.SetAddress("127.0.0.1:23282") c1.Initialize() @@ -695,15 +706,16 @@ motan-service: assert.Nil(t, resp.GetException()) assert.Equal(t, "okay", string(reply)) } + func Test_unixRPCClientCall(t *testing.T) { - t.Parallel() + //t.Parallel() os.Remove("server.sock") startServer(t, "helloService", 0, "server.sock") time.Sleep(time.Second * 3) // start client mesh // start unix server mesh ext := GetDefaultExtFactory() - config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` + serverMeshConfig, _ := config.NewConfigFromReader(bytes.NewReader([]byte(` motan-agent: port: 12821 mport: 12503 @@ -729,9 +741,11 @@ motan-service: proxy.host: unix://./server.sock export: motan2:12281 `))) - agent := NewAgent(ext) - go agent.StartMotanAgentFromConfig(config1) + serverAgent := NewAgent(ext) + go serverAgent.StartMotanAgentFromConfig(serverMeshConfig) time.Sleep(time.Second * 3) + core.SetMport(0) + c1 := NewMeshClient() c1.SetAddress("127.0.0.1:12281") c1.Initialize() @@ -740,10 +754,16 @@ motan-service: resp := c1.BaseCall(req, &reply) assert.Nil(t, resp.GetException()) assert.Equal(t, "Hello jack from motan server", string(reply)) + + // test not found provider endpoint circuit breaker and runtime info + reqNotFound := c1.BuildRequestWithGroup("helloServiceNotFound", "Hello", []interface{}{"jack"}, "helloNotFound") + respNotFound := c1.BaseCall(reqNotFound, &reply) + assert.NotNil(t, respNotFound.GetException()) + assert.Equal(t, core.ProviderNotExistPrefix+"helloNotFound_helloServiceNotFound", respNotFound.GetException().ErrMsg) } func Test_changeDefaultMotanEpAsyncInit(t *testing.T) { - template := ` + serviceAgentConfigTemplate := ` motan-agent: port: 12821 mport: 12503 @@ -770,23 +790,27 @@ motan-service: export: motan2:12281 ` ext := GetDefaultExtFactory() - config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(fmt.Sprintf(template, "false")))) - a := NewAgent(ext) - a.Context = &core.Context{Config: config1} - a.Context.Initialize() - a.initParam() + config1, _ := config.NewConfigFromReader(bytes.NewReader([]byte(fmt.Sprintf(serviceAgentConfigTemplate, "false")))) + serverAgent := NewAgent(ext) + serverAgent.Context = &core.Context{Config: config1} + serverAgent.Context.Initialize() + serverAgent.initParam() assert.Equal(t, endpoint.GetDefaultMotanEPAsynInit(), false) - config2, _ := config.NewConfigFromReader(bytes.NewReader([]byte(fmt.Sprintf(template, "true")))) - a = NewAgent(ext) - a.Context = &core.Context{Config: config2} - a.Context.Initialize() - a.initParam() + config2, _ := config.NewConfigFromReader(bytes.NewReader([]byte(fmt.Sprintf(serviceAgentConfigTemplate, "true")))) + serverAgent = NewAgent(ext) + serverAgent.Context = &core.Context{Config: config2} + serverAgent.Context.Initialize() + serverAgent.initParam() assert.Equal(t, endpoint.GetDefaultMotanEPAsynInit(), true) } func Test_agentRegistryFailback(t *testing.T) { - template := ` + currentFailbackInterval := registry.GetFailbackInterval() + newInterval := uint32(5000) + registry.SetFailbackInterval(newInterval) + + serverAgentTemplate := ` motan-agent: port: 12829 mport: 12604 @@ -814,14 +838,15 @@ motan-service: extFactory.RegistExtRegistry("test-registry", func(url *core.URL) core.Registry { return &testRegistry{url: url} }) - config1, err := config.NewConfigFromReader(bytes.NewReader([]byte(template))) + config1, err := config.NewConfigFromReader(bytes.NewReader([]byte(serverAgentTemplate))) assert.Nil(t, err) - agent := NewAgent(extFactory) - go agent.StartMotanAgentFromConfig(config1) - time.Sleep(time.Second * 10) + serverAgent := NewAgent(extFactory) + go serverAgent.StartMotanAgentFromConfig(config1) + time.Sleep(time.Second * 3) + core.SetMport(0) setRegistryFailSwitcher(true) - m := agent.GetRegistryStatus() + m := serverAgent.GetRegistryStatus() assert.Equal(t, len(m), 1) for _, mm := range m[0] { if mm.Service.Path == "helloService" { @@ -830,8 +855,8 @@ motan-service: } agentStatus := getCurAgentStatus(12604) assert.Equal(t, agentStatus, core.NotRegister) - agent.SetAllServicesAvailable() - m = agent.GetRegistryStatus() + serverAgent.SetAllServicesAvailable() + m = serverAgent.GetRegistryStatus() for _, mm := range m[0] { if mm.Service.Path == "helloService" { assert.Equal(t, mm.Status, core.RegisterFailed) @@ -840,8 +865,8 @@ motan-service: agentStatus = getCurAgentStatus(12604) assert.Equal(t, agentStatus, core.RegisterFailed) setRegistryFailSwitcher(false) - time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) - m = agent.GetRegistryStatus() + time.Sleep(time.Duration(newInterval) * time.Millisecond) + m = serverAgent.GetRegistryStatus() assert.Equal(t, len(m), 1) for _, mm := range m[0] { if mm.Service.Path == "helloService" { @@ -851,8 +876,8 @@ motan-service: agentStatus = getCurAgentStatus(12604) assert.Equal(t, agentStatus, core.RegisterSuccess) setRegistryFailSwitcher(true) - agent.SetAllServicesUnavailable() - m = agent.GetRegistryStatus() + serverAgent.SetAllServicesUnavailable() + m = serverAgent.GetRegistryStatus() assert.Equal(t, len(m), 1) for _, mm := range m[0] { if mm.Service.Path == "helloService" { @@ -862,8 +887,8 @@ motan-service: agentStatus = getCurAgentStatus(12604) assert.Equal(t, agentStatus, core.UnregisterFailed) setRegistryFailSwitcher(false) - time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) - m = agent.GetRegistryStatus() + time.Sleep(time.Duration(newInterval) * time.Millisecond) + m = serverAgent.GetRegistryStatus() assert.Equal(t, len(m), 1) for _, mm := range m[0] { if mm.Service.Path == "helloService" { @@ -872,6 +897,7 @@ motan-service: } agentStatus = getCurAgentStatus(12604) assert.Equal(t, agentStatus, core.UnregisterSuccess) + registry.SetFailbackInterval(currentFailbackInterval) } type testRegistry struct { @@ -880,6 +906,10 @@ type testRegistry struct { registeredServices map[string]*core.URL } +func (t *testRegistry) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (t *testRegistry) Initialize() { t.registeredServices = make(map[string]*core.URL) t.namingServiceStatus = core.NewCopyOnWriteMap() @@ -915,7 +945,6 @@ func (t *testRegistry) Register(serverURL *core.URL) { Registry: t, IsCheck: registry.IsCheck(serverURL), }) - } func (t *testRegistry) UnRegister(serverURL *core.URL) { @@ -991,7 +1020,6 @@ func setRegistryFailSwitcher(b bool) { } else { atomic.StoreInt64(&testRegistryFailSwitcher, 0) } - } func getRegistryFailSwitcher() bool { @@ -1024,3 +1052,30 @@ func getCurAgentStatus(port int64) string { } return res.Status } + +func TestRuntimeHandler(t *testing.T) { + resp, err := http.Get("http://127.0.0.1:8002/runtime/info") + assert.Nil(t, err) + var runtimeInfo map[string]interface{} + bodyBytes, err := ioutil.ReadAll(resp.Body) + assert.Nil(t, err) + err = resp.Body.Close() + assert.Nil(t, err) + err = json.Unmarshal(bodyBytes, &runtimeInfo) + assert.Nil(t, err) + + for _, s := range []string{ + core.RuntimeInstanceTypeKey, + core.RuntimeExportersKey, + core.RuntimeClustersKey, + core.RuntimeHttpClustersKey, + core.RuntimeExtensionFactoryKey, + core.RuntimeServersKey, + core.RuntimeBasicKey, + } { + info, ok := runtimeInfo[s] + assert.True(t, ok) + assert.NotNil(t, info) + t.Logf("key: %s", s) + } +} diff --git a/client.go b/client.go index b00a3c59..e2000de9 100644 --- a/client.go +++ b/client.go @@ -4,13 +4,12 @@ import ( "errors" "flag" "fmt" - "github.com/weibocom/motan-go/config" - vlog "github.com/weibocom/motan-go/log" - "sync" - "github.com/weibocom/motan-go/cluster" + "github.com/weibocom/motan-go/config" motan "github.com/weibocom/motan-go/core" + vlog "github.com/weibocom/motan-go/log" mpro "github.com/weibocom/motan-go/protocol" + "sync" ) var ( @@ -72,13 +71,14 @@ func (c *Client) BaseGo(req motan.Request, reply interface{}, done chan *motan.A rc := req.GetRPCContext(true) rc.ExtFactory = c.extFactory rc.Result = result - rc.AsyncCall = true - rc.Result.Reply = reply - res := c.cluster.Call(req) - if res.GetException() != nil { - result.Error = errors.New(res.GetException().ErrMsg) + rc.Reply = reply + go func() { + res := c.cluster.Call(req) + if res.GetException() != nil { + result.Error = errors.New(res.GetException().ErrMsg) + } result.Done <- result - } + }() return result } @@ -173,3 +173,7 @@ func (m *MCContext) GetRefer(service string) interface{} { // TODO 对client的封装，可以根据idl自动生成代码时支持 return nil } + +func (m *MCContext) GetContext() *motan.Context { + return m.context +} diff --git a/client_test.go b/client_test.go index eff481e0..d114fa77 100644 --- a/client_test.go +++ b/client_test.go @@ -5,6 +5,7 @@ import ( "fmt" assert2 "github.com/stretchr/testify/assert" "github.com/weibocom/motan-go/config" + motan "github.com/weibocom/motan-go/core" "testing" "time" ) @@ -78,6 +79,13 @@ motan-client: err = mclient.BaseCall(req, &reply) // sync call assert.Nil(err) assert.Equal("Hello Ray", reply) + + // test async call + var asyncReply string + asyncResult := mclient.Go("hello", []interface{}{"Ray"}, &asyncReply, make(chan *motan.AsyncResult, 1)) + <-asyncResult.Done + assert.Nil(asyncResult.Error) + assert.Equal("Hello Ray", asyncReply) } func StartServer() { diff --git a/cluster/command.go b/cluster/command.go index f80405f7..cbcefae5 100644 --- a/cluster/command.go +++ b/cluster/command.go @@ -29,6 +29,10 @@ const ( RuleProtocol = "rule" ) +var ( + recordInfoSize = 10 +) + var oldSwitcherMap = make(map[string]bool) //Save the default value before the switcher last called // CommandRegistryWrapper wrapper registry for every cluster @@ -45,6 +49,24 @@ type CommandRegistryWrapper struct { tcCommand *ClientCommand //effective traffic control command degradeCommand *ClientCommand //effective degrade command switcherCommand *ClientCommand + weights map[string]string + commandRecorder *motan.CircularRecorder + notifyRecorder *motan.CircularRecorder +} + +func (c *CommandRegistryWrapper) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{ + motan.RuntimeNameKey: c.GetName(), + } + info[motan.RuntimeWeightKey] = c.weights + if c.staticTcCommand != nil { + info[motan.RuntimeStaticCommandKey] = c.staticTcCommand + } + info[motan.RuntimeAgentCommandKey] = c.agentCommandInfo + info[motan.RuntimeServiceCommandKey] = c.serviceCommandInfo + info[motan.RuntimeCommandHistoryKey] = c.commandRecorder.GetRecords() + info[motan.RuntimeNotifyHistoryKey] = c.notifyRecorder.GetRecords() + return info } type ClientCommand struct { @@ -144,7 +166,7 @@ func ParseCommand(commandInfo string) *Command { } func GetCommandRegistryWrapper(cluster *MotanCluster, registry motan.Registry) motan.Registry { - cmdRegistry := &CommandRegistryWrapper{cluster: cluster, registry: registry} + cmdRegistry := &CommandRegistryWrapper{cluster: cluster, registry: registry, weights: map[string]string{}, commandRecorder: motan.NewCircularRecorder(recordInfoSize), notifyRecorder: motan.NewCircularRecorder(recordInfoSize)} cmdRegistry.ownGroupURLs = make([]*motan.URL, 0) cmdRegistry.otherGroupListener = make(map[string]*serviceListener) cmdRegistry.cluster = cluster @@ -243,6 +265,9 @@ func (c *CommandRegistryWrapper) clear() { for _, l := range c.otherGroupListener { l.unSubscribe(c.registry) } + c.weights = make(map[string]string) + c.notifyRecorder = motan.NewCircularRecorder(recordInfoSize) + c.commandRecorder = motan.NewCircularRecorder(recordInfoSize) c.otherGroupListener = make(map[string]*serviceListener) } @@ -305,6 +330,7 @@ func (c *CommandRegistryWrapper) getResultWithCommand(needNotify bool) []*motan. } if needNotify { c.notifyListener.Notify(c.registry.GetURL(), result) + c.notifyRecorder.AddRecord(result) } vlog.Infof("%s get result with command. tcCommand: %t, degradeCommand:%t, result size %d, will notify:%t", c.cluster.GetURL().GetIdentity(), currentCommand != nil, c.degradeCommand != nil, len(result), needNotify) return result @@ -365,6 +391,8 @@ func buildRuleURL(weight string) *motan.URL { } func (c *CommandRegistryWrapper) processCommand(commandType int, commandInfo string) bool { + c.commandRecorder.AddRecord(commandInfo) + c.mux.Lock() defer c.mux.Unlock() needNotify := false @@ -426,6 +454,7 @@ func (c *CommandRegistryWrapper) processTcCommand(newTcCommand *ClientCommand) { if newTcCommand == nil && c.staticTcCommand == nil { vlog.Infof("%s process command result : no tc command. ", c.cluster.GetURL().GetIdentity()) } else { + tempWeight := map[string]string{} var groups []string if newTcCommand != nil { vlog.Infof("%s process command result : has dynamic tc command. tc command will enable.command : %+v", c.cluster.GetURL().GetIdentity(), newTcCommand) @@ -435,6 +464,9 @@ func (c *CommandRegistryWrapper) processTcCommand(newTcCommand *ClientCommand) { } for _, group := range groups { g := strings.Split(group, ":") + if len(g) >= 2 { + tempWeight[g[0]] = g[1] + } if c.cluster.GetURL().Group == g[0] { // own group already subscribe continue } @@ -446,6 +478,7 @@ func (c *CommandRegistryWrapper) processTcCommand(newTcCommand *ClientCommand) { newListeners[g[0]] = newSubscribe(c, g[0]) } } + c.weights = tempWeight } c.otherGroupListener = newListeners diff --git a/cluster/command_test.go b/cluster/command_test.go index 4ed10ba4..8565f076 100644 --- a/cluster/command_test.go +++ b/cluster/command_test.go @@ -3,6 +3,7 @@ package cluster import ( "bytes" "fmt" + "github.com/stretchr/testify/assert" "strconv" "strings" "testing" @@ -137,6 +138,8 @@ func TestGetResultWithCommand(t *testing.T) { if !hasRule { t.Errorf("notify urls not has rule url! urls:%+v\n", urls) } + // check runtime info + checkDefaultCommandWrapperRuntimeInfo(crw, t) // has static command crw = getDefaultCommandWrapper(true) @@ -152,6 +155,9 @@ func TestGetResultWithCommand(t *testing.T) { crw.processCommand(ServiceCmd, "") validateResult(crw, []string{"test-group1", "test-group2", "test-group3"}, t) + + // check runtime info + checkDefaultCommandWrapperRuntimeInfo(crw, t) } func validateResult(crw *CommandRegistryWrapper, notifyGroups []string, t *testing.T) { @@ -214,6 +220,9 @@ func TestProcessCommand(t *testing.T) { } fmt.Printf("notify:%t, crw:%+v\n", notify, crw) + // check runtime info + checkDefaultCommandWrapperRuntimeInfo(crw, t) + // has static command crw = getDefaultCommandWrapper(true) processServiceCmd(crw, cl, t) @@ -233,6 +242,8 @@ func TestProcessCommand(t *testing.T) { } } + // check runtime info + checkDefaultCommandWrapperRuntimeInfo(crw, t) } func processServiceCmd(crw *CommandRegistryWrapper, cl string, t *testing.T) { @@ -364,6 +375,44 @@ func checkHost(urls []*motan.URL, f func(host string) bool, t *testing.T) { } } +func checkDefaultCommandWrapperRuntimeInfo(crw *CommandRegistryWrapper, t *testing.T) { + info := crw.GetRuntimeInfo() + assert.NotNil(t, info) + + name, ok := info[motan.RuntimeNameKey] + assert.True(t, ok) + assert.Equal(t, crw.GetName(), name) + + weightsInfo, ok := info[motan.RuntimeWeightKey] + assert.True(t, ok) + assert.NotNil(t, weightsInfo) + + staticCommand, ok := info[motan.RuntimeStaticCommandKey] + if crw.staticTcCommand == nil { + assert.False(t, ok) + assert.Nil(t, staticCommand) + } else { + assert.True(t, ok) + assert.NotNil(t, staticCommand) + } + + agentCommandInfo, ok := info[motan.RuntimeAgentCommandKey] + assert.True(t, ok) + assert.NotNil(t, agentCommandInfo) + + serviceCommandInfo, ok := info[motan.RuntimeServiceCommandKey] + assert.True(t, ok) + assert.NotNil(t, serviceCommandInfo) + + commandHistory, ok := info[motan.RuntimeCommandHistoryKey] + assert.True(t, ok) + assert.NotNil(t, commandHistory) + + notifyHistory, ok := info[motan.RuntimeNotifyHistoryKey] + assert.True(t, ok) + assert.NotNil(t, notifyHistory) +} + func getDefaultCommandWrapper(withStaticCommand bool) *CommandRegistryWrapper { cluster := initCluster() if withStaticCommand { diff --git a/cluster/httpCluster.go b/cluster/httpCluster.go index 02ae87c5..fa62fdea 100644 --- a/cluster/httpCluster.go +++ b/cluster/httpCluster.go @@ -107,6 +107,16 @@ func NewHTTPCluster(url *core.URL, proxy bool, context *core.Context, extFactory return c } +func (c *HTTPCluster) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{ + core.RuntimeNameKey: c.GetName(), + } + if c.url != nil { + info[core.RuntimeUrlKey] = c.url.ToExtInfo() + } + return info +} + func (c *HTTPCluster) CanServe(uri string) (string, bool) { service := c.uriConverter.URIToServiceName(uri, nil) if service == "" { diff --git a/cluster/httpCluster_test.go b/cluster/httpCluster_test.go index 014820b5..43f4f139 100644 --- a/cluster/httpCluster_test.go +++ b/cluster/httpCluster_test.go @@ -38,5 +38,18 @@ func TestHTTPCluster_Call(t *testing.T) { request.Method = uri response := httpCluster.Call(request) assert.True(t, response != nil) + + // runtime info + info := httpCluster.GetRuntimeInfo() + assert.NotNil(t, info) + + urlInfo, ok := info[core.RuntimeUrlKey] + assert.True(t, ok) + assert.Equal(t, httpCluster.url.ToExtInfo(), urlInfo) + + name, ok := info[core.RuntimeNameKey] + assert.True(t, ok) + assert.Equal(t, httpCluster.GetName(), name) + httpCluster.Destroy() } diff --git a/cluster/motanCluster.go b/cluster/motanCluster.go index 4bae0690..98b17779 100644 --- a/cluster/motanCluster.go +++ b/cluster/motanCluster.go @@ -268,6 +268,9 @@ func (m *MotanCluster) Destroy() { vlog.Infof("destroy endpoint %s .", e.GetURL().GetIdentity()) e.Destroy() } + if d, ok := m.LoadBalance.(motan.Destroyable); ok { + d.Destroy() + } m.closed = true } } @@ -347,6 +350,61 @@ func (m *MotanCluster) NotifyAgentCommand(commandInfo string) { } } +func (m *MotanCluster) GetRuntimeInfo() map[string]interface{} { + m.notifyLock.Lock() + defer m.notifyLock.Unlock() + + referSize := len(m.Refers) + info := map[string]interface{}{ + motan.RuntimeNameKey: m.GetName(), + } + info[motan.RuntimeRefererSizeKey] = referSize + if m.url != nil { + info[motan.RuntimeUrlKey] = m.url.ToExtInfo() + } + + unavailableRefers := make(map[string]interface{}) + availableRefers := make(map[string]interface{}, referSize) + i := 0 + for _, endpoint := range m.Refers { + ep := endpoint.(*motan.FilterEndPoint) + if ep.IsAvailable() { + availableRefers[fmt.Sprintf("%d-%s", i, ep.URL.GetIdentity())] = ep.Caller.GetRuntimeInfo() + } else { + unavailableRefers[fmt.Sprintf("%d-%s", i, ep.URL.GetIdentity())] = ep.Caller.GetRuntimeInfo() + } + i++ + } + info[motan.RuntimeReferersKey] = map[string]interface{}{ + motan.RuntimeAvailableKey: availableRefers, + motan.RuntimeUnavailableKey: unavailableRefers, + } + registriesInfo := make(map[string]interface{}, len(m.Registries)) + for _, registry := range m.Registries { + registriesInfo[registry.GetURL().GetIdentity()] = registry.GetRuntimeInfo() + } + info[motan.RuntimeRegistriesKey] = registriesInfo + + var filtersInfo []interface{} + for _, filter := range m.Filters { + filtersInfo = append(filtersInfo, filter.GetRuntimeInfo()) + } + if len(filtersInfo) > 0 { + info[motan.RuntimeFiltersKey] = filtersInfo + } + + var clusterFilters []interface{} + clusterFilter := m.clusterFilter + for clusterFilter != nil { + clusterFilters = append(clusterFilters, clusterFilter.GetRuntimeInfo()) + clusterFilter = clusterFilter.GetNext() + } + if len(clusterFilters) > 0 { + info[motan.RuntimeClusterFiltersKey] = clusterFilters + } + return info +} + const ( clusterIdcPlaceHolder = "${idc}" ) diff --git a/cluster/motanCluster_test.go b/cluster/motanCluster_test.go index 20702af1..5c80b5e8 100644 --- a/cluster/motanCluster_test.go +++ b/cluster/motanCluster_test.go @@ -23,6 +23,15 @@ func TestInitFilter(t *testing.T) { cluster.initFilters() checkClusterFilter(cluster.clusterFilter, 4, t) checkEndpointFilter(cluster.Filters, 3, t) + // check runtime info filters + info := cluster.GetRuntimeInfo() + filters, ok := info[motan.RuntimeFiltersKey] + assert.True(t, ok) + assert.Equal(t, 3, len(filters.([]interface{}))) + + clusterFilters, ok := info[motan.RuntimeClusterFiltersKey] + assert.True(t, ok) + assert.Equal(t, 4, len(clusterFilters.([]interface{}))) } func checkClusterFilter(filter motan.ClusterFilter, expectDeep int, t *testing.T) { @@ -94,6 +103,38 @@ func TestNotify(t *testing.T) { if destroyEndpoint.IsAvailable() { t.Fatalf("cluster endpoint should not be available") } + + // check runtime info + info := cluster.GetRuntimeInfo() + assert.NotNil(t, info) + + name, ok := info[motan.RuntimeNameKey] + assert.True(t, ok) + assert.Equal(t, cluster.GetName(), name) + + refersSize, ok := info[motan.RuntimeRefererSizeKey] + assert.True(t, ok) + assert.Equal(t, len(cluster.Refers), refersSize) + + urlInfo, ok := info[motan.RuntimeUrlKey] + assert.True(t, ok) + assert.NotNil(t, urlInfo) + + refers, ok := info[motan.RuntimeReferersKey] + assert.True(t, ok) + assert.NotNil(t, refers) + + available, ok := refers.(map[string]interface{})[motan.RuntimeAvailableKey] + assert.True(t, ok) + assert.NotNil(t, available) + + unavailable, ok := refers.(map[string]interface{})[motan.RuntimeUnavailableKey] + assert.True(t, ok) + assert.NotNil(t, unavailable) + + registries, ok := info[motan.RuntimeRegistriesKey] + assert.True(t, ok) + assert.NotNil(t, registries) } func TestCall(t *testing.T) { diff --git a/config/compatible.yaml b/config/compatible.yaml new file mode 100644 index 00000000..90c549bf --- /dev/null +++ b/config/compatible.yaml @@ -0,0 +1,57 @@ +#only for unit test!! +motan-agent: + port: 9981 + mport: 8002 + #log_dir: "/data1/logs/agentlog/" + log_dir: "./logs" + registry: "vintage" + application: "pc-yf-test" + +motan-server: + testkey:testv + +motan-client: + testkey:testv + +#config of registries +motan-registry: + vintage: + protocol: vintage + host: 10.**.**.** + port: 8090 + registryRetryPeriod: 30000 + registrySessionTimeout: 10000 + requestTimeout: 5000 + consul: + protocol: consul + host: 10.**.**.** + port: 8090 + direct: + protocol: direct + host: 10.**.**.** + port: 8013 + +#conf of basic refers +motan-basicRefer: + mybasicRefer: + group: basic-group + registry: "vintage" + requestTimeout: 1000 + haStrategy: failover + loadbalance: random + filter: "accessLog,metrics" + maxClientConnection: 10 + minClientConnection: 1 + retries: 0 + application: pc + +#conf of refers +motan-refer: + status-rpc-json: + path: com.weibo.api.test.service.TestRpc + group: test-group + registry: vintage + serialization: simple + protocol: motanV1Compatible + version: 0.1 + basicRefer: mybasicRefer diff --git a/config/config.go b/config/config.go index d7b0675b..b9789ef5 100644 --- a/config/config.go +++ b/config/config.go @@ -127,12 +127,17 @@ func (c *Config) Int64(key string) (int64, error) { // String returns the string value for a given key. func (c *Config) String(key string) string { + return c.GetStringWithDefault(key, "") +} + +// String returns the string value for a given key. +func (c *Config) GetStringWithDefault(key string, def string) string { if value, err := c.getData(key); err != nil { - return "" + return def } else if vv, ok := value.(string); ok { return vv } - return "" + return def } // GetSection returns map for the given key diff --git a/core/constants.go b/core/constants.go index 4224b485..5815bc9e 100644 --- a/core/constants.go +++ b/core/constants.go @@ -133,12 +133,117 @@ const ( // errorCodes const ( - ENoEndpoints = 1001 - ENoChannel = 1002 - EUnkonwnMsg = 1003 - EConvertMsg = 1004 + ENoEndpoints = 1001 + ENoChannel = 1002 + EUnkonwnMsg = 1003 + EConvertMsg = 1004 + EProviderNotExist = 404 +) + +// ProviderNotExistPrefix errorMessage +const ( + ProviderNotExistPrefix = "provider not exist serviceKey=" ) const ( DefaultReferVersion = "1.0" ) + +// meta info +const ( + DefaultMetaPrefix = "META_" + EnvMetaPrefixKey = "envMetaPrefix" + URLRegisterMeta = "registerMeta" + DefaultRegisterMeta = false + MetaCacheExpireSecondKey = "metaCacheExpireSecond" + DynamicMetaKey = "dynamicMeta" + DefaultDynamicMeta = true + WeightRefreshPeriodSecondKey = "weightRefreshPeriodSecond" + WeightMetaSuffixKey = "WEIGHT" + ServiceNotSupport = "service not support" +) + +//----------- runtime ------------- + +const ( + // -----------top level keys------------- + + RuntimeInstanceTypeKey = "instanceType" + RuntimeExportersKey = "exporters" + RuntimeClustersKey = "clusters" + RuntimeHttpClustersKey = "httpClusters" + RuntimeExtensionFactoryKey = "extensionFactory" + RuntimeServersKey = "servers" + RuntimeBasicKey = "basic" + + //-----------common keys------------- + + RuntimeUrlKey = "url" + RuntimeIsAvailableKey = "isAvailable" + RuntimeProxyKey = "proxy" + RuntimeAvailableKey = "available" + RuntimeEndpointKey = "endpoint" + RuntimeFiltersKey = "filters" + RuntimeClusterFiltersKey = "clusterFilters" + RuntimeNameKey = "name" + RuntimeIndexKey = "index" + RuntimeTypeKey = "type" + + // -----------exporter keys------------- + + RuntimeProviderKey = "provider" + + // -----------server keys------------- + + RuntimeAgentServerKey = "agentServer" + RuntimeAgentPortServerKey = "agentPortServer" + RuntimeMaxContentLengthKey = "maxContentLength" + RuntimeHeartbeatEnabledKey = "heartbeatEnabled" + RuntimeMessageHandlerKey = "messageHandler" + RuntimeProvidersKey = "providers" + RuntimeMessageHandlerTypeKey = "messageHandlerType" + + // -----------http proxy server------------- + + RuntimeHttpProxyServerKey = "httpProxyServer" + RuntimeDenyKey = "deny" + RuntimeKeepaliveKey = "keepalive" + RuntimeDefaultDomainKey = "defaultDomain" + + // -----------cluster keys------------- + + RuntimeReferersKey = "referers" + RuntimeRefererSizeKey = "refererSize" + RuntimeUnavailableKey = "unavailable" + + // -----------endpoint keys------------- + + RuntimeErrorCountKey = "errorCount" + RuntimeKeepaliveRunningKey = "keepaliveRunning" + RuntimeKeepaliveTypeKey = "keepaliveType" + + // -----------extensionFactory keys------------- + + RuntimeRegistriesKey = "registries" + + // -----------registry keys------------- + + RuntimeRegisteredServiceUrlsKey = "registeredServiceUrls" + RuntimeSubscribedServiceUrlsKey = "subscribedServiceUrls" + RuntimeFailedRegisterUrls = "failedRegisterUrls" + RuntimeFailedUnregisterUrls = "failedUnregisterUrls" + RuntimeFailedSubscribeUrls = "failedSubscribeUrls" + RuntimeFailedUnsubScribeUrls = "failedUnsubscribeUrls" + RuntimeSubscribeInfoKey = "subscribeInfo" + RuntimeAgentCommandKey = "agentCommand" + RuntimeServiceCommandKey = "serviceCommand" + RuntimeStaticCommandKey = "staticCommand" + RuntimeWeightKey = "weight" + RuntimeCommandHistoryKey = "commandHistory" + RuntimeNotifyHistoryKey = "notifyHistory" + + // -----------basic keys------------- + + RuntimeCpuPercentKey = "cpuPercent" + RuntimeRssMemoryKey = "rssMemory" +) diff --git a/core/globalContext.go b/core/globalContext.go index 29d974e4..ec4d4d1f 100644 --- a/core/globalContext.go +++ b/core/globalContext.go @@ -9,6 +9,8 @@ import ( "os" "reflect" "strings" + "sync/atomic" + "unsafe" ) const ( @@ -87,6 +89,22 @@ var ( Recover = flag.Bool("recover", false, "recover from accidental exit") ) +func GetMport() int { + return *(*int)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&Mport)))) +} + +func SetMport(v int) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&Mport)), unsafe.Pointer(&v)) +} + +func GetApplication() string { + return *(*string)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&Application)))) +} + +func SetApplication(v string) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&Application)), unsafe.Pointer(&v)) +} + func AddRelevantFilter(filterStr string) { k := strings.TrimSpace(filterStr) if k != "" { @@ -192,7 +210,7 @@ func (c *Context) Initialize() { c.pool = *Pool } if c.application == "" { - c.application = *Application + c.application = GetApplication() } c.RegistryURLs = make(map[string]*URL) @@ -420,7 +438,6 @@ func (c *Context) basicConfToURLs(section string) map[string]*URL { if len(finalFilters) > 0 { newURL.PutParam(FilterKey, c.FilterSetToStr(finalFilters)) } - newURLs[key] = newURL } return newURLs @@ -536,21 +553,11 @@ func (c *Context) parseRegGroupSuffix(urlMap map[string]*URL) { if regGroupSuffix == "" { return } - filterMap := make(map[string]struct{}, len(urlMap)) for _, url := range urlMap { - filterMap[url.GetIdentityWithRegistry()] = struct{}{} - } - for k, url := range urlMap { if strings.HasSuffix(url.Group, regGroupSuffix) { continue } - newUrl := url.Copy() - newUrl.Group += regGroupSuffix - if _, ok := filterMap[newUrl.GetIdentityWithRegistry()]; ok { - continue - } - filterMap[newUrl.GetIdentityWithRegistry()] = struct{}{} - urlMap[k] = newUrl + url.Group += regGroupSuffix } } @@ -581,6 +588,12 @@ func (c *Context) parseSubGroupSuffix(urlMap map[string]*URL) { func (c *Context) parseRefers() { referUrls := c.basicConfToURLs(refersSection) c.parseSubGroupSuffix(referUrls) + //TODO: to compatible with removed protocol: motanV1Compatible + for k := range referUrls { + if referUrls[k].Protocol == "motanV1Compatible" { + referUrls[k].Protocol = "motan" + } + } c.RefersURLs = referUrls } diff --git a/core/globalContext_test.go b/core/globalContext_test.go index 770bec34..b80943fd 100644 --- a/core/globalContext_test.go +++ b/core/globalContext_test.go @@ -30,6 +30,15 @@ func TestGetContext(t *testing.T) { assert.Equal(t, "motan-demo-rpc", rs.ServiceURLs["mytest-motan2"].Group, "parse serivce key fail") } +func TestCompatible(t *testing.T) { + rs := &Context{ConfigFile: "../config/compatible.yaml"} + rs.Initialize() + assert.NotNil(t, rs.RefersURLs) + for _, j := range rs.RefersURLs { + assert.Equal(t, j.Protocol, "motan") + } +} + func TestNewContext(t *testing.T) { configFile := filepath.Join("testdata", "app.yaml") pool1Context := NewContext(configFile, "app", "app-idc1") @@ -259,11 +268,11 @@ func TestContext_parseRegGroupSuffix(t *testing.T) { }, AssertFunc: func(t *testing.T, urlMap map[string]*URL) { groupMap := countGroup(urlMap) - assert.Equal(t, 1, groupMap["group1"]) - assert.Equal(t, 1, groupMap["group1"+regGroupSuffix]) + assert.Equal(t, 0, groupMap["group1"]) + assert.Equal(t, 2, groupMap["group1"+regGroupSuffix]) assert.Equal(t, 1, groupMap["group2"+regGroupSuffix]) - assert.Equal(t, 1, groupMap["group3"]) - assert.Equal(t, 1, groupMap["group3"+regGroupSuffix]) + assert.Equal(t, 0, groupMap["group3"]) + assert.Equal(t, 2, groupMap["group3"+regGroupSuffix]) assert.Equal(t, 2, groupMap["group4"+regGroupSuffix]) assert.Equal(t, 2, groupMap["group5"+regGroupSuffix]) }, @@ -271,6 +280,7 @@ func TestContext_parseRegGroupSuffix(t *testing.T) { } os.Setenv(RegGroupSuffix, regGroupSuffix) ctx := &Context{} + // replace all groups with regGroupSuffix for _, s := range cases { ctx.parseRegGroupSuffix(s.UrlMap) s.AssertFunc(t, s.UrlMap) diff --git a/core/motan.go b/core/motan.go index ec84c8b3..22936c9c 100644 --- a/core/motan.go +++ b/core/motan.go @@ -1,6 +1,7 @@ package core import ( + "container/ring" "errors" "net" "strconv" @@ -88,6 +89,7 @@ type Cloneable interface { // Caller : can process a motan request. the call maybe process from remote by endpoint, maybe process by some kinds of provider type Caller interface { + RuntimeInfo WithURL Status Call(request Request) Response @@ -160,6 +162,12 @@ type LoadBalance interface { SetWeight(weight string) } +// WeightLoadBalance : weight loadBalance for cluster +type WeightLoadBalance interface { + LoadBalance + NotifyWeightChange() +} + // DiscoverService : discover service for cluster type DiscoverService interface { Subscribe(url *URL, listener NotifyListener) @@ -202,6 +210,7 @@ type SnapshotService interface { // Registry : can subscribe or register service type Registry interface { + RuntimeInfo Name WithURL DiscoverService @@ -235,6 +244,7 @@ type CommandNotifyListener interface { // Filter : filter request or response in a call processing type Filter interface { + RuntimeInfo Name // filter must be prototype NewFilter(url *URL) Filter @@ -266,6 +276,7 @@ type Server interface { WithURL Name Destroyable + RuntimeInfo SetMessageHandler(mh MessageHandler) GetMessageHandler() MessageHandler Open(block bool, proxy bool, handler MessageHandler, extFactory ExtensionFactory) error @@ -274,6 +285,7 @@ type Server interface { // Exporter : export and manage a service. one exporter bind with a service type Exporter interface { + RuntimeInfo Export(server Server, extFactory ExtensionFactory, context *Context) error Unexport() error SetProvider(provider Provider) @@ -293,6 +305,8 @@ type Provider interface { // MessageHandler : handler message(request) for Server type MessageHandler interface { + Name + RuntimeInfo Call(request Request) (res Response) AddProvider(p Provider) error RmProvider(p Provider) @@ -310,6 +324,7 @@ type Serialization interface { // ExtensionFactory : can regiser and get all kinds of extension implements. type ExtensionFactory interface { + RuntimeInfo GetHa(url *URL) HaStrategy GetLB(url *URL) LoadBalance GetFilter(name string) Filter @@ -413,7 +428,6 @@ func (c *RPCContext) Reset() { c.BodySize = 0 c.SerializeNum = 0 c.Serialized = false - c.AsyncCall = false c.Result = nil c.Reply = nil c.FinishHandlers = c.FinishHandlers[:0] @@ -434,10 +448,8 @@ func (c *RPCContext) OnFinish() { // AsyncResult : async call result type AsyncResult struct { - StartTime int64 - Done chan *AsyncResult - Reply interface{} - Error error + Done chan *AsyncResult + Error error } // DeserializableValue : for lazy deserialize @@ -578,7 +590,6 @@ func (req *MotanRequest) Clone() interface{} { GzipSize: req.RPCContext.GzipSize, SerializeNum: req.RPCContext.SerializeNum, Serialized: req.RPCContext.Serialized, - AsyncCall: req.RPCContext.AsyncCall, Result: req.RPCContext.Result, Reply: req.RPCContext.Reply, RequestSendTime: req.RPCContext.RequestSendTime, @@ -748,6 +759,20 @@ type DefaultExtensionFactory struct { newRegistryLock sync.Mutex } +func (d *DefaultExtensionFactory) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + // registries runtime info + d.newRegistryLock.Lock() + defer d.newRegistryLock.Unlock() + registriesInfo := map[string]interface{}{} + for s, registry := range d.registries { + registriesInfo[s] = registry.GetRuntimeInfo() + } + info[RuntimeRegistriesKey] = registriesInfo + + return info +} + func (d *DefaultExtensionFactory) GetHa(url *URL) HaStrategy { haName := url.GetParam(Hakey, "failover") if newHa, ok := d.haFactories[haName]; ok { @@ -920,6 +945,15 @@ func GetLastClusterFilter() ClusterFilter { type lastEndPointFilter struct{} +func (l *lastEndPointFilter) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{ + RuntimeNameKey: l.GetName(), + RuntimeIndexKey: l.GetIndex(), + RuntimeTypeKey: l.GetType(), + } + return info +} + func (l *lastEndPointFilter) GetName() string { return "lastEndPointFilter" } @@ -961,6 +995,14 @@ func (l *lastEndPointFilter) GetType() int32 { type lastClusterFilter struct{} +func (l *lastClusterFilter) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + RuntimeNameKey: l.GetName(), + RuntimeIndexKey: l.GetIndex(), + RuntimeTypeKey: l.GetType(), + } +} + func (l *lastClusterFilter) GetName() string { return "lastClusterFilter" } @@ -1009,6 +1051,13 @@ type FilterEndPoint struct { Caller Caller } +func (f *FilterEndPoint) GetRuntimeInfo() map[string]interface{} { + if f.Caller != nil { + return f.Caller.GetRuntimeInfo() + } + return map[string]interface{}{} +} + func (f *FilterEndPoint) Call(request Request) Response { if request.GetRPCContext(true).Tc != nil { request.GetRPCContext(true).Tc.PutReqSpan(&Span{Name: EpFilterStart, Addr: f.GetURL().GetAddressStr(), Time: time.Now()}) @@ -1197,3 +1246,76 @@ func GetLocalProvider(service string) Provider { } return nil } + +//-----------RuntimeInfo interface------------- + +// RuntimeInfo : output runtime information +type RuntimeInfo interface { + GetRuntimeInfo() map[string]interface{} +} + +// GetRuntimeInfo : call s.GetRuntimeInfo +func GetRuntimeInfo(s interface{}) map[string]interface{} { + if sc, ok := s.(RuntimeInfo); ok { + return sc.GetRuntimeInfo() + } + return map[string]interface{}{} +} + +// -----------CircularRecorder------------- +type circularRecorderItem struct { + timestamp int64 + value interface{} +} + +type CircularRecorder struct { + ring *ring.Ring + lock sync.RWMutex + keyBuf []byte +} + +func NewCircularRecorder(size int) *CircularRecorder { + return &CircularRecorder{ + ring: ring.New(size), + lock: sync.RWMutex{}, + keyBuf: make([]byte, 0, 18), + } +} + +func (c *CircularRecorder) AddRecord(item interface{}) { + if item == nil { + return + } + c.lock.Lock() + defer c.lock.Unlock() + c.ring.Value = &circularRecorderItem{ + timestamp: time.Now().UnixNano() / 1e6, + value: item, + } + c.ring = c.ring.Next() +} + +func (c *CircularRecorder) GetRecords() map[string]interface{} { + c.lock.RLock() + defer c.lock.RUnlock() + records := make(map[string]interface{}) + idx := int64(0) + c.ring.Do(func(i interface{}) { + if i != nil { + item, ok := i.(*circularRecorderItem) + if ok { + records[c.generateRecordKey(idx, item.timestamp)] = item.value + idx++ + } + } + }) + return records +} + +func (c *CircularRecorder) generateRecordKey(idx, timestamp int64) string { + c.keyBuf = c.keyBuf[:0] + c.keyBuf = strconv.AppendInt(c.keyBuf, idx, 10) + c.keyBuf = append(c.keyBuf, ':') + c.keyBuf = strconv.AppendInt(c.keyBuf, timestamp, 10) + return string(c.keyBuf) +} diff --git a/core/motan_test.go b/core/motan_test.go index 388c11c3..7fce15b0 100644 --- a/core/motan_test.go +++ b/core/motan_test.go @@ -2,6 +2,7 @@ package core import ( "strconv" + "strings" "sync" "testing" @@ -190,3 +191,23 @@ func newDiscoverErrorRegistry() *TestRegistry { registry.DiscoverError = true return registry } + +func TestCircularRecorder(t *testing.T) { + size := 10 + r := NewCircularRecorder(size) + assert.NotNil(t, r) + assert.Equal(t, size, r.ring.Len()) + + for i := 0; i < size*2; i++ { + r.AddRecord(i) + } + records := r.GetRecords() + assert.Equal(t, size, r.ring.Len()) + assert.Equal(t, size, len(records)) + + for k, v := range records { + keyArr := strings.Split(k, ":") + idx, _ := strconv.Atoi(keyArr[0]) + assert.Equal(t, idx+size, v.(int)) + } +} diff --git a/core/test.go b/core/test.go index 047683a8..0bf04269 100644 --- a/core/test.go +++ b/core/test.go @@ -15,6 +15,10 @@ type TestFilter struct { next ClusterFilter } +func (t *TestFilter) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (t *TestFilter) GetName() string { return "TestFilter" } @@ -57,6 +61,10 @@ type TestEndPointFilter struct { next EndPointFilter } +func (t *TestEndPointFilter) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (t *TestEndPointFilter) GetName() string { return "TestEndPointFilter" } @@ -99,6 +107,10 @@ type TestProvider struct { URL *URL } +func (t *TestProvider) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (t *TestProvider) SetService(s interface{}) { } @@ -131,6 +143,10 @@ type TestEndPoint struct { available atomic.Value } +func (t *TestEndPoint) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (t *TestEndPoint) GetURL() *URL { return t.URL } @@ -233,6 +249,12 @@ type TestRegistry struct { DiscoverError bool } +func (t *TestRegistry) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + RuntimeNameKey: t.GetName(), + } +} + func (t *TestRegistry) GetName() string { return "testRegistry" } diff --git a/core/url.go b/core/url.go index 11ac3be6..89f9e0f4 100644 --- a/core/url.go +++ b/core/url.go @@ -64,7 +64,7 @@ func (u *URL) IsMatch(service, group, protocol, version string) bool { } // for motan v1 request, parameter protocol should be empty if protocol != "" { - if u.Protocol == "motanV1Compatible" { + if u.Protocol == "motan2" { if protocol != "motan2" && protocol != "motan" { return false } @@ -328,7 +328,7 @@ func (u *URL) MergeParams(params map[string]string) { } func (u *URL) CanServe(other *URL) bool { - if u.Protocol != other.Protocol && u.Protocol != ProtocolLocal { + if !u.CanServeProtocol(other) { vlog.Errorf("can not serve protocol, err : p1:%s, p2:%s", u.Protocol, other.Protocol) return false } @@ -336,9 +336,11 @@ func (u *URL) CanServe(other *URL) bool { vlog.Errorf("can not serve path, err : p1:%s, p2:%s", u.Path, other.Path) return false } - if !IsSame(u.Parameters, other.Parameters, SerializationKey, "") { - vlog.Errorf("can not serve serialization, err : s1:%s, s2:%s", u.Parameters[SerializationKey], other.Parameters[SerializationKey]) - return false + if u.Protocol != "motan2" { + if !IsSame(u.Parameters, other.Parameters, SerializationKey, "") { + vlog.Errorf("can not serve serialization, err : s1:%s, s2:%s", u.Parameters[SerializationKey], other.Parameters[SerializationKey]) + return false + } } // compatible with old version: 0.1 if !(IsSame(u.Parameters, other.Parameters, VersionKey, "0.1") || IsSame(u.Parameters, other.Parameters, VersionKey, DefaultReferVersion)) { @@ -348,6 +350,14 @@ func (u *URL) CanServe(other *URL) bool { return true } +func (u *URL) CanServeProtocol(other *URL) bool { + // motan2 is compatible with motan + if other.Protocol == "motan" && u.Protocol == "motan2" { + return true + } + return u.Protocol == other.Protocol || u.Protocol == ProtocolLocal +} + func IsSame(m1 map[string]string, m2 map[string]string, key string, defaultValue string) bool { if m1 == nil && m2 == nil { return true diff --git a/core/url_test.go b/core/url_test.go index 73ca3a73..fa15f774 100644 --- a/core/url_test.go +++ b/core/url_test.go @@ -103,7 +103,7 @@ func TestCopyAndMerge(t *testing.T) { } } -func TestCanServer(t *testing.T) { +func TestCanServe(t *testing.T) { params1 := make(map[string]string) params2 := make(map[string]string) url1 := &URL{Protocol: "motan", Path: "test/path", Parameters: params1} @@ -146,6 +146,33 @@ func TestCanServer(t *testing.T) { t.Fatalf("url CanServe testFail url1: %+v, url2: %+v\n", url1, url2) } fmt.Printf("url1:%+v, url2:%+v\n", url1, url2) + url1.Path = "" + url2.Path = "" + url1.Protocol = "motan2" + url2.Protocol = "motan" + if !url1.CanServe(url2) { + t.Fatalf("url CanServe testFail url1: %+v, url2: %+v\n", url1, url2) + } + url1.Protocol = "motan" + url2.Protocol = "motan2" + if url1.CanServe(url2) { + t.Fatalf("url CanServe testFail url1: %+v, url2: %+v\n", url1, url2) + } + url1.Protocol = "local" + url2.Protocol = "motan2" + if !url1.CanServe(url2) { + t.Fatalf("url CanServe testFail url1: %+v, url2: %+v\n", url1, url2) + } + url1.Protocol = "abc" + url2.Protocol = "motan2" + if url1.CanServe(url2) { + t.Fatalf("url CanServe testFail url1: %+v, url2: %+v\n", url1, url2) + } + url1.Protocol = "motan" + url2.Protocol = "motan2" + if url1.CanServe(url2) { + t.Fatalf("url CanServe testFail url1: %+v, url2: %+v\n", url1, url2) + } } func TestGetPositiveIntValue(t *testing.T) { diff --git a/core/util.go b/core/util.go index 0438ca69..0ba6f4c3 100644 --- a/core/util.go +++ b/core/util.go @@ -100,6 +100,24 @@ func SliceShuffle(slice []string) []string { return slice } +func EndpointShuffle(slice []EndPoint) []EndPoint { + for i := 0; i < len(slice); i++ { + a := rand.Intn(len(slice)) + b := rand.Intn(len(slice)) + slice[a], slice[b] = slice[b], slice[a] + } + return slice +} + +func ByteSliceShuffle(slice []byte) []byte { + for i := 0; i < len(slice); i++ { + a := rand.Intn(len(slice)) + b := rand.Intn(len(slice)) + slice[a], slice[b] = slice[b], slice[a] + } + return slice +} + func FirstUpper(s string) string { r := []rune(s) @@ -286,3 +304,10 @@ func ClearDirectEnvRegistry() { directRpc = nil initDirectEnv = sync.Once{} } + +func GetNonNegative(originValue int64) int64 { + if originValue > 0 { + return originValue + } + return 0x7fffffffffffffff & originValue +} diff --git a/default.go b/default.go index 52e41cd7..ef77faf7 100644 --- a/default.go +++ b/default.go @@ -82,8 +82,17 @@ func GetDefaultManageHandlers() map[string]http.Handler { defaultManageHandlers["/registry/list"] = dynamicConfigurer defaultManageHandlers["/registry/info"] = dynamicConfigurer + metaInfo := &MetaInfo{} + defaultManageHandlers["/meta/update"] = metaInfo + defaultManageHandlers["/meta/delete"] = metaInfo + defaultManageHandlers["/meta/get"] = metaInfo + defaultManageHandlers["/meta/getAll"] = metaInfo + hotReload := &HotReload{} defaultManageHandlers["/reload/clusters"] = hotReload + + runtimeHandler := &RuntimeHandler{} + defaultManageHandlers["/runtime/info"] = runtimeHandler }) return defaultManageHandlers } diff --git a/endpoint/endpoint.go b/endpoint/endpoint.go index 4ea84c15..45ce1c23 100644 --- a/endpoint/endpoint.go +++ b/endpoint/endpoint.go @@ -10,11 +10,12 @@ import ( // ext name const ( - Grpc = "grpc" - Motan2 = "motan2" - Local = "local" - Mock = "mockEndpoint" - MotanV1Compatible = "motanV1Compatible" + Grpc = "grpc" + Motan2 = "motan2" + // Motan1 endpoint is to support dynamic configuration. Golang cannot build motan1 request + Motan1 = "motan" + Local = "local" + Mock = "mockEndpoint" ) const ( @@ -26,7 +27,7 @@ var idOffset uint64 // id generator offset func RegistDefaultEndpoint(extFactory motan.ExtensionFactory) { extFactory.RegistExtEndpoint(Motan2, func(url *motan.URL) motan.EndPoint { - return &MotanEndpoint{url: url} + return &MotanCommonEndpoint{url: url} }) extFactory.RegistExtEndpoint(Grpc, func(url *motan.URL) motan.EndPoint { @@ -41,7 +42,7 @@ func RegistDefaultEndpoint(extFactory motan.ExtensionFactory) { return &MockEndpoint{URL: url} }) - extFactory.RegistExtEndpoint(MotanV1Compatible, func(url *motan.URL) motan.EndPoint { + extFactory.RegistExtEndpoint(Motan1, func(url *motan.URL) motan.EndPoint { return &MotanCommonEndpoint{url: url} }) } @@ -65,6 +66,12 @@ type MockEndpoint struct { MockResponse motan.Response } +func (m *MockEndpoint) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: m.GetName(), + } +} + func (m *MockEndpoint) GetName() string { return "mockEndpoint" } diff --git a/endpoint/grpcEndpoint.go b/endpoint/grpcEndpoint.go index 997ea7be..258a18b0 100644 --- a/endpoint/grpcEndpoint.go +++ b/endpoint/grpcEndpoint.go @@ -7,8 +7,12 @@ import ( motan "github.com/weibocom/motan-go/core" "github.com/weibocom/motan-go/log" "golang.org/x/net/context" - grpc "google.golang.org/grpc" - metadata "google.golang.org/grpc/metadata" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +const ( + GRPCSerialNum = 1 ) type GrpcEndPoint struct { @@ -17,9 +21,11 @@ type GrpcEndPoint struct { proxy bool } -const ( - GRPCSerialNum = 1 -) +func (g *GrpcEndPoint) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: g.GetName(), + } +} func (g *GrpcEndPoint) Initialize() { grpcconn, err := grpc.Dial((g.url.Host + ":" + strconv.Itoa((int)(g.url.Port))), grpc.WithInsecure(), grpc.WithCodec(&agentCodec{})) diff --git a/endpoint/localEndpoint.go b/endpoint/localEndpoint.go index 4ed863a7..e2b624d8 100644 --- a/endpoint/localEndpoint.go +++ b/endpoint/localEndpoint.go @@ -10,6 +10,12 @@ type LocalEndpoint struct { provider motan.Provider } +func (l *LocalEndpoint) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: l.GetName(), + } +} + func (l *LocalEndpoint) GetName() string { return "localEndpoint" } diff --git a/endpoint/mockDynamicEndpoint.go b/endpoint/mockDynamicEndpoint.go new file mode 100644 index 00000000..4361f450 --- /dev/null +++ b/endpoint/mockDynamicEndpoint.go @@ -0,0 +1,91 @@ +package endpoint + +import ( + motan "github.com/weibocom/motan-go/core" + mpro "github.com/weibocom/motan-go/protocol" + "strconv" + "sync" + "sync/atomic" +) + +type MockDynamicEndpoint struct { + URL *motan.URL + available bool + DynamicWeight int64 + StaticWeight int64 + Count int64 + dynamicMeta sync.Map +} + +func (m *MockDynamicEndpoint) GetName() string { + return "mockEndpoint" +} + +func (m *MockDynamicEndpoint) GetURL() *motan.URL { + return m.URL +} + +func (m *MockDynamicEndpoint) SetURL(url *motan.URL) { + m.URL = url +} + +func (m *MockDynamicEndpoint) IsAvailable() bool { + return m.available +} + +func (m *MockDynamicEndpoint) SetAvailable(a bool) { + m.available = a +} + +func (m *MockDynamicEndpoint) SetProxy(proxy bool) {} + +func (m *MockDynamicEndpoint) SetSerialization(s motan.Serialization) {} + +func (m *MockDynamicEndpoint) Call(request motan.Request) motan.Response { + if isMetaServiceRequest(request) { + resMap := make(map[string]string) + m.dynamicMeta.Range(func(key, value interface{}) bool { + resMap[key.(string)] = value.(string) + return true + }) + atomic.AddInt64(&m.Count, 1) + return &motan.MotanResponse{ProcessTime: 1, Value: resMap} + } + atomic.AddInt64(&m.Count, 1) + return &motan.MotanResponse{ProcessTime: 1, Value: "ok"} +} + +func (m *MockDynamicEndpoint) Destroy() {} + +func (m *MockDynamicEndpoint) GetRuntimeInfo() map[string]interface{} { + return make(map[string]interface{}) +} + +func (m *MockDynamicEndpoint) SetWeight(isDynamic bool, weight int64) { + if isDynamic { + m.DynamicWeight = weight + m.dynamicMeta.Store(motan.DefaultMetaPrefix+motan.WeightMetaSuffixKey, strconv.Itoa(int(weight))) + } else { + m.StaticWeight = weight + m.URL.PutParam(motan.DefaultMetaPrefix+motan.WeightMetaSuffixKey, strconv.Itoa(int(weight))) + } +} + +func NewMockDynamicEndpoint(url *motan.URL) *MockDynamicEndpoint { + return &MockDynamicEndpoint{ + URL: url, + available: true, + } +} + +func NewMockDynamicEndpointWithWeight(url *motan.URL, staticWeight int64) *MockDynamicEndpoint { + res := NewMockDynamicEndpoint(url) + res.StaticWeight = staticWeight + res.URL.PutParam(motan.DefaultMetaPrefix+motan.WeightMetaSuffixKey, strconv.Itoa(int(staticWeight))) + return res +} + +func isMetaServiceRequest(request motan.Request) bool { + return request != nil && "com.weibo.api.motan.runtime.meta.MetaService" == request.GetServiceName() && + "getDynamicMeta" == request.GetMethod() && "y" == request.GetAttachment(mpro.MFrameworkService) +} diff --git a/endpoint/motanCommonEndpoint.go b/endpoint/motanCommonEndpoint.go index 36c4a659..538967ec 100644 --- a/endpoint/motanCommonEndpoint.go +++ b/endpoint/motanCommonEndpoint.go @@ -6,6 +6,7 @@ import ( "github.com/panjf2000/ants/v2" motan "github.com/weibocom/motan-go/core" vlog "github.com/weibocom/motan-go/log" + "github.com/weibocom/motan-go/metrics" mpro "github.com/weibocom/motan-go/protocol" "net" "strconv" @@ -15,6 +16,11 @@ import ( "time" ) +const ( + KeepaliveHeartbeat uint32 = 1 + KeepaliveProfile uint32 = 2 +) + var ( streamPool = sync.Pool{New: func() interface{} { return &Stream{ @@ -31,7 +37,7 @@ type MotanCommonEndpoint struct { channels *ChannelPool destroyed bool destroyCh chan struct{} - available bool + available atomic.Value errorCount uint32 proxy bool errorCountThreshold int64 @@ -45,14 +51,24 @@ type MotanCommonEndpoint struct { heartbeatVersion int gzipSize int - keepaliveRunning bool + keepaliveRunning atomic.Value + keepaliveType uint32 serialization motan.Serialization DefaultVersion int // default encode version } +func (m *MotanCommonEndpoint) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: m.GetName(), + motan.RuntimeErrorCountKey: atomic.LoadUint32(&m.errorCount), + motan.RuntimeKeepaliveRunningKey: m.keepaliveRunning.Load(), + motan.RuntimeKeepaliveTypeKey: atomic.LoadUint32(&m.keepaliveType), + } +} + func (m *MotanCommonEndpoint) setAvailable(available bool) { - m.available = available + m.available.Store(available) } func (m *MotanCommonEndpoint) SetSerialization(s motan.Serialization) { @@ -77,6 +93,7 @@ func (m *MotanCommonEndpoint) Initialize() { m.lazyInit = m.url.GetBoolValue(motan.LazyInit, defaultLazyInit) asyncInitConnection := m.url.GetBoolValue(motan.AsyncInitConnection, GetDefaultMotanEPAsynInit()) m.heartbeatVersion = -1 + m.keepaliveRunning.Store(false) m.DefaultVersion = mpro.Version2 m.gzipSize = int(m.url.GetIntValue(motan.GzipSizeKey, 0)) factory := func() (net.Conn, error) { @@ -136,23 +153,21 @@ func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { if m.channels == nil { vlog.Errorf("motanEndpoint %s error: channels is null", m.url.GetAddressStr()) - m.recordErrAndKeepalive() - return m.defaultErrMotanResponse(request, "motanEndpoint error: channels is null") - } - startTime := time.Now().UnixNano() - if rc.AsyncCall { - rc.Result.StartTime = startTime + resp := m.defaultErrMotanResponse(request, "motanEndpoint error: channels is null") + m.recordErrAndKeepalive(resp.GetException()) + return resp } // get a channel channel, err := m.channels.Get() if err != nil { vlog.Errorf("motanEndpoint %s error: can not get a channel, msg: %s", m.url.GetAddressStr(), err.Error()) - m.recordErrAndKeepalive() - return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ + resp := motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ ErrCode: motan.ENoChannel, ErrMsg: "can not get a channel", ErrType: motan.ServiceException, }) + m.recordErrAndKeepalive(resp.GetException()) + return resp } deadline := m.GetRequestTimeout(request) // do call @@ -160,15 +175,13 @@ func (m *MotanCommonEndpoint) Call(request motan.Request) motan.Response { response, err := channel.Call(request, deadline, rc) if err != nil { vlog.Errorf("motanEndpoint call fail. ep:%s, req:%s, error: %s", m.url.GetAddressStr(), motan.GetReqInfo(request), err.Error()) - m.recordErrAndKeepalive() - return m.defaultErrMotanResponse(request, "channel call error:"+err.Error()) - } - if rc.AsyncCall { - return defaultAsyncResponse + resp := m.defaultErrMotanResponse(request, "channel call error:"+err.Error()) + m.recordErrAndKeepalive(resp.GetException()) + return resp } excep := response.GetException() if excep != nil && excep.ErrType != motan.BizException { - m.recordErrAndKeepalive() + m.recordErrAndKeepalive(excep) } else { // reset errorCount m.resetErr() @@ -217,7 +230,7 @@ func (m *MotanCommonEndpoint) initChannelPoolWithRetry(factory ConnFactory, conf for { select { case <-ticker.C: - channels, err := NewChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) + channels, err = NewChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) if err == nil { m.channels = channels m.setAvailable(true) @@ -236,19 +249,29 @@ func (m *MotanCommonEndpoint) initChannelPoolWithRetry(factory ConnFactory, conf } } -func (m *MotanCommonEndpoint) recordErrAndKeepalive() { +func (m *MotanCommonEndpoint) recordErrAndKeepalive(exception *motan.Exception) { // errorCountThreshold <= 0 means not trigger keepalive if m.errorCountThreshold > 0 { errCount := atomic.AddUint32(&m.errorCount, 1) // ensure trigger keepalive if errCount >= uint32(m.errorCountThreshold) { m.setAvailable(false) + m.setKeepaliveType(exception) vlog.Infoln("Referer disable:" + m.url.GetIdentity()) go m.keepalive() } } } +func (m *MotanCommonEndpoint) setKeepaliveType(exception *motan.Exception) { + if exception != nil && exception.ErrCode == motan.EProviderNotExist && + (strings.Contains(exception.ErrMsg, motan.ProviderNotExistPrefix)) { + atomic.StoreUint32(&m.keepaliveType, KeepaliveProfile) + } else { + atomic.StoreUint32(&m.keepaliveType, KeepaliveHeartbeat) + } +} + func (m *MotanCommonEndpoint) resetErr() { atomic.StoreUint32(&m.errorCount, 0) } @@ -261,16 +284,16 @@ func (m *MotanCommonEndpoint) keepalive() { return } // ensure only one keepalive handler - if m.keepaliveRunning { + if v, ok := m.keepaliveRunning.Load().(bool); ok && v { m.lock.Unlock() return } - m.keepaliveRunning = true + m.keepaliveRunning.Store(true) m.lock.Unlock() defer func() { m.lock.Lock() - m.keepaliveRunning = false + m.keepaliveRunning.Store(false) m.lock.Unlock() }() @@ -280,17 +303,12 @@ func (m *MotanCommonEndpoint) keepalive() { for { select { case <-ticker.C: - if channel, err := m.channels.Get(); err != nil { - vlog.Infof("[keepalive] failed. url:%s, err:%s", m.url.GetIdentity(), err.Error()) + if atomic.LoadUint32(&m.keepaliveType) == KeepaliveProfile { + m.profile() } else { - _, err = channel.HeartBeat(m.heartbeatVersion) - if err == nil { - m.setAvailable(true) - m.resetErr() - vlog.Infof("[keepalive] heartbeat success. url: %s", m.url.GetIdentity()) + if m.heartbeat() { return } - vlog.Infof("[keepalive] heartbeat failed. url:%s, err:%s", m.url.GetIdentity(), err.Error()) } case <-m.destroyCh: return @@ -298,6 +316,29 @@ func (m *MotanCommonEndpoint) keepalive() { } } +func (m *MotanCommonEndpoint) heartbeat() bool { + if channel, err := m.channels.Get(); err != nil { + vlog.Infof("[keepalive] failed. url:%s, err:%s", m.url.GetIdentity(), err.Error()) + } else { + _, err = channel.HeartBeat(m.heartbeatVersion) + if err == nil { + m.setAvailable(true) + m.resetErr() + vlog.Infof("[keepalive] heartbeat success. url: %s", m.url.GetIdentity()) + return true + } + vlog.Infof("[keepalive] heartbeat failed. url:%s, err:%s", m.url.GetIdentity(), err.Error()) + } + return false +} + +func (m *MotanCommonEndpoint) profile() { + metrics.AddGaugeWithKeys(metrics.DefaultRuntimeCircuitBreakerGroup+"-"+motan.GetApplication(), "", + metrics.DefaultRuntimeCircuitBreakerService, + []string{metrics.DefaultStatRole, metrics.DefaultRuntimeErrorApplication, strconv.Itoa(motan.GetMport()) + "-" + m.url.Path}, + "", 1) +} + func (m *MotanCommonEndpoint) defaultErrMotanResponse(request motan.Request, errMsg string) motan.Response { return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ ErrCode: 400, @@ -319,7 +360,10 @@ func (m *MotanCommonEndpoint) SetURL(url *motan.URL) { } func (m *MotanCommonEndpoint) IsAvailable() bool { - return m.available + if m.available.Load() == nil { + return false + } + return m.available.Load().(bool) } type Channel struct { @@ -430,10 +474,6 @@ func (s *Stream) Send() (err error) { if s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Send, Addr: s.channel.address, Time: sendTime}) } - if s.rc.AsyncCall { - // only return send success, it can release in Stream.notify - s.canRelease.Store(true) - } } return nil case <-s.timer.C: @@ -522,24 +562,6 @@ func (s *Stream) notify(msg interface{}, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Decode, Addr: s.channel.address, Time: decodeTime}) s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) } - if s.rc.AsyncCall { - defer func() { - s.RemoveFromChannel() - releaseStream(s) - }() - result := s.rc.Result - if err != nil { - result.Error = err - result.Done <- result - return - } - if err = res.ProcessDeserializable(result.Reply); err != nil { - result.Error = err - } - res.SetProcessTime((time.Now().UnixNano() - result.StartTime) / 1000000) - result.Done <- result - return - } } s.res = res s.recvNotifyCh <- struct{}{} @@ -560,11 +582,7 @@ func (c *Channel) newStream(req motan.Request, rc *motan.RPCContext, deadline ti s.req = req s.deadline = time.Now().Add(deadline) s.rc = rc - if rc != nil && rc.AsyncCall { // release by Stream.Notify - s.canRelease.Store(false) - } else { // release by Channel self - s.canRelease.Store(true) - } + s.canRelease.Store(true) c.streamLock.Lock() c.streams[s.streamId] = s c.streamLock.Unlock() @@ -608,24 +626,17 @@ func (s *Stream) RemoveFromChannel() bool { // Call send request to the server. // -// about return: exception in response will record error count, err will not. +// about return: exception in response will record error Count, err will not. func (c *Channel) Call(req motan.Request, deadline time.Duration, rc *motan.RPCContext) (motan.Response, error) { stream, err := c.newStream(req, rc, deadline) if err != nil { return nil, err } - defer func() { - if rc == nil || !rc.AsyncCall { - releaseStream(stream) - } - }() + defer releaseStream(stream) if err = stream.Send(); err != nil { return nil, err } - if rc != nil && rc.AsyncCall { - return nil, nil - } return stream.Recv() } diff --git a/endpoint/motanCommonEndpoint_test.go b/endpoint/motanCommonEndpoint_test.go index 75c460c5..2957c717 100644 --- a/endpoint/motanCommonEndpoint_test.go +++ b/endpoint/motanCommonEndpoint_test.go @@ -8,12 +8,31 @@ import ( "github.com/weibocom/motan-go/serialize" "net" "runtime" + "strconv" + "sync/atomic" "testing" "time" ) +func TestNotInitCommonEndpoint(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan"} + url.PutParam(motan.TimeOutKey, "100") + url.PutParam(motan.AsyncInitConnection, "false") + ep := &MotanCommonEndpoint{} + ep.SetURL(url) + + ep.SetProxy(true) + ep.SetSerialization(&serialize.SimpleSerialization{}) + assert.Equal(t, false, ep.IsAvailable()) + assert.Nil(t, ep.keepaliveRunning.Load()) + + ep.Initialize() + assert.Equal(t, true, ep.IsAvailable()) + assert.Equal(t, false, ep.keepaliveRunning.Load()) +} + func TestGetV1Name(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url := &motan.URL{Port: 8989, Protocol: "motan"} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} @@ -32,7 +51,7 @@ func TestGetV1Name(t *testing.T) { } func TestV1RecordErrEmptyThreshold(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url := &motan.URL{Port: 8989, Protocol: "motan"} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ClientConnectionKey, "1") url.PutParam(motan.AsyncInitConnection, "false") @@ -54,9 +73,10 @@ func TestV1RecordErrEmptyThreshold(t *testing.T) { } func TestV1RecordErrWithErrThreshold(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} - url.PutParam(motan.TimeOutKey, "100") - url.PutParam(motan.ErrorCountThresholdKey, "5") + errorCountThreshold := 5 + url := &motan.URL{Port: 8989, Protocol: "motan"} + url.PutParam(motan.TimeOutKey, "1100") + url.PutParam(motan.ErrorCountThresholdKey, strconv.Itoa(errorCountThreshold)) url.PutParam(motan.ClientConnectionKey, "1") url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} @@ -65,14 +85,25 @@ func TestV1RecordErrWithErrThreshold(t *testing.T) { ep.SetSerialization(&serialize.SimpleSerialization{}) ep.Initialize() assert.Equal(t, 1, ep.clientConnection) - for j := 0; j < 10; j++ { + for j := 0; j < errorCountThreshold; j++ { request := &motan.MotanRequest{ServiceName: "test", Method: "test"} request.Attachment = motan.NewStringMap(0) - ep.Call(request) - if j < 4 { + request.Attachment.Store("exception", "other_exception") + request.Attachment.Store("sleep_time", "200") + resp := ep.Call(request) + assert.NotNil(t, resp) + assert.NotNil(t, resp.GetException()) + assert.Equal(t, motan.EUnkonwnMsg, resp.GetException().ErrCode) + if j < errorCountThreshold-1 { assert.True(t, ep.IsAvailable()) } else { - assert.False(t, ep.IsAvailable()) + assert.Equal(t, KeepaliveHeartbeat, atomic.LoadUint32(&ep.keepaliveType)) + // wait keepalive goroutine set keepaliveRunning status + time.Sleep(time.Millisecond * 100) + assert.True(t, ep.keepaliveRunning.Load().(bool)) + time.Sleep(ep.keepaliveInterval * 2) + assert.True(t, ep.IsAvailable()) + assert.False(t, ep.keepaliveRunning.Load().(bool)) } } <-ep.channels.channels @@ -83,35 +114,57 @@ func TestV1RecordErrWithErrThreshold(t *testing.T) { time.Sleep(time.Second * 2) assertChanelStreamEmpty(ep, t) - //assert.True(t, ep.IsAvailable()) ep.Destroy() } -func TestMotanCommonEndpoint_SuccessCall(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} - url.PutParam(motan.TimeOutKey, "2000") - url.PutParam(motan.ErrorCountThresholdKey, "1") - url.PutParam(motan.ClientConnectionKey, "1") +func TestNotFoundProviderCircuitBreaker(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan"} + url.PutParam(motan.TimeOutKey, "1200") + url.PutParam(motan.ErrorCountThresholdKey, "5") + url.PutParam(motan.ClientConnectionKey, "10") url.PutParam(motan.AsyncInitConnection, "false") ep := &MotanCommonEndpoint{} ep.SetURL(url) ep.SetSerialization(&serialize.SimpleSerialization{}) ep.Initialize() - assert.Equal(t, 1, ep.clientConnection) - request := &motan.MotanRequest{ServiceName: "test", Method: "test"} - request.Attachment = motan.NewStringMap(0) - res := ep.Call(request) - assert.Nil(t, res.GetException()) - v := res.GetValue() - s, ok := v.(string) + assert.Equal(t, 10, ep.clientConnection) + for j := 0; j < 10; j++ { + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} + request.Attachment = motan.NewStringMap(0) + request.Attachment.Store("exception", "not_found_provider") + res := ep.Call(request) + assert.Equal(t, motan.ProviderNotExistPrefix, res.GetException().ErrMsg) + if j < 4 { + assert.True(t, ep.IsAvailable()) + } else { + assert.False(t, ep.IsAvailable()) + assert.Equal(t, KeepaliveProfile, atomic.LoadUint32(&ep.keepaliveType)) + } + } + + // runtime info + info := ep.GetRuntimeInfo() + name, ok := info[motan.RuntimeNameKey] assert.True(t, ok) - assert.Equal(t, s, "hello") + assert.Equal(t, ep.GetName(), name) + + errorCount, ok := info[motan.RuntimeErrorCountKey] + assert.True(t, ok) + assert.Equal(t, uint32(10), errorCount.(uint32)) + + keepaliveRunning, ok := info[motan.RuntimeKeepaliveRunningKey] + assert.True(t, ok) + assert.Equal(t, ep.keepaliveRunning.Load(), keepaliveRunning) + + keepaliveType, ok := info[motan.RuntimeKeepaliveTypeKey] + assert.Equal(t, ep.keepaliveType, keepaliveType) assertChanelStreamEmpty(ep, t) + ep.Destroy() } -func TestMotanCommonEndpoint_AsyncCall(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} +func TestMotanCommonEndpoint_SuccessCall(t *testing.T) { + url := &motan.URL{Port: 8989, Protocol: "motan"} url.PutParam(motan.TimeOutKey, "2000") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") @@ -121,20 +174,20 @@ func TestMotanCommonEndpoint_AsyncCall(t *testing.T) { ep.SetSerialization(&serialize.SimpleSerialization{}) ep.Initialize() assert.Equal(t, 1, ep.clientConnection) - var resStr string - request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} + request := &motan.MotanRequest{ServiceName: "test", Method: "test"} request.Attachment = motan.NewStringMap(0) res := ep.Call(request) assert.Nil(t, res.GetException()) - resp := <-request.GetRPCContext(false).Result.Done - assert.Nil(t, resp.Error) - assert.Equal(t, resStr, "hello") + v := res.GetValue() + s, ok := v.(string) + assert.True(t, ok) + assert.Equal(t, s, "hello") assertChanelStreamEmpty(ep, t) } func TestMotanCommonEndpoint_ErrorCall(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url := &motan.URL{Port: 8989, Protocol: "motan"} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") @@ -150,6 +203,8 @@ func TestMotanCommonEndpoint_ErrorCall(t *testing.T) { res := ep.Call(request) fmt.Println(res.GetException().ErrMsg) assert.False(t, ep.IsAvailable()) + assert.Equal(t, KeepaliveHeartbeat, atomic.LoadUint32(&ep.keepaliveType)) + time.Sleep(1 * time.Millisecond) beforeNGoroutine := runtime.NumGoroutine() ep.Call(request) @@ -161,7 +216,7 @@ func TestMotanCommonEndpoint_ErrorCall(t *testing.T) { } func TestMotanCommonEndpoint_RequestTimeout(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} + url := &motan.URL{Port: 8989, Protocol: "motan"} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") @@ -189,7 +244,7 @@ func TestMotanCommonEndpoint_RequestTimeout(t *testing.T) { } func TestV1LazyInit(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible", Parameters: map[string]string{"lazyInit": "true"}} + url := &motan.URL{Port: 8989, Protocol: "motan", Parameters: map[string]string{"lazyInit": "true"}} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") @@ -216,7 +271,7 @@ func TestV1LazyInit(t *testing.T) { } func TestV1AsyncInit(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible", Parameters: map[string]string{"asyncInitConnection": "true"}} + url := &motan.URL{Port: 8989, Protocol: "motan", Parameters: map[string]string{"asyncInitConnection": "true"}} url.PutParam(motan.TimeOutKey, "100") url.PutParam(motan.ErrorCountThresholdKey, "1") url.PutParam(motan.ClientConnectionKey, "1") @@ -228,42 +283,6 @@ func TestV1AsyncInit(t *testing.T) { time.Sleep(time.Second * 5) } -// TestMotanCommonEndpoint_AsyncCallNoResponse verify V2Channel streams memory leak when server not reply response -// TODO:: bugs to be fixed -func TestMotanCommonEndpoint_AsyncCallNoResponse(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motanV1Compatible"} - url.PutParam(motan.TimeOutKey, "2000") - url.PutParam(motan.ErrorCountThresholdKey, "1") - url.PutParam(motan.ClientConnectionKey, "1") - url.PutParam(motan.AsyncInitConnection, "false") - ep := &MotanCommonEndpoint{} - ep.SetURL(url) - ep.SetSerialization(&serialize.SimpleSerialization{}) - ep.Initialize() - assert.Equal(t, 1, ep.clientConnection) - var resStr string - request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} - request.Attachment = motan.NewStringMap(0) - // server not reply - request.SetAttachment("no_response", "true") - - res := ep.Call(request) - assert.Nil(t, res.GetException()) - timeoutTimer := time.NewTimer(time.Second * 3) - defer timeoutTimer.Stop() - select { - case <-request.GetRPCContext(false).Result.Done: - t.Errorf("unexpect condition, recv response singnal") - case <-timeoutTimer.C: - t.Logf("expect condition, not recv response singnal") - } - - // Channel.streams can`t release stream - c := <-ep.channels.getChannels() - // it will be zero if server not reply response, bug to be fixed - assert.Equal(t, 1, len(c.streams)) -} - func TestStreamPool(t *testing.T) { var oldStream *Stream // consume stream poll until call New func @@ -315,9 +334,11 @@ func assertChanelStreamEmpty(ep *MotanCommonEndpoint, t *testing.T) { assert.Equal(t, 0, len(c.streams)) c.streamLock.Unlock() - c.heartbeatLock.Lock() - assert.Equal(t, 0, len(c.heartbeats)) - c.heartbeatLock.Unlock() + if v, ok := ep.keepaliveRunning.Load().(bool); ok && !v { + c.heartbeatLock.Lock() + assert.Equal(t, 0, len(c.heartbeats)) + c.heartbeatLock.Unlock() + } } default: return diff --git a/endpoint/motanEndpoint.go b/endpoint/motanEndpoint.go index 69fb0cf3..d1494645 100644 --- a/endpoint/motanEndpoint.go +++ b/endpoint/motanEndpoint.go @@ -29,8 +29,6 @@ var ( ErrRecvRequestTimeout = fmt.Errorf("timeout err: receive request timeout") ErrUnsupportedMessage = fmt.Errorf("stream notify : Unsupported message") - defaultAsyncResponse = &motan.MotanResponse{Attachment: motan.NewStringMap(motan.DefaultAttachmentSize), RPCContext: &motan.RPCContext{AsyncCall: true}} - errPanic = errors.New("panic error") v2StreamPool = sync.Pool{New: func() interface{} { return &V2Stream{ @@ -39,6 +37,7 @@ var ( }} ) +// Deprecated: Use MotanCommonEndpoint instead. type MotanEndpoint struct { url *motan.URL lock sync.Mutex @@ -63,6 +62,12 @@ type MotanEndpoint struct { serialization motan.Serialization } +func (m *MotanEndpoint) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: m.GetName(), + } +} + func (m *MotanEndpoint) setAvailable(available bool) { m.available.Store(available) } @@ -146,9 +151,6 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { m.recordErrAndKeepalive() return m.defaultErrMotanResponse(request, "motanEndpoint error: channels is null") } - if rc.AsyncCall { - rc.Result.StartTime = time.Now().UnixNano() - } // get a channel channel, err := m.channels.Get() if err != nil { @@ -183,9 +185,6 @@ func (m *MotanEndpoint) Call(request motan.Request) motan.Response { m.recordErrAndKeepalive() return m.defaultErrMotanResponse(request, "channel call error:"+err.Error()) } - if rc.AsyncCall { - return defaultAsyncResponse - } recvMsg.Header.SetProxy(m.proxy) recvMsg.Header.RequestID = request.GetRequestID() response, err := mpro.ConvertToResponse(recvMsg, m.serialization) @@ -225,7 +224,7 @@ func (m *MotanEndpoint) initChannelPoolWithRetry(factory ConnFactory, config *Ch for { select { case <-ticker.C: - channels, err := NewV2ChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) + channels, err = NewV2ChannelPool(m.clientConnection, factory, config, m.serialization, m.lazyInit) if err == nil { m.channels = channels m.setAvailable(true) @@ -419,17 +418,12 @@ func (s *V2Stream) Send() error { ready := sendReady{message: s.sendMsg} select { case s.channel.sendCh <- ready: - // read/write data race if s.rc != nil { sendTime := time.Now() s.rc.RequestSendTime = sendTime if s.rc.Tc != nil { s.rc.Tc.PutReqSpan(&motan.Span{Name: motan.Send, Addr: s.channel.address, Time: sendTime}) } - if s.rc.AsyncCall { - // only return send success, it can release in V2Stream.notify - s.canRelease.Store(true) - } } return nil case <-s.timer.C: @@ -479,30 +473,6 @@ func (s *V2Stream) notify(msg *mpro.Message, t time.Time) { s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Receive, Addr: s.channel.address, Time: t}) s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Decode, Time: time.Now()}) } - if s.rc.AsyncCall { - defer func() { - s.RemoveFromChannel() - releaseV2Stream(s) - }() - msg.Header.SetProxy(s.rc.Proxy) - result := s.rc.Result - response, err := mpro.ConvertToResponse(msg, s.channel.serialization) - if err != nil { - vlog.Errorf("convert to response fail. ep: %s, requestid:%d, err:%s", s.channel.address, msg.Header.RequestID, err.Error()) - result.Error = err - result.Done <- result - return - } - if err = response.ProcessDeserializable(result.Reply); err != nil { - result.Error = err - } - response.SetProcessTime((time.Now().UnixNano() - result.StartTime) / 1000000) - if s.rc.Tc != nil { - s.rc.Tc.PutResSpan(&motan.Span{Name: motan.Convert, Addr: s.channel.address, Time: time.Now()}) - } - result.Done <- result - return - } } s.recvMsg = msg @@ -543,11 +513,7 @@ func (c *V2Channel) newStream(msg *mpro.Message, rc *motan.RPCContext) (*V2Strea c.streamLock.Unlock() s.isHeartBeat = false } - if rc != nil && rc.AsyncCall { // release by Stream.Notify - s.canRelease.Store(false) - } else { // release by Channel self - s.canRelease.Store(true) - } + s.canRelease.Store(true) return s, nil } @@ -579,19 +545,12 @@ func (c *V2Channel) Call(msg *mpro.Message, deadline time.Duration, rc *motan.RP if err != nil { return nil, err } - defer func() { - if rc == nil || !rc.AsyncCall { - releaseV2Stream(stream) - } - }() + defer releaseV2Stream(stream) stream.SetDeadline(deadline) if err = stream.Send(); err != nil { return nil, err } - if rc != nil && rc.AsyncCall { - return nil, nil - } return stream.Recv() } diff --git a/endpoint/motanEndpoint_test.go b/endpoint/motanEndpoint_test.go index acb98912..545a0d88 100644 --- a/endpoint/motanEndpoint_test.go +++ b/endpoint/motanEndpoint_test.go @@ -120,30 +120,6 @@ func TestMotanEndpoint_SuccessCall(t *testing.T) { ep.Destroy() } -func TestMotanEndpoint_AsyncCall(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motan2"} - url.PutParam(motan.TimeOutKey, "2000") - url.PutParam(motan.ErrorCountThresholdKey, "1") - url.PutParam(motan.ClientConnectionKey, "1") - url.PutParam(motan.AsyncInitConnection, "false") - ep := &MotanEndpoint{} - ep.SetURL(url) - ep.SetSerialization(&serialize.SimpleSerialization{}) - ep.Initialize() - assert.Equal(t, 1, ep.clientConnection) - var resStr string - request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} - request.Attachment = motan.NewStringMap(0) - res := ep.Call(request) - assert.Nil(t, res.GetException()) - resp := <-request.GetRPCContext(false).Result.Done - assert.Nil(t, resp.Error) - assert.Equal(t, resStr, "hello") - - assertV2ChanelStreamEmpty(ep, t) - ep.Destroy() -} - func TestMotanEndpoint_ErrorCall(t *testing.T) { url := &motan.URL{Port: 8989, Protocol: "motan2"} url.PutParam(motan.TimeOutKey, "100") @@ -241,43 +217,6 @@ func TestAsyncInit(t *testing.T) { time.Sleep(time.Second * 5) } -// TestMotanEndpoint_AsyncCallNoResponse verify V2Channel streams memory leak when server not reply response -// bugs to be fixed -func TestMotanEndpoint_AsyncCallNoResponse(t *testing.T) { - url := &motan.URL{Port: 8989, Protocol: "motan2"} - url.PutParam(motan.TimeOutKey, "2000") - url.PutParam(motan.ErrorCountThresholdKey, "1") - url.PutParam(motan.ClientConnectionKey, "1") - url.PutParam(motan.AsyncInitConnection, "false") - ep := &MotanEndpoint{} - ep.SetURL(url) - ep.SetSerialization(&serialize.SimpleSerialization{}) - ep.Initialize() - assert.Equal(t, 1, ep.clientConnection) - var resStr string - request := &motan.MotanRequest{ServiceName: "test", Method: "test", RPCContext: &motan.RPCContext{AsyncCall: true, Result: &motan.AsyncResult{Reply: &resStr, Done: make(chan *motan.AsyncResult, 5)}}} - request.Attachment = motan.NewStringMap(0) - - // server not reply - request.SetAttachment("no_response", "true") - - res := ep.Call(request) - assert.Nil(t, res.GetException()) - timeoutTimer := time.NewTimer(time.Second * 3) - defer timeoutTimer.Stop() - select { - case <-request.GetRPCContext(false).Result.Done: - t.Errorf("unexpect condition, recv response singnal") - case <-timeoutTimer.C: - t.Logf("expect condition, not recv response singnal") - } - - // Channel.streams cant`t release stream - c := <-ep.channels.getChannels() - // it will be zero if server not reply response, bug to be fixed - assert.Equal(t, 1, len(c.streams)) -} - func TestV2StreamPool(t *testing.T) { var oldStream *V2Stream // consume v2stream poll until call New func @@ -380,15 +319,17 @@ func handle(netListen net.Listener) { } func handleConnection(conn net.Conn, timeout int) { - reader := bufio.NewReader(conn) - decodeBuf := make([]byte, 100) - msg, err := protocol.Decode(reader, &decodeBuf) - if err != nil { - time.Sleep(time.Millisecond * 1000) - conn.Close() - return + for { + reader := bufio.NewReader(conn) + decodeBuf := make([]byte, 100) + msg, err := protocol.Decode(reader, &decodeBuf) + if err != nil { + time.Sleep(time.Millisecond * 1000) + conn.Close() + return + } + processMsg(msg, conn) } - processMsg(msg, conn) } func processMsg(msg *protocol.Message, conn net.Conn) { @@ -403,16 +344,40 @@ func processMsg(msg *protocol.Message, conn net.Conn) { if msg.Header.IsHeartbeat() { res = protocol.BuildHeartbeat(msg.Header.RequestID, protocol.Res) } else { - time.Sleep(time.Millisecond * 1000) - serialization := &serialize.SimpleSerialization{} - resp := &motan.MotanResponse{ - RequestID: lastRequestID, - Value: "hello", - ProcessTime: 1000, + sleepTimeStr, _ := msg.Metadata.Load("sleep_time") + sleepTimeValue, _ := strconv.Atoi(sleepTimeStr) + if sleepTimeValue <= 0 { + sleepTimeValue = 1000 } + time.Sleep(time.Millisecond * time.Duration(sleepTimeValue)) + var resp *motan.MotanResponse + e, _ := msg.Metadata.Load("exception") + switch e { + case "not_found_provider": + resp = motan.BuildExceptionResponse(lastRequestID, + &motan.Exception{ + ErrCode: motan.EProviderNotExist, + ErrMsg: motan.ProviderNotExistPrefix, + ErrType: motan.ServiceException}) + case "other_exception": + resp = motan.BuildExceptionResponse(lastRequestID, + &motan.Exception{ + ErrCode: motan.EUnkonwnMsg, + ErrMsg: "exception", + ErrType: motan.ServiceException}) + default: + resp = &motan.MotanResponse{ + RequestID: lastRequestID, + Value: "hello", + ProcessTime: 1000, + } + } + serialization := &serialize.SimpleSerialization{} + res, err = protocol.ConvertToResMessage(resp, serialization) if err != nil { conn.Close() + return } } res.Header.RequestID = lastRequestID diff --git a/filter/accessLog.go b/filter/accessLog.go index ee980087..2b9cc0c6 100644 --- a/filter/accessLog.go +++ b/filter/accessLog.go @@ -18,6 +18,10 @@ type AccessLogFilter struct { next motan.EndPointFilter } +func (t *AccessLogFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(t) +} + func (t *AccessLogFilter) GetIndex() int { return 1 } diff --git a/filter/circuitBreaker.go b/filter/circuitBreaker.go index 0c986de1..9a379a4d 100644 --- a/filter/circuitBreaker.go +++ b/filter/circuitBreaker.go @@ -24,6 +24,10 @@ type CircuitBreakerFilter struct { includeBizException bool } +func (c *CircuitBreakerFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(c) +} + func (c *CircuitBreakerFilter) GetIndex() int { return 20 } diff --git a/filter/circuitBreaker_test.go b/filter/circuitBreaker_test.go index 687d2a3f..2b929882 100644 --- a/filter/circuitBreaker_test.go +++ b/filter/circuitBreaker_test.go @@ -189,6 +189,10 @@ func TestMockException(t *testing.T) { type mockEndPointFilter struct{} +func (m *mockEndPointFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(m) +} + func (m *mockEndPointFilter) GetName() string { return "mockEndPointFilter" } @@ -226,6 +230,10 @@ func (m *mockEndPointFilter) GetType() int32 { type mockExceptionEPFilter struct{} +func (m *mockExceptionEPFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(m) +} + func (m *mockExceptionEPFilter) GetName() string { return "mockEndPointFilter" } diff --git a/filter/clusterAccessLog.go b/filter/clusterAccessLog.go index b4f359d5..af0328c3 100644 --- a/filter/clusterAccessLog.go +++ b/filter/clusterAccessLog.go @@ -10,6 +10,10 @@ type ClusterAccessLogFilter struct { next motan.ClusterFilter } +func (t *ClusterAccessLogFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(t) +} + func (t *ClusterAccessLogFilter) GetIndex() int { return 1 } diff --git a/filter/clusterCircuitBreaker.go b/filter/clusterCircuitBreaker.go index 78fe169c..b42c78fd 100644 --- a/filter/clusterCircuitBreaker.go +++ b/filter/clusterCircuitBreaker.go @@ -11,6 +11,10 @@ type ClusterCircuitBreakerFilter struct { includeBizException bool } +func (c *ClusterCircuitBreakerFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(c) +} + func (c *ClusterCircuitBreakerFilter) GetIndex() int { return 20 } diff --git a/filter/clusterCircuitBreaker_test.go b/filter/clusterCircuitBreaker_test.go index 6f370018..ce18cf44 100644 --- a/filter/clusterCircuitBreaker_test.go +++ b/filter/clusterCircuitBreaker_test.go @@ -81,6 +81,10 @@ func TestClusterCircuitBreakerOther(t *testing.T) { type mockClusterFilter struct{} +func (c *mockClusterFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(c) +} + func (c *mockClusterFilter) GetIndex() int { return 101 } diff --git a/filter/clusterMetrics.go b/filter/clusterMetrics.go index 6aa2d840..52f9a35e 100644 --- a/filter/clusterMetrics.go +++ b/filter/clusterMetrics.go @@ -11,6 +11,10 @@ type ClusterMetricsFilter struct { next motan.ClusterFilter } +func (c *ClusterMetricsFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(c) +} + func (c *ClusterMetricsFilter) GetIndex() int { return 5 } diff --git a/filter/failfast.go b/filter/failfast.go index 7e03ecfe..e0713074 100644 --- a/filter/failfast.go +++ b/filter/failfast.go @@ -20,6 +20,10 @@ type FailfastFilter struct { eps *endpointStatus } +func (e *FailfastFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(e) +} + type endpointStatus struct { available bool availMutex sync.RWMutex diff --git a/filter/filter.go b/filter/filter.go index 38552ac8..6d9340b1 100644 --- a/filter/filter.go +++ b/filter/filter.go @@ -71,3 +71,11 @@ func getFilterStartTime(caller motan.Caller, request motan.Request) time.Time { return time.Now() } } + +func GetFilterRuntimeInfo(filter motan.Filter) map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: filter.GetName(), + motan.RuntimeIndexKey: filter.GetIndex(), + motan.RuntimeTypeKey: filter.GetType(), + } +} diff --git a/filter/filter_test.go b/filter/filter_test.go new file mode 100644 index 00000000..d3544c09 --- /dev/null +++ b/filter/filter_test.go @@ -0,0 +1,42 @@ +package filter + +import ( + "github.com/stretchr/testify/assert" + "github.com/weibocom/motan-go/core" + "testing" +) + +func TestGetFilterRuntimeInfo(t *testing.T) { + ext := &core.DefaultExtensionFactory{} + ext.Initialize() + RegistDefaultFilters(ext) + + for _, name := range []string{ + AccessLog, + Metrics, + CircuitBreaker, + FailFast, + Trace, + RateLimit, + ClusterAccessLog, + ClusterMetrics, + ClusterCircuitBreaker, + } { + filter := ext.GetFilter(name) + assert.NotNil(t, filter) + info := filter.GetRuntimeInfo() + assert.NotNil(t, info) + + filterName, ok := info[core.RuntimeNameKey] + assert.True(t, ok) + assert.Equal(t, name, filterName) + + index, ok := info[core.RuntimeIndexKey] + assert.True(t, ok) + assert.Equal(t, filter.GetIndex(), index) + + filterType, ok := info[core.RuntimeTypeKey] + assert.True(t, ok) + assert.Equal(t, filter.GetType(), filterType) + } +} diff --git a/filter/metrics.go b/filter/metrics.go index da34c3f4..0770d88a 100644 --- a/filter/metrics.go +++ b/filter/metrics.go @@ -26,6 +26,10 @@ type MetricsFilter struct { next motan.EndPointFilter } +func (m *MetricsFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(m) +} + func (m *MetricsFilter) GetIndex() int { return 2 } diff --git a/filter/rateLimit.go b/filter/rateLimit.go index e98692b1..4651e3bc 100644 --- a/filter/rateLimit.go +++ b/filter/rateLimit.go @@ -29,6 +29,10 @@ type RateLimitFilter struct { url *core.URL } +func (r *RateLimitFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(r) +} + func (r *RateLimitFilter) NewFilter(url *core.URL) core.Filter { ret := &RateLimitFilter{} //init bucket diff --git a/filter/tracing.go b/filter/tracing.go index aa855087..a9014f77 100644 --- a/filter/tracing.go +++ b/filter/tracing.go @@ -119,6 +119,10 @@ type TracingFilter struct { next core.EndPointFilter } +func (t *TracingFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(t) +} + func (t *TracingFilter) SetNext(nextFilter core.EndPointFilter) { t.next = nextFilter } diff --git a/filter/tracing_test.go b/filter/tracing_test.go index 1091dc7b..bb3b6a4d 100644 --- a/filter/tracing_test.go +++ b/filter/tracing_test.go @@ -380,6 +380,10 @@ type MockFilter struct { filter func(caller core.Caller, request core.Request) core.Response } +func (f *MockFilter) GetRuntimeInfo() map[string]interface{} { + return GetFilterRuntimeInfo(f) +} + func (*MockFilter) SetNext(nextFilter core.EndPointFilter) { panic("implement me") } @@ -421,6 +425,10 @@ type Provider struct { url *core.URL } +func (p *Provider) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (p *Provider) SetService(s interface{}) { if f, ok := s.(func(request core.Request) core.Response); ok { p.handler = f @@ -459,6 +467,10 @@ type Referrer struct { call func(caller core.Caller, request core.Request) core.Response } +func (r *Referrer) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{} +} + func (r *Referrer) GetName() string { return r.name } diff --git a/go.mod b/go.mod index e7623234..c2ab501c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/weibocom/motan-go -go 1.11 +go 1.16 require ( github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 @@ -16,15 +16,16 @@ require ( github.com/mitchellh/mapstructure v1.1.2 github.com/opentracing/opentracing-go v1.0.2 github.com/panjf2000/ants/v2 v2.9.0 + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec github.com/shirou/gopsutil/v3 v3.21.9 github.com/smartystreets/goconvey v1.6.4 // indirect - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/valyala/fasthttp v1.2.0 github.com/weibreeze/breeze-go v0.1.1 - go.uber.org/atomic v1.4.0 // indirect + go.uber.org/atomic v1.4.0 go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.10.0 golang.org/x/net v0.0.0-20201224014010-6772e930b67b diff --git a/ha/backupRequestHA.go b/ha/backupRequestHA.go index 8ee949d9..d95fe9dd 100644 --- a/ha/backupRequestHA.go +++ b/ha/backupRequestHA.go @@ -97,10 +97,9 @@ func (br *BackupRequestHA) Call(request motan.Request, loadBalance motan.LoadBal break } // log & clone backup request - pr := request + pr := request.Clone().(motan.Request) if i > 0 { vlog.Infof("[backup request ha] delay %d request id: %d, service: %s, method: %s", delay, request.GetRequestID(), request.GetServiceName(), request.GetMethod()) - pr = request.Clone().(motan.Request) } lastErrorCh = make(chan motan.Response, 1) go func(postRequest motan.Request, endpoint motan.EndPoint, errorCh chan motan.Response) { diff --git a/http/httpProxy.go b/http/httpProxy.go index 33738633..c95cde60 100644 --- a/http/httpProxy.go +++ b/http/httpProxy.go @@ -552,6 +552,7 @@ func MotanRequestToFasthttpRequest(motanRequest core.Request, fasthttpRequest *f } // FasthttpResponseToMotanResponse convert a http response to a motan response +// Notice:: Do not release HttpResponse early using this method // For http mesh server side, the httpServer response to the server agent but client need a motan response // Contrast to request convert, we put all headers to meta, an body maybe just use it with type []byte func FasthttpResponseToMotanResponse(motanResponse core.Response, fasthttpResponse *fasthttp.Response) { diff --git a/http/httpRpc_test.go b/http/httpRpc_test.go index bf33aaf2..4c99b6b2 100644 --- a/http/httpRpc_test.go +++ b/http/httpRpc_test.go @@ -57,6 +57,7 @@ func (s *APITestSuite) SetupTest() { serverConfig, _ := config.NewConfigFromReader(bytes.NewReader([]byte(serverConf))) go serverAgent.StartMotanAgentFromConfig(serverConfig) time.Sleep(time.Second * 3) + motancore.SetMport(0) } func TestAPITestSuite(t *testing.T) { diff --git a/lb/lb.go b/lb/lb.go index 062c5333..48a59161 100644 --- a/lb/lb.go +++ b/lb/lb.go @@ -16,6 +16,7 @@ const ( Random = "random" Roundrobin = "roundrobin" ConsistentHashKey = "consistentHashKey" + WeightRoundRobin = "wrr" ) const ( @@ -39,6 +40,9 @@ func RegistDefaultLb(extFactory motan.ExtensionFactory) { extFactory.RegistExtLb(ConsistentHashKey, NewWeightLbFunc(func(url *motan.URL) motan.LoadBalance { return &ConsistentHashLB{url: url} })) + extFactory.RegistExtLb(WeightRoundRobin, NewWeightLbFunc(func(url *motan.URL) motan.LoadBalance { + return NewWeightRondRobinLb(url) + })) } // WeightedLbWrapper support multi group weighted LB @@ -55,6 +59,25 @@ func NewWeightLbFunc(newLb motan.NewLbFunc) motan.NewLbFunc { } } +func (w *WeightedLbWrapper) Destroy() { + destroyInnerRefers(w.refers) +} + +func destroyInnerRefers(refers innerRefers) { + if v, ok := refers.(*singleGroupRefers); ok { + if vlb, ok := v.lb.(motan.Destroyable); ok { + vlb.Destroy() + } + } + if v, ok := refers.(*weightedRefers); ok { + for _, lb := range v.groupLb { + if vlb, ok := lb.(motan.Destroyable); ok { + vlb.Destroy() + } + } + } +} + func (w *WeightedLbWrapper) OnRefresh(endpoints []motan.EndPoint) { if w.weightString == "" { //not weighted lb vlog.Infof("WeightedLbWrapper: %s - OnRefresh:not have weight", w.url.GetIdentity()) @@ -130,7 +153,9 @@ func (w *WeightedLbWrapper) OnRefresh(endpoints []motan.EndPoint) { } wr.weightRing = motan.SliceShuffle(ring) wr.ringSize = len(wr.weightRing) + oldRefers := w.refers w.refers = wr + destroyInnerRefers(oldRefers) vlog.Infof("WeightedLbWrapper: %s - OnRefresh: weight:%s", w.url.GetIdentity(), w.weightString) } @@ -140,7 +165,9 @@ func (w *WeightedLbWrapper) onRefreshSingleGroup(endpoints []motan.EndPoint) { } else { lb := w.newLb(w.url) lb.OnRefresh(endpoints) + oldRefers := w.refers w.refers = &singleGroupRefers{lb: lb} + destroyInnerRefers(oldRefers) } } diff --git a/lb/weightRoundRobinLb.go b/lb/weightRoundRobinLb.go new file mode 100644 index 00000000..9558344c --- /dev/null +++ b/lb/weightRoundRobinLb.go @@ -0,0 +1,308 @@ +package lb + +import ( + motan "github.com/weibocom/motan-go/core" + vlog "github.com/weibocom/motan-go/log" + "math/rand" + "sync" + "sync/atomic" +) + +type WeightRoundRobinLB struct { + selector Selector + mutex sync.RWMutex + refresher *WeightedEpRefresher +} + +func NewWeightRondRobinLb(url *motan.URL) *WeightRoundRobinLB { + lb := &WeightRoundRobinLB{} + lb.refresher = NewWeightEpRefresher(url, lb) + return lb +} + +func (r *WeightRoundRobinLB) OnRefresh(endpoints []motan.EndPoint) { + r.refresher.RefreshWeightedHolders(endpoints) +} + +func (r *WeightRoundRobinLB) Select(request motan.Request) motan.EndPoint { + selector := r.getSelector() + if selector == nil { + return nil + } + return selector.(Selector).DoSelect(request) +} + +func (r *WeightRoundRobinLB) SelectArray(request motan.Request) []motan.EndPoint { + // cannot use select array, return nil + return nil +} + +func (r *WeightRoundRobinLB) SetWeight(weight string) {} + +func (r *WeightRoundRobinLB) Destroy() { + r.refresher.Destroy() +} + +func (r *WeightRoundRobinLB) getSelector() Selector { + r.mutex.RLock() + defer r.mutex.RUnlock() + return r.selector +} + +func (r *WeightRoundRobinLB) setSelector(s Selector) { + r.mutex.Lock() + defer r.mutex.Unlock() + r.selector = s +} + +func (r *WeightRoundRobinLB) NotifyWeightChange() { + var tempHolders []*WeightedEpHolder + h := r.refresher.weightedEpHolders.Load() + if h == nil { + // fast stop + return + } + tempHolders = h.([]*WeightedEpHolder) + weights := make([]int, len(tempHolders)) + haveSameWeight := true + totalWeight := 0 + for i := 0; i < len(tempHolders); i++ { + weights[i] = int(tempHolders[i].getWeight()) + totalWeight += weights[i] + if weights[i] != weights[0] { + haveSameWeight = false + } + } + // if all eps have the same weight, then use RoundRobinSelector + if haveSameWeight { // use RoundRobinLoadBalance + selector := r.getSelector() + if selector != nil { + if v, ok := selector.(*roundRobinSelector); ok { // reuse the RoundRobinSelector + v.refresh(tempHolders) + return + } + } + // new RoundRobinLoadBalance + r.setSelector(newRoundRobinSelector(tempHolders)) + vlog.Infoln("WeightRoundRobinLoadBalance use RoundRobinSelector. url:" + r.getURLLogInfo()) + return + } + // find the GCD and divide the weights + gcd := findGcd(weights) + if gcd > 1 { + totalWeight = 0 // recalculate totalWeight + for i := 0; i < len(weights); i++ { + weights[i] /= gcd + totalWeight += weights[i] + } + } + // Check whether it is suitable to use WeightedRingSelector + if len(weights) <= wrMaxEpSize && totalWeight <= wrMaxTotalWeight { + r.setSelector(newWeightedRingSelector(tempHolders, totalWeight, weights)) + vlog.Infoln("WeightRoundRobinLoadBalance use WeightedRingSelector. url:" + r.getURLLogInfo()) + return + } + r.setSelector(newSlidingWindowWeightedRoundRobinSelector(tempHolders, weights)) + vlog.Infoln("WeightRoundRobinLoadBalance use SlidingWindowWeightedRoundRobinSelector. url:" + r.getURLLogInfo()) +} + +func (r *WeightRoundRobinLB) getURLLogInfo() string { + url := r.refresher.url + if url == nil { + holders := r.refresher.weightedEpHolders.Load() + if v, ok := holders.([]*WeightedEpHolder); ok { + if len(v) > 0 { + url = v[0].ep.GetURL() + } + } + } + if url == nil { + return "" + } + return url.GetIdentity() +} + +type Selector interface { + DoSelect(request motan.Request) motan.EndPoint +} + +type roundRobinSelector struct { + weightHolders atomic.Value + idx int64 +} + +func newRoundRobinSelector(holders []*WeightedEpHolder) *roundRobinSelector { + res := &roundRobinSelector{} + res.weightHolders.Store(holders) + return res +} + +func (r *roundRobinSelector) DoSelect(request motan.Request) motan.EndPoint { + temp := r.weightHolders.Load() + if temp == nil { + return nil + } + tempHolders := temp.([]*WeightedEpHolder) + ep := tempHolders[int(motan.GetNonNegative(atomic.AddInt64(&r.idx, 1)))%len(tempHolders)].ep + if ep.IsAvailable() { + return ep + } + // if the ep is not available, loop section from random position + start := rand.Intn(len(tempHolders)) + for i := 0; i < len(tempHolders); i++ { + ep = tempHolders[(i+start)%len(tempHolders)].ep + if ep.IsAvailable() { + return ep + } + } + return nil +} + +func (r *roundRobinSelector) refresh(holders []*WeightedEpHolder) { + r.weightHolders.Store(holders) +} + +const ( + wrMaxEpSize = 256 + wrMaxTotalWeight = 256 * 20 +) + +type weightedRingSelector struct { + weightHolders []*WeightedEpHolder + idx int64 + weights []int + weightRing []byte +} + +func newWeightedRingSelector(holders []*WeightedEpHolder, totalWeight int, weights []int) *weightedRingSelector { + wrs := &weightedRingSelector{ + weightHolders: holders, + weightRing: make([]byte, totalWeight), + weights: weights, + } + wrs.initWeightRing() + return wrs +} + +func (r *weightedRingSelector) initWeightRing() { + ringIndex := 0 + for i := 0; i < len(r.weights); i++ { + for j := 0; j < r.weights[i]; j++ { + r.weightRing[ringIndex] = byte(i) + ringIndex++ + } + } + if ringIndex != len(r.weightRing) { + vlog.Warningf("WeightedRingSelector initWeightRing with wrong totalWeight. expect:%d, actual: %d", len(r.weightRing), ringIndex) + } + r.weightRing = motan.ByteSliceShuffle(r.weightRing) + +} + +func (r *weightedRingSelector) DoSelect(request motan.Request) motan.EndPoint { + ep := r.weightHolders[r.getHolderIndex(int(motan.GetNonNegative(atomic.AddInt64(&r.idx, 1))))].ep + if ep.IsAvailable() { + return ep + } + // If the ep is not available, loop selection from random position + start := rand.Intn(len(r.weightRing)) + for i := 0; i < len(r.weightRing); i++ { + // byte could indicate 0~255 + ep = r.weightHolders[r.getHolderIndex(start+i)].getEp() + if ep.IsAvailable() { + return ep + } + } + return nil +} + +func (r *weightedRingSelector) getHolderIndex(ringIndex int) int { + holderIndex := int(r.weightRing[ringIndex%len(r.weightRing)]) + return holderIndex +} + +const ( + swwrDefaultWindowSize = 50 +) + +type slidingWindowWeightedRoundRobinSelector struct { + idx int64 + windowSize int + items []*selectorItem +} + +func newSlidingWindowWeightedRoundRobinSelector(holders []*WeightedEpHolder, weights []int) *slidingWindowWeightedRoundRobinSelector { + windowSize := len(weights) + if windowSize > swwrDefaultWindowSize { + windowSize = swwrDefaultWindowSize + // The window size cannot be divided by the number of referers, which ensures that the starting position + // of the window will gradually change during sliding + for len(weights)%windowSize == 0 { + windowSize-- + } + } + items := make([]*selectorItem, 0, len(holders)) + for i := 0; i < len(weights); i++ { + items = append(items, newSelectorItem(holders[i].getEp(), weights[i])) + } + return &slidingWindowWeightedRoundRobinSelector{ + items: items, + windowSize: windowSize, + } +} + +func (r *slidingWindowWeightedRoundRobinSelector) DoSelect(request motan.Request) motan.EndPoint { + windowStartIndex := motan.GetNonNegative(atomic.AddInt64(&r.idx, int64(r.windowSize))) + totalWeight := 0 + var sMaxWeight int64 = 0 + maxWeightIndex := 0 + // Use SWRR(https://github.com/nginx/nginx/commit/52327e0627f49dbda1e8db695e63a4b0af4448b1) to select referer from the current window. + // In order to reduce costs, do not limit concurrency in the entire selection process, + // and only use atomic updates for the current weight. + // Since concurrent threads will execute Select in different windows, + // the problem of instantaneous requests increase on one node due to concurrency will not be serious. + // And because the referers used on different client sides are shuffled, + // the impact of high instantaneous concurrent selection on the server side will be further reduced. + for i := 0; i < r.windowSize; i++ { + idx := (int(windowStartIndex) + i) % len(r.items) + item := r.items[idx] + if item.ep.IsAvailable() { + currentWeight := atomic.AddInt64(&item.currentWeight, int64(item.weight)) + totalWeight += item.weight + if currentWeight > sMaxWeight { + sMaxWeight = currentWeight + maxWeightIndex = idx + } + } + } + if sMaxWeight > 0 { + item := r.items[maxWeightIndex] + atomic.AddInt64(&item.currentWeight, int64(-totalWeight)) + if item.ep.IsAvailable() { + return item.ep + } + } + // If no suitable node is selected or the node is unavailable, + // then select an available referer from a random index + var idx = int(windowStartIndex) + rand.Intn(r.windowSize) + for i := 1; i < len(r.items); i++ { + item := r.items[(idx+i)%len(r.items)] + if item.ep.IsAvailable() { + return item.ep + } + } + return nil +} + +type selectorItem struct { + ep motan.EndPoint + weight int + currentWeight int64 +} + +func newSelectorItem(ep motan.EndPoint, weight int) *selectorItem { + return &selectorItem{ + weight: weight, + ep: ep, + } +} diff --git a/lb/weightRoundRobinLb_test.go b/lb/weightRoundRobinLb_test.go new file mode 100644 index 00000000..64093acb --- /dev/null +++ b/lb/weightRoundRobinLb_test.go @@ -0,0 +1,358 @@ +package lb + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/endpoint" + "github.com/weibocom/motan-go/meta" + "math" + "math/rand" + "sync/atomic" + "testing" + "time" +) + +func TestDynamicStaticWeight(t *testing.T) { + url := &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 8080, + Path: "mockService", + } + url.PutParam(core.DynamicMetaKey, "false") + // test dynamic weight config + lb := NewWeightRondRobinLb(url) + assert.False(t, lb.refresher.supportDynamicWeight) + lb.Destroy() + + url.PutParam(core.DynamicMetaKey, "true") + lb = NewWeightRondRobinLb(url) + assert.True(t, lb.refresher.supportDynamicWeight) + + meta.ClearMetaCache() + var staticWeight int64 = 9 + eps := buildTestDynamicEps(10, true, staticWeight, url) + eps = core.EndpointShuffle(eps) + lb.OnRefresh(eps) + _, ok := lb.getSelector().(*roundRobinSelector) + assert.True(t, ok) + for _, j := range lb.refresher.weightedEpHolders.Load().([]*WeightedEpHolder) { + // test static weight + assert.Equal(t, j.staticWeight, staticWeight) + assert.Equal(t, j.dynamicWeight, int64(0)) + assert.Equal(t, j.getWeight(), staticWeight) + } + + // test dynamic wight change + lb.refresher.weightedEpHolders.Load().([]*WeightedEpHolder)[3].ep.(*endpoint.MockDynamicEndpoint).SetWeight(true, 22) + meta.ClearMetaCache() + time.Sleep(time.Second * 5) + //assert.Equal(t, int64(22), lb.refresher.weightedEpHolders.Load().([]*WeightedEpHolder)[3].dynamicWeight) + //assert.Equal(t, int64(22), lb.refresher.weightedEpHolders.Load().([]*WeightedEpHolder)[3].getWeight()) + _, ok = lb.getSelector().(*weightedRingSelector) + assert.True(t, ok) + // test close refresh task + lb.Destroy() + time.Sleep(time.Second * 5) + assert.True(t, lb.refresher.isDestroyed.Load()) +} + +func TestGetEpWeight(t *testing.T) { + url := &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 8080, + Path: "mockService", + } + ep := endpoint.NewMockDynamicEndpoint(url) + type test struct { + expectWeight int64 + fromDynamic bool + defaultWeight int64 + setWeight bool + setWeightDynamic bool + setWeightValue int64 + } + testSet := []test{ + // test static weight + {expectWeight: 11, fromDynamic: false, defaultWeight: 11, setWeight: false}, + {expectWeight: 5, fromDynamic: false, defaultWeight: 5, setWeight: false}, + {expectWeight: 8, fromDynamic: false, defaultWeight: 11, setWeight: true, setWeightDynamic: false, setWeightValue: 8}, + // test dynamic weight + {expectWeight: 0, fromDynamic: true, defaultWeight: 0, setWeight: false}, + {expectWeight: 5, fromDynamic: true, defaultWeight: 5, setWeight: false}, + {expectWeight: 15, fromDynamic: true, defaultWeight: 11, setWeight: true, setWeightDynamic: true, setWeightValue: 15}, + // test abnormal weight + {expectWeight: MinEpWeight, fromDynamic: false, defaultWeight: 11, setWeight: true, setWeightDynamic: false, setWeightValue: -8}, + {expectWeight: MaxEpWeight, fromDynamic: false, defaultWeight: 11, setWeight: true, setWeightDynamic: false, setWeightValue: 501}, + {expectWeight: MinEpWeight, fromDynamic: true, defaultWeight: 11, setWeight: true, setWeightDynamic: true, setWeightValue: -1}, + {expectWeight: MaxEpWeight, fromDynamic: true, defaultWeight: 11, setWeight: true, setWeightDynamic: true, setWeightValue: 666}, + } + + for _, j := range testSet { + if j.setWeight { + ep.SetWeight(j.setWeightDynamic, j.setWeightValue) + } + w, e := getEpWeight(ep, j.fromDynamic, j.defaultWeight) + assert.Nil(t, e) + assert.Equal(t, j.expectWeight, w) + meta.ClearMetaCache() + } +} + +func TestNotifyWeightChange(t *testing.T) { + type test struct { + size int + sameWeight bool + maxWeight int + selector string + } + testSet := []test{ + // test RR + {size: 20, sameWeight: true, maxWeight: 8, selector: "roundRobinSelector"}, + {size: 20, sameWeight: true, maxWeight: 0, selector: "roundRobinSelector"}, + {size: 20, sameWeight: true, maxWeight: 101, selector: "roundRobinSelector"}, + {size: 20, sameWeight: true, maxWeight: -10, selector: "roundRobinSelector"}, //abnormal weight + //// test WR + {size: 20, sameWeight: false, maxWeight: 8, selector: "weightedRingSelector"}, + {size: wrMaxEpSize, sameWeight: false, maxWeight: 8, selector: "weightedRingSelector"}, + {size: wrMaxEpSize, sameWeight: false, maxWeight: wrMaxTotalWeight / wrMaxEpSize, selector: "weightedRingSelector"}, + // test SWWRR + {size: wrMaxEpSize + 1, sameWeight: false, maxWeight: 6, selector: "slidingWindowWeightedRoundRobinSelector"}, + {size: 40, sameWeight: false, maxWeight: 800, selector: "slidingWindowWeightedRoundRobinSelector"}, + } + url := &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 8080, + Path: "mockService", + } + url.PutParam(core.DynamicMetaKey, "false") + lb := NewWeightRondRobinLb(url) + for _, j := range testSet { + eps := buildTestDynamicEps(j.size, j.sameWeight, int64(j.maxWeight), url) + eps = core.EndpointShuffle(eps) + lb.OnRefresh(eps) + var ok bool + switch j.selector { + case "roundRobinSelector": + _, ok = lb.getSelector().(*roundRobinSelector) + case "weightedRingSelector": + _, ok = lb.getSelector().(*weightedRingSelector) + case "slidingWindowWeightedRoundRobinSelector": + _, ok = lb.getSelector().(*slidingWindowWeightedRoundRobinSelector) + } + assert.True(t, ok) + meta.ClearMetaCache() + } + lb.Destroy() +} + +func TestRoundRobinSelector(t *testing.T) { + url := &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 8080, + Path: "mockService", + } + url.PutParam(core.DynamicMetaKey, "false") + lb := NewWeightRondRobinLb(url) + round := 100 + // small size + checkRR(t, lb, 20, 8, round, 1, 1, 0, url) + // large size + checkRR(t, lb, 500, 8, round, 1, 1, 0, url) + // some nodes are unavailable + maxRatio := 0.4 + avgRatio := 0.1 + round = 200 + checkRR(t, lb, 20, 8, round, float64(round)*maxRatio, float64(round)*avgRatio, 2, url) + checkRR(t, lb, 100, 8, round, float64(round)*maxRatio, float64(round)*avgRatio, 10, url) + + maxRatio = 0.7 + checkRR(t, lb, 300, 8, round, float64(round)*maxRatio, float64(round)*avgRatio, 50, url) + lb.Destroy() +} + +func checkRR(t *testing.T, lb *WeightRoundRobinLB, size int, initialMaxWeight int64, + round int, expectMaxDelta float64, expectAvgDelta float64, unavailableSize int, url *core.URL) { + eps := buildTestDynamicEpsWithUnavailable(size, true, initialMaxWeight, true, unavailableSize, url) + rand.Seed(time.Now().UnixNano()) + eps = core.EndpointShuffle(eps) + lb.OnRefresh(eps) + _, ok := lb.getSelector().(*roundRobinSelector) + assert.True(t, ok) + processCheck(t, lb, "RR", eps, round, expectMaxDelta, expectAvgDelta, unavailableSize) +} + +func TestWeightRingSelector(t *testing.T) { + url := &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 8080, + Path: "mockService", + } + url.PutParam(core.DynamicMetaKey, "false") + lb := NewWeightRondRobinLb(url) + round := 100 + // small size + checkKWR(t, lb, 51, 49, round, 1, 1, 0, url) + // max node size of WR + checkKWR(t, lb, 256, 15, round, 1, 1, 0, url) + + // same nodes are unavailable + maxRatio := 0.4 + avgRatio := 0.1 + checkKWR(t, lb, 46, 75, round, float64(round)*maxRatio, float64(round)*avgRatio, 5, url) + checkKWR(t, lb, 231, 31, round, float64(round)*maxRatio, float64(round)*avgRatio, 35, url) + maxRatio = 0.6 + //checkKWR(t, lb, 211, 31, round, float64(round)*maxRatio, float64(round)*avgRatio, 45, url) + lb.Destroy() +} + +func checkKWR(t *testing.T, lb *WeightRoundRobinLB, size int, initialMaxWeight int64, + round int, expectMaxDelta float64, expectAvgDelta float64, unavailableSize int, url *core.URL) { + eps := buildTestDynamicEpsWithUnavailable(size, false, initialMaxWeight, true, unavailableSize, url) + rand.Seed(time.Now().UnixNano()) + eps = core.EndpointShuffle(eps) + lb.OnRefresh(eps) + _, ok := lb.getSelector().(*weightedRingSelector) + assert.True(t, ok) + processCheck(t, lb, "WR", eps, round, expectMaxDelta, expectAvgDelta, unavailableSize) +} + +func TestSlidingWindowWeightedRoundRobinSelector(t *testing.T) { + url := &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 8080, + Path: "mockService", + } + url.PutParam(core.DynamicMetaKey, "false") + lb := NewWeightRondRobinLb(url) + round := 100 + // equals default window size, the accuracy is higher than sliding window + size := swwrDefaultWindowSize + checkSWWRR(t, lb, size, int64(wrMaxTotalWeight*3/size), round, 2, 1, 0, url) + // less than default window size + size = swwrDefaultWindowSize - 9 + checkSWWRR(t, lb, size, int64(wrMaxTotalWeight*3/size), round, 2, 1, 0, url) + + // greater than default window size + // sliding windows will reduce the accuracy of WRR, so the threshold should be appropriately increased + maxRatio := 0.5 + avgRatio := 0.1 + round = 200 + size = 270 + checkSWWRR(t, lb, size, 45, round, float64(round)*maxRatio, float64(round)*avgRatio, 0, url) + + // some nodes are unavailable + size = 260 + checkSWWRR(t, lb, size, int64(wrMaxTotalWeight*3/size), round, float64(round)*maxRatio, float64(round)*avgRatio, 10, url) + size = 399 + checkSWWRR(t, lb, size, 67, round, float64(round)*maxRatio, float64(round)*avgRatio, 40, url) + lb.Destroy() +} + +func checkSWWRR(t *testing.T, lb *WeightRoundRobinLB, size int, initialMaxWeight int64, + round int, expectMaxDelta float64, expectAvgDelta float64, unavailableSize int, url *core.URL) { + eps := buildTestDynamicEpsWithUnavailable(size, false, initialMaxWeight, true, unavailableSize, url) + rand.Seed(time.Now().UnixNano()) + eps = core.EndpointShuffle(eps) + lb.OnRefresh(eps) + _, ok := lb.getSelector().(*slidingWindowWeightedRoundRobinSelector) + assert.True(t, ok) + processCheck(t, lb, "SWWRR", eps, round, expectMaxDelta, expectAvgDelta, unavailableSize) +} + +func processCheck(t *testing.T, lb *WeightRoundRobinLB, typ string, eps []core.EndPoint, round int, + expectMaxDelta float64, expectAvgDelta float64, unavailableSize int) { + var totalWeight int64 = 0 + for _, ep := range eps { + if !ep.IsAvailable() { + continue + } + totalWeight += ep.(*endpoint.MockDynamicEndpoint).StaticWeight + } + for i := 0; i < int(totalWeight)*round; i++ { + lb.Select(nil).Call(nil) + } + var maxDelta float64 = 0.0 + var totalDelta float64 = 0.0 + unavailableCount := 0 + for _, ep := range eps { + if !ep.IsAvailable() { + unavailableCount++ + } else { + mep := ep.(*endpoint.MockDynamicEndpoint) + ratio := float64(atomic.LoadInt64(&mep.Count)) / float64(mep.StaticWeight) + delta := math.Abs(ratio - float64(round)) + if delta > maxDelta { + maxDelta = delta + } + totalDelta += delta + if delta > expectMaxDelta { + fmt.Printf("%s: count=%d, staticWeight=%d, ratio=%.2f, delta=%.2f\n", typ, atomic.LoadInt64(&mep.Count), mep.StaticWeight, ratio, delta) + } + assert.True(t, delta <= expectMaxDelta) // check max delta + } + } + // avg delta + avgDelta := totalDelta / float64(len(eps)-unavailableSize) + assert.True(t, avgDelta-float64(round) < expectAvgDelta) + fmt.Printf("%s: avgDeltaPercent=%.2f%%, maxDeltaPercent=%.2f%%, avgDelta=%.2f, maxDelta=%.2f\n", typ, avgDelta*float64(100)/float64(round), maxDelta*float64(100)/float64(round), avgDelta, maxDelta) + if unavailableSize > 0 { + assert.Equal(t, unavailableSize, unavailableCount) + } +} + +func buildTestDynamicEps(size int, sameStaticWeight bool, maxWeight int64, url *core.URL) []core.EndPoint { + return buildTestDynamicEpsWithUnavailable(size, sameStaticWeight, maxWeight, false, 0, url) +} + +func buildTestDynamicEpsWithUnavailable(size int, sameStaticWeight bool, maxWeight int64, adjust bool, unavailableSize int, url *core.URL) []core.EndPoint { + return buildTestEps(size, sameStaticWeight, maxWeight, adjust, unavailableSize, "", url) +} + +func buildTestEps(size int, sameStaticWeight bool, maxWeight int64, adjust bool, unavailableSize int, group string, url *core.URL) []core.EndPoint { + rand.Seed(time.Now().UnixNano()) + var res []core.EndPoint + for i := 0; i < size; i++ { + weight := maxWeight + if !sameStaticWeight { + weight = int64(rand.Float64() * float64(maxWeight)) + } + if adjust { + weight = doAdjust(weight) + } + curUrl := &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 8080 + i, + Path: "mockService", + } + if url.GetParam(core.DynamicMetaKey, "") == "false" { + curUrl.PutParam(core.DynamicMetaKey, "false") + } + ep := endpoint.NewMockDynamicEndpointWithWeight(curUrl, weight) + if i < unavailableSize { + ep.SetAvailable(false) + } + if group != "" { + ep.URL.PutParam(core.GroupKey, group) + } + res = append(res, ep) + } + return res +} + +func doAdjust(w int64) int64 { + if w < MinEpWeight { + return MinEpWeight + } else if w > MaxEpWeight { + return MaxEpWeight + } else { + return w + } +} diff --git a/lb/weightedEpRefresher.go b/lb/weightedEpRefresher.go new file mode 100644 index 00000000..157eb746 --- /dev/null +++ b/lb/weightedEpRefresher.go @@ -0,0 +1,229 @@ +package lb + +import ( + "errors" + motan "github.com/weibocom/motan-go/core" + vlog "github.com/weibocom/motan-go/log" + "github.com/weibocom/motan-go/meta" + "go.uber.org/atomic" + "golang.org/x/net/context" + "strconv" + "sync" + "time" +) + +const ( + defaultEpWeight = 10 + MinEpWeight = 1 + MaxEpWeight = 500 //protective restriction + defaultWeightRefreshPeriodSecond = 3 +) + +// WeightedEpRefresher is held by load balancer who needs dynamic endpoint weights +type WeightedEpRefresher struct { + url *motan.URL + refreshCanceler context.CancelFunc + supportDynamicWeight bool + weightedEpHolders atomic.Value + weightLB motan.WeightLoadBalance + isDestroyed atomic.Bool + mutex sync.Mutex +} + +func NewWeightEpRefresher(url *motan.URL, lb motan.WeightLoadBalance) *WeightedEpRefresher { + refreshPeriod := url.GetIntValue(motan.WeightRefreshPeriodSecondKey, defaultWeightRefreshPeriodSecond) + refreshCtx, refreshCancel := context.WithCancel(context.Background()) + wer := &WeightedEpRefresher{ + url: url, + supportDynamicWeight: url.GetBoolValue(motan.DynamicMetaKey, motan.DefaultDynamicMeta), + refreshCanceler: refreshCancel, + weightLB: lb, + } + go wer.doRefresh(refreshCtx, refreshPeriod) + return wer +} + +func (w *WeightedEpRefresher) Destroy() { + w.refreshCanceler() +} + +func (w *WeightedEpRefresher) notifyWeightChange() { + w.mutex.Lock() + defer w.mutex.Unlock() + vlog.Infoln("weight has changed, url: " + w.url.GetIdentity()) + w.weightLB.NotifyWeightChange() +} + +func (w *WeightedEpRefresher) RefreshWeightedHolders(eps []motan.EndPoint) { + var newHolder []*WeightedEpHolder + var oldHolder []*WeightedEpHolder + if t := w.weightedEpHolders.Load(); t != nil { + oldHolder = t.([]*WeightedEpHolder) + } + allHolder := make([]*WeightedEpHolder, 0, len(eps)) + for _, ep := range eps { + var holder *WeightedEpHolder + if oldHolder != nil { + // Check whether referer can be reused + for _, tempHolder := range oldHolder { + if ep == tempHolder.getEp() { + holder = tempHolder + break + } + } + } + if holder == nil { + staticWeight, _ := getEpWeight(ep, false, defaultEpWeight) + holder = BuildWeightedEpHolder(ep, staticWeight) + newHolder = append(newHolder, holder) + } + allHolder = append(allHolder, holder) + } + // only refresh new holders' dynamic weight + if len(newHolder) != 0 { + refreshDynamicWeight(newHolder, 15*1000) + } + w.weightedEpHolders.Store(allHolder) + w.notifyWeightChange() +} + +func refreshDynamicWeight(holders []*WeightedEpHolder, taskTimeout int64) bool { + needNotify := atomic.NewBool(false) + wg := sync.WaitGroup{} + wg.Add(len(holders)) + for _, h := range holders { + if h.supportDynamicWeight { + go func(holder *WeightedEpHolder) { + defer wg.Done() + oldWeight := holder.dynamicWeight + var err error + dw, err := getEpWeight(holder.getEp(), true, 0) + if err != nil { + if errors.Is(err, meta.ServiceNotSupportError) { + holder.supportDynamicWeight = false + } else { + vlog.Warningf("refresh dynamic weight fail! url:%s, error: %s\n", holder.getEp().GetURL().GetIdentity(), err.Error()) + } + return + } + holder.dynamicWeight = dw + if oldWeight != holder.dynamicWeight { + needNotify.Store(true) + } + }(h) + } else { + wg.Done() + } + } + // just wait certain amount of time + timer := time.NewTimer(time.Millisecond * time.Duration(taskTimeout)) + finishChan := make(chan struct{}) + defer close(finishChan) + go func() { + wg.Wait() + // the chan might be closed + select { + case _, ok := <-finishChan: + if !ok { + // chan has been closed + return + } + default: + finishChan <- struct{}{} + return + } + }() + select { + case <-timer.C: + case <-finishChan: + timer.Stop() + } + return needNotify.Load() +} + +func getEpWeight(ep motan.EndPoint, fromDynamicMeta bool, defaultWeight int64) (int64, error) { + var metaMap map[string]string + var err error + if fromDynamicMeta { + metaMap, err = meta.GetEpDynamicMeta(ep) + } else { + metaMap = meta.GetEpStaticMeta(ep) + } + if err != nil { + return defaultWeight, err + } + weightStr := meta.GetMetaValue(metaMap, motan.WeightMetaSuffixKey) + return adjustWeight(ep, weightStr, defaultWeight), nil +} + +func adjustWeight(ep motan.EndPoint, weight string, defaultWeight int64) int64 { + res := defaultWeight + if weight != "" { + temp, err := strconv.ParseInt(weight, 10, 64) + if err != nil { + vlog.Warningf("WeightedRefererHolder parse weight fail. %s, use default weight %d, org weight: %s, err: %s\n", ep.GetURL().GetIdentity(), defaultWeight, weight, err.Error()) + return defaultWeight + } + if temp < MinEpWeight { + temp = MinEpWeight + } else if temp > MaxEpWeight { + temp = MaxEpWeight + } + res = temp + } + return res +} + +func (w *WeightedEpRefresher) doRefresh(ctx context.Context, refreshPeriod int64) { + ticker := time.NewTicker(time.Second * time.Duration(refreshPeriod)) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + w.isDestroyed.Store(true) + return + case <-ticker.C: + if w.supportDynamicWeight { + w.refreshHolderDynamicWeightTask() + } + } + } +} + +func (w *WeightedEpRefresher) refreshHolderDynamicWeightTask() { + // The reference of holders might be changed during the loop + // Only refresh historical holders + var tempLoaders []*WeightedEpHolder + if t := w.weightedEpHolders.Load(); t != nil { + tempLoaders = t.([]*WeightedEpHolder) + } + if refreshDynamicWeight(tempLoaders, 30*1000) { + w.notifyWeightChange() + } +} + +type WeightedEpHolder struct { + ep motan.EndPoint + staticWeight int64 + supportDynamicWeight bool + dynamicWeight int64 +} + +func BuildWeightedEpHolder(ep motan.EndPoint, staticWeight int64) *WeightedEpHolder { + return &WeightedEpHolder{ + ep: ep, + staticWeight: staticWeight, + supportDynamicWeight: ep.GetURL().GetBoolValue(motan.DynamicMetaKey, motan.DefaultDynamicMeta), + } +} + +func (w *WeightedEpHolder) getWeight() int64 { + if w.dynamicWeight > 0 { + return w.dynamicWeight + } + return w.staticWeight +} + +func (w *WeightedEpHolder) getEp() motan.EndPoint { + return w.ep +} diff --git a/log/log.go b/log/log.go index a3b71c2e..34dd8670 100644 --- a/log/log.go +++ b/log/log.go @@ -373,6 +373,12 @@ func (d *defaultLogger) AccessLog(logObject *AccessLogEntity) { } func (d *defaultLogger) doAccessLog(logObject *AccessLogEntity) { + defer func() { + if err := recover(); err != nil { + log.Printf("doAccessLog failed. err:%v, logObject: %+v", err, logObject) + debug.PrintStack() + } + }() if d.accessStructured { d.accessLogger.Info("", zap.String("filterName", logObject.FilterName), diff --git a/main/agentdemo.go b/main/agentdemo.go index eef08171..17297fc8 100644 --- a/main/agentdemo.go +++ b/main/agentdemo.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/weibocom/motan-go/filter" "net/http" "net/http/httputil" @@ -48,6 +49,10 @@ type MyEndPointFilter struct { next motancore.EndPointFilter } +func (m *MyEndPointFilter) GetRuntimeInfo() map[string]interface{} { + return filter.GetFilterRuntimeInfo(m) +} + func (m *MyEndPointFilter) GetIndex() int { return 20 } diff --git a/manageHandler.go b/manageHandler.go index f8e79085..8cd84f0c 100644 --- a/manageHandler.go +++ b/manageHandler.go @@ -5,6 +5,8 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/weibocom/motan-go/meta" + mserver "github.com/weibocom/motan-go/server" "html/template" "io" "log" @@ -741,6 +743,36 @@ func setLogStatus(jsonEncoder *json.Encoder, logType, available string) { } } +type MetaInfo struct { + agent *Agent +} + +func (m *MetaInfo) SetAgent(agent *Agent) { + m.agent = agent +} + +func (m *MetaInfo) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/meta/update": + if r.Form == nil { + r.ParseMultipartForm(32 << 20) + } + for k, v := range r.Form { + if len(v) > 0 { + meta.PutDynamicMeta(k, v[0]) + } + } + JSONSuccess(w, "ok") + case "/meta/delete": + meta.RemoveDynamicMeta(r.FormValue("key")) + JSONSuccess(w, "ok") + case "/meta/get": + JSONSuccess(w, map[string]string{r.FormValue("key"): meta.GetDynamicMeta()[r.FormValue("key")]}) + case "/meta/getAll": + JSONSuccess(w, map[string]map[string]string{"meta": meta.GetDynamicMeta()}) + } +} + type HotReload struct { agent *Agent } @@ -767,6 +799,143 @@ func (h *HotReload) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } +type RuntimeHandler struct { + agent *Agent +} + +func (h *RuntimeHandler) SetAgent(agent *Agent) { + h.agent = agent +} + +func (h *RuntimeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/runtime/info": + result := map[string]interface{}{ + "result": "ok", + motan.RuntimeInstanceTypeKey: "motan-agent", + } + // cluster info + result[motan.RuntimeClustersKey] = h.getClusterInfo() + + // http cluster info + result[motan.RuntimeHttpClustersKey] = h.getHttpClusterInfo() + + // exporter info + result[motan.RuntimeExportersKey] = h.getExporterInfo() + + // extensionFactory info + result[motan.RuntimeExtensionFactoryKey] = GetDefaultExtFactory().GetRuntimeInfo() + + // servers info + result[motan.RuntimeServersKey] = h.getServersInfo() + + // basic info + result[motan.RuntimeBasicKey] = h.getBasicInfo() + + res, _ := json.Marshal(result) + w.Write(res) + } +} + +func (h *RuntimeHandler) getClusterInfo() map[string]interface{} { + info := make(map[string]interface{}, h.agent.clusterMap.Len()) + h.agent.clusterMap.Range(func(k, v interface{}) bool { + cls, ok := v.(*cluster.MotanCluster) + if !ok { + return true + } + info[k.(string)] = cls.GetRuntimeInfo() + return true + }) + return info +} + +func (h *RuntimeHandler) getHttpClusterInfo() map[string]interface{} { + info := make(map[string]interface{}, h.agent.httpClusterMap.Len()) + h.agent.httpClusterMap.Range(func(k, v interface{}) bool { + cls, ok := v.(*cluster.HTTPCluster) + if !ok { + return true + } + h.addInfos(cls.GetRuntimeInfo(), k.(string), info) + return true + }) + return info +} + +func (h *RuntimeHandler) getExporterInfo() map[string]interface{} { + info := make(map[string]interface{}, h.agent.serviceExporters.Len()) + h.agent.serviceExporters.Range(func(k, v interface{}) bool { + exporter, ok := v.(*mserver.DefaultExporter) + if !ok { + return true + } + h.addInfos(exporter.GetRuntimeInfo(), k.(string), info) + return true + }) + return info +} + +func (h *RuntimeHandler) getServersInfo() map[string]interface{} { + info := map[string]interface{}{} + h.addInfos(h.agent.agentServer.GetRuntimeInfo(), motan.RuntimeAgentServerKey, info) + agentPortServerInfo := map[string]interface{}{} + for port, server := range h.agent.agentPortServer { + agentPortServerInfo[strconv.Itoa(port)] = server.GetRuntimeInfo() + } + h.addInfos(agentPortServerInfo, motan.RuntimeAgentPortServerKey, info) + h.addInfos(h.agent.httpProxyServer.GetRuntimeInfo(), motan.RuntimeHttpProxyServerKey, info) + return info +} + +func (h *RuntimeHandler) getBasicInfo() map[string]interface{} { + info := map[string]interface{}{} + info[motan.RuntimeCpuPercentKey] = GetCpuPercent() + info[motan.RuntimeRssMemoryKey] = GetRssMemory() + return info +} + +func (h *RuntimeHandler) addInfos(info map[string]interface{}, key string, result map[string]interface{}) { + if info != nil && len(info) > 0 { + result[key] = info + } +} + +// JSONSuccess return success JSON data +func JSONSuccess(w http.ResponseWriter, data interface{}) { + JSON(w, "", "ok", data) +} + +// JSONError return Error JSON data +func JSONError(w http.ResponseWriter, msg string) { + JSON(w, msg, "fail", nil) +} + +func JSON(w http.ResponseWriter, errMsg string, result string, data interface{}) { + if errMsg != "" { + d := struct { + Result string `json:"result"` + Error string `json:"error"` + }{ + Result: result, + Error: errMsg, + } + bs, _ := json.Marshal(d) + w.Write(bs) + } else { + d := struct { + Result string `json:"result"` + Data interface{} `json:"data"` + }{ + Result: result, + Data: data, + } + bs, _ := json.Marshal(d) + w.Write(bs) + } + +} + //------------ below code is copied from net/http/pprof ------------- // Cmdline responds with the running program's diff --git a/meshClient.go b/meshClient.go index 88e93a46..147d0043 100644 --- a/meshClient.go +++ b/meshClient.go @@ -119,8 +119,8 @@ func (c *MeshClient) BaseCall(request core.Request, reply interface{}) core.Resp } httpRequest := fasthttp.AcquireRequest() httpResponse := fasthttp.AcquireResponse() + // do not release http response defer fasthttp.ReleaseRequest(httpRequest) - defer fasthttp.ReleaseResponse(httpResponse) httpRequest.Header.Del("Host") httpRequest.SetHost(request.GetServiceName()) httpRequest.URI().SetPath(request.GetMethod()) diff --git a/meta/meta.go b/meta/meta.go new file mode 100644 index 00000000..175cc640 --- /dev/null +++ b/meta/meta.go @@ -0,0 +1,193 @@ +package meta + +import ( + "errors" + "github.com/patrickmn/go-cache" + "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/endpoint" + vlog "github.com/weibocom/motan-go/log" + mpro "github.com/weibocom/motan-go/protocol" + "os" + "strconv" + "strings" + "sync" + "time" +) + +const ( + defaultCacheExpireSecond = 3 + notSupportCacheExpireSecond = 30 +) + +var ( + dynamicMeta = core.NewStringMap(30) + envMeta = make(map[string]string) + envPrefix = core.DefaultMetaPrefix + metaEmptyMap = make(map[string]string) + metaCache = cache.New(time.Second*time.Duration(defaultCacheExpireSecond), 30*time.Second) + notSupportCache = cache.New(time.Second*time.Duration(notSupportCacheExpireSecond), 30*time.Second) + ServiceNotSupportError = errors.New(core.ServiceNotSupport) + notSupportSerializer = map[string]bool{ + "protobuf": true, + "grpc-pb": true, + "grpc-pb-json": true, + } + supportProtocols = map[string]bool{ + "motan": true, + "motan2": true, + } + once = sync.Once{} +) + +func Initialize(ctx *core.Context) { + once.Do(func() { + expireSecond := defaultCacheExpireSecond + if ctx != nil && ctx.Config != nil { + envPrefix = ctx.Config.GetStringWithDefault(core.EnvMetaPrefixKey, core.DefaultMetaPrefix) + expireSecondStr := ctx.Config.GetStringWithDefault(core.MetaCacheExpireSecondKey, "") + if expireSecondStr != "" { + tempCacheExpireSecond, err := strconv.Atoi(expireSecondStr) + if err == nil && tempCacheExpireSecond > 0 { + expireSecond = tempCacheExpireSecond + } + } + } + vlog.Infof("meta cache expire time : %d(s)\n", expireSecond) + metaCache = cache.New(time.Second*time.Duration(expireSecond), 30*time.Second) + vlog.Infof("using meta prefix : %s\n", envPrefix) + // load meta info from env variable + for _, env := range os.Environ() { + if strings.HasPrefix(env, envPrefix) { + kv := strings.Split(env, "=") + envMeta[kv[0]] = kv[1] + } + } + }) +} + +func GetEnvMeta() map[string]string { + return envMeta +} + +func PutDynamicMeta(key, value string) { + dynamicMeta.Store(key, value) +} + +func RemoveDynamicMeta(key string) { + dynamicMeta.Delete(key) +} + +func GetDynamicMeta() map[string]string { + return dynamicMeta.RawMap() +} + +func GetMergedMeta() map[string]string { + mergedMap := make(map[string]string) + for k, v := range envMeta { + mergedMap[k] = v + } + for k, v := range dynamicMeta.RawMap() { + mergedMap[k] = v + } + return mergedMap +} + +func GetMetaValue(meta map[string]string, keySuffix string) string { + var res string + if meta != nil { + if v, ok := meta[envPrefix+keySuffix]; ok { + res = v + } else { + res = meta[core.DefaultMetaPrefix+keySuffix] + } + } + return res +} + +func GetEpDynamicMeta(endpoint core.EndPoint) (map[string]string, error) { + cacheKey := getCacheKey(endpoint.GetURL()) + if v, ok := metaCache.Get(cacheKey); ok { + return v.(map[string]string), nil + } + res, err := getRemoteDynamicMeta(cacheKey, endpoint) + if err != nil { + return nil, err + } + metaCache.Set(cacheKey, res, cache.DefaultExpiration) + return res, nil +} + +// GetEpStaticMeta get remote static meta information from referer url attachments. +// the static meta is init at server start from env. +func GetEpStaticMeta(endpoint core.EndPoint) map[string]string { + res := make(map[string]string) + url := endpoint.GetURL() + if url != nil { + for k, v := range url.Parameters { + if strings.HasPrefix(k, core.DefaultMetaPrefix) || strings.HasPrefix(k, envPrefix) { + res[k] = v + } + } + } + return res +} + +func getRemoteDynamicMeta(cacheKey string, endpoint core.EndPoint) (map[string]string, error) { + if _, ok := notSupportCache.Get(cacheKey); ok || !isSupport(cacheKey, endpoint.GetURL()) { + return nil, ServiceNotSupportError + } + if !endpoint.IsAvailable() { + return nil, errors.New("endpoint unavailable") + } + resp := endpoint.Call(getMetaServiceRequest()) + if resp.GetException() != nil { + if resp.GetException().ErrMsg == core.ServiceNotSupport { + notSupportCache.Set(cacheKey, true, cache.DefaultExpiration) + return nil, ServiceNotSupportError + } + return nil, errors.New(resp.GetException().ErrMsg) + } + reply := make(map[string]string) + err := resp.ProcessDeserializable(&reply) + if err != nil { + return nil, err + } + // multiple serialization might encode empty map into interface{}, not map[string]string + // in this case, return a public empty string map + if res, ok := resp.GetValue().(map[string]string); ok && res != nil { + return res, nil + } + return metaEmptyMap, nil +} + +func getMetaServiceRequest() core.Request { + req := &core.MotanRequest{ + RequestID: endpoint.GenerateRequestID(), + ServiceName: MetaServiceName, + Method: MetaMethodName, + Attachment: core.NewStringMap(core.DefaultAttachmentSize), + Arguments: []interface{}{}, + } + req.SetAttachment(mpro.MFrameworkService, "y") + return req +} + +func getCacheKey(url *core.URL) string { + return url.Host + ":" + url.GetPortStr() +} + +func isSupport(cacheKey string, url *core.URL) bool { + // check dynamicMeta config, protocol and serializer + if url.GetBoolValue(core.DynamicMetaKey, core.DefaultDynamicMeta) && + !notSupportSerializer[url.GetStringParamsWithDefault(core.SerializationKey, "")] && + supportProtocols[url.Protocol] { + return true + } + notSupportCache.Set(cacheKey, true, cache.DefaultExpiration) + return false +} + +func ClearMetaCache() { + metaCache.Flush() + notSupportCache.Flush() +} diff --git a/meta/service.go b/meta/service.go new file mode 100644 index 00000000..326a7843 --- /dev/null +++ b/meta/service.go @@ -0,0 +1,13 @@ +package meta + +const ( + MetaServiceName = "com.weibo.api.motan.runtime.meta.MetaService" + MetaMethodName = "getDynamicMeta" +) + +type MetaService struct { +} + +func (s *MetaService) getDynamicMeta() map[string]string { + return GetDynamicMeta() +} diff --git a/metrics/metrics.go b/metrics/metrics.go index 9180ef35..d2f6e560 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -39,6 +39,10 @@ const ( DefaultStatService = "status" DefaultStatRole = "motan-agent" DefaultStatApplication = "unknown" + + DefaultRuntimeErrorApplication = "runtime-error" + DefaultRuntimeCircuitBreakerGroup = "circuit-breaker" + DefaultRuntimeCircuitBreakerService = "endpoint" ) var ( diff --git a/protocol/motanProtocol.go b/protocol/motanProtocol.go index ea9321ec..789db413 100644 --- a/protocol/motanProtocol.go +++ b/protocol/motanProtocol.go @@ -41,18 +41,19 @@ const ( ) const ( - MPath = "M_p" - MMethod = "M_m" - MException = "M_e" - MProcessTime = "M_pt" - MMethodDesc = "M_md" - MGroup = "M_g" - MProxyProtocol = "M_pp" - MVersion = "M_v" - MModule = "M_mdu" - MSource = "M_s" - MRequestID = "M_rid" - MTimeout = "M_tmo" + MPath = "M_p" + MMethod = "M_m" + MException = "M_e" + MProcessTime = "M_pt" + MMethodDesc = "M_md" + MGroup = "M_g" + MProxyProtocol = "M_pp" + MVersion = "M_v" + MModule = "M_mdu" + MSource = "M_s" + MRequestID = "M_rid" + MTimeout = "M_tmo" + MFrameworkService = "M_fws" ) type Header struct { diff --git a/provider/cgiProvider.go b/provider/cgiProvider.go index e3ce4416..0e5cd243 100644 --- a/provider/cgiProvider.go +++ b/provider/cgiProvider.go @@ -30,6 +30,13 @@ type CgiProvider struct { url *motan.URL } +func (c *CgiProvider) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{ + motan.RuntimeNameKey: c.GetName(), + } + return info +} + func (c *CgiProvider) Initialize() { } diff --git a/provider/httpProvider.go b/provider/httpProvider.go index a5b47ec7..f4175238 100644 --- a/provider/httpProvider.go +++ b/provider/httpProvider.go @@ -43,6 +43,16 @@ type HTTPProvider struct { enableHttpException bool } +func (h *HTTPProvider) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{ + motan.RuntimeNameKey: h.GetName(), + } + if h.url != nil { + info[motan.RuntimeUrlKey] = h.url.ToExtInfo() + } + return info +} + const ( // DefaultMotanMethodConfKey for default motan method conf, when make a http call without a specific motan method DefaultMotanMethodConfKey = "http_default_motan_method" @@ -261,7 +271,7 @@ func (h *HTTPProvider) DoTransparentProxy(request motan.Request, t int64, ip str fillHttpException(resp, http.StatusNotFound, t, err.Error()) return resp } - // sets rewrite + // set rewrite httpReq.URI().SetScheme(h.proxySchema) httpReq.URI().SetPath(rewritePath) request.GetAttachments().Range(func(k, v string) bool { diff --git a/provider/httpProvider_test.go b/provider/httpProvider_test.go index cf587864..af843262 100644 --- a/provider/httpProvider_test.go +++ b/provider/httpProvider_test.go @@ -68,6 +68,14 @@ func TestHTTPProvider_Call(t *testing.T) { body, _ := serialization.SerializeMulti(req.Arguments) req.Arguments = []interface{}{&core.DeserializableValue{Serialization: serialization, Body: body}} assert.Equal(t, "/2/p1/test?a=b", string(provider.Call(req).GetValue().([]interface{})[1].([]byte))) + + // runtime info + info := provider.GetRuntimeInfo() + assert.NotNil(t, info) + + urlInfo, ok := info[core.RuntimeUrlKey] + assert.True(t, ok) + assert.Equal(t, providerURL.ToExtInfo(), urlInfo) } func TestHTTPProvider_Http_Exception(t *testing.T) { diff --git a/provider/httpxProvider.go b/provider/httpxProvider.go index dee3a26d..6d67ba19 100644 --- a/provider/httpxProvider.go +++ b/provider/httpxProvider.go @@ -32,6 +32,14 @@ type HTTPXProvider struct { mixVars []string } +func (h *HTTPXProvider) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + if h.url != nil { + info[motan.RuntimeUrlKey] = h.url.ToExtInfo() + } + return info +} + // Initialize http provider func (h *HTTPXProvider) Initialize() { h.httpClient = &fasthttp.Client{ diff --git a/provider/metaProvider.go b/provider/metaProvider.go new file mode 100644 index 00000000..bde124b8 --- /dev/null +++ b/provider/metaProvider.go @@ -0,0 +1,50 @@ +package provider + +import ( + motan "github.com/weibocom/motan-go/core" + "github.com/weibocom/motan-go/meta" +) + +type MetaProvider struct{} + +func (m *MetaProvider) Initialize() {} + +func (m *MetaProvider) Call(request motan.Request) motan.Response { + resp := &motan.MotanResponse{ + RequestID: request.GetRequestID(), + Value: meta.GetDynamicMeta(), + } + return resp +} + +func (m *MetaProvider) GetPath() string { + return "com.weibo.api.motan.runtime.meta.MetaService" +} + +func (m *MetaProvider) SetService(s interface{}) {} + +func (m *MetaProvider) SetContext(context *motan.Context) {} + +func (m *MetaProvider) GetName() string { + return "metaProvider" +} + +func (m *MetaProvider) GetURL() *motan.URL { + return nil +} + +func (m *MetaProvider) SetURL(url *motan.URL) {} + +func (m *MetaProvider) SetSerialization(s motan.Serialization) {} + +func (m *MetaProvider) SetProxy(proxy bool) {} + +func (m *MetaProvider) Destroy() {} + +func (m *MetaProvider) IsAvailable() bool { + return true +} + +func (m *MetaProvider) GetRuntimeInfo() map[string]interface{} { + return make(map[string]interface{}) +} diff --git a/provider/motanProvider.go b/provider/motanProvider.go index b2c5f860..b1fb5bec 100644 --- a/provider/motanProvider.go +++ b/provider/motanProvider.go @@ -15,6 +15,16 @@ type MotanProvider struct { extFactory motan.ExtensionFactory } +func (m *MotanProvider) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + if m.url != nil { + info[motan.RuntimeUrlKey] = m.url.ToExtInfo() + } + info[motan.RuntimeAvailableKey] = m.available + info[motan.RuntimeEndpointKey] = m.ep.GetRuntimeInfo() + return info +} + const ( ProxyHostKey = "proxy.host" DefaultHost = "127.0.0.1" @@ -29,9 +39,6 @@ func (m *MotanProvider) Initialize() { vlog.Errorln("reverse proxy service port config error!") return } - if protocol == "motan2" || protocol == "motan" { // TODO temp compatible with motan1. remove if MotanCommonEndpoint as default Motan endpoint - protocol = "motanV1Compatible" - } host := m.url.GetParam(ProxyHostKey, DefaultHost) endpointURL := m.url.Copy() endpointURL.Protocol = protocol diff --git a/provider/motanProvider_test.go b/provider/motanProvider_test.go index 2a086518..eab09009 100644 --- a/provider/motanProvider_test.go +++ b/provider/motanProvider_test.go @@ -75,3 +75,39 @@ func TestXForwardedFor(t *testing.T) { providerCorr.Call(request) assert.NotEqual(t, request.GetAttachment(motan.XForwardedForLower), "test") } + +func TestMotanProvider_GetRuntimeInfo(t *testing.T) { + //init factory + factory := &motan.DefaultExtensionFactory{} + factory.Initialize() + endpoint.RegistDefaultEndpoint(factory) + RegistDefaultProvider(factory) + + //init motanProvider + mContext := motan.Context{} + mContext.ConfigFile = confFilePath + mContext.Initialize() + request := &motan.MotanRequest{} + request.SetAttachment(motan.XForwardedForLower, "test") + url := mContext.ServiceURLs[serviceName] + //call correct + providerCorr := MotanProvider{url: url, extFactory: factory} + providerCorr.Initialize() + + // runtime info + info := providerCorr.GetRuntimeInfo() + assert.NotNil(t, info) + + urlInfo, ok := info[motan.RuntimeUrlKey] + assert.True(t, ok) + assert.Equal(t, url.ToExtInfo(), urlInfo) + + available, ok := info[motan.RuntimeAvailableKey] + assert.True(t, ok) + assert.Equal(t, providerCorr.available, available) + + endpointInfo, ok := info[motan.RuntimeEndpointKey] + assert.True(t, ok) + assert.Equal(t, providerCorr.ep.GetRuntimeInfo(), endpointInfo) + assert.NotNil(t, endpointInfo) +} diff --git a/provider/provider.go b/provider/provider.go index 37862de8..8b70cdd8 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -55,6 +55,14 @@ type DefaultProvider struct { url *motan.URL } +func (d *DefaultProvider) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + if d.url != nil { + info[motan.RuntimeUrlKey] = d.url.ToExtInfo() + } + return info +} + func (d *DefaultProvider) Initialize() { d.methods = make(map[string]reflect.Value, 32) if d.service != nil && d.url != nil { @@ -133,6 +141,12 @@ type MockProvider struct { service interface{} } +func (m *MockProvider) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: m.GetName(), + } +} + func (m *MockProvider) GetName() string { return "mockProvider" } diff --git a/registry/consulRegistry.go b/registry/consulRegistry.go index ae1c1168..56e3c3cd 100644 --- a/registry/consulRegistry.go +++ b/registry/consulRegistry.go @@ -13,6 +13,12 @@ type ConsulRegistry struct { heartInterval time.Duration } +func (v *ConsulRegistry) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: v.GetName(), + } +} + func (v *ConsulRegistry) GetURL() *motan.URL { return v.url } diff --git a/registry/directRegistry.go b/registry/directRegistry.go index 9bc46783..6743a04d 100644 --- a/registry/directRegistry.go +++ b/registry/directRegistry.go @@ -13,6 +13,12 @@ type DirectRegistry struct { urls []*motan.URL } +func (d *DirectRegistry) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: d.GetName(), + } +} + func (d *DirectRegistry) GetURL() *motan.URL { return d.url } diff --git a/registry/localRegistry.go b/registry/localRegistry.go index cda744b9..eff8c4da 100644 --- a/registry/localRegistry.go +++ b/registry/localRegistry.go @@ -8,6 +8,12 @@ type LocalRegistry struct { url *motan.URL } +func (d *LocalRegistry) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: d.GetName(), + } +} + func (d *LocalRegistry) GetURL() *motan.URL { return d.url } diff --git a/registry/meshRegistry.go b/registry/meshRegistry.go index fa72c438..d362f46f 100644 --- a/registry/meshRegistry.go +++ b/registry/meshRegistry.go @@ -32,6 +32,12 @@ type MeshRegistry struct { subscribeLock sync.Mutex } +func (r *MeshRegistry) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: r.GetName(), + } +} + func (r *MeshRegistry) Initialize() { r.registeredService = make(map[string]*motan.URL) r.subscribedService = make(map[string]*motan.URL) diff --git a/registry/registry.go b/registry/registry.go index 0dca1d32..b7e716c5 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strconv" "sync" + "sync/atomic" "time" "unsafe" @@ -19,7 +20,6 @@ const ( DefaultHeartbeatInterval = 10 * 1000 //ms DefaultTimeout = 3 * 1000 //ms DefaultSnapshotDir = "./snapshot" - DefaultFailbackInterval = 30 * 1000 //ms ) // ext name @@ -32,9 +32,18 @@ const ( ) var ( - setSnapshotConfLock sync.Mutex + setSnapshotConfLock sync.Mutex + defaultFailbackInterval uint32 = 30000 // ms ) +func GetFailbackInterval() uint32 { + return atomic.LoadUint32(&defaultFailbackInterval) +} + +func SetFailbackInterval(interval uint32) { + atomic.StoreUint32(&defaultFailbackInterval, interval) +} + type SnapshotNodeInfo struct { ExtInfo string `json:"extInfo"` Addr string `json:"address"` diff --git a/registry/zkRegistry.go b/registry/zkRegistry.go index 7fd239cf..6b0f39f7 100644 --- a/registry/zkRegistry.go +++ b/registry/zkRegistry.go @@ -47,6 +47,12 @@ type ZkRegistry struct { subscribedCommandMap map[string]map[motan.CommandNotifyListener]*motan.URL // save all subscribed commands with listeners } +func (z *ZkRegistry) GetRuntimeInfo() map[string]interface{} { + return map[string]interface{}{ + motan.RuntimeNameKey: z.GetName(), + } +} + // Initialize initializes all structure members and handles new session. func (z *ZkRegistry) Initialize() { z.sessionTimeout = time.Duration( diff --git a/server.go b/server.go index 63672ac2..8baa135a 100644 --- a/server.go +++ b/server.go @@ -207,6 +207,10 @@ func (m *MSContext) Initialize() { } } +func (m *MSContext) GetContext() *motan.Context { + return m.context +} + // RegisterService register service with serviceId for config ref. // the type.string will used as serviceId if sid is not set. e.g. 'packageName.structName' func (m *MSContext) RegisterService(s interface{}, sid string) error { @@ -271,7 +275,7 @@ func (m *MSContext) GetRegistryStatus() []map[string]*motan.RegistryStatus { func (m *MSContext) startRegistryFailback() { vlog.Infoln("start MSContext registry failback") - ticker := time.NewTicker(registry.DefaultFailbackInterval * time.Millisecond) + ticker := time.NewTicker(time.Duration(registry.GetFailbackInterval()) * time.Millisecond) defer ticker.Stop() for range ticker.C { m.registryLock.Lock() diff --git a/server/httpProxyServer.go b/server/httpProxyServer.go index 70fa2a6b..494cd6eb 100644 --- a/server/httpProxyServer.go +++ b/server/httpProxyServer.go @@ -55,6 +55,17 @@ type HTTPProxyServer struct { defaultDomain string } +func (s *HTTPProxyServer) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + if s.url != nil { + info[core.RuntimeUrlKey] = s.url.ToExtInfo() + } + info[core.RuntimeDenyKey] = s.deny + info[core.RuntimeKeepaliveKey] = s.keepalive + info[core.RuntimeDefaultDomainKey] = s.defaultDomain + return info +} + func NewHTTPProxyServer(url *core.URL) *HTTPProxyServer { return &HTTPProxyServer{url: url} } diff --git a/server/motanserver.go b/server/motanserver.go index 7fdd21fb..2f73e645 100644 --- a/server/motanserver.go +++ b/server/motanserver.go @@ -55,6 +55,18 @@ type MotanServer struct { heartbeatEnabled bool } +func (m *MotanServer) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + info[motan.RuntimeProxyKey] = m.proxy + info[motan.RuntimeMaxContentLengthKey] = m.maxContextLength + info[motan.RuntimeHeartbeatEnabledKey] = m.heartbeatEnabled + if m.URL != nil { + info[motan.RuntimeUrlKey] = m.URL.ToExtInfo() + } + info[motan.RuntimeMessageHandlerKey] = m.handler.GetRuntimeInfo() + return info +} + func (m *MotanServer) Open(block bool, proxy bool, handler motan.MessageHandler, extFactory motan.ExtensionFactory) error { m.isDestroyed = make(chan bool, 1) m.heartbeatEnabled = true diff --git a/server/server.go b/server/server.go index 78f60db8..cae13134 100644 --- a/server/server.go +++ b/server/server.go @@ -3,6 +3,9 @@ package server import ( "errors" "fmt" + "github.com/weibocom/motan-go/meta" + mpro "github.com/weibocom/motan-go/protocol" + "github.com/weibocom/motan-go/provider" "sync" motan "github.com/weibocom/motan-go/core" @@ -50,6 +53,16 @@ type DefaultExporter struct { // 服务管理单位，负责服务注册、心跳、导出和销毁，内部包含provider，与provider是一对一关系 } +func (d *DefaultExporter) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + if d.url != nil { + info[motan.RuntimeUrlKey] = d.url.ToExtInfo() + } + info[motan.RuntimeProviderKey] = d.provider.GetRuntimeInfo() + info[motan.RuntimeIsAvailableKey] = d.available + return info +} + func (d *DefaultExporter) Export(server motan.Server, extFactory motan.ExtensionFactory, context *motan.Context) (err error) { d.lock.Lock() defer d.lock.Unlock() @@ -64,6 +77,12 @@ func (d *DefaultExporter) Export(server motan.Server, extFactory motan.Extension d.extFactory = extFactory d.server = server d.url = d.provider.GetURL() + // add server side meta info to the url, so these meta info can be passed to the client side through the registration mechanism. + if d.url.GetBoolValue(motan.URLRegisterMeta, motan.DefaultRegisterMeta) { + for k, v := range meta.GetEnvMeta() { + d.url.PutParam(k, v) + } + } d.url.PutParam(motan.NodeTypeKey, motan.NodeTypeService) // node type must be service in export regs, ok := d.url.Parameters[motan.RegistryKey] if !ok { @@ -142,11 +161,33 @@ func (d *DefaultExporter) SetURL(url *motan.URL) { } type DefaultMessageHandler struct { - providers map[string]motan.Provider + providers map[string]motan.Provider + frameworkProviders map[string]motan.Provider +} + +func (d *DefaultMessageHandler) GetName() string { + return "defaultMessageHandler" +} + +func (d *DefaultMessageHandler) GetRuntimeInfo() map[string]interface{} { + info := map[string]interface{}{} + info[motan.RuntimeMessageHandlerTypeKey] = d.GetName() + providersInfo := map[string]interface{}{} + for s, provider := range d.providers { + providersInfo[s] = provider.GetRuntimeInfo() + } + info[motan.RuntimeProvidersKey] = providersInfo + return info } func (d *DefaultMessageHandler) Initialize() { d.providers = make(map[string]motan.Provider) + d.frameworkProviders = make(map[string]motan.Provider) + d.initFrameworkServiceProvider() +} + +func (d *DefaultMessageHandler) initFrameworkServiceProvider() { + d.frameworkProviders[meta.MetaServiceName] = &provider.MetaProvider{} } func (d *DefaultMessageHandler) AddProvider(p motan.Provider) error { @@ -170,14 +211,21 @@ func (d *DefaultMessageHandler) Call(request motan.Request) (res motan.Response) res = motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "provider call panic", ErrType: motan.ServiceException}) vlog.Errorf("provider call panic. req:%s", motan.GetReqInfo(request)) }) + if mfs := request.GetAttachment(mpro.MFrameworkService); mfs != "" { + if fp, ok := d.frameworkProviders[request.GetServiceName()]; ok { + return fp.(motan.Provider).Call(request) + } + //throw specific exception to avoid triggering forced fusing on the client side。 + return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 501, ErrMsg: motan.ServiceNotSupport, ErrType: motan.ServiceException}) + } p := d.providers[request.GetServiceName()] if p != nil { res = p.Call(request) res.GetRPCContext(true).GzipSize = int(p.GetURL().GetIntValue(motan.GzipSizeKey, 0)) return res } - vlog.Errorf("not found provider for %s", motan.GetReqInfo(request)) - return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: 500, ErrMsg: "not found provider for " + request.GetServiceName(), ErrType: motan.ServiceException}) + vlog.Errorf("%s%s%s", motan.ProviderNotExistPrefix, request.GetServiceName(), motan.GetReqInfo(request)) + return motan.BuildExceptionResponse(request.GetRequestID(), &motan.Exception{ErrCode: motan.EProviderNotExist, ErrMsg: motan.ProviderNotExistPrefix + request.GetServiceName(), ErrType: motan.ServiceException}) } type FilterProviderWrapper struct { @@ -185,6 +233,20 @@ type FilterProviderWrapper struct { filter motan.EndPointFilter } +func (f *FilterProviderWrapper) GetRuntimeInfo() map[string]interface{} { + info := f.provider.GetRuntimeInfo() + var filterInfo []interface{} + filter := f.filter + for filter != nil { + filterInfo = append(filterInfo, filter.GetRuntimeInfo()) + filter = filter.GetNext() + } + if len(filterInfo) > 0 { + info[motan.RuntimeFiltersKey] = filterInfo + } + return info +} + func (f *FilterProviderWrapper) SetService(s interface{}) { f.provider.SetService(s) } diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 00000000..fc41bfc7 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,110 @@ +package server + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/weibocom/motan-go/cluster" + "github.com/weibocom/motan-go/core" + "sync" + "testing" +) + +type APITestSuite struct { + suite.Suite + motanServer *MotanServer + motanServerUrl *core.URL + motanServerMessageHandler core.MessageHandler + httpProxyServerUrl *core.URL + httpProxyServer *HTTPProxyServer +} + +func (s *APITestSuite) SetupTest() { + s.motanServerUrl = &core.URL{ + Protocol: "motan2", + Host: "127.0.0.1", + Port: 20243, + Path: "testpath", + Group: "testgroup", + } + s.motanServerMessageHandler = &DefaultMessageHandler{} + server := &MotanServer{URL: s.motanServerUrl} + server.Open(false, false, s.motanServerMessageHandler, nil) + s.motanServer = server + + s.httpProxyServerUrl = &core.URL{ + Host: "127.0.0.1", + Port: 20241, + Path: "test.path", + Group: "test.group", + } + s.httpProxyServer = NewHTTPProxyServer(s.httpProxyServerUrl) + s.httpProxyServer.Open(false, false, &testHttpClusterGetter{}, nil) +} + +func (s *APITestSuite) TearDownTest() { + s.motanServer.Destroy() + s.httpProxyServer.Destroy() +} + +func TestAPITestSuite(t *testing.T) { + suite.Run(t, new(APITestSuite)) +} + +func (s *APITestSuite) TestMotanServerRuntimeInfo() { + info := s.motanServer.GetRuntimeInfo() + assert.NotNil(s.T(), info) + urlInfo, ok := info[core.RuntimeUrlKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), s.motanServerUrl.ToExtInfo(), urlInfo.(string)) + + heartbeatEnabled, ok := info[core.RuntimeHeartbeatEnabledKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), s.motanServer.heartbeatEnabled, heartbeatEnabled) + + proxy, ok := info[core.RuntimeProxyKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), false, proxy) + + maxContentLength, ok := info[core.RuntimeMaxContentLengthKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), core.DefaultMaxContentLength, maxContentLength) + + messageHandlerInfo, ok := info[core.RuntimeMessageHandlerKey] + assert.True(s.T(), ok) + + assert.True(s.T(), ok) + assert.Equal(s.T(), s.motanServerMessageHandler.GetRuntimeInfo(), messageHandlerInfo) +} + +func (s *APITestSuite) TestHttpProxyServerRuntimeInfo() { + info := s.httpProxyServer.GetRuntimeInfo() + assert.NotNil(s.T(), info) + + urlInfo, ok := info[core.RuntimeUrlKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), s.httpProxyServerUrl.ToExtInfo(), urlInfo.(string)) + + deny, ok := info[core.RuntimeDenyKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), s.httpProxyServer.deny, deny) + + keepalive, ok := info[core.RuntimeKeepaliveKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), s.httpProxyServer.keepalive, keepalive) + + domain, ok := info[core.RuntimeDefaultDomainKey] + assert.True(s.T(), ok) + assert.Equal(s.T(), s.httpProxyServer.defaultDomain, domain) +} + +type testHttpClusterGetter struct { + cluster *cluster.HTTPCluster + once sync.Once +} + +func (t *testHttpClusterGetter) GetHTTPCluster(host string) *cluster.HTTPCluster { + t.once.Do(func() { + t.cluster = cluster.NewHTTPCluster(&core.URL{}, false, nil, nil) + }) + return t.cluster +} diff --git a/server_test.go b/server_test.go index 3ee7b110..850ce27c 100644 --- a/server_test.go +++ b/server_test.go @@ -95,6 +95,9 @@ motan-server: } func TestServerRegisterFailBack(t *testing.T) { + currentFailbackInterval := registry.GetFailbackInterval() + newInterval := uint32(1000 * 5) + registry.SetFailbackInterval(newInterval) assert := assert2.New(t) cfgText := ` motan-server: @@ -148,7 +151,7 @@ motan-service: } } setRegistryFailSwitcher(false) - time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) + time.Sleep(time.Duration(newInterval) * time.Millisecond) m = mscontext.GetRegistryStatus() assert.Equal(len(m), 1) for _, mm := range m[0] { @@ -166,7 +169,7 @@ motan-service: } } setRegistryFailSwitcher(false) - time.Sleep(registry.DefaultFailbackInterval * time.Millisecond) + time.Sleep(time.Duration(newInterval) * time.Millisecond) m = mscontext.GetRegistryStatus() assert.Equal(len(m), 1) for _, mm := range m[0] { @@ -174,6 +177,7 @@ motan-service: assert.Equal(mm.Status, motan.UnregisterSuccess) } } + registry.SetFailbackInterval(currentFailbackInterval) } func TestNewMotanServerContextFromConfig(t *testing.T) { diff --git a/version.go b/version.go index 3b5f329f..d37ef2f5 100644 --- a/version.go +++ b/version.go @@ -1,5 +1,5 @@ package motan const ( - Version = "1.0.0" + Version = "1.2.0" )