From 9298c7ebe594f2a1cd9b402ea987e272e857a5eb Mon Sep 17 00:00:00 2001 From: flycash Date: Sun, 26 Jan 2020 21:53:08 +0800 Subject: [PATCH 01/13] Implemented prometheus reporter --- common/constant/time.go | 26 +++++ common/extension/metrics.go | 42 ++++++++ common/extension/metrics_test.go | 46 +++++++++ config/base_config.go | 4 +- config/config_loader.go | 8 ++ config/metric_config.go | 50 +++++++++ config/metric_config_test.go | 32 ++++++ filter/filter_impl/metrics_filter.go | 88 ++++++++++++++++ filter/filter_impl/metrics_filter_test.go | 81 +++++++++++++++ metrics/prometheus/prometheus_reporter.go | 120 ++++++++++++++++++++++ metrics/reporter.go | 35 +++++++ 11 files changed, 531 insertions(+), 1 deletion(-) create mode 100644 common/constant/time.go create mode 100644 common/extension/metrics.go create mode 100644 common/extension/metrics_test.go create mode 100644 config/metric_config.go create mode 100644 config/metric_config_test.go create mode 100644 filter/filter_impl/metrics_filter.go create mode 100644 filter/filter_impl/metrics_filter_test.go create mode 100644 metrics/prometheus/prometheus_reporter.go create mode 100644 metrics/reporter.go diff --git a/common/constant/time.go b/common/constant/time.go new file mode 100644 index 0000000000..be1baaca67 --- /dev/null +++ b/common/constant/time.go @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package constant + +import ( + "time" +) + +var ( + MsToNanoRate = int64(time.Millisecond / time.Nanosecond) +) diff --git a/common/extension/metrics.go b/common/extension/metrics.go new file mode 100644 index 0000000000..8f32341176 --- /dev/null +++ b/common/extension/metrics.go @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package extension + +import ( + "github.com/apache/dubbo-go/metrics" +) + +var( + metricReporterMap = make(map[string]metrics.Reporter, 4) +) + +// set a reporter with the name +func SetMetricReporter(name string, reporter metrics.Reporter) { + metricReporterMap[name] = reporter +} + +// find the reporter with name. +// if not found, it will panic. +// we should know that this method usually is called when system starts, so we should panic +func GetMetricReporter(name string) metrics.Reporter { + reporter, found := metricReporterMap[name] + if !found { + panic("Cannot find the reporter with name: " + name) + } + return reporter +} diff --git a/common/extension/metrics_test.go b/common/extension/metrics_test.go new file mode 100644 index 0000000000..74a02e8935 --- /dev/null +++ b/common/extension/metrics_test.go @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package extension + +import ( + "context" + "testing" + "time" +) + +import ( + "github.com/stretchr/testify/assert" +) + +import ( + "github.com/apache/dubbo-go/protocol" +) + +func TestGetMetricReporter(t *testing.T) { + reporter := &mockReporter{} + name := "mock" + SetMetricReporter(name, reporter) + res := GetMetricReporter(name) + assert.Equal(t, reporter, res) +} + +type mockReporter struct { +} + +func (m mockReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) { +} diff --git a/config/base_config.go b/config/base_config.go index 0949574115..f38ef9b953 100644 --- a/config/base_config.go +++ b/config/base_config.go @@ -40,12 +40,14 @@ type multiConfiger interface { Prefix() string } -// BaseConfig ... +// BaseConfig is the common configuration for provider and consumer type BaseConfig struct { ConfigCenterConfig *ConfigCenterConfig `yaml:"config_center" json:"config_center,omitempty"` configCenterUrl *common.URL prefix string fatherConfig interface{} + + MetricConfig *MetricConfig `yaml:"metrics" json:"metrics,omitempty"` } func (c *BaseConfig) startConfigCenter(ctx context.Context) error { diff --git a/config/config_loader.go b/config/config_loader.go index d6eb7ff524..73bab42b6a 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -33,6 +33,7 @@ import ( var ( consumerConfig *ConsumerConfig providerConfig *ProviderConfig + metricConfig *MetricConfig maxWait = 3 ) @@ -75,6 +76,9 @@ func Load() { if consumerConfig == nil { logger.Warnf("consumerConfig is nil!") } else { + + metricConfig = consumerConfig.MetricConfig + checkApplicationName(consumerConfig.ApplicationConfig) if err := configCenterRefreshConsumer(); err != nil { logger.Errorf("[consumer config center refresh] %#v", err) @@ -131,6 +135,10 @@ func Load() { if providerConfig == nil { logger.Warnf("providerConfig is nil!") } else { + + // so, you should know that the consumer's metric config will be override + metricConfig = providerConfig.MetricConfig + checkApplicationName(providerConfig.ApplicationConfig) if err := configCenterRefreshProvider(); err != nil { logger.Errorf("[provider config center refresh] %#v", err) diff --git a/config/metric_config.go b/config/metric_config.go new file mode 100644 index 0000000000..71071b289b --- /dev/null +++ b/config/metric_config.go @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "github.com/creasty/defaults" +) + +// This is the config struct for all metrics implementation +type MetricConfig struct { + Reporters []string `yaml:"reporters" json:"reporters,omitempty"` +} + +// parse the config from yml +func (c *MetricConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + if err := defaults.Set(c); err != nil { + return err + } + type plain MetricConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return nil +} + +// find the MetricConfig +// if it is nil, create a new one +func GetMetricConfig() *MetricConfig { + if metricConfig == nil { + metricConfig = &MetricConfig{} + } + return metricConfig +} + + diff --git a/config/metric_config_test.go b/config/metric_config_test.go new file mode 100644 index 0000000000..ff8b795506 --- /dev/null +++ b/config/metric_config_test.go @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "testing" +) + +import ( + "github.com/stretchr/testify/assert" +) + + +func TestGetMetricConfig(t *testing.T) { + empty := GetMetricConfig() + assert.NotNil(t, empty) +} diff --git a/filter/filter_impl/metrics_filter.go b/filter/filter_impl/metrics_filter.go new file mode 100644 index 0000000000..e3db21a979 --- /dev/null +++ b/filter/filter_impl/metrics_filter.go @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package filter_impl + +import ( + "context" + "time" + + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/config" + "github.com/apache/dubbo-go/filter" + "github.com/apache/dubbo-go/metrics" + "github.com/apache/dubbo-go/protocol" +) + +const ( + metricFilterName = "metrics" +) + +var ( + metricFilterInstance filter.Filter +) + +// must initialized before using the filter and after loading configuration +func init() { + extension.SetFilter(metricFilterName, newMetricsFilter) +} + +// metricFilter will calculate the invocation's duration and the report to the reporters +// If you want to use this filter to collect the metrics, +// Adding this into your configuration file, like: +// filter: "metrics" +// metrics: +// reporter: +// - "your reporter" # here you should specify the reporter, for example 'prometheus' +// more info please take a look at dubbo-samples projects +type metricsFilter struct { + reporters []metrics.Reporter +} + +// using goroutine to report the duration. +func (p *metricsFilter) Invoke(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation) protocol.Result { + start := time.Now() + res := invoker.Invoke(ctx, invocation) + end := time.Now() + duration := end.Sub(start) + go func() { + for _, reporter := range p.reporters { + reporter.Report(ctx, invoker, invocation, duration) + } + }() + return res +} + +// do nothing and return the result +func (p *metricsFilter) OnResponse(ctx context.Context, res protocol.Result, invoker protocol.Invoker, invocation protocol.Invocation) protocol.Result { + return res +} + +func newMetricsFilter() filter.Filter { + if metricFilterInstance == nil { + reporterNames := config.GetMetricConfig().Reporters + reporters := make([]metrics.Reporter, 0, len(reporterNames)) + for _, name := range reporterNames { + reporters = append(reporters, extension.GetMetricReporter(name)) + } + metricFilterInstance = &metricsFilter{ + reporters: reporters, + } + } + + return metricFilterInstance +} diff --git a/filter/filter_impl/metrics_filter_test.go b/filter/filter_impl/metrics_filter_test.go new file mode 100644 index 0000000000..f461a0d14d --- /dev/null +++ b/filter/filter_impl/metrics_filter_test.go @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package filter_impl + +import ( + "context" + "sync" + "testing" + "time" +) + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/config" + "github.com/apache/dubbo-go/protocol" + "github.com/apache/dubbo-go/protocol/invocation" +) + +func TestMetricsFilter_Invoke(t *testing.T) { + + // prepare the mock reporter + config.GetMetricConfig().Reporters = []string{"mock"} + mk := &mockReporter{} + extension.SetMetricReporter("mock", mk) + + instance := extension.GetFilter(metricFilterName) + + url, _ := common.NewURL(context.Background(), + "dubbo://:20000/UserProvider?app.version=0.0.1&application=BDTService&bean.name=UserProvider"+ + "&cluster=failover&environment=dev&group=&interface=com.ikurento.user.UserProvider&loadbalance=random&methods.GetUser."+ + "loadbalance=random&methods.GetUser.retries=1&methods.GetUser.weight=0&module=dubbogo+user-info+server&name="+ + "BDTService&organization=ikurento.com&owner=ZX®istry.role=3&retries=&"+ + "service.filter=echo%2Ctoken%2Caccesslog×tamp=1569153406&token=934804bf-b007-4174-94eb-96e3e1d60cc7&version=&warmup=100") + invoker := protocol.NewBaseInvoker(url) + + attach := make(map[string]string, 10) + inv := invocation.NewRPCInvocation("MethodName", []interface{}{"OK", "Hello"}, attach) + + ctx := context.Background() + + mk.On("Report", ctx, invoker, inv).Return(true, nil) + + mk.wg.Add(1) + result := instance.Invoke(ctx, invoker, inv) + assert.NotNil(t, result) + mk.AssertNotCalled(t, "Report", 1) + // it will do nothing + result = instance.OnResponse(ctx, nil, invoker, inv) + assert.Nil(t, result) +} + +type mockReporter struct { + mock.Mock + wg sync.WaitGroup +} + +func (m *mockReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) { + m.Called(ctx, invoker, invocation) + m.wg.Done() +} diff --git a/metrics/prometheus/prometheus_reporter.go b/metrics/prometheus/prometheus_reporter.go new file mode 100644 index 0000000000..245e2b0c67 --- /dev/null +++ b/metrics/prometheus/prometheus_reporter.go @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package prometheus + +import ( + "context" + "strconv" + "strings" + "time" +) +import ( + "github.com/prometheus/client_golang/prometheus" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/metrics" + "github.com/apache/dubbo-go/protocol" +) + +const ( + reporterName = "prometheus" + serviceKey = constant.SERVICE_KEY + groupKey = constant.GROUP_KEY + versionKey = constant.VERSION_KEY + methodKey = constant.METHOD_KEY + timeoutKey = constant.TIMEOUT_KEY + remoteKey = "remote" + localKey = "local" + + providerKey = "provider" + consumerKey = "consumer" +) + +func init() { + extension.SetMetricReporter(reporterName, newPrometheus()) +} + +// it will collect the data for Prometheus +type PrometheusReporter struct { + + // report the consumer-side's data + consumerVec *prometheus.SummaryVec + // report the provider-side's data + providerVec *prometheus.SummaryVec +} + +// report the duration to Prometheus +func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) { + url := invoker.GetUrl() + var sumVec *prometheus.SummaryVec + if isProvider(url) { + sumVec = reporter.providerVec + } else { + sumVec = reporter.consumerVec + } + + sumVec.With(prometheus.Labels{ + serviceKey: url.Service(), + groupKey: url.GetParam(groupKey, ""), + versionKey: url.GetParam(versionKey, ""), + methodKey: invocation.MethodName(), + timeoutKey: url.GetParam(timeoutKey, ""), + remoteKey: invocation.AttachmentsByKey(constant.REMOTE_ADDR, ""), + localKey: invocation.AttachmentsByKey(constant.REMOTE_ADDR, ""), + }).Observe(float64(cost.Nanoseconds() / constant.MsToNanoRate)) +} + +func isProvider(url common.URL) bool { + side := url.GetParam(constant.ROLE_KEY, "") + return strings.EqualFold(side, strconv.Itoa(common.PROVIDER)) +} + +func newPrometheus() metrics.Reporter { + // cfg := *config.GetMetricConfig().GetPrometheusConfig() + result := &PrometheusReporter{ + consumerVec: newSummaryVec(consumerKey), + providerVec: newSummaryVec(providerKey), + } + prometheus.MustRegister(result.consumerVec, result.providerVec) + return result +} + +func newSummaryVec(side string) *prometheus.SummaryVec { + + return prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: metrics.NameSpace, + Help: "this is the dubbo's metrics", + Subsystem: side, + Name: serviceKey, + Objectives: map[float64]float64{ + 0.5: 0.01, + 0.75: 0.01, + 0.90: 0.005, + 0.98: 0.002, + 0.99: 0.001, + 0.999: 0.0001, + }, + }, + []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey, remoteKey, localKey}, + ) +} diff --git a/metrics/reporter.go b/metrics/reporter.go new file mode 100644 index 0000000000..b6e4c6c0ec --- /dev/null +++ b/metrics/reporter.go @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metrics + +import ( + "context" + "time" + + "github.com/apache/dubbo-go/protocol" +) + +const ( + NameSpace = "dubbo" +) + +// it will be use to report the invocation's duration +type Reporter interface { + // report the duration of an invocation + Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) +} From 9cb396584d61cf9f71720cf506668eca19df1f2b Mon Sep 17 00:00:00 2001 From: flycash Date: Sun, 26 Jan 2020 22:50:30 +0800 Subject: [PATCH 02/13] Tested prometheus --- common/extension/metrics.go | 2 +- config/config_loader.go | 2 +- config/metric_config.go | 4 +- config/metric_config_test.go | 1 - filter/filter_impl/metrics_filter.go | 3 ++ go.mod | 2 +- .../{prometheus_reporter.go => reporter.go} | 37 +++++++++----- metrics/prometheus/reporter_test.go | 48 +++++++++++++++++++ 8 files changed, 80 insertions(+), 19 deletions(-) rename metrics/prometheus/{prometheus_reporter.go => reporter.go} (75%) create mode 100644 metrics/prometheus/reporter_test.go diff --git a/common/extension/metrics.go b/common/extension/metrics.go index 8f32341176..d78907ee48 100644 --- a/common/extension/metrics.go +++ b/common/extension/metrics.go @@ -21,7 +21,7 @@ import ( "github.com/apache/dubbo-go/metrics" ) -var( +var ( metricReporterMap = make(map[string]metrics.Reporter, 4) ) diff --git a/config/config_loader.go b/config/config_loader.go index 73bab42b6a..fa0d3da433 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -33,7 +33,7 @@ import ( var ( consumerConfig *ConsumerConfig providerConfig *ProviderConfig - metricConfig *MetricConfig + metricConfig *MetricConfig maxWait = 3 ) diff --git a/config/metric_config.go b/config/metric_config.go index 71071b289b..1b0f234f8a 100644 --- a/config/metric_config.go +++ b/config/metric_config.go @@ -23,7 +23,7 @@ import ( // This is the config struct for all metrics implementation type MetricConfig struct { - Reporters []string `yaml:"reporters" json:"reporters,omitempty"` + Reporters []string `yaml:"reporters" json:"reporters,omitempty"` } // parse the config from yml @@ -46,5 +46,3 @@ func GetMetricConfig() *MetricConfig { } return metricConfig } - - diff --git a/config/metric_config_test.go b/config/metric_config_test.go index ff8b795506..fe9d2493f3 100644 --- a/config/metric_config_test.go +++ b/config/metric_config_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/assert" ) - func TestGetMetricConfig(t *testing.T) { empty := GetMetricConfig() assert.NotNil(t, empty) diff --git a/filter/filter_impl/metrics_filter.go b/filter/filter_impl/metrics_filter.go index e3db21a979..e35a97d198 100644 --- a/filter/filter_impl/metrics_filter.go +++ b/filter/filter_impl/metrics_filter.go @@ -72,6 +72,9 @@ func (p *metricsFilter) OnResponse(ctx context.Context, res protocol.Result, inv return res } +// the metricsFilter is singleton. +// it's lazy initialization +// make sure that the configuration had been loaded before invoking this method. func newMetricsFilter() filter.Filter { if metricFilterInstance == nil { reporterNames := config.GetMetricConfig().Reporters diff --git a/go.mod b/go.mod index db6dc92c63..58932cf84a 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/nacos-group/nacos-sdk-go v0.0.0-20190723125407-0242d42e3dbb github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 - github.com/prometheus/client_golang v1.1.0 // indirect + github.com/prometheus/client_golang v1.1.0 github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec github.com/satori/go.uuid v1.2.0 github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect diff --git a/metrics/prometheus/prometheus_reporter.go b/metrics/prometheus/reporter.go similarity index 75% rename from metrics/prometheus/prometheus_reporter.go rename to metrics/prometheus/reporter.go index 245e2b0c67..8b16ff5e71 100644 --- a/metrics/prometheus/prometheus_reporter.go +++ b/metrics/prometheus/reporter.go @@ -22,6 +22,8 @@ import ( "strconv" "strings" "time" + + "github.com/apache/dubbo-go/common/logger" ) import ( "github.com/prometheus/client_golang/prometheus" @@ -49,11 +51,19 @@ const ( consumerKey = "consumer" ) +// should initialize after loading configuration func init() { - extension.SetMetricReporter(reporterName, newPrometheus()) + rpt := &PrometheusReporter{ + consumerVec: newSummaryVec(consumerKey), + providerVec: newSummaryVec(providerKey), + } + prometheus.MustRegister(rpt.consumerVec, rpt.providerVec) + extension.SetMetricReporter(reporterName, rpt) } // it will collect the data for Prometheus +// if you want to use this, you should initialize your prometheus. +// https://prometheus.io/docs/guides/go-application/ type PrometheusReporter struct { // report the consumer-side's data @@ -68,8 +78,12 @@ func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol var sumVec *prometheus.SummaryVec if isProvider(url) { sumVec = reporter.providerVec - } else { + } else if isConsumer(url) { sumVec = reporter.consumerVec + } else { + logger.Warnf("The url is not the consumer's or provider's, "+ + "so the invocation will be ignored. url: %s", url.String()) + return } sumVec.With(prometheus.Labels{ @@ -83,21 +97,20 @@ func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol }).Observe(float64(cost.Nanoseconds() / constant.MsToNanoRate)) } +// whether this url represents the application received the request as server func isProvider(url common.URL) bool { - side := url.GetParam(constant.ROLE_KEY, "") - return strings.EqualFold(side, strconv.Itoa(common.PROVIDER)) + role := url.GetParam(constant.ROLE_KEY, "") + return strings.EqualFold(role, strconv.Itoa(common.PROVIDER)) } -func newPrometheus() metrics.Reporter { - // cfg := *config.GetMetricConfig().GetPrometheusConfig() - result := &PrometheusReporter{ - consumerVec: newSummaryVec(consumerKey), - providerVec: newSummaryVec(providerKey), - } - prometheus.MustRegister(result.consumerVec, result.providerVec) - return result +// whether this url represents the application sent then request as client +func isConsumer(url common.URL) bool { + role := url.GetParam(constant.ROLE_KEY, "") + return strings.EqualFold(role, strconv.Itoa(common.CONSUMER)) } +// create SummaryVec, the Namespace is dubbo +// the objectives is from my experience. func newSummaryVec(side string) *prometheus.SummaryVec { return prometheus.NewSummaryVec( diff --git a/metrics/prometheus/reporter_test.go b/metrics/prometheus/reporter_test.go new file mode 100644 index 0000000000..bc0a14c354 --- /dev/null +++ b/metrics/prometheus/reporter_test.go @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package prometheus + +import ( + "context" + "testing" + "time" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/protocol" + "github.com/apache/dubbo-go/protocol/invocation" +) + +func TestPrometheusReporter_Report(t *testing.T) { + reporter := extension.GetMetricReporter(reporterName) + url, _ := common.NewURL(context.Background(), + "dubbo://:20000/UserProvider?app.version=0.0.1&application=BDTService&bean.name=UserProvider"+ + "&cluster=failover&environment=dev&group=&interface=com.ikurento.user.UserProvider&loadbalance=random&methods.GetUser."+ + "loadbalance=random&methods.GetUser.retries=1&methods.GetUser.weight=0&module=dubbogo+user-info+server&name="+ + "BDTService&organization=ikurento.com&owner=ZX®istry.role=3&retries=&"+ + "service.filter=echo%2Ctoken%2Caccesslog×tamp=1569153406&token=934804bf-b007-4174-94eb-96e3e1d60cc7&version=&warmup=100") + invoker := protocol.NewBaseInvoker(url) + + attach := make(map[string]string, 10) + inv := invocation.NewRPCInvocation("MethodName", []interface{}{"OK", "Hello"}, attach) + + ctx := context.Background() + reporter.Report(ctx, invoker, inv, 100*time.Millisecond) +} From fcae1b74d8d46f19e67e06e187dc6c11e1f6a2ef Mon Sep 17 00:00:00 2001 From: flycash Date: Sun, 26 Jan 2020 23:07:24 +0800 Subject: [PATCH 03/13] Add protocol.Result into Report method --- common/extension/metrics_test.go | 2 +- filter/filter_impl/metrics_filter.go | 2 +- filter/filter_impl/metrics_filter_test.go | 2 +- metrics/prometheus/reporter.go | 2 +- metrics/prometheus/reporter_test.go | 2 +- metrics/reporter.go | 3 ++- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/common/extension/metrics_test.go b/common/extension/metrics_test.go index 74a02e8935..526471e4e6 100644 --- a/common/extension/metrics_test.go +++ b/common/extension/metrics_test.go @@ -42,5 +42,5 @@ func TestGetMetricReporter(t *testing.T) { type mockReporter struct { } -func (m mockReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) { +func (m mockReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration, res protocol.Result) { } diff --git a/filter/filter_impl/metrics_filter.go b/filter/filter_impl/metrics_filter.go index e35a97d198..dda24bc8ad 100644 --- a/filter/filter_impl/metrics_filter.go +++ b/filter/filter_impl/metrics_filter.go @@ -61,7 +61,7 @@ func (p *metricsFilter) Invoke(ctx context.Context, invoker protocol.Invoker, in duration := end.Sub(start) go func() { for _, reporter := range p.reporters { - reporter.Report(ctx, invoker, invocation, duration) + reporter.Report(ctx, invoker, invocation, duration, nil) } }() return res diff --git a/filter/filter_impl/metrics_filter_test.go b/filter/filter_impl/metrics_filter_test.go index f461a0d14d..312b1b9fcd 100644 --- a/filter/filter_impl/metrics_filter_test.go +++ b/filter/filter_impl/metrics_filter_test.go @@ -75,7 +75,7 @@ type mockReporter struct { wg sync.WaitGroup } -func (m *mockReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) { +func (m *mockReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration, res protocol.Result) { m.Called(ctx, invoker, invocation) m.wg.Done() } diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index 8b16ff5e71..253330264f 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -73,7 +73,7 @@ type PrometheusReporter struct { } // report the duration to Prometheus -func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) { +func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration, res protocol.Result) { url := invoker.GetUrl() var sumVec *prometheus.SummaryVec if isProvider(url) { diff --git a/metrics/prometheus/reporter_test.go b/metrics/prometheus/reporter_test.go index bc0a14c354..4f734ac8b5 100644 --- a/metrics/prometheus/reporter_test.go +++ b/metrics/prometheus/reporter_test.go @@ -44,5 +44,5 @@ func TestPrometheusReporter_Report(t *testing.T) { inv := invocation.NewRPCInvocation("MethodName", []interface{}{"OK", "Hello"}, attach) ctx := context.Background() - reporter.Report(ctx, invoker, inv, 100*time.Millisecond) + reporter.Report(ctx, invoker, inv, 100*time.Millisecond, nil) } diff --git a/metrics/reporter.go b/metrics/reporter.go index b6e4c6c0ec..002443575c 100644 --- a/metrics/reporter.go +++ b/metrics/reporter.go @@ -31,5 +31,6 @@ const ( // it will be use to report the invocation's duration type Reporter interface { // report the duration of an invocation - Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration) + Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, + cost time.Duration, res protocol.Result) } From 061eb82ddd422b203d6dbd1d291da218073d7466 Mon Sep 17 00:00:00 2001 From: flycash Date: Sun, 26 Jan 2020 23:11:23 +0800 Subject: [PATCH 04/13] Rearrange imports --- filter/filter_impl/metrics_filter.go | 2 ++ metrics/prometheus/reporter.go | 3 +-- metrics/reporter.go | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/filter/filter_impl/metrics_filter.go b/filter/filter_impl/metrics_filter.go index dda24bc8ad..d07dc2268e 100644 --- a/filter/filter_impl/metrics_filter.go +++ b/filter/filter_impl/metrics_filter.go @@ -20,7 +20,9 @@ package filter_impl import ( "context" "time" +) +import ( "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/config" "github.com/apache/dubbo-go/filter" diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index 253330264f..5a2ec2016a 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -22,8 +22,6 @@ import ( "strconv" "strings" "time" - - "github.com/apache/dubbo-go/common/logger" ) import ( "github.com/prometheus/client_golang/prometheus" @@ -33,6 +31,7 @@ import ( "github.com/apache/dubbo-go/common" "github.com/apache/dubbo-go/common/constant" "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/common/logger" "github.com/apache/dubbo-go/metrics" "github.com/apache/dubbo-go/protocol" ) diff --git a/metrics/reporter.go b/metrics/reporter.go index 002443575c..85ef1dcdf0 100644 --- a/metrics/reporter.go +++ b/metrics/reporter.go @@ -20,7 +20,8 @@ package metrics import ( "context" "time" - +) +import ( "github.com/apache/dubbo-go/protocol" ) From 8ceeefd61cf6bca67140815e2e61b5630b282c70 Mon Sep 17 00:00:00 2001 From: flycash Date: Mon, 27 Jan 2020 12:10:10 +0800 Subject: [PATCH 05/13] Fix review --- config/metric_config.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/config/metric_config.go b/config/metric_config.go index 1b0f234f8a..e3dacdaad6 100644 --- a/config/metric_config.go +++ b/config/metric_config.go @@ -17,27 +17,11 @@ package config -import ( - "github.com/creasty/defaults" -) - // This is the config struct for all metrics implementation type MetricConfig struct { Reporters []string `yaml:"reporters" json:"reporters,omitempty"` } -// parse the config from yml -func (c *MetricConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - if err := defaults.Set(c); err != nil { - return err - } - type plain MetricConfig - if err := unmarshal((*plain)(c)); err != nil { - return err - } - return nil -} - // find the MetricConfig // if it is nil, create a new one func GetMetricConfig() *MetricConfig { From 10d0ac89b832248038b2a35d930a72894b0b8067 Mon Sep 17 00:00:00 2001 From: flycash Date: Mon, 27 Jan 2020 12:53:37 +0800 Subject: [PATCH 06/13] Fix BUG --- filter/filter_impl/metrics_filter.go | 2 +- metrics/prometheus/reporter_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/filter/filter_impl/metrics_filter.go b/filter/filter_impl/metrics_filter.go index d07dc2268e..1e2dc35f2d 100644 --- a/filter/filter_impl/metrics_filter.go +++ b/filter/filter_impl/metrics_filter.go @@ -63,7 +63,7 @@ func (p *metricsFilter) Invoke(ctx context.Context, invoker protocol.Invoker, in duration := end.Sub(start) go func() { for _, reporter := range p.reporters { - reporter.Report(ctx, invoker, invocation, duration, nil) + reporter.Report(ctx, invoker, invocation, duration, res) } }() return res diff --git a/metrics/prometheus/reporter_test.go b/metrics/prometheus/reporter_test.go index 4f734ac8b5..52e8b50c57 100644 --- a/metrics/prometheus/reporter_test.go +++ b/metrics/prometheus/reporter_test.go @@ -23,6 +23,9 @@ import ( "time" ) +import ( + "github.com/stretchr/testify/assert" +) import ( "github.com/apache/dubbo-go/common" "github.com/apache/dubbo-go/common/extension" @@ -43,6 +46,7 @@ func TestPrometheusReporter_Report(t *testing.T) { attach := make(map[string]string, 10) inv := invocation.NewRPCInvocation("MethodName", []interface{}{"OK", "Hello"}, attach) + assert.False(t, isConsumer(url)) ctx := context.Background() reporter.Report(ctx, invoker, inv, 100*time.Millisecond, nil) } From 2d665520727f46b82fbb5212302620f10c06e482 Mon Sep 17 00:00:00 2001 From: flycash Date: Mon, 27 Jan 2020 20:48:11 +0800 Subject: [PATCH 07/13] Support histogram --- config/metric_config.go | 16 ++++++- metrics/prometheus/reporter.go | 67 +++++++++++++++++++++++------ metrics/prometheus/reporter_test.go | 20 +++++++++ 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/config/metric_config.go b/config/metric_config.go index e3dacdaad6..7b9222f29b 100644 --- a/config/metric_config.go +++ b/config/metric_config.go @@ -17,9 +17,14 @@ package config +var ( + defaultHistogramBucket = []float64{10, 50, 100, 200, 500, 1000, 10000} +) + // This is the config struct for all metrics implementation type MetricConfig struct { - Reporters []string `yaml:"reporters" json:"reporters,omitempty"` + Reporters []string `yaml:"reporters" json:"reporters,omitempty"` + HistogramBucket []float64 `yaml:"histogram_bucket" json:"histogram_bucket,omitempty"` } // find the MetricConfig @@ -30,3 +35,12 @@ func GetMetricConfig() *MetricConfig { } return metricConfig } + +// find the histogram bucket +// if it's empty, the default value will be return +func (mc *MetricConfig) GetHistogramBucket() []float64 { + if len(mc.HistogramBucket) == 0 { + mc.HistogramBucket = defaultHistogramBucket + } + return mc.HistogramBucket +} diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index 5a2ec2016a..8914902ab0 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -32,6 +32,7 @@ import ( "github.com/apache/dubbo-go/common/constant" "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/config" "github.com/apache/dubbo-go/metrics" "github.com/apache/dubbo-go/protocol" ) @@ -48,15 +49,28 @@ const ( providerKey = "provider" consumerKey = "consumer" + + // to identify the metric's type + histogramSuffix = "_histogram" + // to identify the metric's type + summarySuffix = "_summary" +) + +var ( + labelNames = []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey, remoteKey, localKey} ) // should initialize after loading configuration func init() { rpt := &PrometheusReporter{ - consumerVec: newSummaryVec(consumerKey), - providerVec: newSummaryVec(providerKey), + consumerSummaryVec: newSummaryVec(consumerKey), + providerSummaryVec: newSummaryVec(providerKey), + + consumerHistogramVec: newHistogramVec(consumerKey), + providerHistogramVec: newHistogramVec(providerKey), } - prometheus.MustRegister(rpt.consumerVec, rpt.providerVec) + prometheus.MustRegister(rpt.consumerSummaryVec, rpt.providerSummaryVec, + rpt.consumerHistogramVec, rpt.providerHistogramVec) extension.SetMetricReporter(reporterName, rpt) } @@ -65,27 +79,35 @@ func init() { // https://prometheus.io/docs/guides/go-application/ type PrometheusReporter struct { - // report the consumer-side's data - consumerVec *prometheus.SummaryVec - // report the provider-side's data - providerVec *prometheus.SummaryVec + // report the consumer-side's summary data + consumerSummaryVec *prometheus.SummaryVec + // report the provider-side's summary data + providerSummaryVec *prometheus.SummaryVec + + // report the provider-side's histogram data + providerHistogramVec *prometheus.HistogramVec + // report the consumer-side's histogram data + consumerHistogramVec *prometheus.HistogramVec } // report the duration to Prometheus func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration, res protocol.Result) { url := invoker.GetUrl() var sumVec *prometheus.SummaryVec + var hisVec *prometheus.HistogramVec if isProvider(url) { - sumVec = reporter.providerVec + sumVec = reporter.providerSummaryVec + hisVec = reporter.providerHistogramVec } else if isConsumer(url) { - sumVec = reporter.consumerVec + sumVec = reporter.consumerSummaryVec + hisVec = reporter.consumerHistogramVec } else { logger.Warnf("The url is not the consumer's or provider's, "+ "so the invocation will be ignored. url: %s", url.String()) return } - sumVec.With(prometheus.Labels{ + labels := prometheus.Labels{ serviceKey: url.Service(), groupKey: url.GetParam(groupKey, ""), versionKey: url.GetParam(versionKey, ""), @@ -93,7 +115,24 @@ func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol timeoutKey: url.GetParam(timeoutKey, ""), remoteKey: invocation.AttachmentsByKey(constant.REMOTE_ADDR, ""), localKey: invocation.AttachmentsByKey(constant.REMOTE_ADDR, ""), - }).Observe(float64(cost.Nanoseconds() / constant.MsToNanoRate)) + } + + costMs := float64(cost.Nanoseconds() / constant.MsToNanoRate) + sumVec.With(labels).Observe(costMs) + hisVec.With(labels).Observe(costMs) +} + +func newHistogramVec(side string) *prometheus.HistogramVec { + mc := config.GetMetricConfig() + return prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metrics.NameSpace, + Subsystem: side, + Name: serviceKey + histogramSuffix, + Help: "This is the dubbo's histogram metrics", + Buckets: mc.GetHistogramBucket(), + }, + labelNames) } // whether this url represents the application received the request as server @@ -115,9 +154,9 @@ func newSummaryVec(side string) *prometheus.SummaryVec { return prometheus.NewSummaryVec( prometheus.SummaryOpts{ Namespace: metrics.NameSpace, - Help: "this is the dubbo's metrics", + Help: "This is the dubbo's summary metrics", Subsystem: side, - Name: serviceKey, + Name: serviceKey + summarySuffix, Objectives: map[float64]float64{ 0.5: 0.01, 0.75: 0.01, @@ -127,6 +166,6 @@ func newSummaryVec(side string) *prometheus.SummaryVec { 0.999: 0.0001, }, }, - []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey, remoteKey, localKey}, + labelNames, ) } diff --git a/metrics/prometheus/reporter_test.go b/metrics/prometheus/reporter_test.go index 52e8b50c57..d1741d16d0 100644 --- a/metrics/prometheus/reporter_test.go +++ b/metrics/prometheus/reporter_test.go @@ -49,4 +49,24 @@ func TestPrometheusReporter_Report(t *testing.T) { assert.False(t, isConsumer(url)) ctx := context.Background() reporter.Report(ctx, invoker, inv, 100*time.Millisecond, nil) + + // consumer side + url, _ = common.NewURL(context.Background(), + "dubbo://:20000/UserProvider?app.version=0.0.1&application=BDTService&bean.name=UserProvider"+ + "&cluster=failover&environment=dev&group=&interface=com.ikurento.user.UserProvider&loadbalance=random&methods.GetUser."+ + "loadbalance=random&methods.GetUser.retries=1&methods.GetUser.weight=0&module=dubbogo+user-info+server&name="+ + "BDTService&organization=ikurento.com&owner=ZX®istry.role=0&retries=&"+ + "service.filter=echo%2Ctoken%2Caccesslog×tamp=1569153406&token=934804bf-b007-4174-94eb-96e3e1d60cc7&version=&warmup=100") + invoker = protocol.NewBaseInvoker(url) + reporter.Report(ctx, invoker, inv, 100*time.Millisecond, nil) + + // invalid role + url, _ = common.NewURL(context.Background(), + "dubbo://:20000/UserProvider?app.version=0.0.1&application=BDTService&bean.name=UserProvider"+ + "&cluster=failover&environment=dev&group=&interface=com.ikurento.user.UserProvider&loadbalance=random&methods.GetUser."+ + "loadbalance=random&methods.GetUser.retries=1&methods.GetUser.weight=0&module=dubbogo+user-info+server&name="+ + "BDTService&organization=ikurento.com&owner=ZX®istry.role=9&retries=&"+ + "service.filter=echo%2Ctoken%2Caccesslog×tamp=1569153406&token=934804bf-b007-4174-94eb-96e3e1d60cc7&version=&warmup=100") + invoker = protocol.NewBaseInvoker(url) + reporter.Report(ctx, invoker, inv, 100*time.Millisecond, nil) } From 9d5943accb6f3cd34da3ac24519d0d85788d44b1 Mon Sep 17 00:00:00 2001 From: flycash Date: Mon, 27 Jan 2020 22:44:10 +0800 Subject: [PATCH 08/13] Lazy initialization --- common/extension/metrics.go | 12 ++++--- common/extension/metrics_test.go | 5 ++- config/application_config.go | 10 ++++++ config/config_loader.go | 13 +++++--- filter/filter_impl/metrics_filter_test.go | 5 ++- metrics/prometheus/reporter.go | 38 +++++++++++++++-------- 6 files changed, 58 insertions(+), 25 deletions(-) diff --git a/common/extension/metrics.go b/common/extension/metrics.go index d78907ee48..e7695c7633 100644 --- a/common/extension/metrics.go +++ b/common/extension/metrics.go @@ -22,21 +22,23 @@ import ( ) var ( - metricReporterMap = make(map[string]metrics.Reporter, 4) + // we couldn't store the instance because the some instance may initialize before loading configuration + // so lazy initialization will be better. + metricReporterMap = make(map[string]func() metrics.Reporter, 4) ) // set a reporter with the name -func SetMetricReporter(name string, reporter metrics.Reporter) { - metricReporterMap[name] = reporter +func SetMetricReporter(name string, reporterFunc func() metrics.Reporter) { + metricReporterMap[name] = reporterFunc } // find the reporter with name. // if not found, it will panic. // we should know that this method usually is called when system starts, so we should panic func GetMetricReporter(name string) metrics.Reporter { - reporter, found := metricReporterMap[name] + reporterFunc, found := metricReporterMap[name] if !found { panic("Cannot find the reporter with name: " + name) } - return reporter + return reporterFunc() } diff --git a/common/extension/metrics_test.go b/common/extension/metrics_test.go index 526471e4e6..6a8a3fe538 100644 --- a/common/extension/metrics_test.go +++ b/common/extension/metrics_test.go @@ -28,13 +28,16 @@ import ( ) import ( + "github.com/apache/dubbo-go/metrics" "github.com/apache/dubbo-go/protocol" ) func TestGetMetricReporter(t *testing.T) { reporter := &mockReporter{} name := "mock" - SetMetricReporter(name, reporter) + SetMetricReporter(name, func() metrics.Reporter { + return reporter + }) res := GetMetricReporter(name) assert.Equal(t, reporter, res) } diff --git a/config/application_config.go b/config/application_config.go index 23ab7d34ac..5db27cf0a2 100644 --- a/config/application_config.go +++ b/config/application_config.go @@ -35,6 +35,16 @@ type ApplicationConfig struct { Environment string `yaml:"environment" json:"environment,omitempty" property:"environment"` } +// find the application config +// if not, we will create one +// Usually applicationConfig will be initialized when system start +func GetApplicationConfig() *ApplicationConfig { + if applicationConfig == nil { + applicationConfig = &ApplicationConfig{} + } + return applicationConfig +} + // Prefix ... func (*ApplicationConfig) Prefix() string { return constant.DUBBO + ".application." diff --git a/config/config_loader.go b/config/config_loader.go index fa0d3da433..c29e7bce5e 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -31,10 +31,11 @@ import ( ) var ( - consumerConfig *ConsumerConfig - providerConfig *ProviderConfig - metricConfig *MetricConfig - maxWait = 3 + consumerConfig *ConsumerConfig + providerConfig *ProviderConfig + metricConfig *MetricConfig + applicationConfig *ApplicationConfig + maxWait = 3 ) // loaded consumer & provider config from xxx.yml, and log config from xxx.xml @@ -78,6 +79,7 @@ func Load() { } else { metricConfig = consumerConfig.MetricConfig + applicationConfig = consumerConfig.ApplicationConfig checkApplicationName(consumerConfig.ApplicationConfig) if err := configCenterRefreshConsumer(); err != nil { @@ -136,8 +138,9 @@ func Load() { logger.Warnf("providerConfig is nil!") } else { - // so, you should know that the consumer's metric config will be override + // so, you should know that the consumer's config will be override metricConfig = providerConfig.MetricConfig + applicationConfig = providerConfig.ApplicationConfig checkApplicationName(providerConfig.ApplicationConfig) if err := configCenterRefreshProvider(); err != nil { diff --git a/filter/filter_impl/metrics_filter_test.go b/filter/filter_impl/metrics_filter_test.go index 312b1b9fcd..83697f0f29 100644 --- a/filter/filter_impl/metrics_filter_test.go +++ b/filter/filter_impl/metrics_filter_test.go @@ -33,6 +33,7 @@ import ( "github.com/apache/dubbo-go/common" "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/config" + "github.com/apache/dubbo-go/metrics" "github.com/apache/dubbo-go/protocol" "github.com/apache/dubbo-go/protocol/invocation" ) @@ -42,7 +43,9 @@ func TestMetricsFilter_Invoke(t *testing.T) { // prepare the mock reporter config.GetMetricConfig().Reporters = []string{"mock"} mk := &mockReporter{} - extension.SetMetricReporter("mock", mk) + extension.SetMetricReporter("mock", func() metrics.Reporter { + return mk + }) instance := extension.GetFilter(metricFilterName) diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index 8914902ab0..eb8e9fe481 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -21,6 +21,7 @@ import ( "context" "strconv" "strings" + "sync" "time" ) import ( @@ -57,21 +58,16 @@ const ( ) var ( - labelNames = []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey, remoteKey, localKey} + labelNames = []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey, remoteKey, localKey} + namespace = config.GetApplicationConfig().Name + reporterInstance *PrometheusReporter + reporterInitOnce sync.Once ) // should initialize after loading configuration func init() { - rpt := &PrometheusReporter{ - consumerSummaryVec: newSummaryVec(consumerKey), - providerSummaryVec: newSummaryVec(providerKey), - consumerHistogramVec: newHistogramVec(consumerKey), - providerHistogramVec: newHistogramVec(providerKey), - } - prometheus.MustRegister(rpt.consumerSummaryVec, rpt.providerSummaryVec, - rpt.consumerHistogramVec, rpt.providerHistogramVec) - extension.SetMetricReporter(reporterName, rpt) + extension.SetMetricReporter(reporterName, newPrometheusReporter) } // it will collect the data for Prometheus @@ -126,7 +122,7 @@ func newHistogramVec(side string) *prometheus.HistogramVec { mc := config.GetMetricConfig() return prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: metrics.NameSpace, + Namespace: namespace, Subsystem: side, Name: serviceKey + histogramSuffix, Help: "This is the dubbo's histogram metrics", @@ -150,10 +146,9 @@ func isConsumer(url common.URL) bool { // create SummaryVec, the Namespace is dubbo // the objectives is from my experience. func newSummaryVec(side string) *prometheus.SummaryVec { - return prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Namespace: metrics.NameSpace, + Namespace: namespace, Help: "This is the dubbo's summary metrics", Subsystem: side, Name: serviceKey + summarySuffix, @@ -169,3 +164,20 @@ func newSummaryVec(side string) *prometheus.SummaryVec { labelNames, ) } + +func newPrometheusReporter() metrics.Reporter { + if reporterInstance == nil { + reporterInitOnce.Do(func() { + reporterInstance = &PrometheusReporter{ + consumerSummaryVec: newSummaryVec(consumerKey), + providerSummaryVec: newSummaryVec(providerKey), + + consumerHistogramVec: newHistogramVec(consumerKey), + providerHistogramVec: newHistogramVec(providerKey), + } + prometheus.MustRegister(reporterInstance.consumerSummaryVec, reporterInstance.providerSummaryVec, + reporterInstance.consumerHistogramVec, reporterInstance.providerHistogramVec) + }) + } + return reporterInstance +} From 83d3ca693a1f2142d6d928033f2b02c4e4b254d0 Mon Sep 17 00:00:00 2001 From: flycash Date: Thu, 30 Jan 2020 18:12:43 +0800 Subject: [PATCH 09/13] Remove remote key and local key --- metrics/prometheus/reporter.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index eb8e9fe481..b7e120bdd4 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -45,8 +45,6 @@ const ( versionKey = constant.VERSION_KEY methodKey = constant.METHOD_KEY timeoutKey = constant.TIMEOUT_KEY - remoteKey = "remote" - localKey = "local" providerKey = "provider" consumerKey = "consumer" @@ -58,7 +56,7 @@ const ( ) var ( - labelNames = []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey, remoteKey, localKey} + labelNames = []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey,} namespace = config.GetApplicationConfig().Name reporterInstance *PrometheusReporter reporterInitOnce sync.Once @@ -109,8 +107,6 @@ func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol versionKey: url.GetParam(versionKey, ""), methodKey: invocation.MethodName(), timeoutKey: url.GetParam(timeoutKey, ""), - remoteKey: invocation.AttachmentsByKey(constant.REMOTE_ADDR, ""), - localKey: invocation.AttachmentsByKey(constant.REMOTE_ADDR, ""), } costMs := float64(cost.Nanoseconds() / constant.MsToNanoRate) From 5e74a2c3498e7f59dc506373375c09a0c66621ab Mon Sep 17 00:00:00 2001 From: flycash Date: Thu, 30 Jan 2020 18:13:17 +0800 Subject: [PATCH 10/13] fmt code --- metrics/prometheus/reporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index b7e120bdd4..e6dfdb4aee 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -56,7 +56,7 @@ const ( ) var ( - labelNames = []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey,} + labelNames = []string{serviceKey, groupKey, versionKey, methodKey, timeoutKey} namespace = config.GetApplicationConfig().Name reporterInstance *PrometheusReporter reporterInitOnce sync.Once From d4ee2bfc64a35980deaf7ffd96964a3e81814e26 Mon Sep 17 00:00:00 2001 From: flycash Date: Fri, 31 Jan 2020 13:01:23 +0800 Subject: [PATCH 11/13] Move GetXXXConfig to config_load file --- config/application_config.go | 10 --------- config/config_loader.go | 39 ++++++++++++++++++++++++++++++++++++ config/consumer_config.go | 9 --------- config/metric_config.go | 9 --------- config/provider_config.go | 9 --------- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/config/application_config.go b/config/application_config.go index 5db27cf0a2..23ab7d34ac 100644 --- a/config/application_config.go +++ b/config/application_config.go @@ -35,16 +35,6 @@ type ApplicationConfig struct { Environment string `yaml:"environment" json:"environment,omitempty" property:"environment"` } -// find the application config -// if not, we will create one -// Usually applicationConfig will be initialized when system start -func GetApplicationConfig() *ApplicationConfig { - if applicationConfig == nil { - applicationConfig = &ApplicationConfig{} - } - return applicationConfig -} - // Prefix ... func (*ApplicationConfig) Prefix() string { return constant.DUBBO + ".application." diff --git a/config/config_loader.go b/config/config_loader.go index c29e7bce5e..875d1f6ddb 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -173,3 +173,42 @@ func GetRPCService(name string) common.RPCService { func RPCService(service common.RPCService) { consumerConfig.References[service.Reference()].Implement(service) } + +// GetMetricConfig find the MetricConfig +// if it is nil, create a new one +func GetMetricConfig() *MetricConfig { + if metricConfig == nil { + metricConfig = &MetricConfig{} + } + return metricConfig +} + +// GetApplicationConfig find the application config +// if not, we will create one +// Usually applicationConfig will be initialized when system start +func GetApplicationConfig() *ApplicationConfig { + if applicationConfig == nil { + applicationConfig = &ApplicationConfig{} + } + return applicationConfig +} + +// GetProviderConfig find the provider config +// if not found, create new one +func GetProviderConfig() ProviderConfig { + if providerConfig == nil { + logger.Warnf("providerConfig is nil!") + return ProviderConfig{} + } + return *providerConfig +} + +// GetConsumerConfig find the consumer config +// if not found, create new one +func GetConsumerConfig() ConsumerConfig { + if consumerConfig == nil { + logger.Warnf("consumerConfig is nil!") + return ConsumerConfig{} + } + return *consumerConfig +} diff --git a/config/consumer_config.go b/config/consumer_config.go index 7756f3b51c..1bfa761fc9 100644 --- a/config/consumer_config.go +++ b/config/consumer_config.go @@ -85,15 +85,6 @@ func SetConsumerConfig(c ConsumerConfig) { consumerConfig = &c } -// GetConsumerConfig ... -func GetConsumerConfig() ConsumerConfig { - if consumerConfig == nil { - logger.Warnf("consumerConfig is nil!") - return ConsumerConfig{} - } - return *consumerConfig -} - // ConsumerInit ... func ConsumerInit(confConFile string) error { if confConFile == "" { diff --git a/config/metric_config.go b/config/metric_config.go index 7b9222f29b..73a3ca1cfe 100644 --- a/config/metric_config.go +++ b/config/metric_config.go @@ -27,15 +27,6 @@ type MetricConfig struct { HistogramBucket []float64 `yaml:"histogram_bucket" json:"histogram_bucket,omitempty"` } -// find the MetricConfig -// if it is nil, create a new one -func GetMetricConfig() *MetricConfig { - if metricConfig == nil { - metricConfig = &MetricConfig{} - } - return metricConfig -} - // find the histogram bucket // if it's empty, the default value will be return func (mc *MetricConfig) GetHistogramBucket() []float64 { diff --git a/config/provider_config.go b/config/provider_config.go index 0bfa78647b..0f5c71a7de 100644 --- a/config/provider_config.go +++ b/config/provider_config.go @@ -76,15 +76,6 @@ func SetProviderConfig(p ProviderConfig) { providerConfig = &p } -// GetProviderConfig ... -func GetProviderConfig() ProviderConfig { - if providerConfig == nil { - logger.Warnf("providerConfig is nil!") - return ProviderConfig{} - } - return *providerConfig -} - // ProviderInit ... func ProviderInit(confProFile string) error { if len(confProFile) == 0 { From 6404fbbbe737470061ae4cccbc0d5062d0fe47d5 Mon Sep 17 00:00:00 2001 From: flycash Date: Fri, 31 Jan 2020 13:11:00 +0800 Subject: [PATCH 12/13] fmt comments --- common/extension/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/extension/metrics.go b/common/extension/metrics.go index e7695c7633..42fca7a2db 100644 --- a/common/extension/metrics.go +++ b/common/extension/metrics.go @@ -27,12 +27,12 @@ var ( metricReporterMap = make(map[string]func() metrics.Reporter, 4) ) -// set a reporter with the name +// SetMetricReporter set a reporter with the name func SetMetricReporter(name string, reporterFunc func() metrics.Reporter) { metricReporterMap[name] = reporterFunc } -// find the reporter with name. +// GetMetricReporter find the reporter with name. // if not found, it will panic. // we should know that this method usually is called when system starts, so we should panic func GetMetricReporter(name string) metrics.Reporter { From 229e027fbe892c9916554dec5ffa0f27d6c07d4b Mon Sep 17 00:00:00 2001 From: flycash Date: Fri, 31 Jan 2020 19:53:11 +0800 Subject: [PATCH 13/13] fix review --- filter/filter_impl/metrics_filter.go | 6 +++--- metrics/prometheus/reporter.go | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/filter/filter_impl/metrics_filter.go b/filter/filter_impl/metrics_filter.go index 1e2dc35f2d..f4734172b7 100644 --- a/filter/filter_impl/metrics_filter.go +++ b/filter/filter_impl/metrics_filter.go @@ -55,7 +55,7 @@ type metricsFilter struct { reporters []metrics.Reporter } -// using goroutine to report the duration. +// Invoke collect the duration of invocation and then report the duration by using goroutine func (p *metricsFilter) Invoke(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation) protocol.Result { start := time.Now() res := invoker.Invoke(ctx, invocation) @@ -69,12 +69,12 @@ func (p *metricsFilter) Invoke(ctx context.Context, invoker protocol.Invoker, in return res } -// do nothing and return the result +// OnResponse do nothing and return the result func (p *metricsFilter) OnResponse(ctx context.Context, res protocol.Result, invoker protocol.Invoker, invocation protocol.Invocation) protocol.Result { return res } -// the metricsFilter is singleton. +// newMetricsFilter the metricsFilter is singleton. // it's lazy initialization // make sure that the configuration had been loaded before invoking this method. func newMetricsFilter() filter.Filter { diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index e6dfdb4aee..1636b14da2 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -68,6 +68,7 @@ func init() { extension.SetMetricReporter(reporterName, newPrometheusReporter) } +// PrometheusReporter // it will collect the data for Prometheus // if you want to use this, you should initialize your prometheus. // https://prometheus.io/docs/guides/go-application/ @@ -84,7 +85,9 @@ type PrometheusReporter struct { consumerHistogramVec *prometheus.HistogramVec } -// report the duration to Prometheus +// Report report the duration to Prometheus +// the role in url must be consumer or provider +// or it will be ignored func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration, res protocol.Result) { url := invoker.GetUrl() var sumVec *prometheus.SummaryVec @@ -139,7 +142,7 @@ func isConsumer(url common.URL) bool { return strings.EqualFold(role, strconv.Itoa(common.CONSUMER)) } -// create SummaryVec, the Namespace is dubbo +// newSummaryVec create SummaryVec, the Namespace is dubbo // the objectives is from my experience. func newSummaryVec(side string) *prometheus.SummaryVec { return prometheus.NewSummaryVec( @@ -161,6 +164,8 @@ func newSummaryVec(side string) *prometheus.SummaryVec { ) } +// newPrometheusReporter create new prometheusReporter +// it will register the metrics into prometheus func newPrometheusReporter() metrics.Reporter { if reporterInstance == nil { reporterInitOnce.Do(func() {