diff --git a/embed/config.go b/embed/config.go index 80410312230..ee5fcce4f08 100644 --- a/embed/config.go +++ b/embed/config.go @@ -112,10 +112,12 @@ type Config struct { // debug - Debug bool `json:"debug"` - LogPkgLevels string `json:"log-package-levels"` - EnablePprof bool - Metrics string `json:"metrics"` + Debug bool `json:"debug"` + LogPkgLevels string `json:"log-package-levels"` + EnablePprof bool + Metrics string `json:"metrics"` + ListenMetricsUrls []url.URL + ListenMetricsUrlsJSON string `json:"listen-metrics-urls"` // ForceNewCluster starts a new cluster even if previously started; unsafe. ForceNewCluster bool `json:"force-new-cluster"` @@ -255,6 +257,14 @@ func (cfg *configYAML) configFromFile(path string) error { cfg.ACUrls = []url.URL(u) } + if cfg.ListenMetricsUrlsJSON != "" { + u, err := types.NewURLs(strings.Split(cfg.ListenMetricsUrlsJSON, ",")) + if err != nil { + plog.Fatalf("unexpected error setting up listen-metrics-urls: %v", err) + } + cfg.ListenMetricsUrls = []url.URL(u) + } + // If a discovery flag is set, clear default initial cluster set by InitialClusterFromName if (cfg.Durl != "" || cfg.DNSCluster != "") && cfg.InitialCluster == defaultInitialCluster { cfg.InitialCluster = "" @@ -285,6 +295,9 @@ func (cfg *Config) Validate() error { if err := checkBindURLs(cfg.LCUrls); err != nil { return err } + if err := checkBindURLs(cfg.ListenMetricsUrls); err != nil { + return err + } // Check if conflicting flags are passed. nSet := 0 diff --git a/embed/etcd.go b/embed/etcd.go index 64fbfac702a..5b7a17ca99a 100644 --- a/embed/etcd.go +++ b/embed/etcd.go @@ -21,6 +21,7 @@ import ( defaultLog "log" "net" "net/http" + "net/url" "path/filepath" "sync" "time" @@ -35,6 +36,7 @@ import ( "github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/rafthttp" "github.com/coreos/pkg/capnslog" + "github.com/prometheus/client_golang/prometheus" ) var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "embed") @@ -55,9 +57,10 @@ const ( // Etcd contains a running etcd server and its listeners. type Etcd struct { - Peers []*peerListener - Clients []net.Listener - Server *etcdserver.EtcdServer + Peers []*peerListener + Clients []net.Listener + metricsListeners []net.Listener + Server *etcdserver.EtcdServer cfg Config stopc chan struct{} @@ -205,6 +208,9 @@ func (e *Etcd) Close() { e.Clients[i].Close() } } + for i := range e.metricsListeners { + e.metricsListeners[i].Close() + } // close rafthttp transports if e.Server != nil { @@ -400,6 +406,25 @@ func (e *Etcd) serve() (err error) { e.errHandler(s.serve(e.Server, &e.cfg.ClientTLSInfo, h, e.errHandler)) }(sctx) } + + if len(e.cfg.ListenMetricsUrls) > 0 { + // TODO: maybe etcdhttp.MetricsPath or get the path from the user-provided URL + metricsMux := http.NewServeMux() + metricsMux.Handle("/metrics", prometheus.Handler()) + + for _, murl := range e.cfg.ListenMetricsUrls { + ml, err := transport.NewListener(murl.Host, murl.Scheme, &e.cfg.ClientTLSInfo) + if err != nil { + return err + } + e.metricsListeners = append(e.metricsListeners, ml) + go func(u url.URL, ln net.Listener) { + plog.Info("listening for metrics on ", u.String()) + e.errHandler(http.Serve(ln, metricsMux)) + }(murl, ml) + } + } + return nil } diff --git a/etcdmain/config.go b/etcdmain/config.go index b541409a412..4bc900bc1ed 100644 --- a/etcdmain/config.go +++ b/etcdmain/config.go @@ -20,12 +20,14 @@ import ( "flag" "fmt" "io/ioutil" + "net/url" "os" "runtime" "strings" "github.com/coreos/etcd/embed" "github.com/coreos/etcd/pkg/flags" + "github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/version" "github.com/ghodss/yaml" ) @@ -131,6 +133,7 @@ func newConfig() *config { fs.StringVar(&cfg.WalDir, "wal-dir", cfg.WalDir, "Path to the dedicated wal directory.") fs.Var(flags.NewURLsValue(embed.DefaultListenPeerURLs), "listen-peer-urls", "List of URLs to listen on for peer traffic.") fs.Var(flags.NewURLsValue(embed.DefaultListenClientURLs), "listen-client-urls", "List of URLs to listen on for client traffic.") + fs.StringVar(&cfg.ListenMetricsUrlsJSON, "listen-metrics-urls", "", "List of URLs to listen on for metrics.") fs.UintVar(&cfg.MaxSnapFiles, "max-snapshots", cfg.MaxSnapFiles, "Maximum number of snapshot files to retain (0 is unlimited).") fs.UintVar(&cfg.MaxWalFiles, "max-wals", cfg.MaxWalFiles, "Maximum number of wal files to retain (0 is unlimited).") fs.StringVar(&cfg.Name, "name", cfg.Name, "Human-readable name for this member.") @@ -262,6 +265,15 @@ func (cfg *config) configFromCmdLine() error { cfg.APUrls = flags.URLsFromFlag(cfg.FlagSet, "initial-advertise-peer-urls") cfg.LCUrls = flags.URLsFromFlag(cfg.FlagSet, "listen-client-urls") cfg.ACUrls = flags.URLsFromFlag(cfg.FlagSet, "advertise-client-urls") + + if len(cfg.ListenMetricsUrlsJSON) > 0 { + u, err := types.NewURLs(strings.Split(cfg.ListenMetricsUrlsJSON, ",")) + if err != nil { + plog.Fatalf("unexpected error setting up listen-metrics-urls: %v", err) + } + cfg.ListenMetricsUrls = []url.URL(u) + } + cfg.ClusterState = cfg.clusterState.String() cfg.Fallback = cfg.fallback.String() cfg.Proxy = cfg.proxy.String() diff --git a/etcdmain/etcd.go b/etcdmain/etcd.go index 2f7f00d61ad..722df51f92f 100644 --- a/etcdmain/etcd.go +++ b/etcdmain/etcd.go @@ -313,7 +313,7 @@ func startProxy(cfg *config) error { go func() { plog.Info("proxy: listening for client requests on ", host) mux := http.NewServeMux() - mux.Handle("/metrics", prometheus.Handler()) + mux.Handle("/metrics", prometheus.Handler()) // v2 proxy just uses the same port mux.Handle("/", ph) plog.Fatal(http.Serve(l, mux)) }() diff --git a/etcdmain/grpc_proxy.go b/etcdmain/grpc_proxy.go index b2cc25c3a22..800ab712d33 100644 --- a/etcdmain/grpc_proxy.go +++ b/etcdmain/grpc_proxy.go @@ -19,6 +19,7 @@ import ( "fmt" "net" "net/http" + "net/url" "os" "time" @@ -40,6 +41,7 @@ import ( var ( grpcProxyListenAddr string + grpcProxyMetricsListenAddr string grpcProxyEndpoints []string grpcProxyDNSCluster string grpcProxyInsecureDiscovery bool @@ -80,6 +82,7 @@ func newGRPCProxyStartCommand() *cobra.Command { cmd.Flags().StringVar(&grpcProxyListenAddr, "listen-addr", "127.0.0.1:23790", "listen address") cmd.Flags().StringVar(&grpcProxyDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster") + cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for /metrics requests on an additional interface") cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records") cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints") cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file") @@ -129,7 +132,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { }() m := cmux.New(l) - cfg, err := newClientCfg() + cfg, cfgtls, err := newClientCfg() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -202,6 +205,27 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { go func() { errc <- m.Serve() }() + if len(grpcProxyMetricsListenAddr) > 0 { + murl, err := url.Parse(grpcProxyMetricsListenAddr) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot parse %q", grpcProxyMetricsListenAddr) + os.Exit(1) + } + ml, err := transport.NewListener(murl.Host, murl.Scheme, cfgtls) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + mux := http.NewServeMux() + mux.Handle("/metrics", prometheus.Handler()) + + go func() { + plog.Info("grpc-proxy: listening for metrics on ", murl.String()) + plog.Fatal(http.Serve(ml, mux)) + }() + } + // grpc-proxy is initialized, ready to serve notifySystemd() @@ -209,7 +233,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { os.Exit(1) } -func newClientCfg() (*clientv3.Config, error) { +func newClientCfg() (*clientv3.Config, *transport.TLSInfo, error) { // set tls if any one tls option set var cfgtls *transport.TLSInfo tlsinfo := transport.TLSInfo{} @@ -235,12 +259,12 @@ func newClientCfg() (*clientv3.Config, error) { if cfgtls != nil { clientTLS, err := cfgtls.ClientConfig() if err != nil { - return nil, err + return nil, nil, err } cfg.TLS = clientTLS } // TODO: support insecure tls - return &cfg, nil + return &cfg, cfgtls, nil } diff --git a/etcdmain/help.go b/etcdmain/help.go index 5a5ea551a00..ad4d30240ee 100644 --- a/etcdmain/help.go +++ b/etcdmain/help.go @@ -66,7 +66,7 @@ member flags: comma-separated whitelist of origins for CORS (cross-origin resource sharing). --quota-backend-bytes '0' raise alarms when backend size exceeds the given quota (0 defaults to low space quota). - --max-txn-ops '128' + --max-txn-ops '128' maximum number of operations permitted in a transaction. --max-request-bytes '1572864' maximum client request size in bytes the server will accept. @@ -172,7 +172,9 @@ profiling flags: --enable-pprof 'false' Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/" --metrics 'basic' - Set level of detail for exported metrics, specify 'extensive' to include histogram metrics. + Set level of detail for exported metrics, specify 'extensive' to include histogram metrics. + --listen-metrics-urls '' + List of URLs to listen on for metrics. auth flags: --auth-token 'simple'