docs,tools/etcd-dump-metrics: automate metrics documentation

docs/index.rst

@@ -7,7 +7,7 @@ This is official etcd documentation.
 Still working in progress...
 
 * :ref:`set-up`: setting up an etcd cluster.
-* :ref:`monitor`: monitoring an etcd cluster.
+* :ref:`operate`: operating an etcd cluster.
 * :ref:`client-architecture`: describes etcd client components.
 
 .. toctree::
@@ -22,12 +22,6 @@ Still working in progress...
 
    operate
 
-.. toctree::
-   :maxdepth: 3
-   :caption: Monitor
-
-   monitor
-
 .. toctree::
    :maxdepth: 3
    :caption: FAQs

docs/metrics-v3.1 → docs/metrics/v3.1.18

@@ -462,9 +462,11 @@ go_memstats_sys_bytes
 # type: "counter"
 # gRPC codes: 
 #  - "OK"
+#  - "Unavailable"
 grpc_server_handled_total{grpc_code="CODE",grpc_method="DeleteRange",grpc_service="etcdserverpb.KV",grpc_type="unary"}
 grpc_server_handled_total{grpc_code="CODE",grpc_method="Put",grpc_service="etcdserverpb.KV",grpc_type="unary"}
 grpc_server_handled_total{grpc_code="CODE",grpc_method="Range",grpc_service="etcdserverpb.KV",grpc_type="unary"}
+grpc_server_handled_total{grpc_code="CODE",grpc_method="Watch",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}
 
 # name: "grpc_server_msg_received_total"
 # description: "Total number of RPC stream messages received on the server."

docs/metrics-v3.2 → docs/metrics/v3.2.23

@@ -544,11 +544,6 @@ http_request_size_bytes{handler="prometheus",quantile="0.99"}
 http_request_size_bytes_sum{handler="prometheus"}
 http_request_size_bytes_count{handler="prometheus"}
 
-# name: "http_requests_total"
-# description: "Total number of HTTP requests made."
-# type: "counter"
-http_requests_total{code="200",handler="prometheus",method="get"}
-
 # name: "http_response_size_bytes"
 # description: "The HTTP response sizes in bytes."
 # type: "summary"

docs/operate.rst

@@ -1,4 +1,4 @@
-.. _monitor:
+.. _operate:
 
 
 Monitor
@@ -13,23 +13,83 @@ List of metrics
 Latest
 ======
 
-.. literalinclude:: metrics-latest
+Download: :download:`latest <metrics/latest>`.
+
+.. literalinclude:: metrics/latest
   :language: BASH
 
 v3.3
 ====
 
-.. literalinclude:: metrics-v3.3
+- :download:`v3.3.9 <metrics/v3.3.9>`
+- :download:`v3.3.8 <metrics/v3.3.8>`
+- :download:`v3.3.7 <metrics/v3.3.7>`
+- :download:`v3.3.6 <metrics/v3.3.6>`
+- :download:`v3.3.5 <metrics/v3.3.5>`
+- :download:`v3.3.4 <metrics/v3.3.4>`
+- :download:`v3.3.3 <metrics/v3.3.3>`
+- :download:`v3.3.2 <metrics/v3.3.2>`
+- :download:`v3.3.1 <metrics/v3.3.1>`
+- :download:`v3.3.0 <metrics/v3.3.0>`
+
+.. literalinclude:: metrics/v3.3.9
   :language: BASH
 
 v3.2
 ====
 
-.. literalinclude:: metrics-v3.2
+- :download:`v3.2.24 <metrics/v3.2.24>`
+- :download:`v3.2.23 <metrics/v3.2.23>`
+- :download:`v3.2.22 <metrics/v3.2.22>`
+- :download:`v3.2.21 <metrics/v3.2.21>`
+- :download:`v3.2.20 <metrics/v3.2.20>`
+- :download:`v3.2.19 <metrics/v3.2.19>`
+- :download:`v3.2.18 <metrics/v3.2.18>`
+- :download:`v3.2.17 <metrics/v3.2.17>`
+- :download:`v3.2.16 <metrics/v3.2.16>`
+- :download:`v3.2.15 <metrics/v3.2.15>`
+- :download:`v3.2.14 <metrics/v3.2.14>`
+- :download:`v3.2.13 <metrics/v3.2.13>`
+- :download:`v3.2.12 <metrics/v3.2.12>`
+- :download:`v3.2.11 <metrics/v3.2.11>`
+- :download:`v3.2.10 <metrics/v3.2.10>`
+- :download:`v3.2.9 <metrics/v3.2.9>`
+- :download:`v3.2.8 <metrics/v3.2.8>`
+- :download:`v3.2.7 <metrics/v3.2.7>`
+- :download:`v3.2.6 <metrics/v3.2.6>`
+- :download:`v3.2.5 <metrics/v3.2.5>`
+- :download:`v3.2.4 <metrics/v3.2.4>`
+- :download:`v3.2.3 <metrics/v3.2.3>`
+- :download:`v3.2.2 <metrics/v3.2.2>`
+- :download:`v3.2.1 <metrics/v3.2.1>`
+- :download:`v3.2.0 <metrics/v3.2.0>`
+
+.. literalinclude:: metrics/v3.2.24
   :language: BASH
 
 v3.1
 ====
 
-.. literalinclude:: metrics-v3.1
+- :download:`v3.1.19 <metrics/v3.1.19>`
+- :download:`v3.1.18 <metrics/v3.1.18>`
+- :download:`v3.1.17 <metrics/v3.1.17>`
+- :download:`v3.1.16 <metrics/v3.1.16>`
+- :download:`v3.1.15 <metrics/v3.1.15>`
+- :download:`v3.1.14 <metrics/v3.1.14>`
+- :download:`v3.1.13 <metrics/v3.1.13>`
+- :download:`v3.1.12 <metrics/v3.1.12>`
+- :download:`v3.1.11 <metrics/v3.1.11>`
+- :download:`v3.1.10 <metrics/v3.1.10>`
+- :download:`v3.1.9 <metrics/v3.1.9>`
+- :download:`v3.1.8 <metrics/v3.1.8>`
+- :download:`v3.1.7 <metrics/v3.1.7>`
+- :download:`v3.1.6 <metrics/v3.1.6>`
+- :download:`v3.1.5 <metrics/v3.1.5>`
+- :download:`v3.1.4 <metrics/v3.1.4>`
+- :download:`v3.1.3 <metrics/v3.1.3>`
+- :download:`v3.1.2 <metrics/v3.1.2>`
+- :download:`v3.1.1 <metrics/v3.1.1>`
+- :download:`v3.1.0 <metrics/v3.1.0>`
+
+.. literalinclude:: metrics/v3.1.19
   :language: BASH

tools/etcd-dump-metrics/README

@@ -2,17 +2,12 @@
 go install -v ./tools/etcd-dump-metrics
 
 # for latest master branch
-etcd-dump-metrics > docs/metrics-latest
+etcd-dump-metrics > docs/metrics/latest
 
-# download etcd v3.3 to ./bin
+# Or download etcd v3.3.9 to ./bin
 goreman start
-etcd-dump-metrics -addr http://localhost:2379/metrics > docs/metrics-v3.3
-
-# download etcd v3.2 to ./bin
-goreman start
-etcd-dump-metrics -addr http://localhost:2379/metrics > docs/metrics-v3.2
-
-# download etcd v3.1 to ./bin
-goreman start
-etcd-dump-metrics -addr http://localhost:2379/metrics > docs/metrics-v3.1
+etcd-dump-metrics --addr http://localhost:2379/metrics > docs/metrics/v3.3.9
 
+# Or download etcd v3.3.9 to temporary directory to fetch metrics
+etcd-dump-metrics --debug --download-ver v3.3.9
+etcd-dump-metrics --download-ver v3.3.9 > docs/metrics/v3.3.9

tools/etcd-dump-metrics/etcd.go

@@ -0,0 +1,94 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/embed"
+
+	"go.uber.org/zap"
+)
+
+func newEmbedURLs(n int) (urls []url.URL) {
+	urls = make([]url.URL, n)
+	for i := 0; i < n; i++ {
+		u, _ := url.Parse(fmt.Sprintf("unix://localhost:%d%06d", os.Getpid(), i))
+		urls[i] = *u
+	}
+	return urls
+}
+
+func setupEmbedCfg(cfg *embed.Config, curls, purls, ics []url.URL) {
+	cfg.Logger = "zap"
+	cfg.LogOutputs = []string{"/dev/null"}
+	// []string{"stderr"} to enable server logging
+	cfg.Debug = false
+
+	var err error
+	cfg.Dir, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("%016X", time.Now().UnixNano()))
+	if err != nil {
+		panic(err)
+	}
+	os.RemoveAll(cfg.Dir)
+
+	cfg.ClusterState = "new"
+	cfg.LCUrls, cfg.ACUrls = curls, curls
+	cfg.LPUrls, cfg.APUrls = purls, purls
+
+	cfg.InitialCluster = ""
+	for i := range ics {
+		cfg.InitialCluster += fmt.Sprintf(",%d=%s", i, ics[i].String())
+	}
+	cfg.InitialCluster = cfg.InitialCluster[1:]
+}
+
+func getCommand(exec, name, dir, cURL, pURL, cluster string) string {
+	s := fmt.Sprintf("%s --name %s --data-dir %s --listen-client-urls %s --advertise-client-urls %s ",
+		exec, name, dir, cURL, cURL)
+	s += fmt.Sprintf("--listen-peer-urls %s --initial-advertise-peer-urls %s ", pURL, pURL)
+	s += fmt.Sprintf("--initial-cluster %s ", cluster)
+	return s + "--initial-cluster-token tkn --initial-cluster-state new"
+}
+
+func write(ep string) {
+	cli, err := clientv3.New(clientv3.Config{Endpoints: []string{strings.Replace(ep, "/metrics", "", 1)}})
+	if err != nil {
+		lg.Panic("failed to create client", zap.Error(err))
+	}
+	defer cli.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err = cli.Put(ctx, "____test", "")
+	if err != nil {
+		lg.Panic("failed to write test key", zap.Error(err))
+	}
+	_, err = cli.Get(ctx, "____test")
+	if err != nil {
+		lg.Panic("failed to read test key", zap.Error(err))
+	}
+	_, err = cli.Delete(ctx, "____test")
+	if err != nil {
+		lg.Panic("failed to delete test key", zap.Error(err))
+	}
+	cli.Watch(ctx, "____test", clientv3.WithCreatedNotify())
+}

tools/etcd-dump-metrics/install_darwin.go

@@ -0,0 +1,58 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build darwin
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+)
+
+const downloadURL = `https://storage.googleapis.com/etcd/%s/etcd-%s-darwin-amd64.zip`
+
+func install(ver, dir string) (string, error) {
+	ep := fmt.Sprintf(downloadURL, ver, ver)
+
+	resp, err := http.Get(ep)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	d, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	zipPath := filepath.Join(dir, "etcd.zip")
+	if err = ioutil.WriteFile(zipPath, d, fileutil.PrivateFileMode); err != nil {
+		return "", err
+	}
+
+	if err = exec.Command("bash", "-c", fmt.Sprintf("unzip %s -d %s", zipPath, dir)).Run(); err != nil {
+		return "", err
+	}
+
+	bp1 := filepath.Join(dir, fmt.Sprintf("etcd-%s-darwin-amd64", ver), "etcd")
+	bp2 := filepath.Join(dir, "etcd")
+	return bp2, os.Rename(bp1, bp2)
+}

tools/etcd-dump-metrics/install_linux.go

@@ -0,0 +1,54 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+)
+
+const downloadURL = `https://storage.googleapis.com/etcd/%s/etcd-%s-linux-amd64.tar.gz`
+
+func install(ver, dir string) (string, error) {
+	ep := fmt.Sprintf(downloadURL, ver, ver)
+
+	resp, err := http.Get(ep)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	d, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	tarPath := filepath.Join(dir, "etcd.tar.gz")
+	if err = ioutil.WriteFile(tarPath, d, fileutil.PrivateFileMode); err != nil {
+		return "", err
+	}
+
+	if err = exec.Command("bash", "-c", fmt.Sprintf("tar xzvf %s -C %s --strip-components=1", tarPath, dir)).Run(); err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "etcd"), nil
+}

tools/etcd-dump-metrics/install_windows.go

@@ -0,0 +1,23 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build windows
+
+package main
+
+import "errors"
+
+func install(ver, dir string) error {
+	return errors.New("windows install is not supported yet")
+}