From 5d3d496095a5d5424b69a516ed532d626eaeee5b Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Wed, 1 May 2019 21:47:19 -0700
Subject: [PATCH 01/10] impr/clients: Handle TLS config and MTLS for logcli and
 promtail

---
 cmd/logcli/client.go          | 40 +++++++++++++++-
 cmd/logcli/main.go            |  6 +++
 docs/logcli.md                | 20 +++++---
 docs/promtail-setup.md        | 28 +++++++++--
 pkg/helpers/tls.go            | 87 +++++++++++++++++++++++++++++++++++
 pkg/promtail/client/client.go | 45 ++++++++++++++++--
 6 files changed, 207 insertions(+), 19 deletions(-)
 create mode 100644 pkg/helpers/tls.go

diff --git a/cmd/logcli/client.go b/cmd/logcli/client.go
index 3acaea6b65eea..980fd71df6264 100644
--- a/cmd/logcli/client.go
+++ b/cmd/logcli/client.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/gorilla/websocket"
 
+	"github.com/grafana/loki/pkg/helpers"
 	"github.com/grafana/loki/pkg/logproto"
 )
 
@@ -60,9 +61,27 @@ func doRequest(path string, out interface{}) error {
 	if err != nil {
 		return err
 	}
+
 	req.SetBasicAuth(*username, *password)
 
-	resp, err := http.DefaultClient.Do(req)
+	tlsConfig, err := helpers.NewTLSConfigFromOptions(
+		url,
+		*tlsCACertPath,
+		*tlsClientCertPath,
+		*tlsClientCertKeyPath,
+		*tlsClientCertKeyPass,
+		*tlsSkipVerify)
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: tlsConfig,
+		},
+	}
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
@@ -86,6 +105,18 @@ func liveTailQueryConn() (*websocket.Conn, error) {
 }
 
 func wsConnect(path string) (*websocket.Conn, error) {
+
+	tlsConfig, err := helpers.NewTLSConfigFromOptions(
+		*addr,
+		*tlsCACertPath,
+		*tlsClientCertPath,
+		*tlsClientCertKeyPath,
+		*tlsClientCertKeyPass,
+		*tlsSkipVerify)
+	if err != nil {
+		return nil, err
+	}
+
 	url := *addr + path
 	if strings.HasPrefix(url, "https") {
 		url = strings.Replace(url, "https", "wss", 1)
@@ -95,7 +126,12 @@ func wsConnect(path string) (*websocket.Conn, error) {
 	fmt.Println(url)
 
 	h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}}
-	c, resp, err := websocket.DefaultDialer.Dial(url, h)
+
+	ws := websocket.Dialer{
+		TLSClientConfig: tlsConfig,
+	}
+
+	c, resp, err := ws.Dial(url, h)
 
 	if err != nil {
 		if resp == nil {
diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index 596bdda6f0f0e..c910840181a96 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -14,6 +14,12 @@ var (
 	username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String()
 	password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String()
 
+	tlsCACertPath        = app.Flag("ca-cert", "Path to the server Certificate Authority.").Default("").Envar("LOKI_CA_CERT_PATH").String()
+	tlsSkipVerify        = app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").Bool()
+	tlsClientCertPath    = app.Flag("cert", "Path to the client certificate.").Default("").Envar("LOKI_CLIENT_CERT_PATH").String()
+	tlsClientCertKeyPath = app.Flag("key", "Path to the client certificate key.").Default("").Envar("LOKI_CLIENT_KEY_PATH").String()
+	tlsClientCertKeyPass = app.Flag("key-pass", "Client certificate key password.").Default("").Envar("LOKI_CLIENT_KEY_PASS").String()
+
 	queryCmd  = app.Command("query", "Run a LogQL query.")
 	queryStr  = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
 	regexpStr = queryCmd.Arg("regex", "").String()
diff --git a/docs/logcli.md b/docs/logcli.md
index d7330a2826bbb..2b8b85cec7b0b 100644
--- a/docs/logcli.md
+++ b/docs/logcli.md
@@ -44,8 +44,8 @@ Common labels: {job="cortex-ops/consul", namespace="cortex-ops"}
 
 ### Configuration
 
-
 Configuration values are considered in the following order (lowest to highest):
+
 - environment value
 - command line
 
@@ -53,17 +53,23 @@ The URLs of the requests are printed to help with integration work.
 
 ### Details
 
-```
+```console
 $ logcli help
 usage: logcli [<flags>] <command> [<args> ...]
 
 A command-line for loki.
 
 Flags:
-  --help         Show context-sensitive help (also try --help-long and --help-man).
-  --addr=""      Server address, need to specify.
-  --username=""  Username for HTTP basic auth.
-  --password=""  Password for HTTP basic auth.
+  --help             Show context-sensitive help (also try --help-long and --help-man).
+  --addr="https://logs-us-west1.grafana.net"
+                     Server address.
+  --username=""      Username for HTTP basic auth.
+  --password=""      Password for HTTP basic auth.
+  --ca-cert=""       Path to the server Certificate Authority.
+  --tls-skip-verify  Server certificate TLS skip verify.
+  --cert=""          Path to the client certificate.
+  --key=""           Path to the client certificate key.
+  --key-pass=""      Client certificate key password.
 
 Commands:
   help [<command>...]
@@ -72,7 +78,7 @@ Commands:
   query [<flags>] <query> [<regex>]
     Run a LogQL query.
 
-  labels <label>
+  labels [<label>]
     Find values for a given label.
 
 $ logcli help query
diff --git a/docs/promtail-setup.md b/docs/promtail-setup.md
index 48fd0580935af..6dc61757f7016 100644
--- a/docs/promtail-setup.md
+++ b/docs/promtail-setup.md
@@ -6,13 +6,14 @@
 
 ## Daemonset method
 
-Daemonset will deploy promtail on every node within the kubernetes cluster.
+Daemonset will deploy `promtail` on every node within the Kubernetes cluster.
 
 Daemonset deployment is great to collect all of the container logs within the
 cluster. It is great solution for single tenant.  All of the logs will send to a
 single Loki server.
 
 ### Example
+
 ```yaml
 ---Daemonset.yaml
 apiVersion: extensions/v1beta1
@@ -89,14 +90,15 @@ roleRef:
 
 ## Sidecar Method
 
-Sidecar method will deploy promtail as a container within a pod that
+Sidecar method will deploy `promtail` as a container within a pod that
 developer/devops create.
 
-This method will deploy promtail as a sidecar container within a pod.
+This method will deploy `promtail` as a sidecar container within a pod.
 In a multi-tenant environment, this enables teams to aggregate logs
 for specific pods and deployments for example for all pods in a namespace.
 
 ### Example
+
 ```yaml
 ---Deployment.yaml
 apiVersion: extensions/v1beta1
@@ -104,7 +106,7 @@ kind: Deployment
 metadata:
   name: my_test_app
   ...
-spec: 
+spec:
   ...
   template:
     spec:
@@ -135,7 +137,7 @@ spec:
 Sometime application create customized log files.  To collect those logs, you
 would need to have a customized `__path__` in your scrape_config.
 
-Right now, the best way to watch and tail custom log path is define log filepath
+Right now, the best way to watch and tail custom log path is define log file path
 as a label for the pod.
 
 #### Example
@@ -166,3 +168,19 @@ scrape_configs:
         replacement: /your_log_file_dir/$1.log
       ...
 ```
+
+### Custom Client options
+
+If you put `loki` behind a reverse proxy to use https transport (either `TLS` or `MTLS`) you can use the following settings.
+
+```yaml
+---promtail_config.yaml
+...
+client:
+  ca: /path/to/ca/cert.pem # This is the Certificate Authority to verify the server certificate. [optional]
+  certificate: /path/to/ca/client-cert.pem # This is the client certificate that will be checked by the server [optional]
+  certificate-key: /path/to/ca/client-cert.key # This is the client certificate key [optional]
+  certificate-key-pass: password # This is the client certificate password key [optional]
+  tls-skip-verify: true # Never use that settings except for development [optional]
+      ...
+```
diff --git a/pkg/helpers/tls.go b/pkg/helpers/tls.go
new file mode 100644
index 0000000000000..3beffd069c5d3
--- /dev/null
+++ b/pkg/helpers/tls.go
@@ -0,0 +1,87 @@
+
+package helpers
+
+import (
+  "io/ioutil"
+  "strings"
+  "crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+)
+
+// NewTLSConfigFromOptions return a tls.Config given parameters provided
+func NewTLSConfigFromOptions(url, ca, cert, certKey, certKeyPass string, skipVerify bool) (*tls.Config, error) {
+
+	tlsConfig := tls.Config{}
+
+	// if url schme is https prefixed
+	if strings.HasPrefix(url, "https://") {
+
+    tlsConfig.InsecureSkipVerify = skipVerify
+
+		// Add any provided CA
+		if ca != "" {
+			CACert, err := ioutil.ReadFile(ca)
+			if err != nil {
+				return nil, err
+			}
+
+			capool, err := x509.SystemCertPool()
+			if err != nil {
+				return nil, fmt.Errorf("unable to read system cert pool: %s", err)
+			}
+
+			capool.AppendCertsFromPEM(CACert)
+
+			tlsConfig.RootCAs = capool
+		}
+
+		// Add any client certs
+		if cert != "" && certKey != "" {
+
+			certPemBytes, err := ioutil.ReadFile(cert)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read pem file: %s", err)
+			}
+
+			keyPemBytes, err := ioutil.ReadFile(certKey)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read key pem file: %s", err)
+			}
+
+			// Parse key
+			keyBlock, rest := pem.Decode(keyPemBytes)
+			if keyBlock == nil {
+				return nil, fmt.Errorf("could not read key data from bytes: '%s'", string(keyPemBytes))
+			}
+
+			if len(rest) > 0 {
+				return nil, fmt.Errorf("multiple private keys found: this is not supported")
+			}
+
+			// Decrypt if needed
+			if x509.IsEncryptedPEMBlock(keyBlock) {
+
+				data, err := x509.DecryptPEMBlock(keyBlock, []byte(certKeyPass))
+				if err != nil {
+					return nil, err
+				}
+
+				keyBlock = &pem.Block{
+					Type:  keyBlock.Type,
+					Bytes: data,
+				}
+			}
+
+			clientCert, err := tls.X509KeyPair(certPemBytes, pem.EncodeToMemory(keyBlock))
+			if err != nil {
+				return nil, err
+			}
+
+			tlsConfig.Certificates = []tls.Certificate{clientCert}
+		}
+
+	}
+	return &tlsConfig, nil
+}
diff --git a/pkg/promtail/client/client.go b/pkg/promtail/client/client.go
index 127431808fb91..0ccbfa3e8f818 100644
--- a/pkg/promtail/client/client.go
+++ b/pkg/promtail/client/client.go
@@ -54,7 +54,14 @@ func init() {
 
 // Config describes configuration for a HTTP pusher client.
 type Config struct {
-	URL       flagext.URLValue
+	URL flagext.URLValue
+
+	TLSCACertPath               string `yaml:"ca,omitempty"`
+	TLSServerSkipVerify         bool   `yaml:"tls-skip-verify,omitempty"`
+	TLSClientCertificate        string `yaml:"certificate,omitempty"`
+	TLSClientCertificateKey     string `yaml:"certificate-key,omitempty"`
+	TLSClientCertificateKeyPass string `yaml:"certificate-key-pass,omitempty"`
+
 	BatchWait time.Duration
 	BatchSize int
 
@@ -67,6 +74,13 @@ type Config struct {
 // RegisterFlags registers flags.
 func (c *Config) RegisterFlags(flags *flag.FlagSet) {
 	flags.Var(&c.URL, "client.url", "URL of log server")
+
+	flags.StringVar(&c.TLSCACertPath, "client.ca", "", "Path to the server Certificate Authority")
+	flags.BoolVar(&c.TLSServerSkipVerify, "client.tls-skip-verify", false, "Server certificate TLS skip verify")
+	flags.StringVar(&c.TLSClientCertificate, "client.certificate", "", "Path to the client certificate")
+	flags.StringVar(&c.TLSClientCertificateKey, "client.certificate-key", "", "Path to the client certificate key")
+	flags.StringVar(&c.TLSClientCertificateKeyPass, "client.certificate-key-pass", "", "Client certificate key password")
+
 	flags.DurationVar(&c.BatchWait, "client.batch-wait", 1*time.Second, "Maximum wait period before sending batch.")
 	flags.IntVar(&c.BatchSize, "client.batch-size-bytes", 100*1024, "Maximum batch size to accrue before sending. ")
 
@@ -80,6 +94,7 @@ func (c *Config) RegisterFlags(flags *flag.FlagSet) {
 type Client struct {
 	logger  log.Logger
 	cfg     Config
+	client  *http.Client
 	quit    chan struct{}
 	entries chan entry
 	wg      sync.WaitGroup
@@ -102,6 +117,26 @@ func New(cfg Config, logger log.Logger) (*Client, error) {
 
 		externalLabels: cfg.ExternalLabels,
 	}
+
+	tlsConfig, err := helpers.NewTLSConfigFromOptions(
+		cfg.URL.String(),
+		cfg.TLSCACertPath,
+		cfg.TLSClientCertificate,
+		cfg.TLSClientCertificateKey,
+		cfg.TLSClientCertificateKeyPass,
+		cfg.TLSServerSkipVerify)
+
+	c.client = &http.Client{
+		Timeout: cfg.Timeout,
+		Transport: &http.Transport{
+			TLSClientConfig: tlsConfig,
+		},
+	}
+	if err != nil {
+		level.Error(c.logger).Log("msg", "error while creating http client", "error", err) //nolint
+		return nil, err
+	}
+
 	c.wg.Add(1)
 	go c.run()
 	return c, nil
@@ -154,7 +189,7 @@ func (c *Client) run() {
 func (c *Client) sendBatch(batch map[model.Fingerprint]*logproto.Stream) {
 	buf, err := encodeBatch(batch)
 	if err != nil {
-		level.Error(c.logger).Log("msg", "error encoding batch", "error", err)
+		level.Error(c.logger).Log("msg", "error encoding batch", "error", err) //nolint
 		return
 	}
 	bufBytes := float64(len(buf))
@@ -178,12 +213,12 @@ func (c *Client) sendBatch(batch map[model.Fingerprint]*logproto.Stream) {
 			break
 		}
 
-		level.Warn(c.logger).Log("msg", "error sending batch, will retry", "status", status, "error", err)
+		level.Warn(c.logger).Log("msg", "error sending batch, will retry", "status", status, "error", err) //nolint
 		backoff.Wait()
 	}
 
 	if err != nil {
-		level.Error(c.logger).Log("msg", "final error sending batch", "status", status, "error", err)
+		level.Error(c.logger).Log("msg", "final error sending batch", "status", status, "error", err) //nolint
 	}
 }
 
@@ -212,7 +247,7 @@ func (c *Client) send(ctx context.Context, buf []byte) (int, error) {
 	req = req.WithContext(ctx)
 	req.Header.Set("Content-Type", contentType)
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := c.client.Do(req)
 	if err != nil {
 		return -1, err
 	}

From 2b10f76fb3a4afb8da36fe9a9985f7b1ffb0967c Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Wed, 1 May 2019 22:04:58 -0700
Subject: [PATCH 02/10] fix/tls: Please gofmt...

---
 pkg/helpers/tls.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/pkg/helpers/tls.go b/pkg/helpers/tls.go
index 3beffd069c5d3..0a93da69affb0 100644
--- a/pkg/helpers/tls.go
+++ b/pkg/helpers/tls.go
@@ -1,13 +1,12 @@
-
 package helpers
 
 import (
-  "io/ioutil"
-  "strings"
-  "crypto/tls"
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
+	"io/ioutil"
+	"strings"
 )
 
 // NewTLSConfigFromOptions return a tls.Config given parameters provided
@@ -18,7 +17,7 @@ func NewTLSConfigFromOptions(url, ca, cert, certKey, certKeyPass string, skipVer
 	// if url schme is https prefixed
 	if strings.HasPrefix(url, "https://") {
 
-    tlsConfig.InsecureSkipVerify = skipVerify
+		tlsConfig.InsecureSkipVerify = skipVerify
 
 		// Add any provided CA
 		if ca != "" {

From e735c9ffcb266b62ddebbfb1255198ce99ae4e1b Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Mon, 6 May 2019 12:16:55 -0700
Subject: [PATCH 03/10] impr/clients: use prometheus HTTPClientConfig for
 logcli and promtail

---
 cmd/logcli/client.go          | 40 ++++++++--------
 cmd/logcli/main.go            |  1 -
 docs/logcli.md                |  1 -
 docs/promtail-setup.md        |  1 -
 pkg/helpers/tls.go            | 86 -----------------------------------
 pkg/promtail/client/client.go | 35 +++++++-------
 6 files changed, 36 insertions(+), 128 deletions(-)
 delete mode 100644 pkg/helpers/tls.go

diff --git a/cmd/logcli/client.go b/cmd/logcli/client.go
index 980fd71df6264..94c1e9b3343ee 100644
--- a/cmd/logcli/client.go
+++ b/cmd/logcli/client.go
@@ -12,8 +12,8 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
+	"github.com/prometheus/common/config"
 
-	"github.com/grafana/loki/pkg/helpers"
 	"github.com/grafana/loki/pkg/logproto"
 )
 
@@ -64,21 +64,19 @@ func doRequest(path string, out interface{}) error {
 
 	req.SetBasicAuth(*username, *password)
 
-	tlsConfig, err := helpers.NewTLSConfigFromOptions(
-		url,
-		*tlsCACertPath,
-		*tlsClientCertPath,
-		*tlsClientCertKeyPath,
-		*tlsClientCertKeyPass,
-		*tlsSkipVerify)
-	if err != nil {
-		return err
+	clientConfig := config.HTTPClientConfig{
+		TLSConfig: config.TLSConfig{
+			CAFile:             *tlsCACertPath,
+			CertFile:           *tlsClientCertPath,
+			KeyFile:            *tlsClientCertKeyPath,
+			ServerName:         url,
+			InsecureSkipVerify: *tlsSkipVerify,
+		},
 	}
 
-	client := &http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: tlsConfig,
-		},
+	client, err := config.NewClientFromConfig(clientConfig, "logcli")
+	if err != nil {
+		return err
 	}
 
 	resp, err := client.Do(req)
@@ -106,13 +104,13 @@ func liveTailQueryConn() (*websocket.Conn, error) {
 
 func wsConnect(path string) (*websocket.Conn, error) {
 
-	tlsConfig, err := helpers.NewTLSConfigFromOptions(
-		*addr,
-		*tlsCACertPath,
-		*tlsClientCertPath,
-		*tlsClientCertKeyPath,
-		*tlsClientCertKeyPass,
-		*tlsSkipVerify)
+	tlsConfig, err := config.NewTLSConfig(&config.TLSConfig{
+		CAFile:             *tlsCACertPath,
+		CertFile:           *tlsClientCertPath,
+		KeyFile:            *tlsClientCertKeyPath,
+		ServerName:         *addr,
+		InsecureSkipVerify: *tlsSkipVerify,
+	})
 	if err != nil {
 		return nil, err
 	}
diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index c910840181a96..229a8f86d97d4 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -18,7 +18,6 @@ var (
 	tlsSkipVerify        = app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").Bool()
 	tlsClientCertPath    = app.Flag("cert", "Path to the client certificate.").Default("").Envar("LOKI_CLIENT_CERT_PATH").String()
 	tlsClientCertKeyPath = app.Flag("key", "Path to the client certificate key.").Default("").Envar("LOKI_CLIENT_KEY_PATH").String()
-	tlsClientCertKeyPass = app.Flag("key-pass", "Client certificate key password.").Default("").Envar("LOKI_CLIENT_KEY_PASS").String()
 
 	queryCmd  = app.Command("query", "Run a LogQL query.")
 	queryStr  = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
diff --git a/docs/logcli.md b/docs/logcli.md
index 2b8b85cec7b0b..c809125120510 100644
--- a/docs/logcli.md
+++ b/docs/logcli.md
@@ -69,7 +69,6 @@ Flags:
   --tls-skip-verify  Server certificate TLS skip verify.
   --cert=""          Path to the client certificate.
   --key=""           Path to the client certificate key.
-  --key-pass=""      Client certificate key password.
 
 Commands:
   help [<command>...]
diff --git a/docs/promtail-setup.md b/docs/promtail-setup.md
index 6dc61757f7016..b15deac6402b8 100644
--- a/docs/promtail-setup.md
+++ b/docs/promtail-setup.md
@@ -180,7 +180,6 @@ client:
   ca: /path/to/ca/cert.pem # This is the Certificate Authority to verify the server certificate. [optional]
   certificate: /path/to/ca/client-cert.pem # This is the client certificate that will be checked by the server [optional]
   certificate-key: /path/to/ca/client-cert.key # This is the client certificate key [optional]
-  certificate-key-pass: password # This is the client certificate password key [optional]
   tls-skip-verify: true # Never use that settings except for development [optional]
       ...
 ```
diff --git a/pkg/helpers/tls.go b/pkg/helpers/tls.go
deleted file mode 100644
index 0a93da69affb0..0000000000000
--- a/pkg/helpers/tls.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package helpers
-
-import (
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-	"io/ioutil"
-	"strings"
-)
-
-// NewTLSConfigFromOptions return a tls.Config given parameters provided
-func NewTLSConfigFromOptions(url, ca, cert, certKey, certKeyPass string, skipVerify bool) (*tls.Config, error) {
-
-	tlsConfig := tls.Config{}
-
-	// if url schme is https prefixed
-	if strings.HasPrefix(url, "https://") {
-
-		tlsConfig.InsecureSkipVerify = skipVerify
-
-		// Add any provided CA
-		if ca != "" {
-			CACert, err := ioutil.ReadFile(ca)
-			if err != nil {
-				return nil, err
-			}
-
-			capool, err := x509.SystemCertPool()
-			if err != nil {
-				return nil, fmt.Errorf("unable to read system cert pool: %s", err)
-			}
-
-			capool.AppendCertsFromPEM(CACert)
-
-			tlsConfig.RootCAs = capool
-		}
-
-		// Add any client certs
-		if cert != "" && certKey != "" {
-
-			certPemBytes, err := ioutil.ReadFile(cert)
-			if err != nil {
-				return nil, fmt.Errorf("unable to read pem file: %s", err)
-			}
-
-			keyPemBytes, err := ioutil.ReadFile(certKey)
-			if err != nil {
-				return nil, fmt.Errorf("unable to read key pem file: %s", err)
-			}
-
-			// Parse key
-			keyBlock, rest := pem.Decode(keyPemBytes)
-			if keyBlock == nil {
-				return nil, fmt.Errorf("could not read key data from bytes: '%s'", string(keyPemBytes))
-			}
-
-			if len(rest) > 0 {
-				return nil, fmt.Errorf("multiple private keys found: this is not supported")
-			}
-
-			// Decrypt if needed
-			if x509.IsEncryptedPEMBlock(keyBlock) {
-
-				data, err := x509.DecryptPEMBlock(keyBlock, []byte(certKeyPass))
-				if err != nil {
-					return nil, err
-				}
-
-				keyBlock = &pem.Block{
-					Type:  keyBlock.Type,
-					Bytes: data,
-				}
-			}
-
-			clientCert, err := tls.X509KeyPair(certPemBytes, pem.EncodeToMemory(keyBlock))
-			if err != nil {
-				return nil, err
-			}
-
-			tlsConfig.Certificates = []tls.Certificate{clientCert}
-		}
-
-	}
-	return &tlsConfig, nil
-}
diff --git a/pkg/promtail/client/client.go b/pkg/promtail/client/client.go
index 0ccbfa3e8f818..577ab729ced27 100644
--- a/pkg/promtail/client/client.go
+++ b/pkg/promtail/client/client.go
@@ -19,6 +19,7 @@ import (
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
 	"github.com/grafana/loki/pkg/helpers"
@@ -56,11 +57,10 @@ func init() {
 type Config struct {
 	URL flagext.URLValue
 
-	TLSCACertPath               string `yaml:"ca,omitempty"`
-	TLSServerSkipVerify         bool   `yaml:"tls-skip-verify,omitempty"`
-	TLSClientCertificate        string `yaml:"certificate,omitempty"`
-	TLSClientCertificateKey     string `yaml:"certificate-key,omitempty"`
-	TLSClientCertificateKeyPass string `yaml:"certificate-key-pass,omitempty"`
+	TLSCACertPath           string `yaml:"ca,omitempty"`
+	TLSServerSkipVerify     bool   `yaml:"tls-skip-verify,omitempty"`
+	TLSClientCertificate    string `yaml:"certificate,omitempty"`
+	TLSClientCertificateKey string `yaml:"certificate-key,omitempty"`
 
 	BatchWait time.Duration
 	BatchSize int
@@ -79,7 +79,6 @@ func (c *Config) RegisterFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&c.TLSServerSkipVerify, "client.tls-skip-verify", false, "Server certificate TLS skip verify")
 	flags.StringVar(&c.TLSClientCertificate, "client.certificate", "", "Path to the client certificate")
 	flags.StringVar(&c.TLSClientCertificateKey, "client.certificate-key", "", "Path to the client certificate key")
-	flags.StringVar(&c.TLSClientCertificateKeyPass, "client.certificate-key-pass", "", "Client certificate key password")
 
 	flags.DurationVar(&c.BatchWait, "client.batch-wait", 1*time.Second, "Maximum wait period before sending batch.")
 	flags.IntVar(&c.BatchSize, "client.batch-size-bytes", 100*1024, "Maximum batch size to accrue before sending. ")
@@ -118,25 +117,25 @@ func New(cfg Config, logger log.Logger) (*Client, error) {
 		externalLabels: cfg.ExternalLabels,
 	}
 
-	tlsConfig, err := helpers.NewTLSConfigFromOptions(
-		cfg.URL.String(),
-		cfg.TLSCACertPath,
-		cfg.TLSClientCertificate,
-		cfg.TLSClientCertificateKey,
-		cfg.TLSClientCertificateKeyPass,
-		cfg.TLSServerSkipVerify)
-
-	c.client = &http.Client{
-		Timeout: cfg.Timeout,
-		Transport: &http.Transport{
-			TLSClientConfig: tlsConfig,
+	clientConfig := config.HTTPClientConfig{
+		TLSConfig: config.TLSConfig{
+			CAFile:             cfg.TLSCACertPath,
+			CertFile:           cfg.TLSClientCertificate,
+			KeyFile:            cfg.TLSClientCertificateKey,
+			ServerName:         cfg.URL.String(),
+			InsecureSkipVerify: cfg.TLSServerSkipVerify,
 		},
 	}
+
+	var err error
+	c.client, err = config.NewClientFromConfig(clientConfig, "logcli")
 	if err != nil {
 		level.Error(c.logger).Log("msg", "error while creating http client", "error", err) //nolint
 		return nil, err
 	}
 
+	c.client.Timeout = cfg.Timeout
+
 	c.wg.Add(1)
 	go c.run()
 	return c, nil

From d7c05b123f158a257068c358d4e4070e25a859b0 Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Mon, 6 May 2019 12:22:40 -0700
Subject: [PATCH 04/10] fix/promtail: Set proper Client config name

---
 pkg/promtail/client/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/promtail/client/client.go b/pkg/promtail/client/client.go
index 577ab729ced27..63ce20ca5b0de 100644
--- a/pkg/promtail/client/client.go
+++ b/pkg/promtail/client/client.go
@@ -128,7 +128,7 @@ func New(cfg Config, logger log.Logger) (*Client, error) {
 	}
 
 	var err error
-	c.client, err = config.NewClientFromConfig(clientConfig, "logcli")
+	c.client, err = config.NewClientFromConfig(clientConfig, "promtail")
 	if err != nil {
 		level.Error(c.logger).Log("msg", "error while creating http client", "error", err) //nolint
 		return nil, err

From 1165379465adb32de4244fe87fe91ad174a4bfdf Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Tue, 7 May 2019 10:09:34 -0700
Subject: [PATCH 05/10] impr/promtail: Use prometheus HTTPClientConfig
 configuration

---
 cmd/promtail/main.go          |  6 +++++
 docs/operations.md            |  4 +--
 docs/promtail-setup.md        | 46 ++++++++++++++++++++++++++++++-----
 pkg/promtail/client/client.go | 22 ++---------------
 4 files changed, 49 insertions(+), 29 deletions(-)

diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go
index 77ece24615fef..901f531dd50e9 100644
--- a/cmd/promtail/main.go
+++ b/cmd/promtail/main.go
@@ -37,6 +37,12 @@ func main() {
 		}
 	}
 
+	err := config.ClientConfig.Client.Validate()
+	if err != nil {
+		level.Error(util.Logger).Log("msg", "error validating client config", "error", err)
+		os.Exit(1)
+	}
+
 	p, err := promtail.New(config)
 	if err != nil {
 		level.Error(util.Logger).Log("msg", "error creating promtail", "error", err)
diff --git a/docs/operations.md b/docs/operations.md
index 883766832a12f..a02effa32db9f 100644
--- a/docs/operations.md
+++ b/docs/operations.md
@@ -5,7 +5,7 @@ This page lists operational aspects of running Loki in alphabetical order:
 ## Authentication
 
 Loki does not have an authentication layer.
-You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy.
+You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy, mutual TLS. See [here](promtail-setup.md#custom-client-options) for more details about supported authentication methods.
 
 ### Multi-tenancy
 
@@ -129,7 +129,6 @@ storage_config:
       dynamodb: dynamodb://region
 ```
 
-
 #### S3
 
 Loki is using S3 as object storage. It stores log within directories based on
@@ -158,4 +157,3 @@ create the table manually you cannot easily erase old data and your index just g
 If you set your DynamoDB table manually, ensure you set the primary index key to `h`
 (string) and use `r` (binary) as the sort key. Also set the "period" attribute in the yaml to zero.
 Make sure adjust your throughput base on your usage.
-
diff --git a/docs/promtail-setup.md b/docs/promtail-setup.md
index b15deac6402b8..a36661a7e5122 100644
--- a/docs/promtail-setup.md
+++ b/docs/promtail-setup.md
@@ -141,6 +141,7 @@ Right now, the best way to watch and tail custom log path is define log file pat
 as a label for the pod.
 
 #### Example
+
 ```yaml
 ---Deployment.yaml
 apiVersion: extensions/v1beta1
@@ -171,15 +172,48 @@ scrape_configs:
 
 ### Custom Client options
 
-If you put `loki` behind a reverse proxy to use https transport (either `TLS` or `MTLS`) you can use the following settings.
+`promtail` client configuration uses the [prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
+Therefor you can configure the following parameters in the `client` section.
 
 ```yaml
 ---promtail_config.yaml
 ...
 client:
-  ca: /path/to/ca/cert.pem # This is the Certificate Authority to verify the server certificate. [optional]
-  certificate: /path/to/ca/client-cert.pem # This is the client certificate that will be checked by the server [optional]
-  certificate-key: /path/to/ca/client-cert.key # This is the client certificate key [optional]
-  tls-skip-verify: true # Never use that settings except for development [optional]
-      ...
+
+  # Sets the `Authorization` header on every promtail request with the
+  # configured username and password.
+  # password and password_file are mutually exclusive.
+  basic_auth:
+    username: <string>
+    password: <secret>
+    password_file: <string>
+
+  # Sets the `Authorization` header on every promtail request with
+  # the configured bearer token. It is mutually exclusive with `bearer_token_file`.
+  bearer_token: <secret>
+
+  # Sets the `Authorization` header on every promtail request with the bearer token
+  # read from the configured file. It is mutually exclusive with `bearer_token`.
+  bearer_token_file: /path/to/bearer/token/file
+
+  # Configures the promtail request's TLS settings.
+  tls_config:
+    # CA certificate to validate API server certificate with.
+    # If not provided Trusted CA from sytem will be used.
+    ca_file: <filename>
+
+    # Certificate and key files for client cert authentication to the server.
+    cert_file: <filename>
+    key_file: <filename>
+
+    # ServerName extension to indicate the name of the server.
+    # https://tools.ietf.org/html/rfc4366#section-3.1
+    server_name: <string>
+
+    # Disable validation of the server certificate.
+    insecure_skip_verify: <boolean>
+
+  # Optional proxy URL.
+  proxy_url: <string>
+
 ```
diff --git a/pkg/promtail/client/client.go b/pkg/promtail/client/client.go
index 63ce20ca5b0de..24f247416537b 100644
--- a/pkg/promtail/client/client.go
+++ b/pkg/promtail/client/client.go
@@ -57,10 +57,7 @@ func init() {
 type Config struct {
 	URL flagext.URLValue
 
-	TLSCACertPath           string `yaml:"ca,omitempty"`
-	TLSServerSkipVerify     bool   `yaml:"tls-skip-verify,omitempty"`
-	TLSClientCertificate    string `yaml:"certificate,omitempty"`
-	TLSClientCertificateKey string `yaml:"certificate-key,omitempty"`
+	Client config.HTTPClientConfig `yaml:",inline"`
 
 	BatchWait time.Duration
 	BatchSize int
@@ -75,11 +72,6 @@ type Config struct {
 func (c *Config) RegisterFlags(flags *flag.FlagSet) {
 	flags.Var(&c.URL, "client.url", "URL of log server")
 
-	flags.StringVar(&c.TLSCACertPath, "client.ca", "", "Path to the server Certificate Authority")
-	flags.BoolVar(&c.TLSServerSkipVerify, "client.tls-skip-verify", false, "Server certificate TLS skip verify")
-	flags.StringVar(&c.TLSClientCertificate, "client.certificate", "", "Path to the client certificate")
-	flags.StringVar(&c.TLSClientCertificateKey, "client.certificate-key", "", "Path to the client certificate key")
-
 	flags.DurationVar(&c.BatchWait, "client.batch-wait", 1*time.Second, "Maximum wait period before sending batch.")
 	flags.IntVar(&c.BatchSize, "client.batch-size-bytes", 100*1024, "Maximum batch size to accrue before sending. ")
 
@@ -117,18 +109,8 @@ func New(cfg Config, logger log.Logger) (*Client, error) {
 		externalLabels: cfg.ExternalLabels,
 	}
 
-	clientConfig := config.HTTPClientConfig{
-		TLSConfig: config.TLSConfig{
-			CAFile:             cfg.TLSCACertPath,
-			CertFile:           cfg.TLSClientCertificate,
-			KeyFile:            cfg.TLSClientCertificateKey,
-			ServerName:         cfg.URL.String(),
-			InsecureSkipVerify: cfg.TLSServerSkipVerify,
-		},
-	}
-
 	var err error
-	c.client, err = config.NewClientFromConfig(clientConfig, "promtail")
+	c.client, err = config.NewClientFromConfig(cfg.Client, "promtail")
 	if err != nil {
 		level.Error(c.logger).Log("msg", "error while creating http client", "error", err) //nolint
 		return nil, err

From 44f3d363e639f01cc82b2d8627bf1bb2fd883024 Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Tue, 7 May 2019 10:23:53 -0700
Subject: [PATCH 06/10] adapt with master

---
 docs/operations.md            |  2 +-
 docs/promtail-setup.md        | 81 ++++++++++++++++++-----------------
 pkg/promtail/client/client.go | 33 +-------------
 pkg/promtail/client/config.go |  3 ++
 pkg/promtail/client/multi.go  |  8 +++-
 5 files changed, 53 insertions(+), 74 deletions(-)

diff --git a/docs/operations.md b/docs/operations.md
index a02effa32db9f..a956818e4a623 100644
--- a/docs/operations.md
+++ b/docs/operations.md
@@ -5,7 +5,7 @@ This page lists operational aspects of running Loki in alphabetical order:
 ## Authentication
 
 Loki does not have an authentication layer.
-You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy, mutual TLS. See [here](promtail-setup.md#custom-client-options) for more details about supported authentication methods.
+You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy, mutual TLS. See [here](promtail-setup.md#custom-clients-options) for more details about supported authentication methods.
 
 ### Multi-tenancy
 
diff --git a/docs/promtail-setup.md b/docs/promtail-setup.md
index a36661a7e5122..c50fc72b13181 100644
--- a/docs/promtail-setup.md
+++ b/docs/promtail-setup.md
@@ -170,50 +170,51 @@ scrape_configs:
       ...
 ```
 
-### Custom Client options
+### Custom Clients options
 
-`promtail` client configuration uses the [prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
-Therefor you can configure the following parameters in the `client` section.
+`promtail` clients configuration uses the [prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
+Therefor you can configure the following parameters in the `clients` section.
 
 ```yaml
 ---promtail_config.yaml
 ...
-client:
-
-  # Sets the `Authorization` header on every promtail request with the
-  # configured username and password.
-  # password and password_file are mutually exclusive.
-  basic_auth:
-    username: <string>
-    password: <secret>
-    password_file: <string>
-
-  # Sets the `Authorization` header on every promtail request with
-  # the configured bearer token. It is mutually exclusive with `bearer_token_file`.
-  bearer_token: <secret>
-
-  # Sets the `Authorization` header on every promtail request with the bearer token
-  # read from the configured file. It is mutually exclusive with `bearer_token`.
-  bearer_token_file: /path/to/bearer/token/file
-
-  # Configures the promtail request's TLS settings.
-  tls_config:
-    # CA certificate to validate API server certificate with.
-    # If not provided Trusted CA from sytem will be used.
-    ca_file: <filename>
-
-    # Certificate and key files for client cert authentication to the server.
-    cert_file: <filename>
-    key_file: <filename>
-
-    # ServerName extension to indicate the name of the server.
-    # https://tools.ietf.org/html/rfc4366#section-3.1
-    server_name: <string>
-
-    # Disable validation of the server certificate.
-    insecure_skip_verify: <boolean>
-
-  # Optional proxy URL.
-  proxy_url: <string>
+clients:
+  - url: http://localhost:3100/api/prom/push
+
+    # Sets the `Authorization` header on every promtail request with the
+    # configured username and password.
+    # password and password_file are mutually exclusive.
+    basic_auth:
+      username: <string>
+      password: <secret>
+      password_file: <string>
+
+    # Sets the `Authorization` header on every promtail request with
+    # the configured bearer token. It is mutually exclusive with `bearer_token_file`.
+    bearer_token: <secret>
+
+    # Sets the `Authorization` header on every promtail request with the bearer token
+    # read from the configured file. It is mutually exclusive with `bearer_token`.
+    bearer_token_file: /path/to/bearer/token/file
+
+    # Configures the promtail request's TLS settings.
+    tls_config:
+      # CA certificate to validate API server certificate with.
+      # If not provided Trusted CA from sytem will be used.
+      ca_file: <filename>
+
+      # Certificate and key files for client cert authentication to the server.
+      cert_file: <filename>
+      key_file: <filename>
+
+      # ServerName extension to indicate the name of the server.
+      # https://tools.ietf.org/html/rfc4366#section-3.1
+      server_name: <string>
+
+      # Disable validation of the server certificate.
+      insecure_skip_verify: <boolean>
+
+    # Optional proxy URL.
+    proxy_url: <string>
 
 ```
diff --git a/pkg/promtail/client/client.go b/pkg/promtail/client/client.go
index 8d2fa2904f7d7..5e59533b7cb03 100644
--- a/pkg/promtail/client/client.go
+++ b/pkg/promtail/client/client.go
@@ -53,40 +53,11 @@ func init() {
 	prometheus.MustRegister(requestDuration)
 }
 
-<<<<<<< HEAD
-// Config describes configuration for a HTTP pusher client.
-type Config struct {
-	URL flagext.URLValue
-
-	Client config.HTTPClientConfig `yaml:",inline"`
-
-	BatchWait time.Duration
-	BatchSize int
-
-	BackoffConfig util.BackoffConfig `yaml:"backoff_config"`
-	// The labels to add to any time series or alerts when communicating with loki
-	ExternalLabels model.LabelSet `yaml:"external_labels,omitempty"`
-	Timeout        time.Duration  `yaml:"timeout"`
-}
-
-// RegisterFlags registers flags.
-func (c *Config) RegisterFlags(flags *flag.FlagSet) {
-	flags.Var(&c.URL, "client.url", "URL of log server")
-
-	flags.DurationVar(&c.BatchWait, "client.batch-wait", 1*time.Second, "Maximum wait period before sending batch.")
-	flags.IntVar(&c.BatchSize, "client.batch-size-bytes", 100*1024, "Maximum batch size to accrue before sending. ")
-
-	flag.IntVar(&c.BackoffConfig.MaxRetries, "client.max-retries", 5, "Maximum number of retires when sending batches.")
-	flag.DurationVar(&c.BackoffConfig.MinBackoff, "client.min-backoff", 100*time.Millisecond, "Initial backoff time between retries.")
-	flag.DurationVar(&c.BackoffConfig.MaxBackoff, "client.max-backoff", 5*time.Second, "Maximum backoff time between retries.")
-	flag.DurationVar(&c.Timeout, "client.timeout", 10*time.Second, "Maximum time to wait for server to respond to a request")
-=======
 // Client pushes entries to Loki and can be stopped
 type Client interface {
 	api.EntryHandler
 	// Stop goroutine sending batch of entries.
 	Stop()
->>>>>>> 53075db577c72a5649fdb50020590382812bf0f9
 }
 
 // Client for pushing logs in snappy-compressed protos over HTTP.
@@ -107,7 +78,7 @@ type entry struct {
 }
 
 // New makes a new Client.
-func New(cfg Config, logger log.Logger) Client {
+func New(cfg Config, logger log.Logger) (Client, error) {
 	c := &client{
 		logger:  log.With(logger, "component", "client", "host", cfg.URL.Host),
 		cfg:     cfg,
@@ -128,7 +99,7 @@ func New(cfg Config, logger log.Logger) Client {
 
 	c.wg.Add(1)
 	go c.run()
-	return c
+	return c, nil
 }
 
 func (c *client) run() {
diff --git a/pkg/promtail/client/config.go b/pkg/promtail/client/config.go
index bcc3d04bd7013..0a55100e57c5d 100644
--- a/pkg/promtail/client/config.go
+++ b/pkg/promtail/client/config.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 )
 
@@ -15,6 +16,8 @@ type Config struct {
 	BatchWait time.Duration
 	BatchSize int
 
+	Client config.HTTPClientConfig `yaml:",inline"`
+
 	BackoffConfig util.BackoffConfig `yaml:"backoff_config"`
 	// The labels to add to any time series or alerts when communicating with loki
 	ExternalLabels model.LabelSet `yaml:"external_labels,omitempty"`
diff --git a/pkg/promtail/client/multi.go b/pkg/promtail/client/multi.go
index d634e3025059e..548912c3addc9 100644
--- a/pkg/promtail/client/multi.go
+++ b/pkg/promtail/client/multi.go
@@ -17,9 +17,13 @@ func NewMulti(logger log.Logger, cfgs ...Config) (Client, error) {
 	if len(cfgs) == 0 {
 		return nil, errors.New("at least one client config should be provided")
 	}
-	var clients []Client
+	clients := []Client{}
 	for _, cfg := range cfgs {
-		clients = append(clients, New(cfg, logger))
+		client, err := New(cfg, logger)
+		if err != nil {
+			return nil, err
+		}
+		clients = append(clients, client)
 	}
 	return MultiClient(clients), nil
 }

From f0472b167f7541d3ec3e9e9d89a1f4a3786c279b Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Wed, 8 May 2019 08:38:00 -0700
Subject: [PATCH 07/10] address review

---
 cmd/promtail/main.go          |   6 --
 docs/operations.md            |   3 +-
 docs/promtail-setup.md        | 109 ++++++++++++++++++++++------------
 pkg/promtail/client/client.go |  13 ++--
 pkg/promtail/client/multi.go  |   3 +-
 5 files changed, 84 insertions(+), 50 deletions(-)

diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go
index 7f193e3230f46..d5c34433c214d 100644
--- a/cmd/promtail/main.go
+++ b/cmd/promtail/main.go
@@ -37,12 +37,6 @@ func main() {
 		}
 	}
 
-	err := config.ClientConfig.Client.Validate()
-	if err != nil {
-		level.Error(util.Logger).Log("msg", "error validating client config", "error", err)
-		os.Exit(1)
-	}
-
 	p, err := promtail.New(config)
 	if err != nil {
 		level.Error(util.Logger).Log("msg", "error creating promtail", "error", err)
diff --git a/docs/operations.md b/docs/operations.md
index a956818e4a623..17190e6ac295b 100644
--- a/docs/operations.md
+++ b/docs/operations.md
@@ -5,7 +5,8 @@ This page lists operational aspects of running Loki in alphabetical order:
 ## Authentication
 
 Loki does not have an authentication layer.
-You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy, mutual TLS. See [here](promtail-setup.md#custom-clients-options) for more details about supported authentication methods.
+You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy.
+See [client options](promtail-setup.md#custom-client-options) for more details about supported authentication methods.
 
 ### Multi-tenancy
 
diff --git a/docs/promtail-setup.md b/docs/promtail-setup.md
index c50fc72b13181..2b513296e5c4c 100644
--- a/docs/promtail-setup.md
+++ b/docs/promtail-setup.md
@@ -170,51 +170,86 @@ scrape_configs:
       ...
 ```
 
-### Custom Clients options
+### Custom Client options
 
-`promtail` clients configuration uses the [prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
-Therefor you can configure the following parameters in the `clients` section.
+`promtail` client configuration uses the [Prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
+Therefor you can configure the following authentication parameters in the `client` or `clients` section.
 
 ```yaml
 ---promtail_config.yaml
 ...
-clients:
-  - url: http://localhost:3100/api/prom/push
-
-    # Sets the `Authorization` header on every promtail request with the
-    # configured username and password.
-    # password and password_file are mutually exclusive.
-    basic_auth:
-      username: <string>
-      password: <secret>
-      password_file: <string>
-
-    # Sets the `Authorization` header on every promtail request with
-    # the configured bearer token. It is mutually exclusive with `bearer_token_file`.
-    bearer_token: <secret>
-
-    # Sets the `Authorization` header on every promtail request with the bearer token
-    # read from the configured file. It is mutually exclusive with `bearer_token`.
-    bearer_token_file: /path/to/bearer/token/file
-
-    # Configures the promtail request's TLS settings.
-    tls_config:
-      # CA certificate to validate API server certificate with.
-      # If not provided Trusted CA from sytem will be used.
-      ca_file: <filename>
 
-      # Certificate and key files for client cert authentication to the server.
-      cert_file: <filename>
-      key_file: <filename>
+# Simple client
+client:
+  [ <client_option> ]
 
-      # ServerName extension to indicate the name of the server.
-      # https://tools.ietf.org/html/rfc4366#section-3.1
-      server_name: <string>
+# Multiple clients
+clients:
+  [ - <client_option> ]
+...
+```
 
-      # Disable validation of the server certificate.
-      insecure_skip_verify: <boolean>
+>Note: Passing the `-client.url` from command line is only valid if you set the `client` section.
 
-    # Optional proxy URL.
-    proxy_url: <string>
+#### `<client_option>`
 
+```yaml
+  # Sets the `url` of loki api push endpoint
+  url: http[s]://<host>:<port>/api/prom/push
+
+  # Sets the `Authorization` header on every promtail request with the
+  # configured username and password.
+  # password and password_file are mutually exclusive.
+  basic_auth:
+    username: <string>
+    password: <secret>
+    password_file: <string>
+
+  # Sets the `Authorization` header on every promtail request with
+  # the configured bearer token. It is mutually exclusive with `bearer_token_file`.
+  bearer_token: <secret>
+
+  # Sets the `Authorization` header on every promtail request with the bearer token
+  # read from the configured file. It is mutually exclusive with `bearer_token`.
+  bearer_token_file: /path/to/bearer/token/file
+
+  # Configures the promtail request's TLS settings.
+  tls_config:
+    # CA certificate to validate API server certificate with.
+    # If not provided Trusted CA from sytem will be used.
+    ca_file: <filename>
+
+    # Certificate and key files for client cert authentication to the server.
+    cert_file: <filename>
+    key_file: <filename>
+
+    # ServerName extension to indicate the name of the server.
+    # https://tools.ietf.org/html/rfc4366#section-3.1
+    server_name: <string>
+
+    # Disable validation of the server certificate.
+    insecure_skip_verify: <boolean>
+
+  # Optional proxy URL.
+  proxy_url: <string>
+
+  # Maximum wait period before sending batch
+  batchwait: 1s
+
+  # Maximum batch size to accrue before sending, unit is byte
+  batchsize: 102400
+
+  # Maximum time to wait for server to respond to a request
+  timeout: 10s
+
+  backoff_config:
+    # Initial backoff time between retries
+    minbackoff: 100ms
+    # Maximum backoff time between retries
+    maxbackoff: 5s
+    # Maximum number of retires when sending batches, 0 means infinite retries
+    maxretries: 5
+
+  # The labels to add to any time series or alerts when communicating with loki
+  external_labels: {}
 ```
diff --git a/pkg/promtail/client/client.go b/pkg/promtail/client/client.go
index 5e59533b7cb03..de069be424c58 100644
--- a/pkg/promtail/client/client.go
+++ b/pkg/promtail/client/client.go
@@ -88,10 +88,13 @@ func New(cfg Config, logger log.Logger) (Client, error) {
 		externalLabels: cfg.ExternalLabels,
 	}
 
-	var err error
+	err := cfg.Client.Validate()
+	if err != nil {
+		return nil, err
+	}
+
 	c.client, err = config.NewClientFromConfig(cfg.Client, "promtail")
 	if err != nil {
-		level.Error(c.logger).Log("msg", "error while creating http client", "error", err) //nolint
 		return nil, err
 	}
 
@@ -149,7 +152,7 @@ func (c *client) run() {
 func (c *client) sendBatch(batch map[model.Fingerprint]*logproto.Stream) {
 	buf, err := encodeBatch(batch)
 	if err != nil {
-		level.Error(c.logger).Log("msg", "error encoding batch", "error", err) //nolint
+		level.Error(c.logger).Log("msg", "error encoding batch", "error", err)
 		return
 	}
 	bufBytes := float64(len(buf))
@@ -173,12 +176,12 @@ func (c *client) sendBatch(batch map[model.Fingerprint]*logproto.Stream) {
 			break
 		}
 
-		level.Warn(c.logger).Log("msg", "error sending batch, will retry", "status", status, "error", err) //nolint
+		level.Warn(c.logger).Log("msg", "error sending batch, will retry", "status", status, "error", err)
 		backoff.Wait()
 	}
 
 	if err != nil {
-		level.Error(c.logger).Log("msg", "final error sending batch", "status", status, "error", err) //nolint
+		level.Error(c.logger).Log("msg", "final error sending batch", "status", status, "error", err)
 	}
 }
 
diff --git a/pkg/promtail/client/multi.go b/pkg/promtail/client/multi.go
index 548912c3addc9..47249948eeb78 100644
--- a/pkg/promtail/client/multi.go
+++ b/pkg/promtail/client/multi.go
@@ -17,7 +17,8 @@ func NewMulti(logger log.Logger, cfgs ...Config) (Client, error) {
 	if len(cfgs) == 0 {
 		return nil, errors.New("at least one client config should be provided")
 	}
-	clients := []Client{}
+
+	clients := make([]Client, 0, len(cfgs))
 	for _, cfg := range cfgs {
 		client, err := New(cfg, logger)
 		if err != nil {

From 0480c6a6124dbc527bccce423aa296c36b46e9fd Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Wed, 15 May 2019 08:48:58 -0700
Subject: [PATCH 08/10] fix conflicts

---
 pkg/promtail/client/config.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pkg/promtail/client/config.go b/pkg/promtail/client/config.go
index cda0dc158b34f..b4652e4120def 100644
--- a/pkg/promtail/client/config.go
+++ b/pkg/promtail/client/config.go
@@ -6,9 +6,8 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
-	"github.com/prometheus/common/config"
-	"github.com/prometheus/common/model"
 	lokiflag "github.com/grafana/loki/pkg/util/flagext"
+	"github.com/prometheus/common/config"
 )
 
 // Config describes configuration for a HTTP pusher client.

From 5953fe5cc450bea858e6794e4e178ec53497d6b5 Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Wed, 15 May 2019 08:50:25 -0700
Subject: [PATCH 09/10] address requested changes

---
 docs/promtail-setup.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/promtail-setup.md b/docs/promtail-setup.md
index 67b7a04ae7958..0d596f7f25f4a 100644
--- a/docs/promtail-setup.md
+++ b/docs/promtail-setup.md
@@ -171,7 +171,7 @@ scrape_configs:
 ### Custom Client options
 
 `promtail` client configuration uses the [Prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
-Therefor you can configure the following authentication parameters in the `client` or `clients` section.
+Therefore you can configure the following authentication parameters in the `client` or `clients` section.
 
 ```yaml
 ---promtail_config.yaml

From 5cc70ab7af4bd32f4745c00f6726a68f2d9c3fee Mon Sep 17 00:00:00 2001
From: CyrilPeponnet <cyril@peponnet.fr>
Date: Wed, 15 May 2019 08:53:18 -0700
Subject: [PATCH 10/10] remove file

---
 tlsconf.yaml | 41 -----------------------------------------
 1 file changed, 41 deletions(-)
 delete mode 100644 tlsconf.yaml

diff --git a/tlsconf.yaml b/tlsconf.yaml
deleted file mode 100644
index 46a5be2df64c6..0000000000000
--- a/tlsconf.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-client:
-  external_labels: {}
-  # The labels to add to any time series or alerts when communicating with loki
-
-  # Maximum wait period before sending batch
-  batchwait: 1s
-  # Maximum batch size to accrue before sending, unit is byte
-  batchsize: 102400
-  # Maximum time to wait for server to respond to a request
-  timeout: 10s
-
-  backoff_config:
-    # Initial backoff time between retries
-    minbackoff: 100ms
-    # Maximum backoff time between retries
-    maxbackoff: 5s
-    # Maximum number of retires when sending batches, 0 means infinite retries
-    maxretries: 5
-
-server:
-  http_listen_port: 3101
-
-positions:
-  filename: /run/promtail/positions.yaml
-
-target_config:
-  # Period to resync directories being watched and files being tailed
-  sync_period: 10s
-
-scrape_configs:
-- job_name: containers
-  entry_parser: docker
-  file_sd_configs:
-  - files:
-    - /etc/promtail/promtail-targets.json
-  relabel_configs:
-  - source_labels: [__address__]
-    target_label: container_id
-  - source_labels: [container_id]
-    target_label: __path__
-    replacement: /var/lib/docker/containers/$1*/*.log