clients/promtail: Add ndjson and plaintext formats to loki_push

README.md

@@ -110,7 +110,7 @@ $ ./loki -config.file=./cmd/loki/loki-local-config.yaml
 To build Promtail on non-Linux platforms, use the following command:
 
 ```bash
-$ go build ./cmd/promtail
+$ go build ./clients/cmd/promtail
 ```
 
 On Linux, Promtail requires the systemd headers to be installed for
@@ -120,21 +120,21 @@ With Journal support on Ubuntu, run with the following commands:
 
 ```bash
 $ sudo apt install -y libsystemd-dev
-$ go build ./cmd/promtail
+$ go build ./clients/cmd/promtail
 ```
 
 With Journal support on CentOS, run with the following commands:
 
 ```bash
 $ sudo yum install -y systemd-devel
-$ go build ./cmd/promtail
+$ go build ./clients/cmd/promtail
 ```
 
 Otherwise, to build Promtail without Journal support, run `go build`
 with CGO disabled:
 
 ```bash
-$ CGO_ENABLED=0 go build ./cmd/promtail
+$ CGO_ENABLED=0 go build ./clients/cmd/promtail
 ```
 
 ## License

clients/pkg/promtail/scrapeconfig/scrapeconfig.go

@@ -245,6 +245,9 @@ type PushTargetConfig struct {
 
 	// If promtail should maintain the incoming log timestamp or replace it with the current time.
 	KeepTimestamp bool `yaml:"use_incoming_timestamp"`
+
+	// Format dictates which format the enpoint should expect. Valid values are `loki`, `ndjson` and `plaintext`.
+	Format string `yaml:"format"`
 }
 
 // DefaultScrapeConfig is the default Config.

clients/pkg/promtail/targets/lokipush/pushtarget.go

@@ -1,7 +1,9 @@
 package lokipush
 
 import (
+	"bufio"
 	"flag"
+	"io"
 	"net/http"
 	"sort"
 	"strings"
@@ -92,7 +94,16 @@ func (t *PushTarget) run() error {
 	}
 
 	t.server = srv
-	t.server.HTTP.Handle("/loki/api/v1/push", http.HandlerFunc(t.handle))
+
+	switch t.config.Format {
+	// NDJSON and Plaintext are both newline delimited formats and will be handled the same.
+	case "ndjson", "plaintext":
+		t.server.HTTP.Handle("/promtail/api/v1", http.HandlerFunc(t.handlePlaintext))
+	case "loki":
+		fallthrough
+	default:
+		t.server.HTTP.Handle("/loki/api/v1/push", http.HandlerFunc(t.handleLoki))
+	}
 
 	go func() {
 		err := srv.Run()
@@ -104,7 +115,7 @@ func (t *PushTarget) run() error {
 	return nil
 }
 
-func (t *PushTarget) handle(w http.ResponseWriter, r *http.Request) {
+func (t *PushTarget) handleLoki(w http.ResponseWriter, r *http.Request) {
 	logger := util_log.WithContext(r.Context(), util_log.Logger)
 	userID, _ := tenant.TenantID(r.Context())
 	req, err := push.ParseRequest(logger, userID, r, nil)
@@ -170,6 +181,40 @@ func (t *PushTarget) handle(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+// handlePlaintext handles newline delimited input such as plaintext or NDJSON.
+func (t *PushTarget) handlePlaintext(w http.ResponseWriter, r *http.Request) {
+	entries := t.handler.Chan()
+	defer r.Body.Close()
+	body := bufio.NewReader(r.Body)
+	for {
+		line, err := body.ReadString('\n')
+		if err != nil && err != io.EOF {
+			level.Warn(t.logger).Log("msg", "failed to read incoming push request", "err", err.Error())
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		line = strings.TrimRight(line, "\r\n")
+		if line == "" {
+			if err == io.EOF {
+				break
+			}
+			continue
+		}
+		entries <- api.Entry{
+			Labels: t.Labels().Clone(),
+			Entry: logproto.Entry{
+				Timestamp: time.Now(),
+				Line:      line,
+			},
+		}
+		if err == io.EOF {
+			break
+		}
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
 // Type returns PushTargetType.
 func (t *PushTarget) Type() target.TargetType {
 	return target.PushTargetType

docs/sources/clients/promtail/configuration.md

@@ -782,9 +782,11 @@ The `loki_push_api` block configures Promtail to expose a [Loki push API](../../
 
 Each job configured with a `loki_push_api` will expose this API and will require a separate port.
 
-Note the `server` configuration is the same as [server](#server)
-
+Note the `server` configuration is the same as [server](#server).
 
+The format of the expected payload can be configured using the `format` configuration:
+- `loki` expects the [Loki Push Format](../../../api#post-lokiapiv1push);
+- `plaintext` and `ndjson` expect newline delimited loglines.
 
 ```yaml
 # The push server configuration options
@@ -797,6 +799,9 @@ labels:
 # If Promtail should pass on the timestamp from the incoming log or not.
 # When false Promtail will assign the current timestamp to the log when it was processed
 [use_incoming_timestamp: <bool> | default = false]
+
+# Defines how promtail should interpret the incoming payload. Valid values are `loki`, `plaintext` and `ndjson`.
+[format: <string> | default = "loki"]
 ```
 
 See [Example Push Config](#example-push-config)