Merge pull request grafana#72 from grafana/main

Update from upstream repository

.github/CODEOWNERS

@@ -14,4 +14,4 @@
 
 # Documentation owner: Grafana Docs Squad
 # Grafana Docs Squad does not own the content, but assists with editing and updates.
-/docs/ @grafana/docs-squad @KMiller-Grafana
+/docs/ @grafana/docs-logs

.github/workflows/helm-ci.yml

@@ -71,7 +71,7 @@ jobs:
         run: ct lint --config "${CT_CONFIGFILE}" --check-version-increment=false
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.4.0
         if: steps.list-changed.outputs.changed == 'true'
 
       - name: Install prometheus operator

ADOPTERS.md

@@ -6,3 +6,4 @@
 * [X4B](https://www.x4b.net)
 * [Heureka Group](https://heureka.group)
 * [Norwegian Refugee Council](https://www.nrc.no/)
+* [Dropbox](https://www.dropbox.com/)

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 ##### Enhancements
 
+* [6360](https://github.com/grafana/loki/pull/6099) **liguozhong**: Hide error message when ctx timeout occurs in s3.getObject
 * [7602](https://github.com/grafana/loki/pull/7602) **vmax**: Add decolorize filter to easily parse colored logs.
 
 ##### Fixes
@@ -21,6 +22,7 @@
 ##### Fixes
 
 ##### Changes
+* [7587](https://github.com/grafana/loki/pull/7587) **mar4uk**: Add go build tag `promtail_journal_enabled` to include/exclude Promtail journald code from binary.
 
 #### Fluent Bit
 
@@ -83,6 +85,8 @@ Check the history of the branch FIXME.
 * [7212](https://github.com/grafana/loki/pull/7212) **Juneezee**: Replaces deprecated `io/ioutil` with `io` and `os`.
 * [7361](https://github.com/grafana/loki/pull/7361) **szczepad**: Renames metric `loki_log_messages_total` to `loki_internal_log_messages_total`
 * [7510](https://github.com/grafana/loki/pull/7510) **slim-bean**: Limited queries (queries without filter expressions) will now be split and sharded.
+* [5400](https://github.com/grafana/loki/pull/5400) **BenoitKnecht**: promtail/server: Disable profiling by default
+
 
 #### Promtail
 
@@ -96,8 +100,10 @@ Check the history of the branch FIXME.
 * [6656](https://github.com/grafana/loki/pull/6656) **carlospeon**: Allow promtail to add matches to the journal reader
 * [7401](https://github.com/grafana/loki/pull/7401) **thepalbi**: Add timeout to GCP Logs push target
 * [7414](https://github.com/grafana/loki/pull/7414) **thepalbi**: Add basic tracing support
+* [7462](https://github.com/grafana/loki/pull/7462) **MarNicGit**: Allow excluding event message from Windows Event Log entries.
 
 ##### Fixes
+* [7394](https://github.com/grafana/loki/pull/7394) **liguozhong**: Fix issue with the Cloudflare target that caused it to stop working after it received an error in the logpull request as explained in issue https://github.com/grafana/loki/issues/6150
 * [6766](https://github.com/grafana/loki/pull/6766) **kavirajk**: fix(logql): Make `LabelSampleExtractor` ignore processing the line if it doesn't contain that specific label. Fixes unwrap behavior explained in the issue https://github.com/grafana/loki/issues/6713
 * [7016](https://github.com/grafana/loki/pull/7016) **chodges15**: Fix issue with dropping logs when a file based SD target's labels are updated
 * [7461](https://github.com/grafana/loki/pull/7461) **MarNicGit**: Promtail: Fix collecting userdata field from Windows Event Log
@@ -402,6 +408,7 @@ to include only the most relevant.
 
 #### Lambda-Promtail
 * [5065](https://github.com/grafana/loki/pull/5065) **AndreZiviani**: lambda-promtail: Add ability to ingest logs from S3
+* [7632](https://github.com/grafana/loki/pull/7632) **changhyuni**: lambda-promtail: Add kinesis data stream to use in terraform
 
 #### Fluent Bit
 * [5223](https://github.com/grafana/loki/pull/5223) **cyriltovena**: fluent-bit: Attempt to unmarshal nested json.
@@ -421,7 +428,6 @@ Check the history of the branch `release-2.5.x`.
 
 * Go Version:     1.17.8
 
-
 # 2.4.1 (2021/11/07)
 
 Release notes for 2.4.1 can be found on the [release notes page](https://grafana.com/docs/loki/latest/release-notes/v2-4/)

MAINTAINERS.md

@@ -1,6 +1,4 @@
 @slim-bean is the main/default maintainer.
 
 Some parts of the codebase have other maintainers:
-- `docs`:
-  - @osg-grafana - [@osg-grafana](https://github.com/osg-grafana) ([Grafana Labs](https://grafana.com/))
-  - @knylander-grafana - [@knylander-grafana](https://github.com/knylander-grafana) ([Grafana Labs](https://grafana.com/))
+- `@grafana/docs-logs`, which includes [@osg-grafana](https://github.com/osg-grafana) ([Grafana Labs](https://grafana.com/)) and [@knylander-grafana](https://github.com/knylander-grafana) ([Grafana Labs](https://grafana.com/))

Makefile

@@ -186,6 +186,9 @@ PROMTAIL_GO_FLAGS = $(DYN_GO_FLAGS)
 PROMTAIL_DEBUG_GO_FLAGS = $(DYN_DEBUG_GO_FLAGS)
 endif
 endif
+ifeq ($(PROMTAIL_JOURNAL_ENABLED), true)
+PROMTAIL_GO_TAGS = promtail_journal_enabled
+endif
 .PHONY: clients/cmd/promtail/promtail clients/cmd/promtail/promtail-debug
 promtail: clients/cmd/promtail/promtail
 promtail-debug: clients/cmd/promtail/promtail-debug
@@ -199,10 +202,10 @@ $(PROMTAIL_GENERATED_FILE): $(PROMTAIL_UI_FILES)
 	GOOS=$(shell go env GOHOSTOS) go generate -x -v ./clients/pkg/promtail/server/ui
 
 clients/cmd/promtail/promtail:
-	CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_GO_FLAGS) -o $@ ./$(@D)
+	CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_GO_FLAGS) --tags=$(PROMTAIL_GO_TAGS) -o $@ ./$(@D)
 
 clients/cmd/promtail/promtail-debug:
-	CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_DEBUG_GO_FLAGS) -o $@ ./$(@D)
+	CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_DEBUG_GO_FLAGS) --tags=$(PROMTAIL_GO_TAGS) -o $@ ./$(@D)
 
 #########
 # Mixin #
@@ -303,6 +306,7 @@ clean:
 	rm -rf clients/cmd/fluent-bit/out_grafana_loki.so
 	rm -rf cmd/migrate/migrate
 	rm -rf cmd/logql-analyzer/logql-analyzer
+	$(MAKE) -BC clients/cmd/fluentd $@
 	go clean ./...
 
 #########
@@ -428,7 +432,7 @@ fluent-bit-plugin:
 	go build $(DYN_GO_FLAGS) -buildmode=c-shared -o clients/cmd/fluent-bit/out_grafana_loki.so ./clients/cmd/fluent-bit/
 
 fluent-bit-image:
-	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluent-bit/Dockerfile .
+	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) --build-arg LDFLAGS="-s -w $(GO_LDFLAGS)" -f clients/cmd/fluent-bit/Dockerfile .
 
 fluent-bit-push:
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG)
@@ -443,15 +447,16 @@ fluent-bit-test:
 # fluentd plugin #
 ##################
 fluentd-plugin:
-	gem install bundler --version 2.3.4
-	bundle config silence_root_warning true
-	bundle config set --local path clients/cmd/fluentd/vendor/bundle
-	bundle install --gemfile=clients/cmd/fluentd/Gemfile
+	$(MAKE) -BC clients/cmd/fluentd $@
+
+fluentd-plugin-push:
+	$(MAKE) -BC clients/cmd/fluentd $@
 
 fluentd-image:
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f clients/cmd/fluentd/Dockerfile .
 
 fluentd-push:
+fluentd-image-push:
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
 
 fluentd-test: LOKI_URL ?= http://loki:3100

README.md

@@ -113,21 +113,22 @@ To build Promtail on non-Linux platforms, use the following command:
 $ go build ./clients/cmd/promtail
 ```
 
-On Linux, Promtail requires the systemd headers to be installed for
-Journal support.
+On Linux, Promtail requires the systemd headers to be installed if
+Journal support is enabled.
+To enable Journal support the go build tag flag `promtail_journal_enabled` should be passed
 
 With Journal support on Ubuntu, run with the following commands:
 
 ```bash
 $ sudo apt install -y libsystemd-dev
-$ go build ./clients/cmd/promtail
+$ go build ./clients/cmd/promtail --tags=promtail_journal_enabled
 ```
 
 With Journal support on CentOS, run with the following commands:
 
 ```bash
 $ sudo yum install -y systemd-devel
-$ go build ./clients/cmd/promtail
+$ go build ./clients/cmd/promtail --tags=promtail_journal_enabled
 ```
 
 Otherwise, to build Promtail without Journal support, run `go build`

clients/cmd/fluent-bit/Dockerfile

@@ -1,10 +1,24 @@
-FROM golang:1.19.2 as build
-COPY . /src/loki
-WORKDIR /src/loki
-RUN make clean && make BUILD_IN_CONTAINER=false fluent-bit-plugin
+FROM golang:1.19.2@sha256:b850621230956a6d960d6d7cfaba6a8a2e8e245b230a928ef66aa0cfd065e229 AS builder
 
-FROM fluent/fluent-bit:1.8
-COPY --from=build /src/loki/clients/cmd/fluent-bit/out_grafana_loki.so /fluent-bit/bin
+COPY . /src
+
+WORKDIR /src
+
+ARG LDFLAGS
+ENV CGO_ENABLED=1
+
+RUN go build \
+    -trimpath -ldflags "${LDFLAGS}" \
+    -tags netgo \
+    -buildmode=c-shared \
+    -o clients/cmd/fluent-bit/out_grafana_loki.so \
+    /src/clients/cmd/fluent-bit
+
+FROM fluent/fluent-bit:1.9.9@sha256:3045036b2ef35eae09a5f40273a0f1fbd70ca4d67e80918bfd0676b16ba43a29
+
+COPY --from=builder /src/clients/cmd/fluent-bit/out_grafana_loki.so /fluent-bit/bin
 COPY clients/cmd/fluent-bit/fluent-bit.conf /fluent-bit/etc/fluent-bit.conf
+
 EXPOSE 2020
+
 CMD ["/fluent-bit/bin/fluent-bit", "-e","/fluent-bit/bin/out_grafana_loki.so", "-c", "/fluent-bit/etc/fluent-bit.conf"]

clients/cmd/fluent-bit/README.md

@@ -4,25 +4,40 @@
 
 This plugin is implemented with [Fluent Bit's Go plugin](https://github.com/fluent/fluent-bit-go) interface. It pushes logs to Loki using a GRPC connection.
 
-> syslog and systemd input plugin have not been tested yet, feedback appreciated.
+> **Warning**
+> `syslog` and `systemd` input plugins have not been tested yet. Feedback appreciated, file [an issue](https://github.com/grafana/loki/issues/new?template=bug_report.md) if you encounter any misbehaviors.
 
 ## Building
 
-Prerequisites:
+**Prerequisites**
 
 * Go 1.16+
 * gcc (for cgo)
 
-To build the output plugin library file (`out_grafana_loki.so`), you can use:
+To [build](https://docs.fluentbit.io/manual/development/golang-output-plugins#build-a-go-plugin) the output plugin library file `out_grafana_loki.so`, in the root directory of Loki source code, you can use:
 
 ```bash
-make fluent-bit-plugin
+$ make fluent-bit-plugin
 ```
 
-You can also build the docker image with the plugin pre-installed using:
+You can also build the Docker image with the plugin pre-installed using:
 
 ```bash
-make fluent-bit-image
+$ make fluent-bit-image
 ```
 
-Finally if you want to test you can use `make fluent-bit-test` to send some logs to your local Loki instance.
+## Running
+
+```bash
+$ fluent-bit -e out_grafana_loki.so -c /etc/fluent-bit.conf
+```
+
+**Testing**
+
+Issue the following command to send `/var/log` logs to your `http://localhost:3100/loki/api/` Loki instance for testing:
+
+```bash
+$ make fluent-bit-test
+```
+
+You can easily override the address by setting  the `$LOKI_URL` environment variable.

clients/cmd/fluentd/Makefile

@@ -0,0 +1,27 @@
+.PHONY: clean
+
+SHELL    = /usr/bin/env bash -o pipefail
+# needs to match the name in fluent-plugin-grafana-loki.gemspec
+NAME    := fluent-plugin-grafana-loki
+# needs to match the version in fluent-plugin-grafana-loki.gemspec
+VERSION := 1.2.19
+
+clean:
+	rm -f $(NAME)-*.gem
+	rm -rf .bundle
+
+.bundle:
+	ruby --version
+	@echo ""
+	ruby -S gem install bundler --version 2.3.4
+	ruby -S bundle config silence_root_warning true
+	ruby -S bundle config set --local path ./vendor/bundle
+
+fluentd-plugin: .bundle
+	ruby -S bundle install --gemfile=Gemfile
+
+$(NAME)-$(VERSION).gem: .bundle
+	ruby -S gem build $(NAME).gemspec
+
+fluentd-plugin-push: $(NAME)-$(VERSION).gem
+	ruby -S gem push $(NAME)-$(VERSION).gem

clients/cmd/fluentd/README.md

@@ -82,6 +82,15 @@ The expected output is:
 ]
 ```
 
+## Build and publish gem
+
+To build and publish a gem to
+[rubygems](https://rubygems.org/gems/fluent-plugin-grafana-loki) you first need
+to update the version in the `fluent-plugin-grafana-loki.gemspec` file.
+Then update the `VERSION` variable in the `Makefile` to match the new version number.
+Create a PR with the changes against the `main` branch und run `make
+fluentd-plugin-push` from the root of the project once the PR has been merged.
+
 ## Copyright
 
 * Copyright(c) 2018- Grafana Labs

clients/cmd/fluentd/fluent-plugin-grafana-loki.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-grafana-loki'
-  spec.version = '1.2.18'
+  spec.version = '1.2.19'
   spec.authors = %w[woodsaj briangann cyriltovena]
   spec.email   = ['awoods@grafana.com', 'brian@grafana.com', 'cyril.tovena@grafana.com']
 

clients/cmd/promtail/Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /src/loki
 # Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression
 RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list
 RUN apt-get update && apt-get install -t bullseye-backports -qy libsystemd-dev
-RUN make clean && make BUILD_IN_CONTAINER=false promtail
+RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail
 
 # Promtail requires debian as the base image to support systemd journal reading
 FROM debian:bullseye-slim

clients/cmd/promtail/Dockerfile.arm32

@@ -5,7 +5,7 @@ WORKDIR /src/loki
 # Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression
 RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list
 RUN apt-get update && apt-get install -t bullseye-backports -qy libsystemd-dev
-RUN make clean && make BUILD_IN_CONTAINER=false promtail
+RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail
 
 # Promtail requires debian as the base image to support systemd journal reading
 FROM debian:bullseye-slim

clients/cmd/promtail/Dockerfile.cross

@@ -10,7 +10,7 @@ FROM --platform=linux/amd64 $BUILD_IMAGE as build
 COPY --from=goenv /goarch /goarm /
 COPY . /src/loki
 WORKDIR /src/loki
-RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false promtail
+RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail
 
 # Promtail requires debian as the base image to support systemd journal reading
 FROM debian:stretch-slim

clients/cmd/promtail/Dockerfile.debug

@@ -6,7 +6,7 @@ FROM grafana/loki-build-image as build
 ARG GOARCH="amd64"
 COPY . /src/loki
 WORKDIR /src/loki
-RUN make clean && make BUILD_IN_CONTAINER=false promtail-debug
+RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail-debug
 
 
 FROM       alpine:3.16.2

clients/pkg/promtail/client/client.go

@@ -94,7 +94,7 @@ func NewMetrics(reg prometheus.Registerer, streamLagLabels []string) *Metrics {
 	}, []string{HostLabel})
 
 	m.countersWithHost = []*prometheus.CounterVec{
-		m.encodedBytes, m.sentBytes, m.droppedBytes, m.sentEntries, m.droppedEntries,
+		m.encodedBytes, m.sentBytes, m.droppedBytes, m.sentEntries, m.droppedEntries, m.batchRetries,
 	}
 
 	streamLagLabelsMerged := []string{HostLabel, ClientLabel}

clients/pkg/promtail/scrapeconfig/scrapeconfig.go

@@ -236,6 +236,9 @@ type WindowsEventsTargetConfig struct {
 	// ExcludeEventData allows to exclude the xml event data.
 	ExcludeEventData bool `yaml:"exclude_event_data"`
 
+	// ExcludeEventMessage allows to exclude the human-friendly message contained in each windows event.
+	ExcludeEventMessage bool `yaml:"exclude_event_message"`
+
 	// ExcludeUserData allows to exclude the user data of each windows event.
 	ExcludeUserData bool `yaml:"exclude_user_data"`