Skip to content

Commit

Permalink
feat: add observe k8s attributes processor (#59)
Browse files Browse the repository at this point in the history
### Description

OB-34553 Add observek8sattributesprocessor

### Checklist
- [ ] Created tests which fail without the change (if possible)
- [ ] Extended the README / documentation, if necessary
  • Loading branch information
obs-gh-alexlew committed Jul 25, 2024
1 parent a70f4c0 commit 06c379d
Show file tree
Hide file tree
Showing 244 changed files with 32,248 additions and 8,620 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/generate-changelog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
# either 'goreleaser' (default) or 'goreleaser-pro'
distribution: goreleaser-pro
# 'latest', 'nightly', or a semver
version: latest
version: 2.0.0
args: changelog
env:
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
6 changes: 2 additions & 4 deletions .github/workflows/release-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: stable
go-version: 1.22.3
- name: Install msitools
run: sudo apt-get install -y wixl
# More assembly might be required: Docker logins, GPG, etc.
Expand All @@ -35,10 +35,8 @@ jobs:
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v6
with:
# either 'goreleaser' (default) or 'goreleaser-pro'
distribution: goreleaser-pro
# 'latest', 'nightly', or a semver
version: latest
version: '2.0.0'
args: release --prepare --clean --snapshot --verbose
env:
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
6 changes: 2 additions & 4 deletions .github/workflows/release-nightly.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: stable
go-version: 1.22.3
- name: Install msitools
run: sudo apt-get install -y wixl
# More assembly might be required: Docker logins, GPG, etc.
Expand All @@ -41,10 +41,8 @@ jobs:
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v6
with:
# either 'goreleaser' (default) or 'goreleaser-pro'
distribution: goreleaser-pro
# 'latest', 'nightly', or a semver
version: latest
version: 2.0.0
args: release --clean --skip=validate --verbose --nightly
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: stable
go-version: 1.22.3
- name: Install msitools
run: sudo apt-get install -y wixl
# More assembly might be required: Docker logins, GPG, etc.
Expand All @@ -40,7 +40,7 @@ jobs:
# either 'goreleaser' (default) or 'goreleaser-pro'
distribution: goreleaser-pro
# 'latest', 'nightly', or a semver
version: latest
version: 2.0.0
args: release --clean --skip=validate --verbose
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/tests-integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: stable
go-version: 1.22.3
- name: Install msitools
run: sudo apt-get install -y wixl
- name: Install qemu
Expand All @@ -41,7 +41,7 @@ jobs:
uses: goreleaser/goreleaser-action@v6
with:
distribution: goreleaser-pro
version: '~> v2'
version: '2.0.0'
args: release --prepare --clean --snapshot --verbose --parallelism 8
env:
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
Expand Down
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# build
agent
output-logs
./observe-agent
/observe-agent
dist/
agent.exe
agent.exe~
Expand Down
4 changes: 2 additions & 2 deletions .goreleaser.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ builds:
goos:
- linux
- darwin
ldflags: -s -w -X observe/agent/build.Version={{.Version}}
ldflags: -s -w -X observe-agent/build.Version={{.Version}}

- id: windows_build
binary: observe-agent
Expand All @@ -34,7 +34,7 @@ builds:
goarch:
- amd64
- '386'
ldflags: -s -w -X observe/agent/build.Version={{.Version}}
ldflags: -s -w -X observe-agent/build.Version={{.Version}}

archives:
- id: default
Expand Down
11 changes: 8 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
## go-test: Runs Go tests across all packages
go-test:
go work vendor
go build ./...
go test -v ./...

go list -f '{{.Dir}}' -m | xargs go test -v ./...
## release: Releases current tag through goreleaser
release:
goreleaser release --clean

## install-ocb: Installs correct version of ocb binary
install-ocb:
curl --proto '=https' --tlsv1.2 -L -o "$(HOME)/bin/ocb" https://github.com/open-telemetry/opentelemetry-collector/releases/download/cmd%2Fbuilder%2Fv0.104.0/ocb_0.104.0_darwin_arm64
curl --proto '=https' --tlsv1.2 -L -o "$(HOME)/bin/ocb" https://github.com/open-telemetry/opentelemetry-collector/releases/download/cmd%2Fbuilder%2Fv0.105.0/ocb_0.105.0_darwin_arm64
@chmod +x "$(HOME)/bin/ocb"

## build-ocb: Builds project using ocb
Expand All @@ -21,3 +22,7 @@ build-ocb:
cp ./ocb-build/go.sum ./cmd/collector/go.sum
go mod tidy && go work vendor
cd ./cmd/collector && go mod tidy && go work vendor

install-tools:
cd ./internal/tools && go install go.opentelemetry.io/collector/cmd/mdatagen

3 changes: 2 additions & 1 deletion builder-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ processors:
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.105.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.105.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.105.0
- gomod: github.com/observeinc/observe-agent/components/processors/observek8sattributesprocessor v0.0.0-00010101000000-000000000000
path: ./components/processors/observek8sattributesprocessor

receivers:
- gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.105.0
Expand Down Expand Up @@ -48,7 +50,6 @@ extensions:

- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.105.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.105.0

connectors:
- gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.105.0

Expand Down
3 changes: 3 additions & 0 deletions cmd/collector/components.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 6 additions & 5 deletions cmd/collector/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@

module observe/otelcol

go 1.21.0

toolchain go1.22.3
go 1.22.3

require (
github.com/observeinc/observe-agent/components/processors/observek8sattributesprocessor v0.0.0-00010101000000-000000000000
github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.105.0
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.105.0
github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.105.0
Expand Down Expand Up @@ -312,8 +311,8 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.29.3 // indirect
k8s.io/apimachinery v0.29.3 // indirect
k8s.io/api v0.30.3 // indirect
k8s.io/apimachinery v0.30.3 // indirect
k8s.io/client-go v0.29.3 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
Expand All @@ -323,3 +322,5 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace github.com/observeinc/observe-agent/components/processors/observek8sattributesprocessor v0.0.0-00010101000000-000000000000 => ../../components/processors/observek8sattributesprocessor
16 changes: 8 additions & 8 deletions cmd/collector/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -570,12 +570,12 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE=
github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.105.0 h1:dCq7P9MfGeBCQwxDu1TlS94KXkVjZhueCm4fSFlEG38=
github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.105.0/go.mod h1:x/H4ulXmk8HU7HzBOHrJL+C4dHMGauV6f5CxFXyP4rE=
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.105.0 h1:I3I7FaUgATms/w1O+aKvg7/cjlQg+J/BCECUHHP80rI=
Expand Down Expand Up @@ -1394,11 +1394,11 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw=
k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80=
k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ=
k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04=
k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU=
k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU=
k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg=
k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0=
Expand Down
2 changes: 1 addition & 1 deletion cmd/collector/otelcollector.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package observeotel

import (
"observe/agent/build"
"observe-agent/build"

"github.com/spf13/cobra"
"go.opentelemetry.io/collector/component"
Expand Down
2 changes: 1 addition & 1 deletion cmd/commands/diagnose/diagnose.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package diagnose
import (
"embed"
"fmt"
"observe/agent/cmd"
"observe-agent/cmd"
"os"
"text/template"

Expand Down
2 changes: 1 addition & 1 deletion cmd/commands/initconfig/initconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"embed"
"fmt"
"html/template"
"observe/agent/cmd"
"observe-agent/cmd"
"os"

"github.com/spf13/cobra"
Expand Down
4 changes: 2 additions & 2 deletions cmd/commands/start/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ Copyright © 2024 NAME HERE <EMAIL ADDRESS>
package start

import (
"observe/agent/cmd"
"observe-agent/cmd"
"observe-agent/cmd/config"
observeotel "observe/otelcol"
"observe/agent/cmd/config"
"os"

"github.com/spf13/cobra"
Expand Down
2 changes: 1 addition & 1 deletion cmd/commands/status/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package status
import (
"embed"
"html/template"
"observe/agent/cmd"
"observe-agent/cmd"
"os"

"github.com/spf13/cobra"
Expand Down
4 changes: 2 additions & 2 deletions cmd/commands/version/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ package version

import (
"fmt"
"observe/agent/build"
"observe/agent/cmd"
"observe-agent/build"
"observe-agent/cmd"

"github.com/spf13/cobra"
)
Expand Down
2 changes: 1 addition & 1 deletion cmd/config/confighandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"path/filepath"
"runtime"

"observe/agent/cmd/connections"
"observe-agent/cmd/connections"

"github.com/spf13/viper"
)
Expand Down
2 changes: 1 addition & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package cmd

import (
"fmt"
"observe/agent/cmd/config"
"observe-agent/cmd/config"
"os"

"github.com/spf13/cobra"
Expand Down
12 changes: 12 additions & 0 deletions components/processors/observek8sattributesprocessor/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Observe K8s Attributes Processor
This processor operates on K8s resource logs from the `k8sobjectsreceiver` and adds additional attributes.


## Caveats
This processor currently expects the `kind` field to be set at the base level of the event. In the case of `watch` events from the `k8sobjectsreceiver`, this field is instead present inside of the `object` field. This processor currently expects this field to be lifted from inside the `object` field to the base level by a transform processor earlier in the pipeline. If that isn't set up, this processor will only calculate status for `pull` events from the `k8sobjectsreceiver`.

## Emitted Attributes

| Attribute Key | Description |
|-----------------------------------|--------------------------------------------------------------|
| `observe_transform.facets.status` | The derived Pod status based on the current Pod description. |
8 changes: 8 additions & 0 deletions components/processors/observek8sattributesprocessor/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package observek8sattributesprocessor

type Config struct {
}

func (cfg *Config) Validate() error {
return nil
}
5 changes: 5 additions & 0 deletions components/processors/observek8sattributesprocessor/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
//go:generate mdatagen metadata.yaml

// package observek8sattributesprocessor implements a processor which
// calculates and attaches additional k8s attributes
package observek8sattributesprocessor
43 changes: 43 additions & 0 deletions components/processors/observek8sattributesprocessor/factory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package observek8sattributesprocessor

import (
"context"

"go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/consumer"
"go.opentelemetry.io/collector/processor"
"go.opentelemetry.io/collector/processor/processorhelper"
)

var processorCapabilities = consumer.Capabilities{MutatesData: true}

func NewFactory() processor.Factory {
return processor.NewFactory(
Type,
createDefaultConfig,
processor.WithLogs(createLogsProcessor, LogsStability),
)
}

func createDefaultConfig() component.Config {
return &Config{}
}

func createLogsProcessor(
ctx context.Context,
set processor.Settings,
cfg component.Config,
nextConsumer consumer.Logs,
) (processor.Logs, error) {
kep := newK8sEventsProcessor(set.Logger, cfg)
return processorhelper.NewLogsProcessor(
ctx,
set,
cfg,
nextConsumer,
kep.processLogs,
processorhelper.WithCapabilities(processorCapabilities),
processorhelper.WithStart(kep.Start),
processorhelper.WithShutdown(kep.Shutdown),
)
}
Loading

0 comments on commit 06c379d

Please sign in to comment.