diff --git a/Dockerfile b/Dockerfile index 6ebc713..fe1f491 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,5 +22,7 @@ LABEL org.opencontainers.image.source=https://github.com/ebpfdev/dev-agent WORKDIR /app COPY --from=0 /build/dev-agent ./ -CMD ["/app/dev-agent"] +EXPOSE 8080 + +ENTRYPOINT ["/app/dev-agent"] diff --git a/README.md b/README.md index 623990c..e4cf759 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This agent provides access to system's eBPF-programs and maps to perform remote ## GraphQL server ```shell -sudo ./phydev server +sudo ./phydev server [--help] ``` GraphQL interface: [http://localhost:8080/](http://localhost:8080/) @@ -14,10 +14,58 @@ Schema: [pkg/graph/schema.graphqls](pkg/graph/schema.graphqls) ![GraphQL interface example](docs/graphql-example.png) +### Prometheus endpoint +Metrics scrape endpoint for Prometheus: [http://localhost:8080/metrics](http://localhost:8080/metrics) + +* program metrics: + * `devagent_ebpf_prog_count` - number of eBPF programs by `type` + * runtime metrics only available with `sysctl -w kernel.bpf_stats_enabled=1`: + * `devagent_ebpf_prog_run_count` - number of times an eBPF program has been run (by `id`, `name`, `tag`, `type`) + * `devagent_ebpf_prog_run_time` - total time spent running eBPF programs (by `id`, `name`, `tag`, `type`) +* map metrics: + * `devagent_ebpf_map_count` - number of eBPF maps by `type` + * if map export is configured (see below): + * `devagent_ebpf_map_entry_count` - number of entries in an eBPF map (by `id`, `name`, `type`) + * `devagent_ebpf_map_entry_value` - value of an eBPF map entry (by `key`, `cpu`, `id`, `name`, `type`) + +You can find example of Grafana dashboard in [grafana-ebpf-dashboard.json](./grafana-ebpf-dashboard.json): +![grafana dashboard with program metrics](docs/grafana-ebpf.png) + +#### Configuring map export + +As an example, I'm running this [bpftrace](https://github.com/iovisor/bpftrace) program: +```shell +sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @SYSCALLNUM[comm] = count(); }' +``` + +You could see the name of created map - `AT_SYSCALLNUM`, and the map content in [ebpf-explorer](https://github.com/ebpfdev/explorer): +![exbpf explorer showing AT_SYSCALNUM page](docs/explorer-syscallnum.png) + +By default, dev-agent doesn't export map entries to Prometheus, as it may introduce some performance issues. + +Instead, you could set an option `--etm -:AT_SYSCALLNUM:string` when running server, which will suggest agent which map entries to expose in /metrics. + +For this HASH_PER_CPU map, it will export 2 metrics: +```text +# HELP devagent_ebpf_map_entry_count Number of entries in an eBPF map +# TYPE devagent_ebpf_map_entry_count gauge +devagent_ebpf_map_entry_count{id="25",name="AT_SYSCALLNUM",type="PerCPUHash"} 764 +# HELP devagent_ebpf_map_entry_value Value of an eBPF map entry +# TYPE devagent_ebpf_map_entry_value gauge +devagent_ebpf_map_entry_value{cpu="0",id="25",key="(anacron)",name="AT_SYSCALLNUM",type="PerCPUHash"} 0 +devagent_ebpf_map_entry_value{cpu="0",id="25",key="(fprintd)",name="AT_SYSCALLNUM",type="PerCPUHash"} 0 +``` + +This is how it may look in Grafana (top 10 processes doing most of syscalls): +![Grafana showing top 10 processes doing most of syscalls](docs/grafana-syscallnum.png) + +Run `./phydev server --help` for more details on this flag. ## CLI commands +These are just for debugging purpose, use [bpftool](https://github.com/libbpf/bpftool) instead + List loaded eBPF programs: ```shell @@ -44,6 +92,13 @@ ID Name FD Type Flags IsPinned KeySize ValueSize 63 open_at_args 29 Hash 0 false 8 128 1024 ``` +## Docker + +Instead of `./phydev`, use docker command: +```shell +docker run -ti --rm --privileged -p 8080:8080 ghcr.io/ebpfdev/dev-agent:v0.0.1 /app/dev-agent server +``` + # Development ## Build diff --git a/cmd/dev-agent/commands/commands.go b/cmd/dev-agent/commands/commands.go index 1f10ef9..a61073f 100644 --- a/cmd/dev-agent/commands/commands.go +++ b/cmd/dev-agent/commands/commands.go @@ -12,7 +12,9 @@ import ( func App() *cli.App { logger := log.Logger.Level(zerolog.InfoLevel) progsRepo := progs.NewWatcher(logger, 1*time.Second) - mapsRepo := maps.NewWatcher(logger, 1*time.Second) + mapsRepo := maps.NewWatcher(&maps.WatcherOpts{ + RefreshInterval: 1 * time.Second, + }, logger) progsCommands := &ProgsCommands{ ProgsRepo: progsRepo, } @@ -39,16 +41,38 @@ func App() *cli.App { Name: "server", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "path-prefix", - Usage: "path prefix for the web ui to access the server", - Value: "/", + Name: "path-prefix", + Category: "Server", + Usage: "path prefix for the web ui to access the server", + Value: "/", }, &cli.BoolFlag{ Name: "skip-welcome", Usage: "skip welcome message", }, + &cli.MultiStringFlag{ + Target: &cli.StringSliceFlag{ + Name: "entries-to-metrics", + Category: "Metrics", + Usage: "(experimental, api may change)\n\tConfigure which map entries should be exposed as metrics, " + + "in the format: id_start-id_end:metric_name_regexp:key_format.\n\t" + + "Example: '-:.+:string' to export any map with non-empty name while treating key as string.\n\t" + + "or '10-:.*:hex' to export any map after ID 10 with key represented in HEX format\n\t" + + "Available key formats: string, number, hex\n\t" + + "If a map matches multiple entries, the first one is used.", + Aliases: []string{"etm"}, + }, + }, }, Action: func(c *cli.Context) error { + for _, etm := range c.StringSlice("entries-to-metrics") { + etmConfig, err := maps.ParseMapExportConfiguration(etm) + if err != nil { + return err + } + mapsRepo.AddExportConfig(etmConfig) + } + return serverCommands.ServerStart(&ServerStartOptions{ PathPrefix: c.String("path-prefix"), SkipWelcome: c.Bool("skip-welcome"), diff --git a/cmd/dev-agent/commands/server.go b/cmd/dev-agent/commands/server.go index ee230ce..1fa677c 100644 --- a/cmd/dev-agent/commands/server.go +++ b/cmd/dev-agent/commands/server.go @@ -8,6 +8,8 @@ import ( "github.com/ebpfdev/dev-agent/pkg/ebpf/progs" "github.com/ebpfdev/dev-agent/pkg/graph" "github.com/ebpfdev/dev-agent/pkg/graph/generated" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" "log" "net/http" @@ -33,9 +35,14 @@ func (sc *ServerCommands) ServerStart(options *ServerStartOptions) error { port = defaultPort } + registry := prometheus.NewRegistry() + sc.ProgsRepo.Run(context.Background()) sc.MapsRepo.Run(context.Background()) + sc.ProgsRepo.RegisterMetrics(registry) + sc.MapsRepo.RegisterMetrics(registry) + resolver := &graph.Resolver{ ProgsRepository: sc.ProgsRepo, MapsRepository: sc.MapsRepo, @@ -47,6 +54,11 @@ func (sc *ServerCommands) ServerStart(options *ServerStartOptions) error { mux.Handle("/", playground.Handler("GraphQL playground", options.PathPrefix+"query")) mux.Handle("/query", srv) + mux.Handle("/metrics", promhttp.HandlerFor( + registry, + promhttp.HandlerOpts{ + EnableOpenMetrics: true, + })) if !options.SkipWelcome { log.Printf("connect to http://localhost:%s/ for GraphQL playground", port) diff --git a/docs/explorer-syscallnum.png b/docs/explorer-syscallnum.png new file mode 100644 index 0000000..67ba39f Binary files /dev/null and b/docs/explorer-syscallnum.png differ diff --git a/docs/grafana-ebpf.png b/docs/grafana-ebpf.png new file mode 100644 index 0000000..1d90171 Binary files /dev/null and b/docs/grafana-ebpf.png differ diff --git a/docs/grafana-syscallnum.png b/docs/grafana-syscallnum.png new file mode 100644 index 0000000..fde198f Binary files /dev/null and b/docs/grafana-syscallnum.png differ diff --git a/go.mod b/go.mod index c896440..38ae957 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/99designs/gqlgen v0.17.31 github.com/cilium/ebpf v0.10.0 + github.com/prometheus/client_golang v1.15.1 github.com/rs/cors v1.9.0 github.com/rs/zerolog v1.29.1 github.com/urfave/cli/v2 v2.25.3 @@ -13,17 +14,25 @@ require ( require ( github.com/agnivade/levenshtein v1.1.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.7.0 // indirect golang.org/x/tools v0.6.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 093834a..2bddcac 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,10 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cilium/ebpf v0.10.0 h1:nk5HPMeoBXtOzbkZBWym+ZWq1GIiHUsBFXxwewXAHLQ= github.com/cilium/ebpf v0.10.0/go.mod h1:DPiVdY/kT534dgc9ERmvP8mWA+9gvwgKfRvk4nNWnoE= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -19,6 +23,12 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -36,11 +46,21 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -62,6 +82,7 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRT github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -72,12 +93,17 @@ golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/grafana-ebpf-dashboard.json b/grafana-ebpf-dashboard.json new file mode 100644 index 0000000..e652f90 --- /dev/null +++ b/grafana-ebpf-dashboard.json @@ -0,0 +1,347 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "iteration": 1685807654472, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(devagent_ebpf_prog_run_time{id=~\"$program\"}[1m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total eBPF load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, rate(devagent_ebpf_prog_run_time{id=~\"$program\"}[1m]) / rate(devagent_ebpf_prog_run_count{id=~\"$program\"}[1m]))", + "interval": "", + "legendFormat": "{{id}} {{name}} {{type}} {{tag}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average duration per run", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(devagent_ebpf_prog_run_count{id=~\"$program\"}[1m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Program runs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "label_values(devagent_ebpf_prog_run_time, id)", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "name": "program", + "options": [], + "query": "label_values(devagent_ebpf_prog_run_time, id)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "eBPF", + "uid": "FdyOEWlVk", + "variables": { + "list": [] + }, + "version": 3 +} diff --git a/pkg/ebpf/maps/capabilities.go b/pkg/ebpf/maps/capabilities.go new file mode 100644 index 0000000..455b826 --- /dev/null +++ b/pkg/ebpf/maps/capabilities.go @@ -0,0 +1,45 @@ +package maps + +import "github.com/cilium/ebpf" + +func IsPerCPU(mt ebpf.MapType) bool { + return mt == ebpf.PerCPUHash || mt == ebpf.PerCPUArray || mt == ebpf.LRUCPUHash || mt == ebpf.PerCPUCGroupStorage +} + +var lookupSupported = map[ebpf.MapType]bool{ + ebpf.UnspecifiedMap: false, + ebpf.Hash: true, + ebpf.Array: true, + ebpf.ProgramArray: false, + ebpf.PerfEventArray: false, + ebpf.PerCPUHash: true, + ebpf.PerCPUArray: true, + ebpf.StackTrace: false, + ebpf.CGroupArray: false, + ebpf.LRUHash: false, + ebpf.LRUCPUHash: false, + ebpf.LPMTrie: false, + ebpf.ArrayOfMaps: false, + ebpf.HashOfMaps: false, + ebpf.DevMap: false, + ebpf.SockMap: false, + ebpf.CPUMap: false, + ebpf.XSKMap: false, + ebpf.SockHash: false, + ebpf.CGroupStorage: false, + ebpf.ReusePortSockArray: false, + ebpf.PerCPUCGroupStorage: false, + ebpf.Queue: false, + ebpf.Stack: false, + ebpf.SkStorage: false, + ebpf.DevMapHash: false, + ebpf.StructOpsMap: false, + ebpf.RingBuf: false, + ebpf.InodeStorage: false, + ebpf.TaskStorage: false, +} + +func IsLookupSupported(mt ebpf.MapType) bool { + supported, ok := lookupSupported[mt] + return ok && supported +} diff --git a/pkg/ebpf/maps/display.go b/pkg/ebpf/maps/display.go new file mode 100644 index 0000000..22a9faa --- /dev/null +++ b/pkg/ebpf/maps/display.go @@ -0,0 +1,40 @@ +package maps + +import ( + "fmt" + "github.com/ebpfdev/dev-agent/pkg/ebpf/util" +) + +type DisplayFormat = string + +const ( + DisplayFormatHex DisplayFormat = "hex" + DisplayFormatString DisplayFormat = "string" + DisplayFormatNumber DisplayFormat = "number" +) + +func FormatBytes(format DisplayFormat, value []byte) string { + switch format { + case DisplayFormatString: + // drop trailing zeros + for i := len(value) - 1; i >= 0; i-- { + if value[i] == 0 { + value = value[:i] + } else { + break + } + } + return string(value) + case DisplayFormatHex: + return fmt.Sprintf("%x", value) + case DisplayFormatNumber: + if len(value) <= 8 { + buf := make([]byte, 8) + copy(buf, value) + return fmt.Sprintf("%d", int64(util.GetEndian().Uint64(buf))) + } + return fmt.Sprintf("%x", value) + default: + return fmt.Sprintf("%x", value) + } +} diff --git a/pkg/ebpf/maps/entries.go b/pkg/ebpf/maps/entries.go new file mode 100644 index 0000000..182df98 --- /dev/null +++ b/pkg/ebpf/maps/entries.go @@ -0,0 +1,91 @@ +package maps + +import ( + "github.com/cilium/ebpf" + sortp "sort" +) + +type MapEntries struct { + Entries []*MapEntry +} + +type MapEntry struct { + Key []byte + CPUValues [][]byte + Value []byte +} + +func GetEntries(id ebpf.MapID, sort bool) (*MapEntries, error) { + emap, err := ebpf.NewMapFromID(id) + if err != nil { + return nil, err + } + + entries := make([]*MapEntry, 0) + + if !IsLookupSupported(emap.Type()) { + return &MapEntries{[]*MapEntry{}}, nil + } + + var key []byte + mapIterator := emap.Iterate() + if IsPerCPU(emap.Type()) { + var bufSlice [][]byte + for mapIterator.Next(&key, &bufSlice) { + values := make([][]byte, len(bufSlice)) + for i, value := range bufSlice { + values[i] = value[:] + } + entries = append(entries, &MapEntry{ + Key: key[:], + CPUValues: values, + }) + } + } else { + var buf []byte + for mapIterator.Next(&key, &buf) { + entries = append(entries, &MapEntry{ + Key: key[:], + Value: buf[:], + }) + } + } + + if sort { + sortp.SliceStable(entries, func(i, j int) bool { + iKey := entries[i].Key + jKey := entries[j].Key + for k := 0; k < len(iKey) && k < len(jKey); k++ { + if iKey[k] < jKey[k] { + return true + } + if iKey[k] > jKey[k] { + return false + } + } + return len(iKey) < len(jKey) + }) + } + + return &MapEntries{entries}, mapIterator.Err() +} + +func CountEntries(id ebpf.MapID) (int, error) { + count := 0 + emap, err := ebpf.NewMapFromID(id) + if err != nil { + return 0, err + } + + if !IsLookupSupported(emap.Type()) { + return 0, nil + } + + var key []byte + var buf []byte + mapIterator := emap.Iterate() + for mapIterator.Next(&key, &buf) { + count++ + } + return count, mapIterator.Err() +} diff --git a/pkg/ebpf/maps/entries_export.go b/pkg/ebpf/maps/entries_export.go new file mode 100644 index 0000000..c61e0bb --- /dev/null +++ b/pkg/ebpf/maps/entries_export.go @@ -0,0 +1,52 @@ +package maps + +import ( + "github.com/cilium/ebpf" + "github.com/ebpfdev/dev-agent/pkg/ebpf/util" + "strconv" +) + +func (pw *mapsWatcher) exportMapEntries(id ebpf.MapID, name string, typ ebpf.MapType, config *MapExportConfiguration) { + mapEntries, err := GetEntries(id, false) + if err != nil { + pw.log.Err(err).Msgf("failed to get map entries for map %d", id) + return + } + + pw.mapEntriesCount. + WithLabelValues(strconv.Itoa(int(id)), name, typ.String()). + Set(float64(len(mapEntries.Entries))) + + for _, entry := range mapEntries.Entries { + key := FormatBytes(config.KeyFormat, entry.Key) + + if len(entry.CPUValues) > 0 { + for cpu, value := range entry.CPUValues { + if len(value) <= 8 { + buf := make([]byte, 8) + copy(buf, value) + pw.mapEntryValues. + WithLabelValues( + strconv.Itoa(int(id)), + name, + typ.String(), + key, + strconv.Itoa(cpu)). + Set(float64(util.GetEndian().Uint64(buf))) + } + } + } else { + buf := make([]byte, 8) + copy(buf, entry.Value) + pw.mapEntryValues. + WithLabelValues( + strconv.Itoa(int(id)), + name, + typ.String(), + key, + ""). + Set(float64(util.GetEndian().Uint64(buf))) + } + } + +} diff --git a/pkg/ebpf/maps/entries_export_config.go b/pkg/ebpf/maps/entries_export_config.go new file mode 100644 index 0000000..ce6dd8f --- /dev/null +++ b/pkg/ebpf/maps/entries_export_config.go @@ -0,0 +1,96 @@ +package maps + +import ( + "fmt" + "github.com/cilium/ebpf" + "regexp" + "strconv" + "strings" +) + +type MapExportConfiguration struct { + StartID int + EndID int + MetricNameRegexp regexp.Regexp + KeyFormat DisplayFormat +} + +func (c *MapExportConfiguration) MatchMap(id ebpf.MapID, name string) bool { + if c.StartID >= 0 && int(id) < c.StartID { + return false + } + if c.EndID >= 0 && int(id) > c.EndID { + return false + } + return c.MetricNameRegexp.MatchString(name) +} + +func ParseMapExportConfiguration(config string) (*MapExportConfiguration, error) { + parts := strings.Split(config, ":") + if len(parts) < 3 { + return nil, fmt.Errorf("invalid format: %s, should be -::", config) + } + + rangeStr := parts[0] + keyFormat := parts[len(parts)-1] + metricNameRegexp := strings.Join(parts[1:len(parts)-1], ":") + + idStart, idEnd, err := parseRange(rangeStr) + if err != nil { + return nil, err + } + metricNameRegexpCompiled, err := regexp.Compile(metricNameRegexp) + if err != nil { + return nil, err + } + keyFormatParsed, err := parseDisplayFormat(keyFormat) + if err != nil { + return nil, err + } + + return &MapExportConfiguration{ + StartID: idStart, + EndID: idEnd, + MetricNameRegexp: *metricNameRegexpCompiled, + KeyFormat: keyFormatParsed, + }, nil +} + +func parseDisplayFormat(s string) (DisplayFormat, error) { + switch strings.ToLower(s) { + case "string": + return DisplayFormatString, nil + case "number": + return DisplayFormatNumber, nil + case "hex": + return DisplayFormatHex, nil + default: + return DisplayFormatHex, fmt.Errorf("invalid format: %s, should be string, number or hex", s) + } +} + +func parseRange(rangeStr string) (int, int, error) { + var startID, endID int + var err error + rangeParts := strings.SplitN(rangeStr, "-", 2) + if len(rangeParts) != 2 { + return -1, -1, fmt.Errorf("ID range is invalid: %s", rangeStr) + } + if rangeParts[0] == "" { + startID = -1 + } else { + startID, err = strconv.Atoi(rangeParts[0]) + if err != nil { + return -1, -1, fmt.Errorf("start ID is invalid: %s", rangeStr) + } + } + if rangeParts[1] == "" { + endID = -1 + } else { + endID, err = strconv.Atoi(rangeParts[1]) + if err != nil { + return startID, -1, fmt.Errorf("end ID is invalid: %s", rangeStr) + } + } + return startID, endID, nil +} diff --git a/pkg/ebpf/maps/modify.go b/pkg/ebpf/maps/modify.go new file mode 100644 index 0000000..99531c7 --- /dev/null +++ b/pkg/ebpf/maps/modify.go @@ -0,0 +1,26 @@ +package maps + +// +//import "github.com/cilium/ebpf" +// +//func CreateMap() { +// ebpf.MapSpec{ +// Name: "", +// Type: 0, +// KeySize: 0, +// ValueSize: 0, +// MaxEntries: 0, +// Flags: 0, +// Pinning: 0, +// NumaNode: 0, +// Contents: nil, +// Freeze: false, +// InnerMap: nil, +// Extra: nil, +// Key: nil, +// Value: nil, +// } +// +// ebpf.NewMapWithOptions() +// +//} diff --git a/pkg/ebpf/maps/state.go b/pkg/ebpf/maps/state.go new file mode 100644 index 0000000..c45f3d1 --- /dev/null +++ b/pkg/ebpf/maps/state.go @@ -0,0 +1,202 @@ +package maps + +import ( + "context" + "errors" + "github.com/cilium/ebpf" + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" + "os" + "time" +) + +type mapsWatcher struct { + log zerolog.Logger + refreshInterval time.Duration + maps []*MapInfo + error error + isRunning bool + + mapsCount *prometheus.GaugeVec + mapEntriesCount *prometheus.GaugeVec + mapEntryValues *prometheus.GaugeVec + exportConfigs []*MapExportConfiguration +} + +type MapsWatcher interface { + Run(ctx context.Context) + GetMaps() ([]*MapInfo, error) + GetMap(id ebpf.MapID) (*MapInfo, error) + RegisterMetrics(registry *prometheus.Registry) + AddExportConfig(config *MapExportConfiguration) +} + +type WatcherOpts struct { + RefreshInterval time.Duration +} + +func NewWatcher(opts *WatcherOpts, logger zerolog.Logger) MapsWatcher { + mapsCount := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "devagent", + Subsystem: "ebpf", + Name: "map_count", + Help: "Number of eBPF maps", + }, []string{"type"}) + mapEntriesCount := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "devagent", + Subsystem: "ebpf", + Name: "map_entry_count", + Help: "Number of entries in an eBPF map", + }, []string{"id", "name", "type"}) + mapEntryValues := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "devagent", + Subsystem: "ebpf", + Name: "map_entry_value", + Help: "Value of an eBPF map entry", + }, []string{"id", "name", "type", "key", "cpu"}) + + return &mapsWatcher{ + log: logger, + refreshInterval: opts.RefreshInterval, + mapsCount: mapsCount, + mapEntriesCount: mapEntriesCount, + mapEntryValues: mapEntryValues, + } +} + +func (pw *mapsWatcher) AddExportConfig(config *MapExportConfiguration) { + pw.exportConfigs = append(pw.exportConfigs, config) +} + +func (pw *mapsWatcher) RegisterMetrics(registry *prometheus.Registry) { + err := registry.Register(pw.mapsCount) + if err != nil { + pw.log.Err(err).Msg("Failed to register map_count metric") + } + err = registry.Register(pw.mapEntriesCount) + if err != nil { + pw.log.Err(err).Msg("Failed to register map_entry_count metric") + } + err = registry.Register(pw.mapEntryValues) + if err != nil { + pw.log.Err(err).Msg("Failed to register map_entry_value metric") + } +} + +func (pw *mapsWatcher) Run(ctx context.Context) { + if pw.isRunning { + return + } + go func() { + pw.isRunning = true + ticker := time.NewTicker(pw.refreshInterval) + ctx.Done() + for { + select { + case <-ticker.C: + pw.maps, pw.error = pw.fetchMaps() + case <-ctx.Done(): + pw.isRunning = false + return + } + } + }() +} + +type MapInfo struct { + ID ebpf.MapID + Error error + Name string + Type ebpf.MapType + Flags uint32 + IsPinned bool + KeySize uint32 + ValueSize uint32 + MaxEntries uint32 +} + +func (pw *mapsWatcher) GetMaps() ([]*MapInfo, error) { + if pw.maps == nil { + return pw.fetchMaps() + } + return pw.maps, pw.error +} + +func (pw *mapsWatcher) GetMap(id ebpf.MapID) (*MapInfo, error) { + maps, err := pw.GetMaps() + if err != nil { + return nil, err + } + for _, m := range maps { + if m.ID == id { + return m, nil + } + } + return nil, errors.New("map not found") +} + +func (pw *mapsWatcher) fetchMaps() ([]*MapInfo, error) { + var currID ebpf.MapID = 0 + var err error + var maps []*MapInfo + pw.log.Debug().Msg("fetching maps") + + // maps count by type + mapsCount := make(map[ebpf.MapType]int) + defer func() { + for k, v := range mapsCount { + pw.mapsCount.WithLabelValues(k.String()).Set(float64(v)) + } + }() + + for true { + currID, err = ebpf.MapGetNextID(currID) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + break + } + pw.log.Err(err).Msg("failed to get next map ID") + return maps, err + } + emap, err2 := ebpf.NewMapFromID(currID) + if err2 != nil { + maps = append(maps, mapInfoErr(currID, err2)) + continue + } + + mapsCount[emap.Type()]++ + + info, err2 := emap.Info() + name := "" + if info != nil { + name = info.Name + } + maps = append(maps, &MapInfo{ + ID: currID, + Error: err2, + Name: name, + Type: emap.Type(), + Flags: emap.Flags(), + IsPinned: emap.IsPinned(), + KeySize: emap.KeySize(), + ValueSize: emap.ValueSize(), + MaxEntries: emap.MaxEntries(), + }) + _ = emap.Close() + + for _, config := range pw.exportConfigs { + if config.MatchMap(currID, name) { + pw.exportMapEntries(currID, name, emap.Type(), config) + break + } + } + } + return maps, nil +} + +func mapInfoErr(id ebpf.MapID, err error) *MapInfo { + return &MapInfo{ + ID: id, + Error: err, + } +} diff --git a/pkg/ebpf/maps/watcher.go b/pkg/ebpf/maps/watcher.go deleted file mode 100644 index dc71a44..0000000 --- a/pkg/ebpf/maps/watcher.go +++ /dev/null @@ -1,130 +0,0 @@ -package maps - -import ( - "context" - "errors" - "github.com/cilium/ebpf" - "github.com/rs/zerolog" - "os" - "time" -) - -type mapsWatcher struct { - log zerolog.Logger - checkInterval time.Duration - maps []*MapInfo - error error - isRunning bool -} - -type MapsWatcher interface { - Run(ctx context.Context) - GetMaps() ([]*MapInfo, error) - GetMap(id ebpf.MapID) (*MapInfo, error) -} - -func NewWatcher(logger zerolog.Logger, checkInterval time.Duration) MapsWatcher { - return &mapsWatcher{ - log: logger, - checkInterval: checkInterval, - } -} - -func (pw *mapsWatcher) Run(ctx context.Context) { - if pw.isRunning { - return - } - go func() { - pw.isRunning = true - ticker := time.NewTicker(pw.checkInterval) - ctx.Done() - for { - select { - case <-ticker.C: - pw.maps, pw.error = pw.fetchMaps() - case <-ctx.Done(): - pw.isRunning = false - return - } - } - }() -} - -type MapInfo struct { - ID ebpf.MapID - Error error - Name string - Type ebpf.MapType - Flags uint32 - IsPinned bool - KeySize uint32 - ValueSize uint32 - MaxEntries uint32 -} - -func (pw *mapsWatcher) GetMaps() ([]*MapInfo, error) { - if pw.maps == nil { - return pw.fetchMaps() - } - return pw.maps, pw.error -} - -func (pw *mapsWatcher) GetMap(id ebpf.MapID) (*MapInfo, error) { - maps, err := pw.GetMaps() - if err != nil { - return nil, err - } - for _, m := range maps { - if m.ID == id { - return m, nil - } - } - return nil, errors.New("map not found") -} - -func (pw *mapsWatcher) fetchMaps() ([]*MapInfo, error) { - var currID ebpf.MapID = 0 - var err error - var maps []*MapInfo - pw.log.Debug().Msg("fetching maps") - for true { - currID, err = ebpf.MapGetNextID(currID) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - break - } - pw.log.Err(err).Msg("failed to get next map ID") - return maps, err - } - emap, err2 := ebpf.NewMapFromID(currID) - if err2 != nil { - maps = append(maps, mapInfoErr(currID, err2)) - continue - } - info, err2 := emap.Info() - name := "" - if info != nil { - name = info.Name - } - maps = append(maps, &MapInfo{ - ID: currID, - Error: err2, - Name: name, - Type: emap.Type(), - Flags: emap.Flags(), - IsPinned: emap.IsPinned(), - KeySize: emap.KeySize(), - ValueSize: emap.ValueSize(), - MaxEntries: emap.MaxEntries(), - }) - _ = emap.Close() - } - return maps, nil -} - -func mapInfoErr(id ebpf.MapID, err error) *MapInfo { - return &MapInfo{ - ID: id, - Error: err, - } -} diff --git a/pkg/ebpf/progs/state.go b/pkg/ebpf/progs/state.go new file mode 100644 index 0000000..bc9d9ba --- /dev/null +++ b/pkg/ebpf/progs/state.go @@ -0,0 +1,199 @@ +package progs + +import ( + "context" + "errors" + "github.com/cilium/ebpf" + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "os" + "strconv" + "time" +) + +type progWatcher struct { + log zerolog.Logger + refreshInterval time.Duration + progs []ProgInfo + error error + isRunning bool + progRunCount *prometheus.GaugeVec + progRunTime *prometheus.GaugeVec + progsCount *prometheus.GaugeVec +} + +type ProgWatcher interface { + Run(ctx context.Context) + GetProgs() ([]ProgInfo, error) + GetProg(id ebpf.ProgramID) (*ProgInfo, error) + RegisterMetrics(registry *prometheus.Registry) +} + +func NewWatcher(logger zerolog.Logger, refreshInterval time.Duration) ProgWatcher { + progRunCount := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "devagent", + Subsystem: "ebpf", + Name: "prog_run_count", + Help: "Number of times an eBPF program has been run", + }, []string{"id", "type", "tag", "name"}) + progRunTime := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "devagent", + Subsystem: "ebpf", + Name: "prog_run_time", + Help: "Total time spent running eBPF programs", + }, []string{"id", "type", "tag", "name"}) + progsCount := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "devagent", + Subsystem: "ebpf", + Name: "prog_count", + Help: "Number of eBPF programs", + }, []string{"type"}) + + return &progWatcher{ + log: logger, + refreshInterval: refreshInterval, + progRunCount: progRunCount, + progRunTime: progRunTime, + progsCount: progsCount, + } +} + +func (pw *progWatcher) RegisterMetrics(registry *prometheus.Registry) { + err := registry.Register(pw.progRunCount) + if err != nil { + log.Err(err).Msg("failed to register prog_run_count metric") + } + err = registry.Register(pw.progRunTime) + if err != nil { + log.Err(err).Msg("failed to register prog_run_time metric") + } + err = registry.Register(pw.progsCount) + if err != nil { + log.Err(err).Msg("failed to register prog_count metric") + } +} + +func (pw *progWatcher) Run(ctx context.Context) { + if pw.isRunning { + return + } + go func() { + pw.isRunning = true + ticker := time.NewTicker(pw.refreshInterval) + ctx.Done() + for { + select { + case <-ticker.C: + pw.progs, pw.error = pw.fetchProgs() + case <-ctx.Done(): + pw.isRunning = false + return + } + } + }() +} + +type ProgInfo struct { + ID ebpf.ProgramID + Info *ebpf.ProgramInfo + Error error + VerifierLog string + Type ebpf.ProgramType + IsPinned bool +} + +func (pw *progWatcher) GetProgs() ([]ProgInfo, error) { + if pw.progs == nil { + return pw.fetchProgs() + } + return pw.progs, pw.error +} + +func (pw *progWatcher) GetProg(id ebpf.ProgramID) (*ProgInfo, error) { + progs, err := pw.GetProgs() + if err != nil { + return nil, err + } + for _, prog := range progs { + if prog.ID == id { + return &prog, nil + } + } + return nil, errors.New("program not found") +} + +func (pw *progWatcher) fetchProgs() ([]ProgInfo, error) { + var currID ebpf.ProgramID = 0 + var err error + var progs []ProgInfo + pw.log.Debug().Msg("fetching progs") + + // progs count by type + var progsCount = map[ebpf.ProgramType]uint64{} + defer func() { + for progType, count := range progsCount { + pw.progsCount.WithLabelValues(progType.String()).Set(float64(count)) + } + }() + + for true { + currID, err = ebpf.ProgramGetNextID(currID) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + break + } + pw.log.Err(err).Msg("failed to get next program ID") + return progs, err + } + prog, err2 := ebpf.NewProgramFromID(currID) + if err2 != nil { + progs = append(progs, progInfoErr(currID, err2)) + continue + } + info, err2 := prog.Info() + + runCount := uint64(0) + runTime := time.Duration(0) + var labelValues []string + if info != nil { + runCount, _ = info.RunCount() + runTime, _ = info.Runtime() + labelValues = []string{ + strconv.Itoa(int(currID)), + prog.Type().String(), + info.Tag, + info.Name, + } + } else { + labelValues = []string{ + strconv.Itoa(int(currID)), + prog.Type().String(), + "", + "", + } + } + + pw.progRunCount.WithLabelValues(labelValues...).Set(float64(runCount)) + pw.progRunTime.WithLabelValues(labelValues...).Set(runTime.Seconds()) + progsCount[prog.Type()]++ + + progs = append(progs, ProgInfo{ + ID: currID, + Type: prog.Type(), + IsPinned: prog.IsPinned(), + Info: info, + VerifierLog: prog.VerifierLog, + Error: err2, + }) + } + return progs, nil +} + +func progInfoErr(id ebpf.ProgramID, err error) ProgInfo { + return ProgInfo{ + ID: id, + Info: nil, + Error: err, + } +} diff --git a/pkg/ebpf/progs/watcher.go b/pkg/ebpf/progs/watcher.go deleted file mode 100644 index 2975e31..0000000 --- a/pkg/ebpf/progs/watcher.go +++ /dev/null @@ -1,120 +0,0 @@ -package progs - -import ( - "context" - "errors" - "github.com/cilium/ebpf" - "github.com/rs/zerolog" - "os" - "time" -) - -type progWatcher struct { - log zerolog.Logger - checkInterval time.Duration - progs []ProgInfo - error error - isRunning bool -} - -type ProgWatcher interface { - Run(ctx context.Context) - GetProgs() ([]ProgInfo, error) - GetProg(id ebpf.ProgramID) (*ProgInfo, error) -} - -func NewWatcher(logger zerolog.Logger, checkInterval time.Duration) ProgWatcher { - return &progWatcher{ - log: logger, - checkInterval: checkInterval, - } -} - -func (pw *progWatcher) Run(ctx context.Context) { - if pw.isRunning { - return - } - go func() { - pw.isRunning = true - ticker := time.NewTicker(pw.checkInterval) - ctx.Done() - for { - select { - case <-ticker.C: - pw.progs, pw.error = pw.fetchProgs() - case <-ctx.Done(): - pw.isRunning = false - return - } - } - }() -} - -type ProgInfo struct { - ID ebpf.ProgramID - Info *ebpf.ProgramInfo - Error error - VerifierLog string - Type ebpf.ProgramType - IsPinned bool -} - -func (pw *progWatcher) GetProgs() ([]ProgInfo, error) { - if pw.progs == nil { - return pw.fetchProgs() - } - return pw.progs, pw.error -} - -func (pw *progWatcher) GetProg(id ebpf.ProgramID) (*ProgInfo, error) { - progs, err := pw.GetProgs() - if err != nil { - return nil, err - } - for _, prog := range progs { - if prog.ID == id { - return &prog, nil - } - } - return nil, errors.New("program not found") -} - -func (pw *progWatcher) fetchProgs() ([]ProgInfo, error) { - var currID ebpf.ProgramID = 0 - var err error - var progs []ProgInfo - pw.log.Debug().Msg("fetching progs") - for true { - currID, err = ebpf.ProgramGetNextID(currID) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - break - } - pw.log.Err(err).Msg("failed to get next program ID") - return progs, err - } - prog, err2 := ebpf.NewProgramFromID(currID) - if err2 != nil { - progs = append(progs, progInfoErr(currID, err2)) - continue - } - info, err2 := prog.Info() - progs = append(progs, ProgInfo{ - ID: currID, - Type: prog.Type(), - IsPinned: prog.IsPinned(), - Info: info, - VerifierLog: prog.VerifierLog, - Error: err2, - }) - } - return progs, nil -} - -func progInfoErr(id ebpf.ProgramID, err error) ProgInfo { - return ProgInfo{ - ID: id, - Info: nil, - Error: err, - } -} diff --git a/pkg/ebpf/util/endianess.go b/pkg/ebpf/util/endianess.go new file mode 100644 index 0000000..653aece --- /dev/null +++ b/pkg/ebpf/util/endianess.go @@ -0,0 +1,26 @@ +package util + +import ( + "encoding/binary" + "unsafe" +) + +var nativeEndian binary.ByteOrder + +func init() { + buf := [2]byte{} + *(*uint16)(unsafe.Pointer(&buf[0])) = uint16(0xABCD) + + switch buf { + case [2]byte{0xCD, 0xAB}: + nativeEndian = binary.LittleEndian + case [2]byte{0xAB, 0xCD}: + nativeEndian = binary.BigEndian + default: + panic("Could not determine native endianness.") + } +} + +func GetEndian() binary.ByteOrder { + return nativeEndian +} diff --git a/pkg/graph/helpers.go b/pkg/graph/helpers.go index a3e859b..e373cd6 100644 --- a/pkg/graph/helpers.go +++ b/pkg/graph/helpers.go @@ -1,13 +1,9 @@ package graph import ( - "encoding/binary" - "fmt" - "github.com/cilium/ebpf" "github.com/ebpfdev/dev-agent/pkg/ebpf/maps" "github.com/ebpfdev/dev-agent/pkg/ebpf/progs" "github.com/ebpfdev/dev-agent/pkg/graph/model" - "unsafe" ) func progInfoToModel(prog *progs.ProgInfo) *model.Program { @@ -69,83 +65,20 @@ func mapInfoToModel(m *maps.MapInfo) *model.Map { KeySize: &keySize, ValueSize: &valueSize, MaxEntries: &maxEntries, - IsPerCPU: isPerCPU(m.Type), - IsLookupSupported: isLookupSupported(m.Type), - } -} - -var nativeEndian binary.ByteOrder - -func init() { - buf := [2]byte{} - *(*uint16)(unsafe.Pointer(&buf[0])) = uint16(0xABCD) - - switch buf { - case [2]byte{0xCD, 0xAB}: - nativeEndian = binary.LittleEndian - case [2]byte{0xAB, 0xCD}: - nativeEndian = binary.BigEndian - default: - panic("Could not determine native endianness.") + IsPerCPU: maps.IsPerCPU(m.Type), + IsLookupSupported: maps.IsLookupSupported(m.Type), } } func formatValue(format model.MapEntryFormat, value []byte) string { switch format { case model.MapEntryFormatString: - return string(value) + return maps.FormatBytes(maps.DisplayFormatString, value) case model.MapEntryFormatHex: - return fmt.Sprintf("%x", value) + return maps.FormatBytes(maps.DisplayFormatHex, value) case model.MapEntryFormatNumber: - if len(value) == 8 { - buf := make([]byte, 8) - copy(buf, value) - return fmt.Sprintf("%d", int64(nativeEndian.Uint64(buf))) - } - return fmt.Sprintf("%x", value) + return maps.FormatBytes(maps.DisplayFormatNumber, value) default: - return fmt.Sprintf("%x", value) + return maps.FormatBytes(maps.DisplayFormatHex, value) } } - -func isPerCPU(mt ebpf.MapType) bool { - return mt == ebpf.PerCPUHash || mt == ebpf.PerCPUArray || mt == ebpf.LRUCPUHash || mt == ebpf.PerCPUCGroupStorage -} - -var lookupSupported = map[ebpf.MapType]bool{ - ebpf.UnspecifiedMap: false, - ebpf.Hash: true, - ebpf.Array: true, - ebpf.ProgramArray: false, - ebpf.PerfEventArray: false, - ebpf.PerCPUHash: true, - ebpf.PerCPUArray: true, - ebpf.StackTrace: false, - ebpf.CGroupArray: false, - ebpf.LRUHash: false, - ebpf.LRUCPUHash: false, - ebpf.LPMTrie: false, - ebpf.ArrayOfMaps: false, - ebpf.HashOfMaps: false, - ebpf.DevMap: false, - ebpf.SockMap: false, - ebpf.CPUMap: false, - ebpf.XSKMap: false, - ebpf.SockHash: false, - ebpf.CGroupStorage: false, - ebpf.ReusePortSockArray: false, - ebpf.PerCPUCGroupStorage: false, - ebpf.Queue: false, - ebpf.Stack: false, - ebpf.SkStorage: false, - ebpf.DevMapHash: false, - ebpf.StructOpsMap: false, - ebpf.RingBuf: false, - ebpf.InodeStorage: false, - ebpf.TaskStorage: false, -} - -func isLookupSupported(mt ebpf.MapType) bool { - supported, ok := lookupSupported[mt] - return ok && supported -} diff --git a/pkg/graph/schema.resolvers.go b/pkg/graph/schema.resolvers.go index dab2a20..6c05d0f 100644 --- a/pkg/graph/schema.resolvers.go +++ b/pkg/graph/schema.resolvers.go @@ -17,45 +17,32 @@ import ( // Entries is the resolver for the entries field. func (r *mapResolver) Entries(ctx context.Context, obj *model.Map, offset *int, limit *int, keyFormat *model.MapEntryFormat, valueFormat *model.MapEntryFormat) ([]*model.MapEntry, error) { - emap, err := ebpf.NewMapFromID(ebpf.MapID(obj.ID)) + mapEntries, err := maps.GetEntries(ebpf.MapID(obj.ID), false) if err != nil { return nil, err } - entries := make([]*model.MapEntry, 0) - - if !isLookupSupported(emap.Type()) { - return entries, nil - } - - var key []byte - mapIterator := emap.Iterate() - if isPerCPU(emap.Type()) { - var bufSlice [][]byte - for mapIterator.Next(&key, &bufSlice) { - values := make([]string, len(bufSlice)) - for i, value := range bufSlice { - values[i] = formatValue(*valueFormat, value[:]) - } - entries = append(entries, &model.MapEntry{ - Key: formatValue(*keyFormat, key[:]), - CPUValues: values, - Value: &values[0], - }) + modelEntries := make([]*model.MapEntry, 0) + for _, mapEntry := range mapEntries.Entries { + modelEntry := &model.MapEntry{ + Key: formatValue(*keyFormat, mapEntry.Key), } - } else { - var buf []byte - for mapIterator.Next(&key, &buf) { - value := formatValue(*valueFormat, buf[:]) - entries = append(entries, &model.MapEntry{ - Key: formatValue(*keyFormat, key[:]), - Value: &value, - }) + if len(mapEntry.Value) > 0 { + value := formatValue(*valueFormat, mapEntry.Value) + modelEntry.Value = &value } + if len(mapEntry.CPUValues) > 0 { + values := make([]string, len(mapEntry.CPUValues)) + for i, value := range mapEntry.CPUValues { + values[i] = formatValue(*valueFormat, value) + } + modelEntry.CPUValues = values + } + modelEntries = append(modelEntries, modelEntry) } - sort.SliceStable(entries, func(i, j int) bool { - return entries[i].Key < entries[j].Key + sort.SliceStable(modelEntries, func(i, j int) bool { + return modelEntries[i].Key < modelEntries[j].Key }) offsetStart := 0 @@ -68,36 +55,18 @@ func (r *mapResolver) Entries(ctx context.Context, obj *model.Map, offset *int, } offsetEnd := offsetStart + limitValue - if offsetEnd > len(entries) { - offsetEnd = len(entries) + if offsetEnd > len(modelEntries) { + offsetEnd = len(modelEntries) } - result := entries[offsetStart:offsetEnd] + result := modelEntries[offsetStart:offsetEnd] - return result, mapIterator.Err() + return result, nil } // EntriesCount is the resolver for the entriesCount field. func (r *mapResolver) EntriesCount(ctx context.Context, obj *model.Map) (int, error) { - emap, err := ebpf.NewMapFromID(ebpf.MapID(obj.ID)) - if err != nil { - return 0, err - } - - if !isLookupSupported(emap.Type()) { - return 0, nil - } - - count := 0 - - var key []byte - var value []byte - - mapIterator := emap.Iterate() - for mapIterator.Next(&key, &value) { - count++ - } - return count, mapIterator.Err() + return maps.CountEntries(ebpf.MapID(obj.ID)) } // Programs is the resolver for the programs field.