Skip to content

Commit

Permalink
Merge pull request #929 from njhale/pprof-option
Browse files Browse the repository at this point in the history
Add optional profiling
  • Loading branch information
openshift-merge-robot authored Jul 2, 2019
2 parents 2bd381f + 358120e commit 49690b7
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 3 deletions.
43 changes: 43 additions & 0 deletions Documentation/design/profiling.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Profiling OLM Operators

OLM's `olm` and `catalog` commands support serving profiling samples via the `--profiling` option.

```sh
# run olm operator with profiling enabled
$ go run cmd/olm/main.go --profiling --kubeconfig ~/.kube/config
```

Samples are in a format recognized by [pprof](https://golang.org/pkg/net/http/pprof) and an index of available profile types is made available at `https://127.0.0.1:8080/debug/pprof`.

If profiling is enabled, but operators are running on a kubernetes cluster, a convienient way to expose samples locally is with `kubectl port-forward`:

```sh
# forward traffic from 127.0.0.1:8080 to port 8080 of catalog operator pods
$ kubectl -n <olm-namespace> port-forward deployments/catalog-operator
```

When profiling is enabled, `go tool pprof` can be used to export and visualize samples:

```sh
# assuming a catalog operator's samples are accessible at 127.0.0.1:8080:
# show in-use heap memory in top format
$ go tool pprof -top http://127.0.0.1:8080/debug/pprof/heap
Fetching profile over HTTP from http://127.0.0.1:8080/debug/pprof/heap
Saved profile in /Users/nhale/pprof/pprof.catalog.alloc_objects.alloc_space.inuse_objects.inuse_space.013.pb.gz
File: catalog
Type: inuse_space
Time: Jun 27, 2019 at 12:27pm (EDT)
Showing nodes accounting for 2202.74kB, 100% of 2202.74kB total
flat flat% sum% cum cum%
650.62kB 29.54% 29.54% 650.62kB 29.54% bufio.NewWriterSize
520.04kB 23.61% 53.15% 520.04kB 23.61% golang.org/x/net/http2.NewFramer.func1
520.04kB 23.61% 76.75% 520.04kB 23.61% sync.(*Map).LoadOrStore
512.03kB 23.25% 100% 512.03kB 23.25% github.com/modern-go/reflect2.newUnsafeStructField
...

# save in-use objects graph to svg file
$ go tool pprof -sample_index=inuse_objects -svg http://127.0.0.1:8080/debug/pprof/heap
Fetching profile over HTTP from http://127.0.0.1:8080/debug/pprof/heap
Saved profile in /Users/<user>/pprof/pprof.catalog.alloc_objects.alloc_space.inuse_objects.inuse_space.01.pb.gz
Generating report in profile001.svg
```
15 changes: 13 additions & 2 deletions cmd/catalog/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,16 @@ import (
"time"

configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
utilclock "k8s.io/apimachinery/pkg/util/clock"
"k8s.io/client-go/tools/clientcmd"

"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/catalog"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
Expand Down Expand Up @@ -61,6 +62,9 @@ var (

tlsCertPath = flag.String(
"tls-cert", "", "Path to use for certificate key (requires tls-key)")

profiling = flag.Bool(
"profiling", false, "serve profiling data (on port 8080)")
)

func init() {
Expand Down Expand Up @@ -114,6 +118,13 @@ func main() {
healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})

// Serve profiling if enabled
if *profiling {
logger.Infof("profiling enabled")
profile.RegisterHandlers(healthMux)
}

go http.ListenAndServe(":8080", healthMux)

metricsMux := http.NewServeMux()
Expand Down
13 changes: 12 additions & 1 deletion cmd/olm/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"time"

configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
Expand All @@ -19,6 +18,8 @@ import (
"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client"
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
Expand Down Expand Up @@ -59,6 +60,9 @@ var (

tlsCertPath = flag.String(
"tls-cert", "", "Path to use for certificate key (requires tls-key)")

profiling = flag.Bool(
"profiling", false, "serve profiling data (on port 8080)")
)

func init() {
Expand Down Expand Up @@ -114,6 +118,13 @@ func main() {
healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})

// Serve profiling if enabled
if *profiling {
logger.Infof("profiling enabled")
profile.RegisterHandlers(healthMux)
}

go func() {
err := http.ListenAndServe(":8080", healthMux)
if err != nil {
Expand Down
64 changes: 64 additions & 0 deletions pkg/lib/profile/profile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package profile

import (
"net/http"
"net/http/pprof"
)

type profileConfig struct {
pprof bool
cmdline bool
profile bool
symbol bool
trace bool
}

// Option applies a configuration option to the given config.
type Option func(p *profileConfig)

func (p *profileConfig) apply(options []Option) {
if len(options) == 0 {
// If no options are given, default to all
p.pprof = true
p.cmdline = true
p.profile = true
p.symbol = true
p.trace = true

return
}

for _, o := range options {
o(p)
}
}

func defaultProfileConfig() *profileConfig {
// Initialize config
return &profileConfig{}
}

// RegisterHandlers registers profile Handlers with the given ServeMux.
//
// The Handlers registered are determined by the given options.
// If no options are given, all available handlers are registered by default.
func RegisterHandlers(mux *http.ServeMux, options ...Option) {
config := defaultProfileConfig()
config.apply(options)

if config.pprof {
mux.HandleFunc("/debug/pprof/", pprof.Index)
}
if config.cmdline {
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
}
if config.profile {
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
}
if config.symbol {
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
}
if config.trace {
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}
}

0 comments on commit 49690b7

Please sign in to comment.