forked from cockroachdb/cockroach
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This commit adds the ability to capture execution traces from the past few seconds of execution when something seems wrong. Often when a timer fires and we detect something is wrong, the relevant information is already lost. The new flight recorder in go golang/go#63185 creates a ring buffer that enables capturing these traces. This commit adds the capability to capture traces but doesn't enable it anywhere. There is a small performance cost of having the flight recorder always enabled, so some performance testing is required to determine if we need to protect this behind a cluster setting. Epic: none Release note: None
- Loading branch information
1 parent
3aa4b7f
commit ade6dc3
Showing
13 changed files
with
148 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| // Copyright 2024 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the CockroachDB Software License | ||
| // included in the /LICENSE file. | ||
|
|
||
| package tracing | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/util/timeutil" | ||
| "github.com/cockroachdb/errors" | ||
| "golang.org/x/exp/trace" | ||
| ) | ||
|
|
||
| const timeFormat = "2006-01-02T15_04_05.000" | ||
| const min_record_interval = time.Minute | ||
|
|
||
| type FlightRecorder struct { | ||
| fr *trace.FlightRecorder | ||
| outputDir string | ||
| // lastSnapshot prevents the flight recorder from being called too frequently. | ||
| lastSnapshot time.Time | ||
| } | ||
|
|
||
| // NewFlightRecorder creates and starts a trace.FlightRecorder | ||
| func NewFlightRecorder(ctx context.Context, outputDir string) (*FlightRecorder, error) { | ||
| if outputDir == "" { | ||
| return nil, errors.Newf("output directory not set: %s", outputDir) | ||
| } | ||
|
|
||
| if err := os.MkdirAll(outputDir, 0755); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| fr := FlightRecorder{ | ||
| fr: trace.NewFlightRecorder(), | ||
| outputDir: outputDir, | ||
| } | ||
|
|
||
| // TODO: Only start if the cluster setting is enabled. | ||
| if err := fr.fr.Start(); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &fr, nil | ||
| } | ||
|
|
||
| func (fr *FlightRecorder) MaybeSnapshot() bool { | ||
| now := timeutil.Now() | ||
| // Don't take the snapshot if there was already one in the past interval. | ||
| if now.Sub(fr.lastSnapshot) < min_record_interval { | ||
| return false | ||
| } | ||
| fr.lastSnapshot = now | ||
|
|
||
| // Attempt to grab the snapshot. | ||
| var b bytes.Buffer | ||
| if _, err := fr.fr.WriteTo(&b); err != nil { | ||
| return false | ||
| } | ||
| filename := fmt.Sprintf( | ||
| "%s/trace.%s", | ||
| fr.outputDir, | ||
| now.Format(timeFormat), | ||
| ) | ||
| // Write it to a file. | ||
| if err := os.WriteFile(filename, b.Bytes(), 0o755); err != nil { | ||
| return false | ||
| } | ||
| return true | ||
| } | ||
|
|
||
| func (fr *FlightRecorder) Close() { | ||
| // We don't care about any errors from stopping the flight recorder as we | ||
| // are likely shutting down. | ||
| _ = fr.fr.Stop() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters