Health check the OVS process and restart if it dies #16742
+231
−62
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| package node | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/golang/glog" | ||
|
|
||
| utilruntime "k8s.io/apimachinery/pkg/util/runtime" | ||
| utilwait "k8s.io/apimachinery/pkg/util/wait" | ||
|
|
||
| "github.com/openshift/origin/pkg/util/ovs/ovsclient" | ||
| ) | ||
|
|
||
| const ( | ||
| ovsDialTimeout = 5 * time.Second | ||
| ovsHealthcheckInterval = 30 * time.Second | ||
| ovsRecoveryTimeout = 10 * time.Second | ||
| ovsDialDefaultNetwork = "unix" | ||
| ovsDialDefaultAddress = "/var/run/openvswitch/db.sock" | ||
| ) | ||
|
|
||
| // waitForOVS polls until the OVS server responds to a connection and an 'echo' | ||
| // command. | ||
| func waitForOVS(network, addr string) error { | ||
| return utilwait.PollImmediate(time.Second, time.Minute, func() (bool, error) { | ||
| c, err := ovsclient.DialTimeout(network, addr, ovsDialTimeout) | ||
| if err != nil { | ||
| glog.V(2).Infof("waiting for OVS to start: %v", err) | ||
| return false, nil | ||
| } | ||
| defer c.Close() | ||
| if err := c.Ping(); err != nil { | ||
| glog.V(2).Infof("waiting for OVS to start, ping failed: %v", err) | ||
| return false, nil | ||
| } | ||
| return true, nil | ||
| }) | ||
| } | ||
|
|
||
| // runOVSHealthCheck runs two background loops - one that waits for disconnection | ||
| // from the OVS server and then checks healthFn, and one that periodically checks | ||
| // healthFn. If healthFn returns false in either of these two cases while the OVS | ||
| // server is responsive the node process will terminate. | ||
| func runOVSHealthCheck(network, addr string, healthFn func() bool) { | ||
| // this loop holds an open socket connection to OVS until it times out, then | ||
| // checks for health | ||
| go utilwait.Until(func() { | ||
| c, err := ovsclient.DialTimeout(network, addr, ovsDialTimeout) | ||
| if err != nil { | ||
| utilruntime.HandleError(fmt.Errorf("SDN healthcheck unable to connect to OVS server: %v", err)) | ||
| return | ||
| } | ||
| defer c.Close() | ||
|
|
||
| err = c.WaitForDisconnect() | ||
| utilruntime.HandleError(fmt.Errorf("SDN healthcheck disconnected from OVS server: %v", err)) | ||
|
|
||
| err = utilwait.PollImmediate(100*time.Millisecond, ovsRecoveryTimeout, func() (bool, error) { | ||
| c, err := ovsclient.DialTimeout(network, addr, ovsDialTimeout) | ||
| if err != nil { | ||
| glog.V(2).Infof("SDN healthcheck unable to reconnect to OVS server: %v", err) | ||
| return false, nil | ||
| } | ||
| defer c.Close() | ||
| if err := c.Ping(); err != nil { | ||
| glog.V(2).Infof("SDN healthcheck unable to ping OVS server: %v", err) | ||
| return false, nil | ||
| } | ||
| if !healthFn() { | ||
| return false, fmt.Errorf("OVS health check failed") | ||
| } | ||
| return true, nil | ||
| }) | ||
| if err != nil { | ||
| // If OVS restarts and our health check fails, we exit | ||
| // TODO: make openshift-sdn able to reconcile without a restart | ||
| glog.Fatalf("SDN healthcheck detected unhealthy OVS server, restarting: %v", err) | ||
| } | ||
| }, ovsDialTimeout, utilwait.NeverStop) | ||
|
|
||
| // this loop periodically verifies we can still connect to the OVS server and | ||
| // is an upper bound on the time we wait before detecting a failed OVS configuartion | ||
| go utilwait.Until(func() { | ||
| c, err := ovsclient.DialTimeout(network, addr, ovsDialTimeout) | ||
| if err != nil { | ||
| glog.V(2).Infof("SDN healthcheck unable to reconnect to OVS server: %v", err) | ||
| return | ||
| } | ||
| defer c.Close() | ||
| if err := c.Ping(); err != nil { | ||
| glog.V(2).Infof("SDN healthcheck unable to ping OVS server: %v", err) | ||
| return | ||
| } | ||
| if !healthFn() { | ||
| glog.Fatalf("SDN healthcheck detected unhealthy OVS server, restarting: %v", err) | ||
| } | ||
| glog.V(4).Infof("SDN healthcheck succeeded") | ||
| }, ovsHealthcheckInterval, utilwait.NeverStop) | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't know anything about the OVS raw protocol, but if it eventually times out idle connections then this might result in spurious errors in the logs.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's possible to configure OVS to do timeouts, but at least ootb on our deployed systems it does not. I also have the 5s disconnect. I think if in practice we see this error showing up we would increase the timeout on connections significantly and still be ok. It's effectively a deadman switch (and it works really well at it).