Add panic to waitctx timeout

pkg/opni/commands/agent.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/go-logr/zapr"
 	upgraderesponder "github.com/longhorn/upgrade-responder/client"
@@ -98,7 +99,7 @@ func BuildAgentCmd() *cobra.Command {
 			})
 		}
 
-		waitctx.Wait(ctx)
+		waitctx.WaitWithTimeout(ctx, 60*time.Second, 10*time.Second)
 		return nil
 	}
 

pkg/opni/commands/gateway.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"sync/atomic"
+	"time"
 
 	"github.com/hashicorp/go-plugin"
 	"github.com/rancher/opni/pkg/config"
@@ -154,7 +155,7 @@ func BuildGatewayCmd() *cobra.Command {
 		lg.Info(style.Style("waiting for servers to shut down"))
 		fCancel()
 		cancel()
-		waitctx.Wait(ctx)
+		waitctx.WaitWithTimeout(ctx, 60*time.Second, 10*time.Second)
 
 		atomic.StoreUint32(&plugin.Killed, 0)
 		lg.Info(style.Style("--- reloading ---"))
@@ -165,6 +166,7 @@ func BuildGatewayCmd() *cobra.Command {
 		Use:   "gateway",
 		Short: "Run the Opni Monitoring Gateway",
 		RunE: func(cmd *cobra.Command, args []string) error {
+			defer waitctx.RecoverTimeout()
 			for {
 				if err := run(); err != nil {
 					return err

pkg/util/waitctx/context.go

@@ -13,8 +13,10 @@ import (
 )
 
 type waitCtxDataKeyType struct{}
+type timeout struct{}
 
 var waitCtxDataKey waitCtxDataKeyType
+var waitTimeout timeout
 
 type waitCtxData struct {
 	wg      sync.WaitGroup
@@ -112,6 +114,8 @@ func (restrictive) Wait(ctx RestrictiveContext, notifyAfter ...time.Duration) {
 	return
 }
 
+// WaitWithTimeout follows the same pattern as wait, but with a timeout.  If the timeout expires
+// WaitWithTimeout will panic.
 func (restrictive) WaitWWithTimeout(ctx RestrictiveContext, timeout time.Duration, notifyAfter ...time.Duration) {
 	data := ctx.Value(waitCtxDataKey)
 	if data == nil {
@@ -146,7 +150,7 @@ func (restrictive) WaitWWithTimeout(ctx RestrictiveContext, timeout time.Duratio
 	case <-done:
 		return
 	case <-time.After(timeout):
-		return
+		panic(waitTimeout)
 	}
 }
 
@@ -216,6 +220,22 @@ func (w permissive) Go(ctx PermissiveContext, fn func()) {
 	}()
 }
 
+func RecoverTimeout() {
+	r := recover()
+	if r == nil {
+		return
+	}
+	switch r.(type) {
+	case timeout:
+		fmt.Println("timeout expired, exiting process")
+		os.Exit(0)
+	default:
+		fmt.Println(r)
+		fmt.Println(debug.Stack())
+		os.Exit(1)
+	}
+}
+
 var (
 	Restrictive = restrictive{}
 	Permissive  = permissive{}