From f89fe470335f1a260f15ccd8fe4ec22fe3653856 Mon Sep 17 00:00:00 2001
From: Vince Prignano <vincepri@vmware.com>
Date: Wed, 24 Mar 2021 09:51:57 -0700
Subject: [PATCH] :bug: Event recorder should not send events when broadcaster
 is stopped

This change solves a race condition when the manager is shutting down
and reconcilers are still in the process of sending events through the
recorder. If the shutdown timeout is hit too early, the event recorder
panics because we're still trying to send events through the recorder.

To avoid this issue, we introduce a RWMutex between the provider and
event recorder. When the broadcaster is shutdown, events are now
dropped.

Signed-off-by: Vince Prignano <vincepri@vmware.com>
---
 pkg/internal/recorder/recorder.go | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/pkg/internal/recorder/recorder.go b/pkg/internal/recorder/recorder.go
index c699f04ec0..cb8b7b6d63 100644
--- a/pkg/internal/recorder/recorder.go
+++ b/pkg/internal/recorder/recorder.go
@@ -38,6 +38,9 @@ type EventBroadcasterProducer func() (caster record.EventBroadcaster, stopWithPr
 // Provider is a recorder.Provider that records events to the k8s API server
 // and to a logr Logger.
 type Provider struct {
+	lock    sync.RWMutex
+	stopped bool
+
 	// scheme to specify when creating a recorder
 	scheme *runtime.Scheme
 	// logger is the logger to use when logging diagnostic event info
@@ -70,7 +73,10 @@ func (p *Provider) Stop(shutdownCtx context.Context) {
 		// an invocation of getBroadcaster.
 		broadcaster := p.getBroadcaster()
 		if p.stopBroadcaster {
+			p.lock.Lock()
 			broadcaster.Shutdown()
+			p.stopped = true
+			p.lock.Unlock()
 		}
 		close(doneCh)
 	}()
@@ -144,13 +150,28 @@ func (l *lazyRecorder) ensureRecording() {
 
 func (l *lazyRecorder) Event(object runtime.Object, eventtype, reason, message string) {
 	l.ensureRecording()
-	l.rec.Event(object, eventtype, reason, message)
+
+	l.prov.lock.RLock()
+	if !l.prov.stopped {
+		l.rec.Event(object, eventtype, reason, message)
+	}
+	l.prov.lock.RUnlock()
 }
 func (l *lazyRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) {
 	l.ensureRecording()
-	l.rec.Eventf(object, eventtype, reason, messageFmt, args...)
+
+	l.prov.lock.RLock()
+	if !l.prov.stopped {
+		l.rec.Eventf(object, eventtype, reason, messageFmt, args...)
+	}
+	l.prov.lock.RUnlock()
 }
 func (l *lazyRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) {
 	l.ensureRecording()
-	l.rec.AnnotatedEventf(object, annotations, eventtype, reason, messageFmt, args...)
+
+	l.prov.lock.RLock()
+	if !l.prov.stopped {
+		l.rec.AnnotatedEventf(object, annotations, eventtype, reason, messageFmt, args...)
+	}
+	l.prov.lock.RUnlock()
 }