diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go
index 17846a1abf..745b750341 100644
--- a/pkg/internal/controller/controller.go
+++ b/pkg/internal/controller/controller.go
@@ -124,7 +124,6 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc
 // Start implements controller.Controller
 func (c *Controller) Start(stop <-chan struct{}) error {
 	c.mu.Lock()
-	defer c.mu.Unlock()
 
 	// TODO(pwittrock): Reconsider HandleCrash
 	defer utilruntime.HandleCrash()
@@ -142,6 +141,7 @@ func (c *Controller) Start(stop <-chan struct{}) error {
 		// Leaving it here because that could happen in the future
 		err := fmt.Errorf("failed to wait for %s caches to sync", c.Name)
 		log.Error(err, "Could not wait for Cache to sync", "Controller", c.Name)
+		c.mu.Unlock()
 		return err
 	}
 
@@ -160,6 +160,7 @@ func (c *Controller) Start(stop <-chan struct{}) error {
 	}
 
 	c.Started = true
+	c.mu.Unlock()
 
 	<-stop
 	log.Info("Stopping workers", "Controller", c.Name)