From 48c0a8e8681f9c61489bf04587b00b6847c94a38 Mon Sep 17 00:00:00 2001
From: Vince Prignano <vincepri@redhat.com>
Date: Tue, 31 Jan 2023 10:27:41 -0800
Subject: [PATCH] pkg/certwatcher: Start should retry for 10s when adding files

This fixes a flake in CI, but it could also come in handy when running
the certwatcher against volume mounted certificates. Ideally the timeout
is going to be configurable at some point, for now, let's just retry for
a fixed number of seconds, before returning an error.

Signed-off-by: Vince Prignano <vincepri@redhat.com>
---
 go.mod                         |  2 +-
 pkg/certwatcher/certwatcher.go | 34 ++++++++++++++++++++++++++--------
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index da91bf7c52..65f0af8006 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/onsi/ginkgo/v2 v2.8.0
 	github.com/onsi/gomega v1.26.0
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0
 	go.uber.org/goleak v1.2.0
@@ -50,7 +51,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
diff --git a/pkg/certwatcher/certwatcher.go b/pkg/certwatcher/certwatcher.go
index 1030013db3..2fc15e85ea 100644
--- a/pkg/certwatcher/certwatcher.go
+++ b/pkg/certwatcher/certwatcher.go
@@ -20,8 +20,13 @@ import (
 	"context"
 	"crypto/tls"
 	"sync"
+	"time"
 
 	"github.com/fsnotify/fsnotify"
+	"github.com/pkg/errors"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
 	logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
 )
@@ -72,11 +77,24 @@ func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate,
 
 // Start starts the watch on the certificate and key files.
 func (cw *CertWatcher) Start(ctx context.Context) error {
-	files := []string{cw.certPath, cw.keyPath}
-
-	for _, f := range files {
-		if err := cw.watcher.Add(f); err != nil {
-			return err
+	files := sets.New(cw.certPath, cw.keyPath)
+
+	{
+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		defer cancel()
+		var watchErr error
+		if err := wait.PollImmediateUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (done bool, err error) {
+			for _, f := range files.UnsortedList() {
+				if err := cw.watcher.Add(f); err != nil {
+					watchErr = err
+					return false, nil //nolint:nilerr // We want to keep trying.
+				}
+				// We've added the watch, remove it from the set.
+				files.Delete(f)
+			}
+			return true, nil
+		}); err != nil {
+			return errors.Wrapf(kerrors.NewAggregate([]error{err, watchErr}), "failed to add watches")
 		}
 	}
 
@@ -154,13 +172,13 @@ func (cw *CertWatcher) handleEvent(event fsnotify.Event) {
 }
 
 func isWrite(event fsnotify.Event) bool {
-	return event.Op&fsnotify.Write == fsnotify.Write
+	return event.Op.Has(fsnotify.Write)
 }
 
 func isCreate(event fsnotify.Event) bool {
-	return event.Op&fsnotify.Create == fsnotify.Create
+	return event.Op.Has(fsnotify.Create)
 }
 
 func isRemove(event fsnotify.Event) bool {
-	return event.Op&fsnotify.Remove == fsnotify.Remove
+	return event.Op.Has(fsnotify.Remove)
 }