pkg/certwatcher: Start should retry for 10s when adding files

This fixes a flake in CI, but it could also come in handy when running
the certwatcher against volume mounted certificates. Ideally the timeout
is going to be configurable at some point, for now, let's just retry for
a fixed number of seconds, before returning an error.

Signed-off-by: Vince Prignano <vincepri@redhat.com>

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/onsi/ginkgo/v2 v2.8.0
 	github.com/onsi/gomega v1.26.0
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0
 	go.uber.org/goleak v1.2.0
@@ -50,7 +51,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

pkg/certwatcher/certwatcher.go

@@ -20,8 +20,13 @@ import (
 	"context"
 	"crypto/tls"
 	"sync"
+	"time"
 
 	"github.com/fsnotify/fsnotify"
+	"github.com/pkg/errors"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
 	logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
 )
@@ -72,11 +77,24 @@ func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate,
 
 // Start starts the watch on the certificate and key files.
 func (cw *CertWatcher) Start(ctx context.Context) error {
-	files := []string{cw.certPath, cw.keyPath}
-
-	for _, f := range files {
-		if err := cw.watcher.Add(f); err != nil {
-			return err
+	files := sets.New(cw.certPath, cw.keyPath)
+
+	{
+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		defer cancel()
+		var watchErr error
+		if err := wait.PollImmediateUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (done bool, err error) {
+			for _, f := range files.UnsortedList() {
+				if err := cw.watcher.Add(f); err != nil {
+					watchErr = err
+					return false, nil //nolint:nilerr // We want to keep trying.
+				}
+				// We've added the watch, remove it from the set.
+				files.Delete(f)
+			}
+			return true, nil
+		}); err != nil {
+			return errors.Wrapf(kerrors.NewAggregate([]error{err, watchErr}), "failed to add watches")
 		}
 	}
 
@@ -154,13 +172,13 @@ func (cw *CertWatcher) handleEvent(event fsnotify.Event) {
 }
 
 func isWrite(event fsnotify.Event) bool {
-	return event.Op&fsnotify.Write == fsnotify.Write
+	return event.Op.Has(fsnotify.Write)
 }
 
 func isCreate(event fsnotify.Event) bool {
-	return event.Op&fsnotify.Create == fsnotify.Create
+	return event.Op.Has(fsnotify.Create)
 }
 
 func isRemove(event fsnotify.Event) bool {
-	return event.Op&fsnotify.Remove == fsnotify.Remove
+	return event.Op.Has(fsnotify.Remove)
 }