Use backoff in time machine test to reduce flakiness (hopefully) (#1625)

ee/agent/timemachine/timemachine_darwin_test.go

@@ -5,6 +5,7 @@ package timemachine
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/kolide/launcher/ee/agent/types/mocks"
 	"github.com/kolide/launcher/ee/allowedcmd"
+	"github.com/kolide/launcher/pkg/backoff"
 	"github.com/stretchr/testify/require"
 )
 
@@ -85,22 +87,34 @@ func TestAddExclusions(t *testing.T) {
 	// we've seen some flake in CI here where the exclusions have not been
 	// updated by the time we perform assertions, so sleep for a bit to give
 	// OS some time to catch up
-	time.Sleep(3 * time.Second)
+	time.Sleep(1 * time.Second)
 
 	// ensure the files are included / excluded as expected
 	for fileName, shouldBeExcluded := range shouldBeExcluded {
-		cmd, err := allowedcmd.Tmutil(context.TODO(), "isexcluded", filepath.Join(testDir, fileName))
-		require.NoError(t, err)
-
-		out, err := cmd.CombinedOutput()
-		require.NoError(t, err)
-
-		if shouldBeExcluded {
-			require.Contains(t, string(out), "[Excluded]")
-			continue
-		}
-
-		// should be included
-		require.Contains(t, string(out), "[Included]")
+		// Allow for a couple retries for the file to show up as excluded appropriately --
+		// we've seen this be a little flaky in CI
+		err := backoff.WaitFor(func() error {
+			cmd, err := allowedcmd.Tmutil(context.TODO(), "isexcluded", filepath.Join(testDir, fileName))
+			if err != nil {
+				return fmt.Errorf("creating tmutil command: %w", err)
+			}
+
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("running tmutil isexcluded: %w", err)
+			}
+
+			if shouldBeExcluded && !strings.Contains(string(out), "[Excluded]") {
+				return fmt.Errorf("output `%s` does not contain [Excluded], but file should be excluded", string(out))
+			}
+
+			if !shouldBeExcluded && !strings.Contains(string(out), "[Included]") {
+				return fmt.Errorf("output `%s` does not contain [Included], but file should be included", string(out))
+			}
+
+			return nil
+		}, 1*time.Second, 200*time.Millisecond)
+
+		require.NoError(t, err, "file not handled as expected")
 	}
 }