Better recovery

internal/appconfig/remote.go

@@ -37,7 +37,7 @@ func getAppV2ConfigFromMachines(ctx context.Context, appName string) (*Config, e
 	if err != nil {
 		return nil, fmt.Errorf("error listing active machines for %s app: %w", appName, err)
 	}
-	machineSet := machine.NewMachineSet(flapsClient, io, activeMachines)
+	machineSet := machine.NewMachineSet(flapsClient, io, activeMachines, true)
 	appConfig, warnings, err := FromAppAndMachineSet(ctx, appName, machineSet)
 	if err != nil {
 		return nil, fmt.Errorf("failed to grab app config from existing machines, error: %w", err)

internal/command/deploy/deploy.go

@@ -167,6 +167,11 @@ var CommonFlags = flag.Set{
 		Shorthand:   "s",
 		Description: "Signal to stop the machine with for bluegreen strategy (default: SIGINT)",
 	},
+	flag.String{
+		Name:        "deploy-retries",
+		Description: "Number of times to retry a deployment if it fails",
+		Default:     "auto",
+	},
 }
 
 type Command struct {
@@ -316,10 +321,14 @@ func DeployWithConfig(ctx context.Context, appConfig *appconfig.Config, userID i
 
 	// Fetch an image ref or build from source to get the final image reference to deploy
 	img, err := determineImage(ctx, appConfig, usingWireguard, recreateBuilder)
-	if err != nil && usingWireguard && httpFailover {
-		span.SetAttributes(attribute.String("builder.failover_error", err.Error()))
-		span.AddEvent("using http failover")
-		img, err = determineImage(ctx, appConfig, false, recreateBuilder)
+	if err != nil {
+		noBuilder := strings.Contains(err.Error(), "Could not find App")
+		recreateBuilder = recreateBuilder || noBuilder
+		if noBuilder || (usingWireguard && httpFailover) {
+			span.SetAttributes(attribute.String("builder.failover_error", err.Error()))
+			span.AddEvent("using http failover")
+			img, err = determineImage(ctx, appConfig, false, recreateBuilder)
+		}
 	}
 
 	if err != nil {
@@ -477,6 +486,28 @@ func deployToMachines(
 
 	status.FlyctlVersion = buildinfo.Info().Version.String()
 
+	retriesFlag := flag.GetString(ctx, "deploy-retries")
+	deployRetries := 0
+
+	switch retriesFlag {
+	case "auto":
+		ldClient := launchdarkly.ClientFromContext(ctx)
+		retries := ldClient.GetFeatureFlagValue("deploy-retries", 0.0).(float64)
+		deployRetries = int(retries)
+
+	default:
+		var invalidRetriesErr error = fmt.Errorf("--deploy-retries must be set to a positive integer, 0, or 'auto'")
+		retries, err := strconv.Atoi(retriesFlag)
+		if err != nil {
+			return invalidRetriesErr
+		}
+		if retries < 0 {
+			return invalidRetriesErr
+		}
+
+		deployRetries = retries
+	}
+
 	md, err := NewMachineDeployment(ctx, MachineDeploymentArgs{
 		AppCompact:            app,
 		DeploymentImage:       img.Tag,
@@ -504,6 +535,7 @@ func deployToMachines(
 		MaxConcurrent:         maxConcurrent,
 		VolumeInitialSize:     flag.GetInt(ctx, "volume-initial-size"),
 		ProcessGroups:         processGroups,
+		DeployRetries:         deployRetries,
 	})
 	if err != nil {
 		sentry.CaptureExceptionWithAppInfo(ctx, err, "deploy", app)

internal/command/deploy/machinebasedtest.go

@@ -3,7 +3,6 @@ package deploy
 import (
 	"context"
 	"fmt"
-	"strconv"
 	"time"
 
 	"github.com/cenkalti/backoff"
@@ -21,7 +20,26 @@ type createdTestMachine struct {
 	err  error
 }
 
-func (md *machineDeployment) runTestMachines(ctx context.Context, machineToTest *fly.Machine) (err error) {
+type machineTestErr struct {
+	testMachineLogs string
+	exitCode        int
+	machineID       string
+}
+
+func (e machineTestErr) Error() string {
+	return fmt.Sprintf("Error test command machine %s exited with non-zero status of %d", e.machineID, e.exitCode)
+}
+
+func (e machineTestErr) Description() string {
+	var desc string
+	desc += fmt.Sprintf("Error: test command failed running on machine %s with exit code %d.\n", e.machineID, e.exitCode)
+	desc += fmt.Sprintf("Check its logs: here's the last 100 lines below, or run 'fly logs -i %s':\n\n", e.machineID)
+	desc += e.testMachineLogs
+	return desc
+
+}
+
+func (md *machineDeployment) runTestMachines(ctx context.Context, machineToTest *fly.Machine, sl statuslogger.StatusLine) (err error) {
 	ctx, span := tracing.GetTracer().Start(ctx, "run_test_machine")
 	var (
 		flaps = md.flapsClient
@@ -34,9 +52,13 @@ func (md *machineDeployment) runTestMachines(ctx context.Context, machineToTest
 		span.End()
 	}()
 
+	if sl == nil {
+		return fmt.Errorf("bug: status logger is nil")
+	}
+
 	processGroup := machineToTest.ProcessGroup()
 	machineChecks := lo.FlatMap(md.appConfig.AllServices(), func(svc appconfig.Service, _ int) []*appconfig.ServiceMachineCheck {
-		matchesProcessGroup := lo.Contains(svc.Processes, processGroup)
+		matchesProcessGroup := lo.Contains(svc.Processes, processGroup) || len(svc.Processes) == 0
 		if matchesProcessGroup {
 			return svc.MachineChecks
 		} else {
@@ -55,11 +77,11 @@ func (md *machineDeployment) runTestMachines(ctx context.Context, machineToTest
 		var err error
 		defer func() {
 			if err != nil {
-				statuslogger.Failed(ctx, err)
+				sl.Failed(err)
 			}
 		}()
 
-		mach, err = md.createTestMachine(ctx, machineCheck, machineToTest)
+		mach, err = md.createTestMachine(ctx, machineCheck, machineToTest, sl)
 		return createdTestMachine{mach, err}
 	})
 
@@ -74,13 +96,13 @@ func (md *machineDeployment) runTestMachines(ctx context.Context, machineToTest
 	machineSet := machine.NewMachineSet(flaps, io, lo.FilterMap(machines, func(m createdTestMachine, _ int) (*fly.Machine, bool) {
 		if m.err != nil {
 			tracing.RecordError(span, m.err, "failed to create test machine")
-			statuslogger.LogStatus(ctx, statuslogger.StatusFailure, fmt.Sprintf("failed to create test machine: %s", m.err))
+			sl.LogStatus(statuslogger.StatusFailure, fmt.Sprintf("failed to create test machine: %s", m.err))
 		}
 		return m.mach, m.err == nil
-	}))
+	}), false)
 
 	// FIXME: consolidate this wait stuff with deploy waits? Especially once we improve the output
-	err = md.waitForTestMachinesToFinish(ctx, machineSet)
+	err = md.waitForTestMachinesToFinish(ctx, machineSet, sl)
 	if err != nil {
 		tracing.RecordError(span, err, "failed to wait for test cmd machine")
 		return err
@@ -89,7 +111,7 @@ func (md *machineDeployment) runTestMachines(ctx context.Context, machineToTest
 	for _, testMachine := range machineSet.GetMachines() {
 		md.waitForLogs(ctx, testMachine.Machine(), 10*time.Second)
 
-		statuslogger.Logf(ctx, "Checking test command machine %s", md.colorize.Bold(testMachine.Machine().ID))
+		sl.Logf("Checking test command machine %s", md.colorize.Bold(testMachine.Machine().ID))
 		lastExitEvent, err := testMachine.WaitForEventType(ctx, "exit", md.releaseCmdTimeout, true)
 		if err != nil {
 			return fmt.Errorf("error finding the test command machine %s exit event: %w", testMachine.Machine().ID, err)
@@ -100,27 +122,22 @@ func (md *machineDeployment) runTestMachines(ctx context.Context, machineToTest
 		}
 
 		if exitCode != 0 {
-			statuslogger.LogStatus(ctx, statuslogger.StatusFailure, "test command failed")
+			sl.LogStatus(statuslogger.StatusFailure, "test command failed")
 			// Preemptive cleanup of the logger so that the logs have a clean place to write to
 
-			fmt.Fprintf(md.io.ErrOut, "Error: test command failed running on machine %s with exit code %s.\n",
-				md.colorize.Bold(testMachine.Machine().ID), md.colorize.Red(strconv.Itoa(exitCode)))
-			fmt.Fprintf(md.io.ErrOut, "Check its logs: here's the last 100 lines below, or run 'fly logs -i %s':\n",
-				testMachine.Machine().ID)
 			testLogs, _, err := md.apiClient.GetAppLogs(ctx, md.app.Name, "", md.appConfig.PrimaryRegion, testMachine.Machine().ID)
-			if fly.IsNotAuthenticatedError(err) {
-				fmt.Fprintf(md.io.ErrOut, "Warn: not authorized to retrieve app logs (this can happen when using deploy tokens), so we can't show you what failed. Use `fly logs -i %s` or open the monitoring dashboard to see them: https://fly.io/apps/%s/monitoring?region=&instance=%s\n", testMachine.Machine().ID, md.appConfig.AppName, testMachine.Machine().ID)
-			} else {
-				if err != nil {
-					return fmt.Errorf("Error getting test command logs: %w", err)
-				}
+			if err == nil {
+				var logs string
 				for _, l := range testLogs {
-					fmt.Fprintf(md.io.ErrOut, "  %s\n", l.Message)
+					logs += l.Message + "\n"
 				}
+
+				return machineTestErr{machineID: testMachine.Machine().ID, exitCode: exitCode, testMachineLogs: logs}
 			}
+
 			return fmt.Errorf("Error test command machine %s exited with non-zero status of %d", testMachine.Machine().ID, exitCode)
 		}
-		statuslogger.LogfStatus(ctx,
+		sl.LogfStatus(
 			statuslogger.StatusSuccess,
 			"Test machine %s completed successfully",
 			md.colorize.Bold(testMachine.Machine().ID),
@@ -151,7 +168,7 @@ func (md *machineDeployment) waitForLogs(ctx context.Context, mach *fly.Machine,
 	}, backoff.WithContext(b, ctx))
 }
 
-func (md *machineDeployment) createTestMachine(ctx context.Context, svc *appconfig.ServiceMachineCheck, machineToTest *fly.Machine) (*fly.Machine, error) {
+func (md *machineDeployment) createTestMachine(ctx context.Context, svc *appconfig.ServiceMachineCheck, machineToTest *fly.Machine, sl statuslogger.StatusLine) (*fly.Machine, error) {
 	ctx, span := tracing.GetTracer().Start(ctx, "create_test_machine")
 	defer span.End()
 
@@ -165,7 +182,7 @@ func (md *machineDeployment) createTestMachine(ctx context.Context, svc *appconf
 		return nil, fmt.Errorf("error creating a test machine: %w", err)
 	}
 
-	statuslogger.Logf(ctx, "Created test machine %s", md.colorize.Bold(testMachine.ID))
+	sl.Logf("Created test machine %s", md.colorize.Bold(testMachine.ID))
 	return testMachine, nil
 }
 
@@ -195,7 +212,7 @@ func (md *machineDeployment) launchInputForTestMachine(svc *appconfig.ServiceMac
 	}, nil
 }
 
-func (md *machineDeployment) waitForTestMachinesToFinish(ctx context.Context, testMachines machine.MachineSet) error {
+func (md *machineDeployment) waitForTestMachinesToFinish(ctx context.Context, testMachines machine.MachineSet, sl statuslogger.StatusLine) error {
 	io := iostreams.FromContext(ctx)
 	colorize := io.ColorScheme()
 
@@ -224,7 +241,7 @@ func (md *machineDeployment) waitForTestMachinesToFinish(ctx context.Context, te
 		return m, err == nil
 	})
 	for _, mach := range machs {
-		statuslogger.Logf(ctx, "Test Machine %s: %s", colorize.Bold(mach.ID), mach.State)
+		sl.Logf("Test Machine %s: %s", colorize.Bold(mach.ID), mach.State)
 	}
 
 	return nil

internal/command/deploy/machines.go

@@ -72,6 +72,7 @@ type MachineDeploymentArgs struct {
 	VolumeInitialSize     int
 	RestartPolicy         *fly.MachineRestartPolicy
 	RestartMaxRetries     int
+	DeployRetries         int
 }
 
 type machineDeployment struct {
@@ -111,6 +112,7 @@ type machineDeployment struct {
 	processGroups         map[string]bool
 	maxConcurrent         int
 	volumeInitialSize     int
+	deployRetries         int
 }
 
 func NewMachineDeployment(ctx context.Context, args MachineDeploymentArgs) (_ MachineDeployment, err error) {
@@ -235,6 +237,7 @@ func NewMachineDeployment(ctx context.Context, args MachineDeploymentArgs) (_ Ma
 		maxConcurrent:         maxConcurrent,
 		volumeInitialSize:     args.VolumeInitialSize,
 		processGroups:         args.ProcessGroups,
+		deployRetries:         args.DeployRetries,
 	}
 	if err := md.setStrategy(); err != nil {
 		tracing.RecordError(span, err, "failed to set strategy")
@@ -375,12 +378,12 @@ func (md *machineDeployment) setMachinesForDeployment(ctx context.Context) error
 		}
 	}
 
-	md.machineSet = machine.NewMachineSet(md.flapsClient, md.io, machines)
+	md.machineSet = machine.NewMachineSet(md.flapsClient, md.io, machines, true)
 	var releaseCmdSet []*fly.Machine
 	if releaseCmdMachine != nil {
 		releaseCmdSet = []*fly.Machine{releaseCmdMachine}
 	}
-	md.releaseCommandMachine = machine.NewMachineSet(md.flapsClient, md.io, releaseCmdSet)
+	md.releaseCommandMachine = machine.NewMachineSet(md.flapsClient, md.io, releaseCmdSet, true)
 	return nil
 }