Reapply "Even more remote heartbeat improvements (#3594)" (#3619)

* Restart remote builder machine if it's stopped

* Fix tests

* Reapply "Even more remote heartbeat improvements (#3594)"

This reverts commit 4fb6bd4. Ben T made
some changes that should let this work again. We'll need to add a
preflight test to ensure that this doesn't break in the future, however.

* Add the recreate builder flag

This allows users forcing recreating the remote builder, even if it's valid

* Add a test for deploying and recreating the remote builder

* Catch and trace errors

.github/workflows/checks.yml

@@ -42,4 +42,3 @@ jobs:
         with:
           version: v1.54
           working-directory: .
-          skip-pkg-cache: true

internal/build/imgsrc/docker.go

@@ -53,14 +53,14 @@ type dockerClientFactory struct {
 	appName   string
 }
 
-func newDockerClientFactory(daemonType DockerDaemonType, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, connectOverWireguard bool) *dockerClientFactory {
+func newDockerClientFactory(daemonType DockerDaemonType, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, connectOverWireguard, recreateBuilder bool) *dockerClientFactory {
 	remoteFactory := func() *dockerClientFactory {
 		terminal.Debug("trying remote docker daemon")
 		return &dockerClientFactory{
 			mode:   daemonType,
 			remote: true,
 			buildFn: func(ctx context.Context, build *build) (*dockerclient.Client, error) {
-				return newRemoteDockerClient(ctx, apiClient, appName, streams, build, cachedDocker, connectOverWireguard)
+				return newRemoteDockerClient(ctx, apiClient, appName, streams, build, cachedDocker, connectOverWireguard, recreateBuilder)
 			},
 			apiClient: apiClient,
 			appName:   appName,
@@ -203,7 +203,7 @@ func logClearLinesAbove(streams *iostreams.IOStreams, count int) {
 	}
 }
 
-func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, build *build, cachedClient *dockerclient.Client, connectOverWireguard bool) (c *dockerclient.Client, err error) {
+func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appName string, streams *iostreams.IOStreams, build *build, cachedClient *dockerclient.Client, connectOverWireguard, recreateBuilder bool) (c *dockerclient.Client, err error) {
 	ctx, span := tracing.GetTracer().Start(ctx, "build_remote_docker_client", trace.WithAttributes(
 		attribute.Bool("connect_over_wireguard", connectOverWireguard),
 	))
@@ -223,8 +223,8 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appNam
 
 	var host string
 	var app *fly.App
-	var machine *fly.GqlMachine
-	machine, app, err = remoteBuilderMachine(ctx, apiClient, appName)
+	var machine *fly.Machine
+	machine, app, err = remoteBuilderMachine(ctx, apiClient, appName, recreateBuilder)
 	if err != nil {
 		tracing.RecordError(span, err, "failed to init remote builder machine")
 		return nil, err
@@ -271,7 +271,7 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appNam
 			}
 
 			fmt.Fprintln(streams.Out, streams.ColorScheme().Yellow("🔧 creating fresh remote builder, (this might take a while ...)"))
-			machine, app, err = remoteBuilderMachine(ctx, apiClient, appName)
+			machine, app, err = remoteBuilderMachine(ctx, apiClient, appName, false)
 			if err != nil {
 				tracing.RecordError(span, err, "failed to init remote builder machine")
 				return nil, err
@@ -321,13 +321,7 @@ func newRemoteDockerClient(ctx context.Context, apiClient flyutil.Client, appNam
 		)
 	}
 
-	for _, ip := range machine.IPs.Nodes {
-		terminal.Debugf("checking ip %+v\n", ip)
-		if ip.Kind == "privatenet" {
-			host = "tcp://[" + ip.IP + "]:2375"
-			break
-		}
-	}
+	host = "tcp://[" + machine.PrivateIP + "]:2375"
 
 	if !connectOverWireguard {
 		oldHost := host
@@ -700,20 +694,14 @@ func ResolveDockerfile(cwd string) string {
 	return ""
 }
 
-func EagerlyEnsureRemoteBuilder(ctx context.Context, apiClient flyutil.Client, orgSlug string) {
+func EagerlyEnsureRemoteBuilder(ctx context.Context, apiClient flyutil.Client, org *fly.Organization, recreateBuilder bool) {
 	// skip if local docker is available
 	if _, err := NewLocalDockerClient(); err == nil {
 		return
 	}
 
-	org, err := apiClient.GetOrganizationBySlug(ctx, orgSlug)
-	if err != nil {
-		terminal.Debugf("error resolving organization for slug %s: %s", orgSlug, err)
-		return
-	}
-
 	region := os.Getenv("FLY_REMOTE_BUILDER_REGION")
-	_, app, err := apiClient.EnsureRemoteBuilder(ctx, org.ID, "", region)
+	_, app, err := EnsureBuilder(ctx, org, region, recreateBuilder)
 	if err != nil {
 		terminal.Debugf("error ensuring remote builder for organization: %s", err)
 		return
@@ -722,13 +710,18 @@ func EagerlyEnsureRemoteBuilder(ctx context.Context, apiClient flyutil.Client, o
 	terminal.Debugf("remote builder %s is being prepared", app.Name)
 }
 
-func remoteBuilderMachine(ctx context.Context, apiClient flyutil.Client, appName string) (*fly.GqlMachine, *fly.App, error) {
+func remoteBuilderMachine(ctx context.Context, apiClient flyutil.Client, appName string, recreateBuilder bool) (*fly.Machine, *fly.App, error) {
 	if v := os.Getenv("FLY_REMOTE_BUILDER_HOST"); v != "" {
 		return nil, nil, nil
 	}
 
 	region := os.Getenv("FLY_REMOTE_BUILDER_REGION")
-	return apiClient.EnsureRemoteBuilder(ctx, "", appName, region)
+	org, err := apiClient.GetOrganizationByApp(ctx, appName)
+	if err != nil {
+		return nil, nil, err
+	}
+	builderMachine, builderApp, err := EnsureBuilder(ctx, org, region, recreateBuilder)
+	return builderMachine, builderApp, err
 }
 
 func (d *dockerClientFactory) IsRemote() bool {