generate explicit port bindings

chain/cosmos/chain_node.go

@@ -62,6 +62,8 @@ type ChainNode struct {
 	// Ports set during StartContainer.
 	hostRPCPort  string
 	hostGRPCPort string
+
+	preStartListeners dockerutil.Listeners
 }
 
 // ChainNodes is a collection of ChainNode
@@ -935,6 +937,13 @@ func (tn *ChainNode) CreateNodeContainer(ctx context.Context) error {
 			zap.String("image", imageRef),
 		)
 
+	pb, listeners, err := dockerutil.GeneratePortBindings(sentryPorts)
+	if err != nil {
+		return fmt.Errorf("failed to generate port bindings: %w", err)
+	}
+
+	tn.preStartListeners = listeners
+
 	cc, err := tn.DockerClient.ContainerCreate(
 		ctx,
 		&container.Config{
@@ -951,6 +960,7 @@ func (tn *ChainNode) CreateNodeContainer(ctx context.Context) error {
 		},
 		&container.HostConfig{
 			Binds:           tn.Bind(),
+			PortBindings:    pb,
 			PublishAllPorts: true,
 			AutoRemove:      false,
 			DNS:             []string{},
@@ -964,14 +974,19 @@ func (tn *ChainNode) CreateNodeContainer(ctx context.Context) error {
 		tn.Name(),
 	)
 	if err != nil {
+		tn.preStartListeners.CloseAll()
 		return err
 	}
 	tn.containerID = cc.ID
 	return nil
 }
 
 func (tn *ChainNode) StartContainer(ctx context.Context) error {
-	if err := dockerutil.StartContainer(ctx, tn.DockerClient, tn.containerID); err != nil {
+	dockerutil.LockPortAssignment()
+	tn.preStartListeners.CloseAll()
+	err := dockerutil.StartContainer(ctx, tn.DockerClient, tn.containerID)
+	dockerutil.UnlockPortAssignment()
+	if err != nil {
 		return err
 	}
 

chain/internal/tendermint/tendermint_node.go

@@ -39,6 +39,8 @@ type TendermintNode struct {
 	Image        ibc.DockerImage
 
 	containerID string
+
+	preStartListeners dockerutil.Listeners
 }
 
 // TendermintNodes is a collection of TendermintNode
@@ -220,6 +222,13 @@ func (tn *TendermintNode) CreateNodeContainer(ctx context.Context, additionalFla
 	cmd = append(cmd, additionalFlags...)
 	fmt.Printf("{%s} -> '%s'\n", tn.Name(), strings.Join(cmd, " "))
 
+	pb, listeners, err := dockerutil.GeneratePortBindings(sentryPorts)
+	if err != nil {
+		return fmt.Errorf("failed to generate port bindings: %w", err)
+	}
+
+	tn.preStartListeners = listeners
+
 	cc, err := tn.DockerClient.ContainerCreate(
 		ctx,
 		&container.Config{
@@ -236,6 +245,7 @@ func (tn *TendermintNode) CreateNodeContainer(ctx context.Context, additionalFla
 		},
 		&container.HostConfig{
 			Binds:           tn.Bind(),
+			PortBindings:    pb,
 			PublishAllPorts: true,
 			AutoRemove:      false,
 			DNS:             []string{},
@@ -249,6 +259,7 @@ func (tn *TendermintNode) CreateNodeContainer(ctx context.Context, additionalFla
 		tn.Name(),
 	)
 	if err != nil {
+		tn.preStartListeners.CloseAll()
 		return err
 	}
 	tn.containerID = cc.ID
@@ -261,7 +272,11 @@ func (tn *TendermintNode) StopContainer(ctx context.Context) error {
 }
 
 func (tn *TendermintNode) StartContainer(ctx context.Context) error {
-	if err := dockerutil.StartContainer(ctx, tn.DockerClient, tn.containerID); err != nil {
+	dockerutil.LockPortAssignment()
+	tn.preStartListeners.CloseAll()
+	err := dockerutil.StartContainer(ctx, tn.DockerClient, tn.containerID)
+	dockerutil.UnlockPortAssignment()
+	if err != nil {
 		return err
 	}
 

chain/penumbra/penumbra_app_node.go

@@ -34,6 +34,8 @@ type PenumbraAppNode struct {
 	// Set during StartContainer.
 	hostRPCPort  string
 	hostGRPCPort string
+
+	preStartListeners dockerutil.Listeners
 }
 
 const (
@@ -216,6 +218,13 @@ func (p *PenumbraAppNode) CreateNodeContainer(ctx context.Context) error {
 	cmd := []string{"pd", "start", "--host", "0.0.0.0", "--home", p.HomeDir()}
 	fmt.Printf("{%s} -> '%s'\n", p.Name(), strings.Join(cmd, " "))
 
+	pb, listeners, err := dockerutil.GeneratePortBindings(exposedPorts)
+	if err != nil {
+		return fmt.Errorf("failed to generate port bindings: %w", err)
+	}
+
+	p.preStartListeners = listeners
+
 	cc, err := p.DockerClient.ContainerCreate(
 		ctx,
 		&container.Config{
@@ -233,6 +242,7 @@ func (p *PenumbraAppNode) CreateNodeContainer(ctx context.Context) error {
 		},
 		&container.HostConfig{
 			Binds:           p.Bind(),
+			PortBindings:    pb,
 			PublishAllPorts: true,
 			AutoRemove:      false,
 			DNS:             []string{},
@@ -246,6 +256,7 @@ func (p *PenumbraAppNode) CreateNodeContainer(ctx context.Context) error {
 		p.Name(),
 	)
 	if err != nil {
+		p.preStartListeners.CloseAll()
 		return err
 	}
 	p.containerID = cc.ID
@@ -258,7 +269,11 @@ func (p *PenumbraAppNode) StopContainer(ctx context.Context) error {
 }
 
 func (p *PenumbraAppNode) StartContainer(ctx context.Context) error {
-	if err := dockerutil.StartContainer(ctx, p.DockerClient, p.containerID); err != nil {
+	dockerutil.LockPortAssignment()
+	p.preStartListeners.CloseAll()
+	err := dockerutil.StartContainer(ctx, p.DockerClient, p.containerID)
+	dockerutil.UnlockPortAssignment()
+	if err != nil {
 		return err
 	}
 

chain/polkadot/parachain_node.go

@@ -44,6 +44,8 @@ type ParachainNode struct {
 	api         *gsrpc.SubstrateAPI
 	hostWsPort  string
 	hostRpcPort string
+
+	preStartListeners dockerutil.Listeners
 }
 
 type ParachainNodes []*ParachainNode
@@ -252,6 +254,13 @@ func (pn *ParachainNode) CreateNodeContainer(ctx context.Context) error {
 			zap.String("container", pn.Name()),
 		)
 
+	pb, listeners, err := dockerutil.GeneratePortBindings(exposedPorts)
+	if err != nil {
+		return fmt.Errorf("failed to generate port bindings: %w", err)
+	}
+
+	pn.preStartListeners = listeners
+
 	cc, err := pn.DockerClient.ContainerCreate(
 		ctx,
 		&container.Config{
@@ -269,6 +278,7 @@ func (pn *ParachainNode) CreateNodeContainer(ctx context.Context) error {
 		},
 		&container.HostConfig{
 			Binds:           pn.Bind(),
+			PortBindings:    pb,
 			PublishAllPorts: true,
 			AutoRemove:      false,
 			DNS:             []string{},
@@ -282,6 +292,7 @@ func (pn *ParachainNode) CreateNodeContainer(ctx context.Context) error {
 		pn.Name(),
 	)
 	if err != nil {
+		pn.preStartListeners.CloseAll()
 		return err
 	}
 	pn.containerID = cc.ID
@@ -296,7 +307,11 @@ func (pn *ParachainNode) StopContainer(ctx context.Context) error {
 
 // StartContainer starts the container after it is built by CreateNodeContainer.
 func (pn *ParachainNode) StartContainer(ctx context.Context) error {
-	if err := dockerutil.StartContainer(ctx, pn.DockerClient, pn.containerID); err != nil {
+	dockerutil.LockPortAssignment()
+	pn.preStartListeners.CloseAll()
+	err := dockerutil.StartContainer(ctx, pn.DockerClient, pn.containerID)
+	dockerutil.UnlockPortAssignment()
+	if err != nil {
 		return err
 	}
 

chain/polkadot/relay_chain_node.go

@@ -46,6 +46,8 @@ type RelayChainNode struct {
 	api         *gsrpc.SubstrateAPI
 	hostWsPort  string
 	hostRpcPort string
+
+	preStartListeners dockerutil.Listeners
 }
 
 type RelayChainNodes []*RelayChainNode
@@ -228,6 +230,13 @@ func (p *RelayChainNode) CreateNodeContainer(ctx context.Context) error {
 			zap.String("container", p.Name()),
 		)
 
+	pb, listeners, err := dockerutil.GeneratePortBindings(exposedPorts)
+	if err != nil {
+		return fmt.Errorf("failed to generate port bindings: %w", err)
+	}
+
+	p.preStartListeners = listeners
+
 	cc, err := p.DockerClient.ContainerCreate(
 		ctx,
 		&container.Config{
@@ -245,6 +254,7 @@ func (p *RelayChainNode) CreateNodeContainer(ctx context.Context) error {
 		},
 		&container.HostConfig{
 			Binds:           p.Bind(),
+			PortBindings:    pb,
 			PublishAllPorts: true,
 			AutoRemove:      false,
 			DNS:             []string{},
@@ -258,6 +268,7 @@ func (p *RelayChainNode) CreateNodeContainer(ctx context.Context) error {
 		p.Name(),
 	)
 	if err != nil {
+		p.preStartListeners.CloseAll()
 		return err
 	}
 	p.containerID = cc.ID
@@ -272,7 +283,11 @@ func (p *RelayChainNode) StopContainer(ctx context.Context) error {
 
 // StartContainer starts the container after it is built by CreateNodeContainer.
 func (p *RelayChainNode) StartContainer(ctx context.Context) error {
-	if err := dockerutil.StartContainer(ctx, p.DockerClient, p.containerID); err != nil {
+	dockerutil.LockPortAssignment()
+	p.preStartListeners.CloseAll()
+	err := dockerutil.StartContainer(ctx, p.DockerClient, p.containerID)
+	dockerutil.UnlockPortAssignment()
+	if err != nil {
 		return err
 	}
 

internal/dockerutil/ports.go

@@ -0,0 +1,80 @@
+package dockerutil
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/docker/go-connections/nat"
+)
+
+var mu sync.Mutex
+
+type Listeners []net.Listener
+
+func (l Listeners) CloseAll() {
+	for _, listener := range l {
+		listener.Close()
+	}
+}
+
+func LockPortAssignment() {
+	mu.Lock()
+}
+
+func UnlockPortAssignment() {
+	mu.Unlock()
+}
+
+// openListenerOnFreePort opens the next free port
+func openListenerOnFreePort() (*net.TCPListener, error) {
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+	if err != nil {
+		return nil, err
+	}
+
+	LockPortAssignment()
+	defer UnlockPortAssignment()
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+
+	return l, nil
+}
+
+// nextAvailablePort generates a docker PortBinding by finding the next available port.
+// The listener will be closed in the case of an error, otherwise it will be left open.
+// This allows multiple nextAvailablePort calls to find multiple available ports
+// before closing them so they are available for the PortBinding.
+func nextAvailablePort() (nat.PortBinding, *net.TCPListener, error) {
+	l, err := openListenerOnFreePort()
+	if err != nil {
+		l.Close()
+		return nat.PortBinding{}, nil, err
+	}
+
+	return nat.PortBinding{
+		HostIP:   "0.0.0.0",
+		HostPort: fmt.Sprint(l.Addr().(*net.TCPAddr).Port),
+	}, l, nil
+}
+
+// GeneratePortBindings will find open ports on the local
+// machine and create a PortBinding for every port in the portSet.
+func GeneratePortBindings(portSet nat.PortSet) (nat.PortMap, []net.Listener, error) {
+	m := make(nat.PortMap)
+	listeners := make(Listeners, 0, len(portSet))
+
+	for p := range portSet {
+		pb, l, err := nextAvailablePort()
+		if err != nil {
+			listeners.CloseAll()
+			return nat.PortMap{}, nil, err
+		}
+		listeners = append(listeners, l)
+		m[p] = []nat.PortBinding{pb}
+	}
+
+	return m, listeners, nil
+}