Backport of Basic gobased API gateway spinup test into release/1.15.x (…

…#16423)

* backport of commit 60a62ad

* backport of commit 560cd55

* backport of commit bd3ff14

* backport of commit b8a4c83

* backport of commit 184b6bb

* backport of commit cc213b6

* backport of commit e706bc7

* backport of commit 064a3c0

* backport of commit b8aa15e

* updated GetPort interface

* fix getport

* fix syntax issues

* fix gofmt error

* removed deprecated random calls, removed unused port variable

* clean up duplicate comment

* Update test/integration/consul-container/test/gateways/gateway_endpoint_test.go

* parralelize this test

* delete extra line

* clean up unneeded comments

---------

Co-authored-by: Sarah Alsmiller <sarah.alsmiller@hashicorp.com>
Co-authored-by: Andrew Stucki <andrew.stucki@hashicorp.com>
Co-authored-by: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com>

test/integration/consul-container/libs/cluster/agent.go

@@ -22,6 +22,7 @@ type Agent interface {
 	GetConfig() Config
 	GetInfo() AgentInfo
 	GetDatacenter() string
+	GetNetwork() string
 	IsServer() bool
 	RegisterTermination(func() error)
 	Terminate() error

test/integration/consul-container/libs/cluster/cluster.go

@@ -63,7 +63,7 @@ func NewN(t TestingT, conf Config, count int) (*Cluster, error) {
 //
 // The provided TestingT is used to register a cleanup function to terminate
 // the cluster.
-func New(t TestingT, configs []Config) (*Cluster, error) {
+func New(t TestingT, configs []Config, ports ...int) (*Cluster, error) {
 	id, err := shortid.Generate()
 	if err != nil {
 		return nil, fmt.Errorf("could not cluster id: %w", err)
@@ -99,7 +99,7 @@ func New(t TestingT, configs []Config) (*Cluster, error) {
 		_ = cluster.Terminate()
 	})
 
-	if err := cluster.Add(configs, true); err != nil {
+	if err := cluster.Add(configs, true, ports...); err != nil {
 		return nil, fmt.Errorf("could not start or join all agents: %w", err)
 	}
 
@@ -115,7 +115,7 @@ func (c *Cluster) AddN(conf Config, count int, join bool) error {
 }
 
 // Add starts an agent with the given configuration and joins it with the existing cluster
-func (c *Cluster) Add(configs []Config, serfJoin bool) (xe error) {
+func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) {
 	if c.Index == 0 && !serfJoin {
 		return fmt.Errorf("the first call to Cluster.Add must have serfJoin=true")
 	}
@@ -135,6 +135,7 @@ func (c *Cluster) Add(configs []Config, serfJoin bool) (xe error) {
 			context.Background(),
 			conf,
 			c,
+			ports...,
 		)
 		if err != nil {
 			return fmt.Errorf("could not add container index %d: %w", idx, err)

test/integration/consul-container/libs/cluster/container.go

@@ -73,7 +73,7 @@ func (c *consulContainerNode) ClaimAdminPort() (int, error) {
 }
 
 // NewConsulContainer starts a Consul agent in a container with the given config.
-func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (Agent, error) {
+func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, ports ...int) (Agent, error) {
 	network := cluster.NetworkName
 	index := cluster.Index
 	if config.ScratchDir == "" {
@@ -128,7 +128,7 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (A
 		addtionalNetworks: []string{"bridge", network},
 		hostname:          fmt.Sprintf("agent-%d", index),
 	}
-	podReq, consulReq := newContainerRequest(config, opts)
+	podReq, consulReq := newContainerRequest(config, opts, ports...)
 
 	// Do some trickery to ensure that partial completion is correctly torn
 	// down, but successful execution is not.
@@ -291,6 +291,10 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster) (A
 	return node, nil
 }
 
+func (c *consulContainerNode) GetNetwork() string {
+	return c.network
+}
+
 func (c *consulContainerNode) GetName() string {
 	if c.container == nil {
 		return c.consulReq.Name // TODO: is this safe to do all the time?
@@ -501,7 +505,7 @@ type containerOpts struct {
 	addtionalNetworks []string
 }
 
-func newContainerRequest(config Config, opts containerOpts) (podRequest, consulRequest testcontainers.ContainerRequest) {
+func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRequest, consulRequest testcontainers.ContainerRequest) {
 	skipReaper := isRYUKDisabled()
 
 	pod := testcontainers.ContainerRequest{
@@ -541,6 +545,10 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR
 		pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", basePort+i))
 	}
 
+	for _, port := range ports {
+		pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", port))
+	}
+
 	// For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready".
 	// If we only wait until the log stream starts, subsequent commands to agents will fail.
 	// TODO: optimize the wait strategy

test/integration/consul-container/libs/service/connect.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"path/filepath"
@@ -58,6 +59,10 @@ func (g ConnectContainer) GetAddrs() (string, []int) {
 	return g.ip, g.appPort
 }
 
+func (g ConnectContainer) GetPort(port int) (int, error) {
+	return 0, errors.New("not implemented")
+}
+
 func (g ConnectContainer) Restart() error {
 	_, err := g.GetStatus()
 	if err != nil {

test/integration/consul-container/libs/service/examples.go

@@ -68,6 +68,10 @@ func (g exampleContainer) GetAddrs() (string, []int) {
 	return "", nil
 }
 
+func (g exampleContainer) GetPort(port int) (int, error) {
+	return 0, nil
+}
+
 func (g exampleContainer) Restart() error {
 	return fmt.Errorf("Restart Unimplemented by ConnectContainer")
 }
@@ -121,7 +125,7 @@ func (c exampleContainer) GetStatus() (string, error) {
 	return state.Status, err
 }
 
-func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort int, node libcluster.Agent) (Service, error) {
+func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort int, node libcluster.Agent, containerArgs ...string) (Service, error) {
 	namePrefix := fmt.Sprintf("%s-service-example-%s", node.GetDatacenter(), name)
 	containerName := utils.RandName(namePrefix)
 
@@ -135,18 +139,22 @@ func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort
 		grpcPortStr = strconv.Itoa(grpcPort)
 	)
 
+	command := []string{
+		"server",
+		"-http-port", httpPortStr,
+		"-grpc-port", grpcPortStr,
+		"-redirect-port", "-disabled",
+	}
+
+	command = append(command, containerArgs...)
+
 	req := testcontainers.ContainerRequest{
 		Image:      hashicorpDockerProxy + "/fortio/fortio",
 		WaitingFor: wait.ForLog("").WithStartupTimeout(10 * time.Second),
 		AutoRemove: false,
 		Name:       containerName,
-		Cmd: []string{
-			"server",
-			"-http-port", httpPortStr,
-			"-grpc-port", grpcPortStr,
-			"-redirect-port", "-disabled",
-		},
-		Env: map[string]string{"FORTIO_NAME": name},
+		Cmd:        command,
+		Env:        map[string]string{"FORTIO_NAME": name},
 	}
 
 	info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{httpPortStr, grpcPortStr})

test/integration/consul-container/libs/service/gateway.go

@@ -20,12 +20,13 @@ import (
 
 // gatewayContainer
 type gatewayContainer struct {
-	ctx         context.Context
-	container   testcontainers.Container
-	ip          string
-	port        int
-	adminPort   int
-	serviceName string
+	ctx          context.Context
+	container    testcontainers.Container
+	ip           string
+	port         int
+	adminPort    int
+	serviceName  string
+	portMappings map[int]int
 }
 
 var _ Service = (*gatewayContainer)(nil)
@@ -105,6 +106,15 @@ func (g gatewayContainer) GetAdminAddr() (string, int) {
 	return "localhost", g.adminPort
 }
 
+func (g gatewayContainer) GetPort(port int) (int, error) {
+	p, ok := g.portMappings[port]
+	if !ok {
+		return 0, fmt.Errorf("port does not exist")
+	}
+	return p, nil
+
+}
+
 func (g gatewayContainer) Restart() error {
 	_, err := g.container.State(g.ctx)
 	if err != nil {
@@ -130,7 +140,7 @@ func (g gatewayContainer) GetStatus() (string, error) {
 	return state.Status, err
 }
 
-func NewGatewayService(ctx context.Context, name string, kind string, node libcluster.Agent) (Service, error) {
+func NewGatewayService(ctx context.Context, name string, kind string, node libcluster.Agent, ports ...int) (Service, error) {
 	nodeConfig := node.GetConfig()
 	if nodeConfig.ScratchDir == "" {
 		return nil, fmt.Errorf("node ScratchDir is required")
@@ -207,21 +217,33 @@ func NewGatewayService(ctx context.Context, name string, kind string, node libcl
 		adminPortStr = strconv.Itoa(adminPort)
 	)
 
-	info, err := cluster.LaunchContainerOnNode(ctx, node, req, []string{
+	extraPorts := []string{}
+	for _, port := range ports {
+		extraPorts = append(extraPorts, strconv.Itoa(port))
+	}
+
+	info, err := cluster.LaunchContainerOnNode(ctx, node, req, append(
+		extraPorts,
 		portStr,
 		adminPortStr,
-	})
+	))
 	if err != nil {
 		return nil, err
 	}
 
+	portMappings := make(map[int]int)
+	for _, port := range ports {
+		portMappings[port] = info.MappedPorts[strconv.Itoa(port)].Int()
+	}
+
 	out := &gatewayContainer{
-		ctx:         ctx,
-		container:   info.Container,
-		ip:          info.IP,
-		port:        info.MappedPorts[portStr].Int(),
-		adminPort:   info.MappedPorts[adminPortStr].Int(),
-		serviceName: name,
+		ctx:          ctx,
+		container:    info.Container,
+		ip:           info.IP,
+		port:         info.MappedPorts[portStr].Int(),
+		adminPort:    info.MappedPorts[adminPortStr].Int(),
+		serviceName:  name,
+		portMappings: portMappings,
 	}
 
 	return out, nil

test/integration/consul-container/libs/service/helpers.go

@@ -35,7 +35,7 @@ type ServiceOpts struct {
 }
 
 // createAndRegisterStaticServerAndSidecar register the services and launch static-server containers
-func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int, svc *api.AgentServiceRegistration) (Service, Service, error) {
+func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int, svc *api.AgentServiceRegistration, containerArgs ...string) (Service, Service, error) {
 	// Do some trickery to ensure that partial completion is correctly torn
 	// down, but successful execution is not.
 	var deferClean utils.ResettableDefer
@@ -46,7 +46,7 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int
 	}
 
 	// Create a service and proxy instance
-	serverService, err := NewExampleService(context.Background(), svc.ID, svc.Port, grpcPort, node)
+	serverService, err := NewExampleService(context.Background(), svc.ID, svc.Port, grpcPort, node, containerArgs...)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -68,7 +68,7 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, grpcPort int
 	return serverService, serverConnectProxy, nil
 }
 
-func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) {
+func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts *ServiceOpts, containerArgs ...string) (Service, Service, error) {
 	// Register the static-server service and sidecar first to prevent race with sidecar
 	// trying to get xDS before it's ready
 	req := &api.AgentServiceRegistration{
@@ -88,7 +88,7 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 		},
 		Meta: serviceOpts.Meta,
 	}
-	return createAndRegisterStaticServerAndSidecar(node, serviceOpts.GRPCPort, req)
+	return createAndRegisterStaticServerAndSidecar(node, serviceOpts.GRPCPort, req, containerArgs...)
 }
 
 func CreateAndRegisterStaticServerAndSidecarWithChecks(node libcluster.Agent, serviceOpts *ServiceOpts) (Service, Service, error) {

test/integration/consul-container/libs/service/service.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+
 	"github.com/hashicorp/consul/api"
 )
 
@@ -13,6 +14,7 @@ type Service interface {
 	Export(partition, peer string, client *api.Client) error
 	GetAddr() (string, int)
 	GetAddrs() (string, []int)
+	GetPort(port int) (int, error)
 	// GetAdminAddr returns the external admin address
 	GetAdminAddr() (string, int)
 	GetLogs() (string, error)