NSX-T ALB Virtual Service Transparent mode and Pool Firewall Group membership (VCD 10.4.1+)

.changes/v2.20.0/560-improvements.md

@@ -0,0 +1,5 @@
+* Add new field `TransparentModeEnabled` to `types.NsxtAlbVirtualService` which allows to preserve
+  client IP for NSX-T ALB Virtual Service (VCD 10.4.1+) [GH-560]
+* Add new field `MemberGroupRef` to `types.NsxtAlbPool` which allows to define NSX-T ALB Pool
+  membership by using Edge Firewall Group (`NsxtFirewallGroup`) instead of plain IPs (VCD 10.4.1+)
+  [GH-560]

govcd/api.go

@@ -687,6 +687,13 @@ func combinedTaskErrorMessage(task *types.Task, err error) string {
 	return extendedError
 }
 
+// addrOf is a generic function to return the address of a variable
+// Note. It is mainly meant for converting literal values to pointers (e.g. `addrOf(true)`)
+// and not getting the address of a variable (e.g. `addrOf(variable)`)
+func addrOf[T any](variable T) *T {
+	return &variable
+}
+
 func takeBoolPointer(value bool) *bool {
 	return &value
 }

govcd/api_vcd_test.go

@@ -193,6 +193,7 @@ type TestConfig struct {
 			NsxtAlbControllerUser     string `yaml:"nsxtAlbControllerUser"`
 			NsxtAlbControllerPassword string `yaml:"nsxtAlbControllerPassword"`
 			NsxtAlbImportableCloud    string `yaml:"nsxtAlbImportableCloud"`
+			NsxtAlbServiceEngineGroup string `yaml:"nsxtAlbServiceEngineGroup"`
 		} `yaml:"nsxt"`
 	} `yaml:"vcd"`
 	Logging struct {
@@ -1880,6 +1881,9 @@ func skipNoNsxtAlbConfiguration(vcd *TestVCD, check *C) {
 	if vcd.config.VCD.Nsxt.NsxtAlbImportableCloud == "" {
 		check.Skip(generalMessage + "No NSX-T ALB Controller Importable Cloud Name")
 	}
+	if vcd.config.VCD.Nsxt.NsxtAlbServiceEngineGroup == "" {
+		check.Skip(generalMessage + "No NSX-T ALB Service Engine Group name specified in configuration")
+	}
 }
 
 // skipOpenApiEndpointTest is a helper to skip tests for particular unsupported OpenAPI endpoints
@@ -1901,16 +1905,16 @@ func skipOpenApiEndpointTest(vcd *TestVCD, check *C, endpoint string) {
 // newOrgUserConnection creates a new Org User and returns a connection to it.
 // Attention: Set the user to use only lowercase letters. If you put upper case letters the function fails on waiting
 // because VCD creates the user with lowercase letters.
-func newOrgUserConnection(adminOrg *AdminOrg, userName, password, href string, insecure bool) (*VCDClient, error) {
+func newOrgUserConnection(adminOrg *AdminOrg, userName, password, href string, insecure bool) (*VCDClient, *OrgUser, error) {
 	u, err := url.ParseRequestURI(href)
 	if err != nil {
-		return nil, fmt.Errorf("[newOrgUserConnection] unable to pass url: %s", err)
+		return nil, nil, fmt.Errorf("[newOrgUserConnection] unable to pass url: %s", err)
 	}
 
 	_, err = adminOrg.GetUserByName(userName, false)
 	if err == nil {
 		// user exists
-		return nil, fmt.Errorf("user %s already exists", userName)
+		return nil, nil, fmt.Errorf("user %s already exists", userName)
 	}
 	_, err = adminOrg.CreateUserSimple(OrgUserConfiguration{
 		Name:            userName,
@@ -1924,14 +1928,23 @@ func newOrgUserConnection(adminOrg *AdminOrg, userName, password, href string, i
 		Description:     "Test user created by newOrgUserConnection",
 	})
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
+
 	AddToCleanupList(userName, "user", adminOrg.AdminOrg.Name, "newOrgUserConnection")
+
 	_ = adminOrg.Refresh()
 	vcdClient := NewVCDClient(*u, insecure)
 	err = vcdClient.Authenticate(userName, password, adminOrg.AdminOrg.Name)
 	if err != nil {
-		return nil, fmt.Errorf("[newOrgUserConnection] unable to authenticate: %s", err)
+		return nil, nil, fmt.Errorf("[newOrgUserConnection] unable to authenticate: %s", err)
 	}
-	return vcdClient, nil
+
+	// return newUser delete function
+	newUser, err := adminOrg.GetUserByName(userName, false)
+	if err != nil {
+		return nil, nil, fmt.Errorf("[newOrgUserConnection] unable to retrieve newly created user: %s", err)
+	}
+
+	return vcdClient, newUser, nil
 }

govcd/catalog_test.go

@@ -855,7 +855,7 @@ func (vcd *TestVCD) Test_GetCatalogByXSharedCatalogOrgUser(check *C) {
 	// Create an Org Admin user and test that it can find catalog as well
 	adminOrg, err := vcd.client.GetAdminOrgByName(vcd.config.VCD.Org)
 	check.Assert(err, IsNil)
-	orgAdminClient, err := newOrgUserConnection(adminOrg, "test-user", "CHANGE-ME", vcd.config.Provider.Url, true)
+	orgAdminClient, _, err := newOrgUserConnection(adminOrg, "test-user", "CHANGE-ME", vcd.config.Provider.Url, true)
 	check.Assert(err, IsNil)
 	orgAsOrgUser, err := orgAdminClient.GetOrgByName(vcd.config.VCD.Org)
 	check.Assert(err, IsNil)

govcd/nsxt_alb_pool_test.go

@@ -23,7 +23,7 @@ func (vcd *TestVCD) Test_AlbPool(check *C) {
 	// Setup Org user and connection
 	adminOrg, err := vcd.client.GetAdminOrgByName(vcd.config.VCD.Org)
 	check.Assert(err, IsNil)
-	orgUserVcdClient, err := newOrgUserConnection(adminOrg, "alb-pool-testing", "CHANGE-ME", vcd.config.Provider.Url, true)
+	orgUserVcdClient, orgUser, err := newOrgUserConnection(adminOrg, "alb-pool-testing", "CHANGE-ME", vcd.config.Provider.Url, true)
 	check.Assert(err, IsNil)
 
 	// defer prerequisite teardown
@@ -40,6 +40,10 @@ func (vcd *TestVCD) Test_AlbPool(check *C) {
 	testAdvancedPoolConfig(check, edge, vcd, orgUserVcdClient)
 	testPoolWithCertNoPrivateKey(check, vcd, edge.EdgeGateway.ID, orgUserVcdClient)
 	testPoolWithCertAndPrivateKey(check, vcd, edge.EdgeGateway.ID, orgUserVcdClient)
+
+	// Cleanup Org user
+	err = orgUser.Delete(true)
+	check.Assert(err, IsNil)
 }
 
 func testMinimalPoolConfig(check *C, edge *NsxtEdgeGateway, vcd *TestVCD, client *VCDClient) {
@@ -255,6 +259,12 @@ func setupAlbPoolPrerequisites(check *C, vcd *TestVCD) (*NsxtAlbController, *Nsx
 		albSettingsConfig.SupportedFeatureSet = "PREMIUM"
 	}
 
+	// Enable Transparent mode on VCD >= 10.4.1
+	if vcd.client.Client.APIVCDMaxVersionIs(">= 37.1") {
+		printVerbose("# Enabling Transparent mode on Edge Gateway (VCD 10.4.1+)\n")
+		albSettingsConfig.TransparentModeEnabled = addrOf(true)
+	}
+
 	enabledSettings, err := edge.UpdateAlbSettings(albSettingsConfig)
 	if err != nil {
 		fmt.Printf("# error occured while enabling ALB on Edge Gateway. Cleaning up Service Engine Group, ALB Cloud and ALB Controller: %s", err)

govcd/nsxt_alb_service_engine_groups_test.go

@@ -102,15 +102,15 @@ func spawnAlbControllerCloudServiceEngineGroup(vcd *TestVCD, check *C, seGroupRe
 
 	albController, createdAlbCloud := spawnAlbControllerAndCloud(vcd, check)
 
-	importableSeGroups, err := vcd.client.GetAllAlbImportableServiceEngineGroups(createdAlbCloud.NsxtAlbCloud.ID, nil)
+	importableSeGroup, err := vcd.client.GetAlbImportableServiceEngineGroupByName(createdAlbCloud.NsxtAlbCloud.ID, vcd.config.VCD.Nsxt.NsxtAlbServiceEngineGroup)
 	check.Assert(err, IsNil)
-	check.Assert(len(importableSeGroups) > 0, Equals, true)
+
 	albSeGroup := &types.NsxtAlbServiceEngineGroup{
 		Name:            check.TestName() + "SE-group",
 		Description:     "Service Engine Group created by " + check.TestName(),
 		ReservationType: seGroupReservationType,
 		ServiceEngineGroupBacking: types.ServiceEngineGroupBacking{
-			BackingId: importableSeGroups[0].NsxtAlbImportableServiceEngineGroups.ID,
+			BackingId: importableSeGroup.NsxtAlbImportableServiceEngineGroups.ID,
 			LoadBalancerCloudRef: &types.OpenApiReference{
 				ID: createdAlbCloud.NsxtAlbCloud.ID,
 			},

govcd/nsxt_alb_virtual_service_test.go

@@ -23,23 +23,53 @@ func (vcd *TestVCD) Test_AlbVirtualService(check *C) {
 	// Setup Org user and connection
 	adminOrg, err := vcd.client.GetAdminOrgByName(vcd.config.VCD.Org)
 	check.Assert(err, IsNil)
-	orgUserVcdClient, err := newOrgUserConnection(adminOrg, "alb-virtual-service-testing", "CHANGE-ME", vcd.config.Provider.Url, true)
+	orgUserVcdClient, orgUser, err := newOrgUserConnection(adminOrg, "alb-virtual-service-testing", "CHANGE-ME", vcd.config.Provider.Url, true)
 	check.Assert(err, IsNil)
 
+	printVerbose("# Running tests as Sysadmin user\n")
 	// Run tests with System user
 	testMinimalVirtualServiceConfigHTTP(check, edge, albPool, seGroup, vcd, vcd.client)
 	testVirtualServiceConfigWithCertHTTPS(check, edge, albPool, seGroup, vcd, vcd.client)
 	testMinimalVirtualServiceConfigL4(check, edge, albPool, seGroup, vcd, vcd.client)
 	testMinimalVirtualServiceConfigL4TLS(check, edge, albPool, seGroup, vcd, vcd.client)
 
+	printVerbose("# Running tests as Org user\n")
 	// Run tests with Org admin user
 	testMinimalVirtualServiceConfigHTTP(check, edge, albPool, seGroup, vcd, orgUserVcdClient)
 	testVirtualServiceConfigWithCertHTTPS(check, edge, albPool, seGroup, vcd, orgUserVcdClient)
 	testMinimalVirtualServiceConfigL4(check, edge, albPool, seGroup, vcd, orgUserVcdClient)
 	testMinimalVirtualServiceConfigL4TLS(check, edge, albPool, seGroup, vcd, orgUserVcdClient)
 
+	// Test 10.4.1 Transparent mode on VCD >= 10.4.1
+	if vcd.client.Client.APIVCDMaxVersionIs(">= 37.1") {
+		printVerbose("# Running 10.4.1+ tests as Sysadmin user\n")
+
+		printVerbose("## Creating ALB Pool with Member Group (VCD 10.4.1+) as Sysadmin\n")
+		ipSet, poolWithMemberGroup := setupAlbPoolFirewallGroupMembers(check, vcd, edge)
+
+		testMinimalVirtualServiceConfigHTTPTransparent(check, edge, poolWithMemberGroup, seGroup, vcd, vcd.client, true)
+		testMinimalVirtualServiceConfigHTTPTransparent(check, edge, poolWithMemberGroup, seGroup, vcd, vcd.client, false)
+
+		printVerbose("# Running 10.4.1+ tests as Org user\n")
+
+		printVerbose("## Creating ALB Pool with Member Group (VCD 10.4.1+) as Org user\n")
+		testMinimalVirtualServiceConfigHTTPTransparent(check, edge, poolWithMemberGroup, seGroup, vcd, orgUserVcdClient, true)
+		testMinimalVirtualServiceConfigHTTPTransparent(check, edge, poolWithMemberGroup, seGroup, vcd, orgUserVcdClient, false)
+
+		// cleanup ipset and pool membership
+		err = poolWithMemberGroup.Delete()
+		check.Assert(err, IsNil)
+
+		err = ipSet.Delete()
+		check.Assert(err, IsNil)
+	}
+
 	// teardown prerequisites
 	tearDownAlbVirtualServicePrerequisites(check, albPool, seGroupAssignment, edge, seGroup, cloud, controller)
+
+	// cleanup Org user
+	err = orgUser.Delete(true)
+	check.Assert(err, IsNil)
 }
 
 func testMinimalVirtualServiceConfigHTTP(check *C, edge *NsxtEdgeGateway, pool *NsxtAlbPool, seGroup *NsxtAlbServiceEngineGroup, vcd *TestVCD, client *VCDClient) {
@@ -94,6 +124,63 @@ func testMinimalVirtualServiceConfigHTTP(check *C, edge *NsxtEdgeGateway, pool *
 	testAlbVirtualServiceConfig(check, vcd, "MinimalHTTP", virtualServiceConfig, virtualServiceConfigUpdated, client)
 }
 
+func testMinimalVirtualServiceConfigHTTPTransparent(check *C, edge *NsxtEdgeGateway, poolWithMemberGroup *NsxtAlbPool, seGroup *NsxtAlbServiceEngineGroup, vcd *TestVCD, client *VCDClient, trueOnCreate bool) {
+	createTransparentMode := trueOnCreate
+	updateTransparentMode := !createTransparentMode
+
+	virtualServiceConfig := &types.NsxtAlbVirtualService{
+		Name:                   check.TestName(),
+		Enabled:                addrOf(true),
+		TransparentModeEnabled: &createTransparentMode,
+		ApplicationProfile: types.NsxtAlbVirtualServiceApplicationProfile{
+			SystemDefined: true,
+			Type:          "HTTP",
+		},
+		GatewayRef:            types.OpenApiReference{ID: edge.EdgeGateway.ID},
+		LoadBalancerPoolRef:   types.OpenApiReference{ID: poolWithMemberGroup.NsxtAlbPool.ID},
+		ServiceEngineGroupRef: types.OpenApiReference{ID: seGroup.NsxtAlbServiceEngineGroup.ID},
+		ServicePorts: []types.NsxtAlbVirtualServicePort{
+			{
+				PortStart: addrOf(80),
+			},
+		},
+		VirtualIpAddress: edge.EdgeGateway.EdgeGatewayUplinks[0].Subnets.Values[0].PrimaryIP,
+	}
+
+	virtualServiceConfigUpdated := &types.NsxtAlbVirtualService{
+		Name:                   check.TestName(),
+		Description:            "Updated",
+		Enabled:                addrOf(true),
+		TransparentModeEnabled: &updateTransparentMode,
+		ApplicationProfile: types.NsxtAlbVirtualServiceApplicationProfile{
+			SystemDefined: true,
+			Type:          "HTTP",
+		},
+		GatewayRef:            types.OpenApiReference{ID: edge.EdgeGateway.ID},
+		LoadBalancerPoolRef:   types.OpenApiReference{ID: poolWithMemberGroup.NsxtAlbPool.ID},
+		ServiceEngineGroupRef: types.OpenApiReference{ID: seGroup.NsxtAlbServiceEngineGroup.ID},
+		ServicePorts: []types.NsxtAlbVirtualServicePort{
+			{
+				PortStart:  addrOf(443),
+				PortEnd:    addrOf(449),
+				SslEnabled: addrOf(false),
+			},
+			{
+				PortStart:  addrOf(2000),
+				PortEnd:    addrOf(2010),
+				SslEnabled: addrOf(false),
+			},
+		},
+		// Use Primary IP of Edge Gateway as virtual service IP
+		VirtualIpAddress: edge.EdgeGateway.EdgeGatewayUplinks[0].Subnets.Values[0].PrimaryIP,
+		//HealthStatus:          "",
+		//HealthMessage:         "",
+		//DetailedHealthMessage: "",
+	}
+
+	testAlbVirtualServiceConfig(check, vcd, fmt.Sprintf("MinimalHTTPWithTransparentModeOnCreate%t", createTransparentMode), virtualServiceConfig, virtualServiceConfigUpdated, client)
+}
+
 func testMinimalVirtualServiceConfigL4(check *C, edge *NsxtEdgeGateway, pool *NsxtAlbPool, seGroup *NsxtAlbServiceEngineGroup, vcd *TestVCD, client *VCDClient) {
 	virtualServiceConfig := &types.NsxtAlbVirtualService{
 		Name:    check.TestName(),
@@ -363,6 +450,41 @@ func setupAlbVirtualServicePrerequisites(check *C, vcd *TestVCD) (*NsxtAlbContro
 	return controller, cloud, seGroup, edge, assignedSeGroup, albPool
 }
 
+func setupAlbPoolFirewallGroupMembers(check *C, vcd *TestVCD, edge *NsxtEdgeGateway) (*NsxtFirewallGroup, *NsxtAlbPool) {
+	// creates ip set
+	ipSetConfig := &types.NsxtFirewallGroup{
+		Name:        check.TestName(),
+		OwnerRef:    &types.OpenApiReference{ID: edge.EdgeGateway.ID},
+		Description: "Test IP Set",
+		Type:        "IP_SET",
+		IpAddresses: []string{"1.1.1.1"},
+	}
+
+	ipSet, err := vcd.nsxtVdc.CreateNsxtFirewallGroup(ipSetConfig)
+	check.Assert(err, IsNil)
+
+	// add ip set to cleanup list
+	openApiEndpoint := types.OpenApiPathVersion1_0_0 + types.OpenApiEndpointFirewallGroups + ipSet.NsxtFirewallGroup.ID
+	PrependToCleanupListOpenApi(ipSet.NsxtFirewallGroup.Name, check.TestName(), openApiEndpoint)
+
+	poolConfig := &types.NsxtAlbPool{
+		Name:       check.TestName() + "-member-group",
+		Enabled:    takeBoolPointer(true),
+		GatewayRef: types.OpenApiReference{ID: edge.EdgeGateway.ID},
+		MemberGroupRef: &types.OpenApiReference{
+			ID: ipSet.NsxtFirewallGroup.ID,
+		},
+	}
+
+	albPool, err := vcd.client.CreateNsxtAlbPool(poolConfig)
+	check.Assert(err, IsNil)
+
+	openApiEndpoint = types.OpenApiPathVersion1_0_0 + types.OpenApiEndpointAlbPools + albPool.NsxtAlbPool.ID
+	PrependToCleanupListOpenApi(albPool.NsxtAlbPool.Name, check.TestName(), openApiEndpoint)
+
+	return ipSet, albPool
+}
+
 func tearDownAlbVirtualServicePrerequisites(check *C, albPool *NsxtAlbPool, assignment *NsxtAlbServiceEngineGroupAssignment, edge *NsxtEdgeGateway, seGroup *NsxtAlbServiceEngineGroup, cloud *NsxtAlbCloud, controller *NsxtAlbController) {
 	err := albPool.Delete()
 	check.Assert(err, IsNil)

govcd/nsxt_distributed_firewall_test.go

@@ -4,12 +4,13 @@ package govcd
 
 import (
 	"fmt"
-	"github.com/vmware/go-vcloud-director/v2/util"
 	"os"
 	"strconv"
 	"strings"
 	"text/tabwriter"
 
+	"github.com/vmware/go-vcloud-director/v2/util"
+
 	"github.com/vmware/go-vcloud-director/v2/types/v56"
 	. "gopkg.in/check.v1"
 )
@@ -41,7 +42,7 @@ func (vcd *TestVCD) Test_NsxtDistributedFirewallRules(check *C) {
 	// Prep Org admin user and run firewall tests
 	userName := strings.ToLower(check.TestName())
 	fmt.Printf("# Running Distributed Firewall tests as Org Admin user '%s'\n", userName)
-	orgUserVcdClient, err := newOrgUserConnection(adminOrg, userName, "CHANGE-ME", vcd.config.Provider.Url, true)
+	orgUserVcdClient, _, err := newOrgUserConnection(adminOrg, userName, "CHANGE-ME", vcd.config.Provider.Url, true)
 	check.Assert(err, IsNil)
 	orgUserOrgAdmin, err := orgUserVcdClient.GetAdminOrgById(adminOrg.AdminOrg.ID)
 	check.Assert(err, IsNil)

govcd/nsxt_edgegateway.go

@@ -126,6 +126,14 @@ func (vdcGroup *VdcGroup) GetNsxtEdgeGatewayByName(name string) (*NsxtEdgeGatewa
 	return returnSingleNsxtEdgeGateway(name, allEdges)
 }
 
+// GetAllNsxtEdgeGateways allows to retrieve all NSX-T Edge Gateways
+func (vcdClient *VCDClient) GetAllNsxtEdgeGateways(queryParameters url.Values) ([]*NsxtEdgeGateway, error) {
+	if vcdClient == nil {
+		return nil, fmt.Errorf("vcdClient is empty")
+	}
+	return getAllNsxtEdgeGateways(&vcdClient.Client, queryParameters)
+}
+
 // GetAllNsxtEdgeGateways allows to retrieve all NSX-T edge gateways for Org Admins
 func (adminOrg *AdminOrg) GetAllNsxtEdgeGateways(queryParameters url.Values) ([]*NsxtEdgeGateway, error) {
 	return getAllNsxtEdgeGateways(adminOrg.client, queryParameters)

govcd/sample_govcd_test_config.yaml

@@ -94,6 +94,16 @@ vcd:
       nsxtImportSegment: vcd-org-vdc-imported-network-backing
       # Existing NSX-T Edge Cluster name
       nsxtEdgeCluster: existing-nsxt-edge-cluster
+      # AVI Controller URL 
+      nsxtAlbControllerUrl: https://unknown-hostname.com
+      # AVI Controller username
+      nsxtAlbControllerUser: admin
+      # AVI Controller password
+      nsxtAlbControllerPassword: CHANGE-ME
+      # AVI Controller importable Cloud name 
+      nsxtAlbImportableCloud: NSXT AVI Cloud
+      # Service Engine Group name within (Should be configured in Active Standby mode)
+      nsxtAlbServiceEngineGroup: active-standby-service-engine-group
     # An Org catalog, possibly containing at least one item
     catalog:
         name: mycat

govcd/security_tags_test.go

@@ -95,7 +95,7 @@ func (vcd *TestVCD) Test_SecurityTags(check *C) {
 
 	userName := strings.ToLower(check.TestName())
 	fmt.Printf("# Running Get Security Tag Values test as Org Admin user '%s'\n", userName)
-	orgUserVcdClient, err := newOrgUserConnection(adminOrg, userName, "CHANGE-ME", vcd.config.Provider.Url, true)
+	orgUserVcdClient, _, err := newOrgUserConnection(adminOrg, userName, "CHANGE-ME", vcd.config.Provider.Url, true)
 	check.Assert(err, IsNil)
 	check.Assert(orgUserVcdClient, NotNil)
 

govcd/system_test.go

@@ -609,7 +609,7 @@ func (vcd *TestVCD) Test_QueryOrgVdcStorageProfileByID(check *C) {
 	adminOrg, err := vcd.client.GetAdminOrgByName(vcd.config.VCD.Org)
 	check.Assert(err, IsNil)
 
-	orgUserVcdClient, err := newOrgUserConnection(adminOrg, "query-org-vdc-storage-profile-by-id", "CHANGE-ME", vcd.config.Provider.Url, true)
+	orgUserVcdClient, _, err := newOrgUserConnection(adminOrg, "query-org-vdc-storage-profile-by-id", "CHANGE-ME", vcd.config.Provider.Url, true)
 	check.Assert(err, IsNil)
 
 	ref, err := vcd.vdc.FindStorageProfileReference(vcd.config.VCD.StorageProfile.SP1)

govcd/vdc_group_test.go

@@ -212,7 +212,7 @@ func (vcd *TestVCD) Test_NsxtVdcGroupWithOrgAdmin(check *C) {
 	check.Assert(adminOrg, NotNil)
 
 	skipIfNeededRightsMissing(check, adminOrg)
-	orgAdminClient, err := newOrgUserConnection(adminOrg, "test-user2", "CHANGE-ME", vcd.config.Provider.Url, true)
+	orgAdminClient, _, err := newOrgUserConnection(adminOrg, "test-user2", "CHANGE-ME", vcd.config.Provider.Url, true)
 	check.Assert(err, IsNil)
 	check.Assert(orgAdminClient, NotNil)