Skip to content

Commit

Permalink
Fix LB reconciliation to support multiple NSG (#260)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shyamradhakrishnan
shyamradhakrishnan committed May 9, 2023
1 parent f82ad97 commit e09e016
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 36 deletions.
8 changes: 3 additions & 5 deletions cloud/scope/load_balancer_reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,6 @@ func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.Lo
return nil, nil, errors.New("control plane endpoint subnet not provided")
}

if len(controlPlaneEndpointSubnets) > 1 {
return nil, nil, errors.New("cannot have more than 1 control plane endpoint subnet")
}
lbDetails := loadbalancer.CreateLoadBalancerDetails{
CompartmentId: common.String(s.GetCompartmentId()),
DisplayName: common.String(lb.Name),
Expand All @@ -182,14 +179,15 @@ func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.Lo
FreeformTags: s.GetFreeFormTags(),
DefinedTags: s.GetDefinedTags(),
}

nsgs := make([]string, 0)
for _, nsg := range s.OCIClusterAccessor.GetNetworkSpec().Vcn.NetworkSecurityGroup.List {
if nsg.Role == infrastructurev1beta2.ControlPlaneEndpointRole {
if nsg.ID != nil {
lbDetails.NetworkSecurityGroupIds = []string{*nsg.ID}
nsgs = append(nsgs, *nsg.ID)
}
}
}
lbDetails.NetworkSecurityGroupIds = nsgs

s.Logger.Info("Creating load balancer...")
lbResponse, err := s.LoadBalancerClient.CreateLoadBalancer(ctx, loadbalancer.CreateLoadBalancerRequest{
Expand Down
126 changes: 109 additions & 17 deletions cloud/scope/load_balancer_reconciler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,9 +217,8 @@ func TestLBReconciliation(t *testing.T) {
},
},
{
name: "more than one cp subnet",
errorExpected: true,
matchError: errors.New("cannot have more than 1 control plane endpoint subnet"),
name: "create load balancer more than one subnet",
errorExpected: false,
testSpecificSetup: func(clusterScope *ClusterScope, lbClient *mock_lb.MockLoadBalancerClient) {
clusterScope.OCIClusterAccessor.GetNetworkSpec().Vcn.Subnets = []*infrastructurev1beta2.Subnet{
{
Expand All @@ -231,11 +230,89 @@ func TestLBReconciliation(t *testing.T) {
ID: common.String("s2"),
},
}
clusterScope.OCIClusterAccessor.GetNetworkSpec().Vcn.NetworkSecurityGroup = infrastructurev1beta2.NetworkSecurityGroup{
List: []*infrastructurev1beta2.NSG{
{
Role: infrastructurev1beta2.ControlPlaneEndpointRole,
ID: common.String("nsg1"),
},
{
Role: infrastructurev1beta2.ControlPlaneEndpointRole,
ID: common.String("nsg2"),
},
},
}
definedTags, definedTagsInterface := getDefinedTags()
ociClusterAccessor.OCICluster.Spec.DefinedTags = definedTags
lbClient.EXPECT().ListLoadBalancers(gomock.Any(), gomock.Eq(loadbalancer.ListLoadBalancersRequest{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
})).Return(loadbalancer.ListLoadBalancersResponse{
Items: []loadbalancer.LoadBalancer{},
}, nil)
lbClient.EXPECT().CreateLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.CreateLoadBalancerRequest{
CreateLoadBalancerDetails: loadbalancer.CreateLoadBalancerDetails{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetIds: []string{"s1", "s2"},
NetworkSecurityGroupIds: []string{"nsg1", "nsg2"},
IsPrivate: common.Bool(false),
ShapeName: common.String("flexible"),
ShapeDetails: &loadbalancer.ShapeDetails{MaximumBandwidthInMbps: common.Int(100),
MinimumBandwidthInMbps: common.Int(10)},
Listeners: map[string]loadbalancer.ListenerDetails{
APIServerLBListener: {
Protocol: common.String("TCP"),
Port: common.Int(int(6443)),
DefaultBackendSetName: common.String(APIServerLBBackendSetName),
},
},
BackendSets: map[string]loadbalancer.BackendSetDetails{
APIServerLBBackendSetName: loadbalancer.BackendSetDetails{
Policy: common.String("ROUND_ROBIN"),

HealthChecker: &loadbalancer.HealthCheckerDetails{
Port: common.Int(6443),
Protocol: common.String("TCP"),
},
Backends: []loadbalancer.BackendDetails{},
},
},
FreeformTags: tags,
DefinedTags: definedTagsInterface,
},
OpcRetryToken: ociutil.GetOPCRetryToken("%s-%s", "create-lb", string("resource_uid")),
})).
Return(loadbalancer.ListLoadBalancersResponse{}, nil)
Return(loadbalancer.CreateLoadBalancerResponse{
OpcWorkRequestId: common.String("opc-wr-id"),
}, nil)
lbClient.EXPECT().GetWorkRequest(gomock.Any(), gomock.Eq(loadbalancer.GetWorkRequestRequest{
WorkRequestId: common.String("opc-wr-id"),
})).Return(loadbalancer.GetWorkRequestResponse{
WorkRequest: loadbalancer.WorkRequest{
LoadBalancerId: common.String("lb-id"),
LifecycleState: loadbalancer.WorkRequestLifecycleStateSucceeded,
},
}, nil)

lbClient.EXPECT().GetLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.GetLoadBalancerRequest{
LoadBalancerId: common.String("lb-id"),
})).
Return(loadbalancer.GetLoadBalancerResponse{
LoadBalancer: loadbalancer.LoadBalancer{
Id: common.String("lb-id"),
FreeformTags: tags,
IsPrivate: common.Bool(false),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
IpAddresses: []loadbalancer.IpAddress{
{
IpAddress: common.String("2.2.2.2"),
IsPublic: common.Bool(true),
},
},
},
}, nil)

},
},
{
Expand All @@ -248,6 +325,18 @@ func TestLBReconciliation(t *testing.T) {
ID: common.String("s1"),
},
}
clusterScope.OCIClusterAccessor.GetNetworkSpec().Vcn.NetworkSecurityGroup = infrastructurev1beta2.NetworkSecurityGroup{
List: []*infrastructurev1beta2.NSG{
{
Role: infrastructurev1beta2.ControlPlaneEndpointRole,
ID: common.String("nsg1"),
},
{
Role: infrastructurev1beta2.ControlPlaneEndpointRole,
ID: common.String("nsg2"),
},
},
}
definedTags, definedTagsInterface := getDefinedTags()
ociClusterAccessor.OCICluster.Spec.DefinedTags = definedTags
lbClient.EXPECT().ListLoadBalancers(gomock.Any(), gomock.Eq(loadbalancer.ListLoadBalancersRequest{
Expand All @@ -258,11 +347,12 @@ func TestLBReconciliation(t *testing.T) {
}, nil)
lbClient.EXPECT().CreateLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.CreateLoadBalancerRequest{
CreateLoadBalancerDetails: loadbalancer.CreateLoadBalancerDetails{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetIds: []string{"s1"},
IsPrivate: common.Bool(false),
ShapeName: common.String("flexible"),
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetIds: []string{"s1"},
NetworkSecurityGroupIds: []string{"nsg1", "nsg2"},
IsPrivate: common.Bool(false),
ShapeName: common.String("flexible"),
ShapeDetails: &loadbalancer.ShapeDetails{MaximumBandwidthInMbps: common.Int(100),
MinimumBandwidthInMbps: common.Int(10)},
Listeners: map[string]loadbalancer.ListenerDetails{
Expand Down Expand Up @@ -340,10 +430,11 @@ func TestLBReconciliation(t *testing.T) {
}, nil)
lbClient.EXPECT().CreateLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.CreateLoadBalancerRequest{
CreateLoadBalancerDetails: loadbalancer.CreateLoadBalancerDetails{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetIds: []string{"s1"},
ShapeName: common.String("flexible"),
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetIds: []string{"s1"},
NetworkSecurityGroupIds: make([]string, 0),
ShapeName: common.String("flexible"),
ShapeDetails: &loadbalancer.ShapeDetails{MaximumBandwidthInMbps: common.Int(100),
MinimumBandwidthInMbps: common.Int(10)},
IsPrivate: common.Bool(false),
Expand Down Expand Up @@ -391,10 +482,11 @@ func TestLBReconciliation(t *testing.T) {
Return(loadbalancer.ListLoadBalancersResponse{}, nil)
lbClient.EXPECT().CreateLoadBalancer(gomock.Any(), gomock.Eq(loadbalancer.CreateLoadBalancerRequest{
CreateLoadBalancerDetails: loadbalancer.CreateLoadBalancerDetails{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetIds: []string{"s1"},
ShapeName: common.String("flexible"),
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetIds: []string{"s1"},
NetworkSecurityGroupIds: make([]string, 0),
ShapeName: common.String("flexible"),
ShapeDetails: &loadbalancer.ShapeDetails{MaximumBandwidthInMbps: common.Int(100),
MinimumBandwidthInMbps: common.Int(10)},
IsPrivate: common.Bool(false),
Expand Down
5 changes: 3 additions & 2 deletions cloud/scope/network_load_balancer_reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,15 @@ func (s *ClusterScope) CreateNLB(ctx context.Context, lb infrastructurev1beta2.L
FreeformTags: s.GetFreeFormTags(),
DefinedTags: s.GetDefinedTags(),
}

nsgs := make([]string, 0)
for _, nsg := range s.OCIClusterAccessor.GetNetworkSpec().Vcn.NetworkSecurityGroup.List {
if nsg.Role == infrastructurev1beta2.ControlPlaneEndpointRole {
if nsg.ID != nil {
nlbDetails.NetworkSecurityGroupIds = []string{*nsg.ID}
nsgs = append(nsgs, *nsg.ID)
}
}
}
nlbDetails.NetworkSecurityGroupIds = nsgs

s.Logger.Info("Creating network load balancer")
nlbResponse, err := s.NetworkLoadBalancerClient.CreateNetworkLoadBalancer(ctx, networkloadbalancer.CreateNetworkLoadBalancerRequest{
Expand Down
39 changes: 27 additions & 12 deletions cloud/scope/network_load_balancer_reconciler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,18 @@ func TestNLBReconciliation(t *testing.T) {
ID: common.String("s1"),
},
}
clusterScope.OCIClusterAccessor.GetNetworkSpec().Vcn.NetworkSecurityGroup = infrastructurev1beta2.NetworkSecurityGroup{
List: []*infrastructurev1beta2.NSG{
{
Role: infrastructurev1beta2.ControlPlaneEndpointRole,
ID: common.String("nsg1"),
},
{
Role: infrastructurev1beta2.ControlPlaneEndpointRole,
ID: common.String("nsg2"),
},
},
}
definedTags, definedTagsInterface := getDefinedTags()
ociClusterAccessor.OCICluster.Spec.DefinedTags = definedTags
nlbClient.EXPECT().ListNetworkLoadBalancers(gomock.Any(), gomock.Eq(networkloadbalancer.ListNetworkLoadBalancersRequest{
Expand All @@ -259,10 +271,11 @@ func TestNLBReconciliation(t *testing.T) {
Return(networkloadbalancer.ListNetworkLoadBalancersResponse{}, nil)
nlbClient.EXPECT().CreateNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.CreateNetworkLoadBalancerRequest{
CreateNetworkLoadBalancerDetails: networkloadbalancer.CreateNetworkLoadBalancerDetails{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetId: common.String("s1"),
IsPrivate: common.Bool(false),
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetId: common.String("s1"),
IsPrivate: common.Bool(false),
NetworkSecurityGroupIds: []string{"nsg1", "nsg2"},
Listeners: map[string]networkloadbalancer.ListenerDetails{
APIServerLBListener: {
Protocol: networkloadbalancer.ListenerProtocolsTcp,
Expand Down Expand Up @@ -342,10 +355,11 @@ func TestNLBReconciliation(t *testing.T) {
Return(networkloadbalancer.ListNetworkLoadBalancersResponse{}, nil)
nlbClient.EXPECT().CreateNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.CreateNetworkLoadBalancerRequest{
CreateNetworkLoadBalancerDetails: networkloadbalancer.CreateNetworkLoadBalancerDetails{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetId: common.String("s1"),
IsPrivate: common.Bool(false),
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetId: common.String("s1"),
IsPrivate: common.Bool(false),
NetworkSecurityGroupIds: make([]string, 0),
Listeners: map[string]networkloadbalancer.ListenerDetails{
APIServerLBListener: {
Protocol: networkloadbalancer.ListenerProtocolsTcp,
Expand Down Expand Up @@ -394,10 +408,11 @@ func TestNLBReconciliation(t *testing.T) {
Return(networkloadbalancer.ListNetworkLoadBalancersResponse{}, nil)
nlbClient.EXPECT().CreateNetworkLoadBalancer(gomock.Any(), gomock.Eq(networkloadbalancer.CreateNetworkLoadBalancerRequest{
CreateNetworkLoadBalancerDetails: networkloadbalancer.CreateNetworkLoadBalancerDetails{
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetId: common.String("s1"),
IsPrivate: common.Bool(false),
CompartmentId: common.String("compartment-id"),
DisplayName: common.String(fmt.Sprintf("%s-%s", "cluster", "apiserver")),
SubnetId: common.String("s1"),
NetworkSecurityGroupIds: make([]string, 0),
IsPrivate: common.Bool(false),
Listeners: map[string]networkloadbalancer.ListenerDetails{
APIServerLBListener: {
Protocol: networkloadbalancer.ListenerProtocolsTcp,
Expand Down

0 comments on commit e09e016

Please sign in to comment.