Feature: PowerVS cluster creation with dynamic resource creation (#1608)

* api changes to support infra creation

* controller changes to support infra creation

* scope changes to create infra creation

* package files changes to support infra creation

* docs changes to support infra creation

* go module changes

* Fix test failures

Signed-off-by: Prajyot-Parab <prajyot.parab2@ibm.com>

---------

Signed-off-by: Prajyot-Parab <prajyot.parab2@ibm.com>

api/v1beta1/ibmpowervs_conversion.go

@@ -195,7 +195,7 @@ func Convert_v1beta2_IBMPowerVSMachineSpec_To_v1beta1_IBMPowerVSMachineSpec(in *
 }
 
 func Convert_v1beta2_IBMPowerVSClusterSpec_To_v1beta1_IBMPowerVSClusterSpec(in *infrav1beta2.IBMPowerVSClusterSpec, out *IBMPowerVSClusterSpec, s apiconversion.Scope) error {
-	if in.ServiceInstance.ID != nil {
+	if in.ServiceInstance != nil && in.ServiceInstance.ID != nil {
 		out.ServiceInstanceID = *in.ServiceInstance.ID
 	}
 	return autoConvert_v1beta2_IBMPowerVSClusterSpec_To_v1beta1_IBMPowerVSClusterSpec(in, out, s)

api/v1beta2/conditions_consts.go

@@ -75,6 +75,44 @@ const (
 )
 
 const (
-	// LoadBalancerReadyCondition reports on current status of the load balancer. Ready indicates the load balancer is in a active state.
+	// ServiceInstanceReadyCondition reports on the successful reconciliation of a Power VS workspace.
+	ServiceInstanceReadyCondition capiv1beta1.ConditionType = "ServiceInstanceReady"
+	// ServiceInstanceReconciliationFailedReason used when an error occurs during workspace reconciliation.
+	ServiceInstanceReconciliationFailedReason = "ServiceInstanceReconciliationFailed"
+
+	// NetworkReadyCondition reports on the successful reconciliation of a Power VS network.
+	NetworkReadyCondition capiv1beta1.ConditionType = "NetworkReady"
+	// NetworkReconciliationFailedReason used when an error occurs during network reconciliation.
+	NetworkReconciliationFailedReason = "NetworkReconciliationFailed"
+
+	// VPCReadyCondition reports on the successful reconciliation of a VPC.
+	VPCReadyCondition capiv1beta1.ConditionType = "VPCReady"
+	// VPCReconciliationFailedReason used when an error occurs during VPC reconciliation.
+	VPCReconciliationFailedReason = "VPCReconciliationFailed"
+
+	// VPCSubnetReadyCondition reports on the successful reconciliation of a VPC subnet.
+	VPCSubnetReadyCondition capiv1beta1.ConditionType = "VPCSubnetReady"
+	// VPCSubnetReconciliationFailedReason used when an error occurs during VPC subnet reconciliation.
+	VPCSubnetReconciliationFailedReason = "VPCSubnetReconciliationFailed"
+
+	// TransitGatewayReadyCondition reports on the successful reconciliation of a Power VS transit gateway.
+	TransitGatewayReadyCondition capiv1beta1.ConditionType = "TransitGatewayReady"
+	// TransitGatewayReconciliationFailedReason used when an error occurs during transit gateway reconciliation.
+	TransitGatewayReconciliationFailedReason = "TransitGatewayReconciliationFailed"
+
+	// LoadBalancerReadyCondition reports on the successful reconciliation of a Power VS network.
 	LoadBalancerReadyCondition capiv1beta1.ConditionType = "LoadBalancerReady"
+	// LoadBalancerReconciliationFailedReason used when an error occurs during loadbalancer reconciliation.
+	LoadBalancerReconciliationFailedReason = "LoadBalancerReconciliationFailed"
+
+	// COSInstanceReadyCondition reports on the successful reconciliation of a COS instance.
+	COSInstanceReadyCondition capiv1beta1.ConditionType = "COSInstanceCreated"
+	// COSInstanceReconciliationFailedReason used when an error occurs during COS instance reconciliation.
+	COSInstanceReconciliationFailedReason = "COSInstanceCreationFailed"
+)
+
+const (
+	// CreateInfrastructureAnnotation is the name of an annotation that indicates if
+	// Power VS infrastructure should be created as a part of cluster creation.
+	CreateInfrastructureAnnotation = "powervs.cluster.x-k8s.io/create-infra"
 )

api/v1beta2/ibmpowervscluster_types.go

@@ -37,9 +37,21 @@ type IBMPowerVSClusterSpec struct {
 	ServiceInstanceID string `json:"serviceInstanceID"`
 
 	// Network is the reference to the Network to use for this cluster.
-	// when the field is omitted, A DHCP service will be created in the Power VS server workspace and its private network will be used.
+	// when the field is omitted, A DHCP service will be created in the Power VS workspace and its private network will be used.
+	// the DHCP service created network will have the following name format
+	// 1. in the case of DHCPServer.Name is not set the name will be DHCPSERVER<CLUSTER_NAME>_Private.
+	// 2. if DHCPServer.Name is set the name will be DHCPSERVER<DHCPServer.Name>_Private.
+	// when Network.ID is set, its expected that there exist a network in PowerVS workspace with id or else system will give error.
+	// when Network.Name is set, system will first check for network with Name in PowerVS workspace, if not exist network will be created by DHCP service.
+	// Network.RegEx is not yet supported and system will ignore the value.
 	Network IBMPowerVSResourceReference `json:"network"`
 
+	// dhcpServer is contains the configuration to be used while creating a new DHCP server in PowerVS workspace.
+	// when the field is omitted, CLUSTER_NAME will be used as DHCPServer.Name and DHCP server will be created.
+	// it will automatically create network with name DHCPSERVER<DHCPServer.Name>_Private in PowerVS workspace.
+	// +optional
+	DHCPServer *DHCPServer `json:"dhcpServer,omitempty"`
+
 	// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
 	// +optional
 	ControlPlaneEndpoint capiv1beta1.APIEndpoint `json:"controlPlaneEndpoint"`
@@ -50,48 +62,108 @@ type IBMPowerVSClusterSpec struct {
 	// supported serviceInstance identifier in PowerVSResource are Name and ID and that can be obtained from IBM Cloud UI or IBM Cloud cli.
 	// More detail about Power VS service instance.
 	// https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-creating-power-virtual-server
-	// when omitted system will dynamically create the service instance
+	// when omitted system will dynamically create the service instance with name CLUSTER_NAME-serviceInstance.
+	// when ServiceInstance.ID is set, its expected that there exist a service instance in PowerVS workspace with id or else system will give error.
+	// when ServiceInstance.Name is set, system will first check for service instance with Name in PowerVS workspace, if not exist system will create new instance.
+	// ServiceInstance.Regex is not yet supported not yet supported and system will ignore the value.
 	// +optional
 	ServiceInstance *IBMPowerVSResourceReference `json:"serviceInstance,omitempty"`
 
 	// zone is the name of Power VS zone where the cluster will be created
 	// possible values can be found here https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-creating-power-virtual-server.
-	// when omitted syd04 will be set as default zone.
-	// +kubebuilder:default=dal10
+	// when powervs.cluster.x-k8s.io/create-infra=true annotation is set on IBMPowerVSCluster resource,
+	// 1. it is expected to set the zone, not setting will result in webhook error.
+	// 2. the zone should have PER capabilities, or else system will give error.
 	// +optional
 	Zone *string `json:"zone,omitempty"`
 
 	// resourceGroup name under which the resources will be created.
-	// when omitted default resource group of the account will be used.
+	// when powervs.cluster.x-k8s.io/create-infra=true annotation is set on IBMPowerVSCluster resource,
+	// 1. it is expected to set the ResourceGroup.Name, not setting will result in webhook error.
+	// ServiceInstance.ID and ServiceInstance.Regex is not yet supported and system will ignore the value.
 	// +optional
-	ResourceGroup *string `json:"resourceGroup,omitempty"`
+	ResourceGroup *IBMPowerVSResourceReference `json:"resourceGroup,omitempty"`
 
 	// vpc contains information about IBM Cloud VPC resources.
+	// when omitted system will dynamically create the VPC with name CLUSTER_NAME-vpc.
+	// when VPC.ID is set, its expected that there exist a VPC with ID or else system will give error.
+	// when VPC.Name is set, system will first check for VPC with Name, if not exist system will create new VPC.
+	// when powervs.cluster.x-k8s.io/create-infra=true annotation is set on IBMPowerVSCluster resource,
+	// 1. it is expected to set the VPC.Region, not setting will result in webhook error.
 	// +optional
 	VPC *VPCResourceReference `json:"vpc,omitempty"`
 
 	// vpcSubnets contains information about IBM Cloud VPC Subnet resources.
+	// when omitted system will create the subnets in all the zone corresponding to VPC.Region, with name CLUSTER_NAME-vpcsubnet-ZONE_NAME.
+	// possible values can be found here https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-creating-power-virtual-server.
+	// when VPCSubnets[].ID is set, its expected that there exist a subnet with ID or else system will give error.
+	// when VPCSubnets[].Zone is not set, a random zone is picked from available zones of VPC.Region.
+	// when VPCSubnets[].Name is not set, system will set name as CLUSTER_NAME-vpcsubnet-INDEX.
+	// if subnet with name VPCSubnets[].Name not found, system will create new subnet in VPCSubnets[].Zone.
 	// +optional
 	VPCSubnets []Subnet `json:"vpcSubnets,omitempty"`
 
 	// transitGateway contains information about IBM Cloud TransitGateway
 	// IBM Cloud TransitGateway helps in establishing network connectivity between IBM Cloud Power VS and VPC infrastructure
 	// more information about TransitGateway can be found here https://www.ibm.com/products/transit-gateway.
+	// when TransitGateway.ID is set, its expected that there exist a TransitGateway with ID or else system will give error.
+	// when TransitGateway.Name is set, system will first check for TransitGateway with Name, if not exist system will create new TransitGateway.
 	// +optional
 	TransitGateway *TransitGateway `json:"transitGateway,omitempty"`
 
-	// loadBalancers is optional configuration for configuring loadbalancers to control plane or data plane nodes
+	// loadBalancers is optional configuration for configuring loadbalancers to control plane or data plane nodes.
+	// when omitted system will create a public loadbalancer with name CLUSTER_NAME-loadbalancer.
 	// when specified a vpc loadbalancer will be created and controlPlaneEndpoint will be set with associated hostname of loadbalancer.
-	// when omitted user is expected to set controlPlaneEndpoint.
+	// ControlPlaneEndpoint will be set with associated hostname of public loadbalancer.
+	// when LoadBalancers[].ID is set, its expected that there exist a loadbalancer with ID or else system will give error.
+	// when LoadBalancers[].Name is set, system will first check for loadbalancer with Name, if not exist system will create new loadbalancer.
 	// +optional
 	LoadBalancers []VPCLoadBalancerSpec `json:"loadBalancers,omitempty"`
 
 	// cosInstance contains options to configure a supporting IBM Cloud COS bucket for this
 	// cluster - currently used for nodes requiring Ignition
 	// (https://coreos.github.io/ignition/) for bootstrapping (requires
 	// BootstrapFormatIgnition feature flag to be enabled).
+	// when powervs.cluster.x-k8s.io/create-infra=true annotation is set on IBMPowerVSCluster resource and Ignition is set, then
+	// 1. CosInstance.Name should be set not setting will result in webhook error.
+	// 2. CosInstance.BucketName should be set not setting will result in webhook error.
+	// 3. CosInstance.BucketRegion should be set not setting will result in webhook error.
 	// +optional
 	CosInstance *CosInstance `json:"cosInstance,omitempty"`
+
+	// Ignition defined options related to the bootstrapping systems where Ignition is used.
+	// +optional
+	Ignition *Ignition `json:"ignition,omitempty"`
+}
+
+// Ignition defines options related to the bootstrapping systems where Ignition is used.
+type Ignition struct {
+	// Version defines which version of Ignition will be used to generate bootstrap data.
+	//
+	// +optional
+	// +kubebuilder:default="2.3"
+	// +kubebuilder:validation:Enum="2.3";"2.4";"3.0";"3.1";"3.2";"3.3";"3.4"
+	Version string `json:"version,omitempty"`
+}
+
+// DHCPServer contains the DHCP server configurations.
+type DHCPServer struct {
+	// Optional cidr for DHCP private network
+	Cidr *string `json:"cidr,omitempty"`
+
+	// Optional DNS Server for DHCP service
+	// +kubebuilder:default="1.1.1.1"
+	DNSServer *string `json:"dnsServer,omitempty"`
+
+	// Optional name of DHCP Service. Only alphanumeric characters and dashes are allowed.
+	Name *string `json:"name,omitempty"`
+
+	// Optional id of the existing DHCPServer
+	ID *string `json:"id,omitempty"`
+
+	// Optional indicates if SNAT will be enabled for DHCP service
+	// +kubebuilder:default=true
+	Snat *bool `json:"snat,omitempty"`
 }
 
 // ResourceReference identifies a resource with id.
@@ -109,6 +181,9 @@ type IBMPowerVSClusterStatus struct {
 	// +kubebuilder:default=false
 	Ready bool `json:"ready"`
 
+	// ResourceGroup is the reference to the Power VS resource group under which the resources will be created.
+	ResourceGroup *ResourceReference `json:"resourceGroupID,omitempty"`
+
 	// serviceInstance is the reference to the Power VS service on which the server instance(VM) will be created.
 	ServiceInstance *ResourceReference `json:"serviceInstance,omitempty"`
 
@@ -166,40 +241,38 @@ type IBMPowerVSClusterList struct {
 
 // TransitGateway holds the TransitGateway information.
 type TransitGateway struct {
+	// name of resource.
+	// +optional
 	Name *string `json:"name,omitempty"`
-	ID   *string `json:"id,omitempty"`
+	// id of resource.
+	// +optional
+	ID *string `json:"id,omitempty"`
 }
 
 // VPCResourceReference is a reference to a specific VPC resource by ID or Name
 // Only one of ID or Name may be specified. Specifying more than one will result in
 // a validation error.
 type VPCResourceReference struct {
-	// ID of resource
+	// id of resource.
 	// +kubebuilder:validation:MinLength=1
 	// +optional
 	ID *string `json:"id,omitempty"`
 
-	// Name of resource
+	// name of resource.
 	// +kubebuilder:validation:MinLength=1
 	// +optional
 	Name *string `json:"name,omitempty"`
 
-	// IBM Cloud VPC region
+	// region of IBM Cloud VPC.
+	// when powervs.cluster.x-k8s.io/create-infra=true annotation is set on IBMPowerVSCluster resource,
+	// it is expected to set the region, not setting will result in webhook error.
 	Region *string `json:"region,omitempty"`
 }
 
 // CosInstance represents IBM Cloud COS instance.
 type CosInstance struct {
-	// PresignedURLDuration defines the duration for which presigned URLs are valid.
-	//
-	// This is used to generate presigned URLs for S3 Bucket objects, which are used by
-	// control-plane and worker nodes to fetch bootstrap data.
-	//
-	// When enabled, the IAM instance profiles specified are not used.
-	// +optional
-	PresignedURLDuration *metav1.Duration `json:"presignedURLDuration,omitempty"`
-
-	// Name defines name of IBM cloud COS instance to be created.
+	// name defines name of IBM cloud COS instance to be created.
+	// when IBMPowerVSCluster.Ignition is set
 	// +kubebuilder:validation:MinLength:=3
 	// +kubebuilder:validation:MaxLength:=63
 	// +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`
@@ -222,6 +295,14 @@ func (r *IBMPowerVSCluster) SetConditions(conditions capiv1beta1.Conditions) {
 	r.Status.Conditions = conditions
 }
 
+// Set sets the details of the resource.
+func (rf *ResourceReference) Set(resource ResourceReference) {
+	rf.ID = resource.ID
+	if !*rf.ControllerCreated {
+		rf.ControllerCreated = resource.ControllerCreated
+	}
+}
+
 func init() {
 	SchemeBuilder.Register(&IBMPowerVSCluster{}, &IBMPowerVSClusterList{})
 }

api/v1beta2/ibmpowervscluster_webhook.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1beta2
 
 import (
+	"strconv"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -74,6 +76,11 @@ func (r *IBMPowerVSCluster) validateIBMPowerVSCluster() (admission.Warnings, err
 	if err := r.validateIBMPowerVSClusterNetwork(); err != nil {
 		allErrs = append(allErrs, err)
 	}
+
+	if err := r.validateIBMPowerVSClusterCreateInfraPrereq(); err != nil {
+		allErrs = append(allErrs, err)
+	}
+
 	if len(allErrs) == 0 {
 		return nil, nil
 	}
@@ -89,3 +96,56 @@ func (r *IBMPowerVSCluster) validateIBMPowerVSClusterNetwork() *field.Error {
 	}
 	return nil
 }
+
+func (r *IBMPowerVSCluster) validateIBMPowerVSClusterCreateInfraPrereq() *field.Error {
+	annotations := r.GetAnnotations()
+	if len(annotations) == 0 {
+		return nil
+	}
+
+	value, found := annotations[CreateInfrastructureAnnotation]
+	if !found {
+		return nil
+	}
+
+	createInfra, err := strconv.ParseBool(value)
+	if err != nil {
+		return field.Invalid(field.NewPath("annotations"), r.Annotations, "value of powervs.cluster.x-k8s.io/create-infra should be boolean")
+	}
+
+	if !createInfra {
+		return nil
+	}
+
+	if r.Spec.Zone == nil {
+		return field.Invalid(field.NewPath("spec.zone"), r.Spec.Zone, "value of zone is empty")
+	}
+
+	if r.Spec.VPC.Region == nil {
+		return field.Invalid(field.NewPath("spec.vpc.region"), r.Spec.VPC.Region, "value of VPC region is empty")
+	}
+
+	if r.Spec.ResourceGroup == nil {
+		return field.Invalid(field.NewPath("spec.resourceGroup"), r.Spec.ResourceGroup, "value of resource group is empty")
+	}
+
+	if r.Spec.Ignition == nil {
+		return nil
+	}
+
+	// TODO(Phase 1): If ignition is set and these resources are not set, auto create them.
+	// If ignition is set, make sure to check that CosInstanceName, BucketName and region is set
+	if r.Spec.CosInstance == nil {
+		return field.Invalid(field.NewPath("spec.cosInstance"), r.Spec.CosInstance, "ignition is set but value of cosInstance is empty")
+	}
+	if r.Spec.CosInstance.Name == "" {
+		return field.Invalid(field.NewPath("spec.cosInstance.name"), r.Spec.CosInstance, "ignition is set but value of cosInstance name is empty")
+	}
+	if r.Spec.CosInstance.BucketName == "" {
+		return field.Invalid(field.NewPath("spec.cosInstance.bucketName"), r.Spec.CosInstance, "ignition is set but value of bucketName is empty")
+	}
+	if r.Spec.CosInstance.BucketRegion == "" {
+		return field.Invalid(field.NewPath("spec.cosInstance.bucketRegion"), r.Spec.CosInstance, "ignition is set but value of bucketRegion is empty")
+	}
+	return nil
+}