feat: support multi region

.gitignore

@@ -11,6 +11,7 @@ bin
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+cover.html
 testbin/
 out/
 

Makefile

@@ -48,7 +48,7 @@ ARTIFACTS ?= $(ROOT_DIR)/_artifacts
 KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
 KUBETEST_FAST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance-fast.yaml)
 GINKGO_FOCUS ?= Workload cluster creation
-GINKGO_SKIP ?= "Bare Metal"
+GINKGO_SKIP ?= "Bare Metal|Multi-Region"
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
@@ -106,7 +106,8 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test: manifests generate fmt vet ## Run tests.
 	mkdir -p ${ENVTEST_ASSETS_DIR}
 	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
-	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out
+	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out && go tool cover -html=cover.out -o cover.html
+
 
 ##@ Build
 
@@ -241,6 +242,7 @@ serve-book: build-book ## Build and serve the book with live-reloading enabled
 .PHONY: generate-e2e-templates ## Generate OCI infrastructure templates for e2e test suite.
 generate-e2e-templates: kustomize
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template.yaml
+	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation.yaml
 	$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation.yaml
@@ -256,7 +258,7 @@ generate-e2e-templates: kustomize
 .PHONY: test-e2e-run
 test-e2e-run: generate-e2e-templates ginkgo $(ENVSUBST) ## Run e2e tests
 	envsubst < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
-    $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \
+    $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip=$(GINKGO_SKIP) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \
     	-e2e.artifacts-folder="$(ARTIFACTS)" \
     	-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
     	-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)

api/v1beta1/ocicluster_types.go

@@ -48,6 +48,10 @@ type OCIClusterSpec struct {
 	// Compartment to create the cluster network.
 	CompartmentId string `mandatory:"true" json:"compartmentId"`
 
+	// Region the cluster operates in. It must be one of available regions in Region Identifier format.
+	// See https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm
+	Region string `json:"region,omitempty"`
+
 	// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`

cloud/scope/clients.go

@@ -0,0 +1,154 @@
+/*
+Copyright (c) 2022, Oracle and/or its affiliates.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scope
+
+import (
+	"sync"
+
+	"github.com/go-logr/logr"
+	"github.com/oracle/cluster-api-provider-oci/cloud/services/compute"
+	identityClient "github.com/oracle/cluster-api-provider-oci/cloud/services/identity"
+	nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
+	"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
+	"github.com/oracle/oci-go-sdk/v63/common"
+	"github.com/oracle/oci-go-sdk/v63/core"
+	"github.com/oracle/oci-go-sdk/v63/identity"
+	"github.com/oracle/oci-go-sdk/v63/networkloadbalancer"
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2/klogr"
+)
+
+// OCIClients is the struct of all the needed OCI clients
+type OCIClients struct {
+	ComputeClient      compute.ComputeClient
+	VCNClient          vcn.Client
+	LoadBalancerClient nlb.NetworkLoadBalancerClient
+	IdentityClient     identityClient.Client
+}
+
+// ClientProvider defines the regional clients
+type ClientProvider struct {
+	*logr.Logger
+	ociClients            map[string]OCIClients
+	ociClientsLock        *sync.RWMutex
+	ociAuthConfigProvider common.ConfigurationProvider
+}
+
+// NewClientProvider builds the ClientProvider with a client for the given region
+func NewClientProvider(ociAuthConfigProvider common.ConfigurationProvider) (*ClientProvider, error) {
+	log := klogr.New()
+
+	if ociAuthConfigProvider == nil {
+		return nil, errors.New("ConfigurationProvider can not be nil")
+	}
+
+	provider := ClientProvider{
+		Logger:                &log,
+		ociAuthConfigProvider: ociAuthConfigProvider,
+		ociClients:            map[string]OCIClients{},
+		ociClientsLock:        new(sync.RWMutex),
+	}
+
+	return &provider, nil
+}
+
+// GetOrBuildClient if the OCIClients exist for the region they are returned, if not clients will build them
+func (c *ClientProvider) GetOrBuildClient(region string) (OCIClients, error) {
+	if len(region) <= 0 {
+		return OCIClients{}, errors.New("ClientProvider.GetOrBuildClient region can not be empty")
+	}
+
+	c.ociClientsLock.RLock()
+	clients, regionalClientsExists := c.ociClients[region]
+	c.ociClientsLock.RUnlock()
+
+	if regionalClientsExists {
+		return clients, nil
+	}
+
+	c.ociClientsLock.Lock()
+	defer c.ociClientsLock.Unlock()
+	regionalClient, err := createClients(region, c.ociAuthConfigProvider, c.Logger)
+	if err != nil {
+		return regionalClient, err
+	}
+	c.ociClients[region] = regionalClient
+
+	return regionalClient, nil
+}
+
+func createClients(region string, oCIAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (OCIClients, error) {
+	vcnClient, err := createVncClient(region, oCIAuthConfigProvider, logger)
+	lbClient, err := createLbClient(region, oCIAuthConfigProvider, logger)
+	identityClient, err := createIdentityClient(region, oCIAuthConfigProvider, logger)
+	computeClient, err := createComputeClient(region, oCIAuthConfigProvider, logger)
+
+	if err != nil {
+		return OCIClients{}, err
+	}
+
+	return OCIClients{
+		VCNClient:          vcnClient,
+		LoadBalancerClient: lbClient,
+		IdentityClient:     identityClient,
+		ComputeClient:      computeClient,
+	}, err
+}
+
+func createVncClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.VirtualNetworkClient, error) {
+	vcnClient, err := core.NewVirtualNetworkClientWithConfigurationProvider(ociAuthConfigProvider)
+	if err != nil {
+		logger.Error(err, "unable to create OCI VCN Client")
+		return nil, err
+	}
+	vcnClient.SetRegion(region)
+
+	return &vcnClient, nil
+}
+
+func createLbClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*networkloadbalancer.NetworkLoadBalancerClient, error) {
+	lbClient, err := networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider)
+	if err != nil {
+		logger.Error(err, "unable to create OCI LB Client")
+		return nil, err
+	}
+	lbClient.SetRegion(region)
+
+	return &lbClient, nil
+}
+
+func createIdentityClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*identity.IdentityClient, error) {
+	identityClient, err := identity.NewIdentityClientWithConfigurationProvider(ociAuthConfigProvider)
+	if err != nil {
+		logger.Error(err, "unable to create OCI Identity Client")
+		return nil, err
+	}
+	identityClient.SetRegion(region)
+
+	return &identityClient, nil
+}
+
+func createComputeClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.ComputeClient, error) {
+	computeClient, err := core.NewComputeClientWithConfigurationProvider(ociAuthConfigProvider)
+	if err != nil {
+		logger.Error(err, "unable to create OCI Compute Client")
+		return nil, err
+	}
+	computeClient.SetRegion(region)
+
+	return &computeClient, nil
+}

cloud/scope/clients_mock.go

@@ -0,0 +1,116 @@
+/*
+Copyright (c) 2022, Oracle and/or its affiliates.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scope
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"sync"
+
+	"github.com/oracle/cluster-api-provider-oci/cloud/config"
+	"github.com/oracle/cluster-api-provider-oci/cloud/services/compute"
+	"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
+	"github.com/oracle/oci-go-sdk/v63/identity"
+	"github.com/oracle/oci-go-sdk/v63/networkloadbalancer"
+	"k8s.io/klog/v2/klogr"
+)
+
+type MockOCIClients struct {
+	VCNClient          vcn.Client
+	ComputeClient      compute.ComputeClient
+	LoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient
+	IdentityClient     *identity.IdentityClient
+}
+
+var (
+	MockTestRegion = "us-lexington-1"
+)
+
+func MockNewClientProvider(mockClients MockOCIClients) (*ClientProvider, error) {
+
+	clientsInject := map[string]OCIClients{MockTestRegion: {
+		VCNClient:          mockClients.VCNClient,
+		LoadBalancerClient: mockClients.LoadBalancerClient,
+		IdentityClient:     mockClients.IdentityClient,
+		ComputeClient:      mockClients.ComputeClient,
+	}}
+
+	authConfig, err := MockAuthConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	ociAuthConfigProvider, err := config.NewConfigurationProvider(&authConfig)
+	if err != nil {
+		fmt.Printf("expected ociAuthConfigProvider to be created %s \n", err)
+		return nil, err
+	}
+	log := klogr.New()
+	clientProvider := ClientProvider{
+		Logger:                &log,
+		ociClients:            clientsInject,
+		ociClientsLock:        new(sync.RWMutex),
+		ociAuthConfigProvider: ociAuthConfigProvider,
+	}
+
+	return &clientProvider, nil
+}
+
+func MockAuthConfig() (config.AuthConfig, error) {
+	privateKey, err := generatePrivateKeyPEM()
+	if err != nil {
+		fmt.Println("error generating a private key")
+		return config.AuthConfig{}, err
+	}
+
+	authConfig := config.AuthConfig{
+		UseInstancePrincipals: false,
+		Region:                MockTestRegion,
+		Fingerprint:           "mock-finger-print",
+		PrivateKey:            privateKey,
+		UserID:                "ocid1.tenancy.oc1..<unique_ID>",
+		TenancyID:             "ocid1.tenancy.oc1..<unique_ID>",
+	}
+
+	return authConfig, nil
+}
+
+func generatePrivateKeyPEM() (string, error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		return "", err
+	}
+
+	privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey)
+	privateKeyBlock := &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: privateKeyBytes,
+	}
+
+	var privateKeyBuf bytes.Buffer
+	err = pem.Encode(&privateKeyBuf, privateKeyBlock)
+	if err != nil {
+		fmt.Printf("error when encode private pem: %s \n", err)
+		return "", err
+	}
+
+	return privateKeyBuf.String(), err
+}