From 209b7490ab8936106590201315c82ac62a1cc51b Mon Sep 17 00:00:00 2001 From: Joe Kratzat Date: Tue, 8 Mar 2022 16:04:42 -0500 Subject: [PATCH] feat: add support for clusters in multiple regions Adding `ClientProvider` allows clusters to use different `OCIClients` groups to interact with the regional APIs. --- .gitignore | 1 + Makefile | 8 +- api/v1beta1/ocicluster_types.go | 4 + cloud/ociutil/ociutil.go | 29 + cloud/ociutil/ociutil_test.go | 101 +++ cloud/scope/clients.go | 169 ++++ cloud/scope/clients_mock.go | 117 +++ cloud/scope/clients_test.go | 117 +++ cloud/scope/cluster.go | 20 +- cloud/scope/vcn_reconciler.go | 1 + .../mock_nlb/client_mock.go | 2 +- ...tructure.cluster.x-k8s.io_ociclusters.yaml | 4 + controllers/ocicluster_controller.go | 31 +- controllers/ocicluster_controller_test.go | 6 + controllers/ocimachine_controller.go | 31 +- controllers/ocimachine_controller_test.go | 43 +- docs/src/gs/create-workload-cluster.md | 22 +- main.go | 65 +- scripts/ci-e2e.sh | 3 + test/e2e/cluster_test.go | 29 +- test/e2e/config/e2e_conf.yaml | 1 + .../cluster-template-alternative-region.yaml | 790 ++++++++++++++++++ .../cluster.yaml | 17 + .../kustomization.yaml | 9 + .../md.yaml | 9 + test/e2e/e2e_suite_test.go | 43 +- 26 files changed, 1550 insertions(+), 122 deletions(-) create mode 100644 cloud/scope/clients.go create mode 100644 cloud/scope/clients_mock.go create mode 100644 cloud/scope/clients_test.go create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/cluster.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/md.yaml diff --git a/.gitignore b/.gitignore index 5b304658d..1f86a0fbc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ bin # Output of the go coverage tool, specifically when used with LiteIDE *.out +cover.html testbin/ out/ diff --git a/Makefile b/Makefile index afbbc526c..af4b77b8c 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ ARTIFACTS ?= $(ROOT_DIR)/_artifacts KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml) KUBETEST_FAST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance-fast.yaml) GINKGO_FOCUS ?= Workload cluster creation -GINKGO_SKIP ?= "Bare Metal" +GINKGO_SKIP ?= "Bare Metal|Multi-Region" # Image URL to use all building/pushing image targets IMG ?= controller:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) @@ -106,7 +106,8 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin test: manifests generate fmt vet ## Run tests. mkdir -p ${ENVTEST_ASSETS_DIR} test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh - source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out + source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out; go tool cover -html=cover.out -o cover.html + ##@ Build @@ -241,6 +242,7 @@ serve-book: build-book ## Build and serve the book with live-reloading enabled .PHONY: generate-e2e-templates ## Generate OCI infrastructure templates for e2e test suite. generate-e2e-templates: kustomize $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template.yaml + $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation.yaml $(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation.yaml @@ -256,7 +258,7 @@ generate-e2e-templates: kustomize .PHONY: test-e2e-run test-e2e-run: generate-e2e-templates ginkgo $(ENVSUBST) ## Run e2e tests envsubst < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ - $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \ + $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip=$(GINKGO_SKIP) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) diff --git a/api/v1beta1/ocicluster_types.go b/api/v1beta1/ocicluster_types.go index d9d8ee2aa..dcf74d182 100644 --- a/api/v1beta1/ocicluster_types.go +++ b/api/v1beta1/ocicluster_types.go @@ -48,6 +48,10 @@ type OCIClusterSpec struct { // Compartment to create the cluster network. CompartmentId string `mandatory:"true" json:"compartmentId"` + // Region the cluster operates in. It must be one of available regions in Region Identifier format. + // See https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm + Region string `json:"region,omitempty"` + // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. // +optional ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` diff --git a/cloud/ociutil/ociutil.go b/cloud/ociutil/ociutil.go index 02a874d12..b992cbee5 100644 --- a/cloud/ociutil/ociutil.go +++ b/cloud/ociutil/ociutil.go @@ -19,6 +19,7 @@ package ociutil import ( "context" "fmt" + "github.com/oracle/cluster-api-provider-oci/cloud/config" nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer" "net/http" "time" @@ -112,3 +113,31 @@ func BuildClusterTags(clusterUUID string) map[string]string { tags["ClusterUUID"] = clusterUUID return tags } + +func ValidateAuthConfig(authConfig *config.AuthConfig) error { + if authConfig == nil { + return errors.New("authConfig can't be nil") + } + + if len(authConfig.Region) <= 0 { + return errors.New("authConfig.region can not be empty") + } + + if len(authConfig.TenancyID) <= 0 { + return errors.New("authConfig.tenancyId can not be empty") + } + + if len(authConfig.UserID) <= 0 { + return errors.New("authConfig.userId can not be empty") + } + + if len(authConfig.PrivateKey) <= 0 { + return errors.New("authConfig.privateKey can not be empty") + } + + if len(authConfig.Fingerprint) <= 0 { + return errors.New("authConfig.fingerprint can not be empty") + } + + return nil +} diff --git a/cloud/ociutil/ociutil_test.go b/cloud/ociutil/ociutil_test.go index b3cd14b45..84df071e8 100644 --- a/cloud/ociutil/ociutil_test.go +++ b/cloud/ociutil/ociutil_test.go @@ -21,7 +21,9 @@ import ( "reflect" "testing" + "github.com/oracle/cluster-api-provider-oci/cloud/config" "github.com/oracle/oci-go-sdk/v63/core" + "github.com/pkg/errors" ) func TestGetCloudProviderConfig(t *testing.T) { @@ -83,3 +85,102 @@ func TestAddToDefaultClusterTags(t *testing.T) { } } } + +func TestValidateAuthConfig(t *testing.T) { + const valid string = "valid" + const invalid string = "invalid" + testCases := []struct { + name string + authConfigIn *config.AuthConfig + expected string + expectedErr error + }{ + { + name: "AuthConfig is nil", + authConfigIn: nil, + expected: invalid, + expectedErr: errors.New("authConfig can't be nil"), + }, + { + name: "AuthConfig is missing region", + authConfigIn: &config.AuthConfig{ + TenancyID: "test.tenant", + UserID: "test.user", + PrivateKey: "private-key", + Fingerprint: "00:00:00:00", + }, + expected: invalid, + expectedErr: errors.New("authConfig.region can not be empty"), + }, + { + name: "AuthConfig is missing Tenancy", + authConfigIn: &config.AuthConfig{ + Region: "us-bozeman-1", + UserID: "test.user", + PrivateKey: "private-key", + Fingerprint: "00:00:00:00", + }, + expected: invalid, + expectedErr: errors.New("authConfig.tenancyId can not be empty"), + }, + { + name: "AuthConfig is missing User", + authConfigIn: &config.AuthConfig{ + Region: "us-bozeman-1", + TenancyID: "test.tenant", + PrivateKey: "private-key", + Fingerprint: "00:00:00:00", + }, + expected: invalid, + expectedErr: errors.New("authConfig.userId can not be empty"), + }, + { + name: "AuthConfig is missing Private Key", + authConfigIn: &config.AuthConfig{ + Region: "us-bozeman-1", + TenancyID: "test.tenant", + UserID: "test.user", + Fingerprint: "00:00:00:00", + }, + expected: invalid, + expectedErr: errors.New("authConfig.privateKey can not be empty"), + }, + { + name: "AuthConfig is valid Fingerprint", + authConfigIn: &config.AuthConfig{ + Region: "us-bozeman-1", + TenancyID: "test.tenant", + UserID: "test.user", + PrivateKey: "private-key", + }, + expected: invalid, + expectedErr: errors.New("authConfig.fingerprint can not be empty"), + }, + { + name: "AuthConfig is valid", + authConfigIn: &config.AuthConfig{ + Region: "us-bozeman-1", + TenancyID: "test.tenant", + UserID: "test.user", + PrivateKey: "private-key", + Fingerprint: "00:00:00:00", + }, + expected: valid, + expectedErr: nil, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + err := ValidateAuthConfig(tt.authConfigIn) + if tt.expectedErr != nil { + if !reflect.DeepEqual(err.Error(), tt.expectedErr.Error()) { + t.Errorf("Test (%s) \n Expected %q, \n Actual %q", tt.name, tt.expectedErr, err) + } + } else { + if tt.expected != valid { + t.Errorf("Test (%s) \n Expected %q, \n", tt.name, tt.expected) + } + } + }) + } +} diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go new file mode 100644 index 000000000..38ac05f6c --- /dev/null +++ b/cloud/scope/clients.go @@ -0,0 +1,169 @@ +/* +Copyright (c) 2021, 2022 Oracle and/or its affiliates. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "sync" + + "github.com/go-logr/logr" + "github.com/oracle/cluster-api-provider-oci/cloud/config" + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" + "github.com/oracle/cluster-api-provider-oci/cloud/services/compute" + "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" + "github.com/oracle/oci-go-sdk/v63/common" + "github.com/oracle/oci-go-sdk/v63/core" + "github.com/oracle/oci-go-sdk/v63/identity" + "github.com/oracle/oci-go-sdk/v63/networkloadbalancer" + "github.com/pkg/errors" + "k8s.io/klog/v2/klogr" +) + +// OCIClients is the struct of all the needed OCI clients +type OCIClients struct { + ComputeClient compute.ComputeClient + VCNClient vcn.Client + LoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient + IdentityClient *identity.IdentityClient +} + +// ClientProvider defines the regional clients +type ClientProvider struct { + *logr.Logger + ociClients map[string]OCIClients + ociClientsLock *sync.RWMutex + ociAuthConfigProvider common.ConfigurationProvider + authConfig *config.AuthConfig +} + +// NewClientProvider builds the ClientProvider with a client for the given region +func NewClientProvider(authConfig *config.AuthConfig) (*ClientProvider, error) { + if err := ociutil.ValidateAuthConfig(authConfig); err != nil { + return nil, errors.Wrapf(err, "NewClientProvider authConfig is invalid") + } + + log := klogr.New() + + ociAuthConfigProvider, err := config.NewConfigurationProvider(authConfig) + if err != nil { + log.Error(err, "authentication provider could not be initialised") + return nil, err + } + + provider := ClientProvider{ + Logger: &log, + ociAuthConfigProvider: ociAuthConfigProvider, + ociClients: map[string]OCIClients{}, + ociClientsLock: new(sync.RWMutex), + authConfig: authConfig, + } + _, err = provider.GetOrBuildClient(authConfig.Region) + if err != nil { + return nil, err + } + + return &provider, nil +} + +// GetOrBuildClient if the OCIClients exist for the region they are returned, if not clients will build them +func (c *ClientProvider) GetOrBuildClient(region string) (OCIClients, error) { + if len(region) <= 0 { + return OCIClients{}, errors.New("ClientProvider.GetOrBuildClient region can not be empty") + } + + c.ociClientsLock.RLock() + clients, regionalClientsExists := c.ociClients[region] + c.ociClientsLock.RUnlock() + + if regionalClientsExists { + return clients, nil + } + + regionalAuth := c.authConfig + regionalAuth.Region = region + ociAuthConfigProvider, err := config.NewConfigurationProvider(regionalAuth) + if err != nil { + return OCIClients{}, err + } + + c.ociClientsLock.Lock() + defer c.ociClientsLock.Unlock() + regionalClient, err := createClients(ociAuthConfigProvider, c.Logger) + if err != nil { + return regionalClient, err + } + c.ociClients[region] = regionalClient + + return regionalClient, nil +} + +func createClients(oCIAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (OCIClients, error) { + vcnClient, err := createVncClient(oCIAuthConfigProvider, logger) + lbClient, err := createLbClient(oCIAuthConfigProvider, logger) + identityClient, err := createIdentityClient(oCIAuthConfigProvider, logger) + computeClient, err := createComputeClient(oCIAuthConfigProvider, logger) + + if err != nil { + return OCIClients{}, err + } + + return OCIClients{ + VCNClient: vcnClient, + LoadBalancerClient: lbClient, + IdentityClient: identityClient, + ComputeClient: computeClient, + }, err +} + +func createVncClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.VirtualNetworkClient, error) { + vcnClient, err := core.NewVirtualNetworkClientWithConfigurationProvider(ociAuthConfigProvider) + if err != nil { + logger.Error(err, "unable to create OCI VCN Client") + return nil, err + } + + return &vcnClient, nil +} + +func createLbClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*networkloadbalancer.NetworkLoadBalancerClient, error) { + lbClient, err := networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider) + if err != nil { + logger.Error(err, "unable to create OCI LB Client") + return nil, err + } + + return &lbClient, nil +} + +func createIdentityClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*identity.IdentityClient, error) { + identityClient, err := identity.NewIdentityClientWithConfigurationProvider(ociAuthConfigProvider) + if err != nil { + logger.Error(err, "unable to create OCI Identity Client") + return nil, err + } + + return &identityClient, nil +} + +func createComputeClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.ComputeClient, error) { + computeClient, err := core.NewComputeClientWithConfigurationProvider(ociAuthConfigProvider) + if err != nil { + logger.Error(err, "unable to create OCI Compute Client") + return nil, err + } + + return &computeClient, nil +} diff --git a/cloud/scope/clients_mock.go b/cloud/scope/clients_mock.go new file mode 100644 index 000000000..cd0da512f --- /dev/null +++ b/cloud/scope/clients_mock.go @@ -0,0 +1,117 @@ +/* +Copyright (c) 2021, 2022 Oracle and/or its affiliates. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "sync" + + "github.com/oracle/cluster-api-provider-oci/cloud/config" + "github.com/oracle/cluster-api-provider-oci/cloud/services/compute" + "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" + "github.com/oracle/oci-go-sdk/v63/identity" + "github.com/oracle/oci-go-sdk/v63/networkloadbalancer" + "k8s.io/klog/v2/klogr" +) + +type MockOCIClients struct { + VCNClient vcn.Client + ComputeClient compute.ComputeClient + LoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient + IdentityClient *identity.IdentityClient +} + +var ( + MockTestRegion = "us-lexington-1" +) + +func MockNewClientProvider(mockClients MockOCIClients) (*ClientProvider, error) { + + clientsInject := map[string]OCIClients{MockTestRegion: { + VCNClient: mockClients.VCNClient, + LoadBalancerClient: mockClients.LoadBalancerClient, + IdentityClient: mockClients.IdentityClient, + ComputeClient: mockClients.ComputeClient, + }} + + authConfig, err := MockAuthConfig() + if err != nil { + return nil, err + } + + ociAuthConfigProvider, err := config.NewConfigurationProvider(&authConfig) + if err != nil { + fmt.Printf("expected ociAuthConfigProvider to be created %s \n", err) + return nil, err + } + log := klogr.New() + clientProvider := ClientProvider{ + Logger: &log, + ociClients: clientsInject, + ociClientsLock: new(sync.RWMutex), + ociAuthConfigProvider: ociAuthConfigProvider, + authConfig: &authConfig, + } + + return &clientProvider, nil +} + +func MockAuthConfig() (config.AuthConfig, error) { + privateKey, err := generatePrivateKeyPEM() + if err != nil { + fmt.Println("error generating a private key") + return config.AuthConfig{}, err + } + + authConfig := config.AuthConfig{ + UseInstancePrincipals: false, + Region: MockTestRegion, + Fingerprint: "mock-finger-print", + PrivateKey: privateKey, + UserID: "ocid1.tenancy.oc1..", + TenancyID: "ocid1.tenancy.oc1..", + } + + return authConfig, nil +} + +func generatePrivateKeyPEM() (string, error) { + privateKey, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + return "", err + } + + privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey) + privateKeyBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + } + + var privateKeyBuf bytes.Buffer + err = pem.Encode(&privateKeyBuf, privateKeyBlock) + if err != nil { + fmt.Printf("error when encode private pem: %s \n", err) + return "", err + } + + return privateKeyBuf.String(), err +} diff --git a/cloud/scope/clients_test.go b/cloud/scope/clients_test.go new file mode 100644 index 000000000..812318cbf --- /dev/null +++ b/cloud/scope/clients_test.go @@ -0,0 +1,117 @@ +/* +Copyright (c) 2021, 2022 Oracle and/or its affiliates. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "github.com/oracle/cluster-api-provider-oci/cloud/config" + "reflect" + "testing" + + "github.com/golang/mock/gomock" + "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn/mock_vcn" +) + +func TestClients_NewClientProvider(t *testing.T) { + + authConfig, err := MockAuthConfig() + if err != nil { + t.Error(err) + } + + clientProvider, err := NewClientProvider(&authConfig) + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + + if reflect.DeepEqual(clientProvider, ClientProvider{}) { + t.Errorf("clientProvider can not be an empty struct") + } +} + +func TestClients_NewClientProviderWithBadAuthConfig(t *testing.T) { + + authConfig := config.AuthConfig{} + + clientProvider, err := NewClientProvider(&authConfig) + if err == nil { + t.Errorf("Expected error:%v to not equal nil", err) + } + + if clientProvider != nil { + t.Errorf("Expected clientProvider:%v to equal nil", clientProvider) + } +} + +func TestClients_BuildNewClients(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + vcnClient := mock_vcn.NewMockClient(mockCtrl) + + clientProvider, err := MockNewClientProvider(MockOCIClients{ + VCNClient: vcnClient, + }) + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + + clients, err := clientProvider.GetOrBuildClient(MockTestRegion) + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + vcn := clients.VCNClient + + if vcn != vcnClient { + t.Errorf("Expected %v to equal %v", vcnClient, vcn) + } + + // build clients for a region not in our provider list yet + clients, err = clientProvider.GetOrBuildClient("us-austin-1") + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + vcn = clients.VCNClient + if vcn == vcnClient { + t.Errorf("Expected %v to NOT equal %v", vcnClient, vcn) + } +} + +func TestClients_ReuseClients(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + vcnClient := mock_vcn.NewMockClient(mockCtrl) + + clientProvider, err := MockNewClientProvider(MockOCIClients{ + VCNClient: vcnClient, + }) + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + + firstClients, err := clientProvider.GetOrBuildClient(MockTestRegion) + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + + secondClients, err := clientProvider.GetOrBuildClient(MockTestRegion) + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + + if &secondClients.VCNClient == &firstClients.VCNClient { + t.Errorf("Expected %v to equal %v", secondClients.VCNClient, firstClients.VCNClient) + } +} diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index a4ebd2e4f..7914c73da 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -19,6 +19,7 @@ package scope import ( "context" "fmt" + "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" "reflect" "strconv" @@ -27,7 +28,6 @@ import ( "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" identityClent "github.com/oracle/cluster-api-provider-oci/cloud/services/identity" nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer" - "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" "github.com/oracle/oci-go-sdk/v63/common" "github.com/oracle/oci-go-sdk/v63/identity" "github.com/pkg/errors" @@ -45,14 +45,15 @@ const ( // ClusterScopeParams defines the params need to create a new ClusterScope type ClusterScopeParams struct { - Client client.Client - Logger *logr.Logger - Cluster *clusterv1.Cluster - OCICluster *infrastructurev1beta1.OCICluster - VCNClient vcn.Client - LoadBalancerClient nlb.NetworkLoadBalancerClient - IdentityClient identityClent.Client - Region string + Client client.Client + Logger *logr.Logger + Cluster *clusterv1.Cluster + OCICluster *infrastructurev1beta1.OCICluster + VCNClient vcn.Client + LoadBalancerClient nlb.NetworkLoadBalancerClient + IdentityClient identityClent.Client + Region string + OCIAuthConfigProvider common.ConfigurationProvider } type ClusterScope struct { @@ -87,6 +88,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { if err != nil { return nil, errors.Wrap(err, "failed to init patch helper") } + return &ClusterScope{ Logger: params.Logger, client: params.Client, diff --git a/cloud/scope/vcn_reconciler.go b/cloud/scope/vcn_reconciler.go index 8f19201df..7eeb4cfd2 100644 --- a/cloud/scope/vcn_reconciler.go +++ b/cloud/scope/vcn_reconciler.go @@ -29,6 +29,7 @@ import ( func (s *ClusterScope) ReconcileVCN(ctx context.Context) error { desiredVCN := s.VCNSpec() + var err error vcn, err := s.GetVCN(ctx) if err != nil { diff --git a/cloud/services/networkloadbalancer/mock_nlb/client_mock.go b/cloud/services/networkloadbalancer/mock_nlb/client_mock.go index 7718caa58..73f8e8e3b 100644 --- a/cloud/services/networkloadbalancer/mock_nlb/client_mock.go +++ b/cloud/services/networkloadbalancer/mock_nlb/client_mock.go @@ -12,7 +12,7 @@ import ( networkloadbalancer "github.com/oracle/oci-go-sdk/v63/networkloadbalancer" ) -// MockNetworkLoadBalancerClient is a mock of NetworkLoadBalancerClient interface. +// MockNetworkLoadBalancerClient is a mock of LoadBalancerClient interface. type MockNetworkLoadBalancerClient struct { ctrl *gomock.Controller recorder *MockNetworkLoadBalancerClientMockRecorder diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml index cc3056b7b..6d9144c1d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ociclusters.yaml @@ -845,6 +845,10 @@ spec: type: array type: object type: object + region: + description: Region the cluster operates in. It must be one of available + regions in Region Identifier format. See https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm + type: string required: - compartmentId type: object diff --git a/controllers/ocicluster_controller.go b/controllers/ocicluster_controller.go index febb8a652..ba09cf8e1 100644 --- a/controllers/ocicluster_controller.go +++ b/controllers/ocicluster_controller.go @@ -25,9 +25,6 @@ import ( "github.com/oracle/cluster-api-provider-oci/api/v1beta1" infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" "github.com/oracle/cluster-api-provider-oci/cloud/scope" - nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer" - "github.com/oracle/oci-go-sdk/v63/core" - "github.com/oracle/oci-go-sdk/v63/identity" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -52,12 +49,10 @@ import ( // OCIClusterReconciler reconciles a OciCluster object type OCIClusterReconciler struct { client.Client - Scheme *runtime.Scheme - VCNClient core.VirtualNetworkClient - NetworkLoadBalancerClient nlb.NetworkLoadBalancerClient - Recorder record.EventRecorder - IdentityClient identity.IdentityClient - Region string + Scheme *runtime.Scheme + Recorder record.EventRecorder + Region string + ClientProvider *scope.ClientProvider } //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=ociclusters,verbs=get;list;watch;create;update;patch;delete @@ -84,6 +79,12 @@ func (r *OCIClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } return ctrl.Result{}, err } + if len(ociCluster.Spec.Region) > 0 { + r.Region = ociCluster.Spec.Region + } + if len(r.Region) <= 0 { + return ctrl.Result{}, errors.New("OCIClusterReconciler Region can't be nil") + } // Fetch the Cluster. cluster, err := util.GetOwnerCluster(ctx, r.Client, ociCluster.ObjectMeta) @@ -96,14 +97,20 @@ func (r *OCIClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } var clusterScope scope.ClusterScopeClient + + clients, err := r.ClientProvider.GetOrBuildClient(r.Region) + if err != nil { + logger.Error(err, "Couldn't get the clients for region") + } + clusterScope, err = scope.NewClusterScope(scope.ClusterScopeParams{ Client: r.Client, Logger: &logger, Cluster: cluster, OCICluster: ociCluster, - VCNClient: r.VCNClient, - LoadBalancerClient: r.NetworkLoadBalancerClient, - IdentityClient: r.IdentityClient, + VCNClient: clients.VCNClient, + LoadBalancerClient: clients.LoadBalancerClient, + IdentityClient: clients.IdentityClient, Region: r.Region, }) diff --git a/controllers/ocicluster_controller_test.go b/controllers/ocicluster_controller_test.go index 6e0afc7cc..93f9f102d 100644 --- a/controllers/ocicluster_controller_test.go +++ b/controllers/ocicluster_controller_test.go @@ -21,6 +21,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +var ( + MockTestRegion = "us-austin-1" +) + func TestOCIClusterReconciler_Reconcile(t *testing.T) { var ( r OCIClusterReconciler @@ -63,6 +67,7 @@ func TestOCIClusterReconciler_Reconcile(t *testing.T) { Client: client, Scheme: runtime.NewScheme(), Recorder: recorder, + Region: MockTestRegion, } req = reconcile.Request{ NamespacedName: types.NamespacedName{ @@ -105,6 +110,7 @@ func TestOCIClusterReconciler_reconcile(t *testing.T) { Client: client, Scheme: runtime.NewScheme(), Recorder: recorder, + Region: MockTestRegion, } cs.EXPECT().GetOCICluster().Return(ociCluster) } diff --git a/controllers/ocimachine_controller.go b/controllers/ocimachine_controller.go index fa2cf4152..0020cb7b4 100644 --- a/controllers/ocimachine_controller.go +++ b/controllers/ocimachine_controller.go @@ -19,15 +19,13 @@ package controllers import ( "context" "fmt" - "github.com/oracle/oci-go-sdk/v63/common" "time" "github.com/go-logr/logr" infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" "github.com/oracle/cluster-api-provider-oci/cloud/scope" - "github.com/oracle/cluster-api-provider-oci/cloud/services/compute" - nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer" + "github.com/oracle/oci-go-sdk/v63/common" "github.com/oracle/oci-go-sdk/v63/core" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -53,11 +51,10 @@ import ( // OCIMachineReconciler reconciles a OciMachine object type OCIMachineReconciler struct { client.Client - Scheme *runtime.Scheme - ComputeClient compute.ComputeClient - Recorder record.EventRecorder - VCNClient core.VirtualNetworkClient - NetworkLoadBalancerClient nlb.NetworkLoadBalancerClient + Scheme *runtime.Scheme + Recorder record.EventRecorder + ClientProvider *scope.ClientProvider + Region string } //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=ocimachines,verbs=get;list;watch;create;update;patch;delete @@ -119,17 +116,29 @@ func (r *OCIMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + if len(ociCluster.Spec.Region) > 0 { + r.Region = ociCluster.Spec.Region + } + if len(r.Region) <= 0 { + return ctrl.Result{}, errors.New("OCIMachineReconciler Region can't be nil") + } + + clients, err := r.ClientProvider.GetOrBuildClient(r.Region) + if err != nil { + logger.Error(err, "Couldn't get the clients for region") + } + // Create the machine scope machineScope, err := scope.NewMachineScope(scope.MachineScopeParams{ Client: r.Client, - ComputeClient: r.ComputeClient, + ComputeClient: clients.ComputeClient, Logger: &logger, Cluster: cluster, OCICluster: ociCluster, Machine: machine, OCIMachine: ociMachine, - VCNClient: r.VCNClient, - NetworkLoadBalancerClient: r.NetworkLoadBalancerClient, + VCNClient: clients.VCNClient, + NetworkLoadBalancerClient: clients.LoadBalancerClient, }) if err != nil { return ctrl.Result{}, errors.Errorf("failed to create scope: %+v", err) diff --git a/controllers/ocimachine_controller_test.go b/controllers/ocimachine_controller_test.go index 542214972..6b19c584e 100644 --- a/controllers/ocimachine_controller_test.go +++ b/controllers/ocimachine_controller_test.go @@ -106,6 +106,11 @@ func TestMachineReconciliation(t *testing.T) { }, } + clientProvider, err := scope.MockNewClientProvider(scope.MockOCIClients{}) + if err != nil { + t.Error(err) + } + for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) @@ -114,9 +119,11 @@ func TestMachineReconciliation(t *testing.T) { client := fake.NewClientBuilder().WithObjects(tc.objects...).Build() r = OCIMachineReconciler{ - Client: client, - Scheme: runtime.NewScheme(), - Recorder: recorder, + Client: client, + Scheme: runtime.NewScheme(), + Recorder: recorder, + ClientProvider: clientProvider, + Region: MockTestRegion, } req = reconcile.Request{ NamespacedName: types.NamespacedName{ @@ -178,10 +185,9 @@ func TestNormalReconciliationFunction(t *testing.T) { recorder = record.NewFakeRecorder(2) r = OCIMachineReconciler{ - Client: client, - Scheme: runtime.NewScheme(), - Recorder: recorder, - ComputeClient: computeClient, + Client: client, + Scheme: runtime.NewScheme(), + Recorder: recorder, } g.Expect(err).To(BeNil()) } @@ -585,11 +591,19 @@ func TestMachineReconciliationDelete(t *testing.T) { now := metav1.NewTime(time.Now()) ociMachine.DeletionTimestamp = &now client := fake.NewClientBuilder().WithObjects(getSecret(), getMachine(), ociMachine, getCluster(), getOCICluster()).Build() - r = OCIMachineReconciler{ - Client: client, - Scheme: runtime.NewScheme(), - Recorder: recorder, + clientProvider, err := scope.MockNewClientProvider(scope.MockOCIClients{ ComputeClient: computeClient, + }) + if err != nil { + t.Errorf("Expected %v to equal nil", err) + } + + r = OCIMachineReconciler{ + Client: client, + Scheme: runtime.NewScheme(), + Recorder: recorder, + ClientProvider: clientProvider, + Region: scope.MockTestRegion, } g.Expect(err).To(BeNil()) } @@ -667,10 +681,9 @@ func TestMachineReconciliationDeletionNormal(t *testing.T) { recorder = record.NewFakeRecorder(2) r = OCIMachineReconciler{ - Client: client, - Scheme: runtime.NewScheme(), - Recorder: recorder, - ComputeClient: computeClient, + Client: client, + Scheme: runtime.NewScheme(), + Recorder: recorder, } g.Expect(err).To(BeNil()) } diff --git a/docs/src/gs/create-workload-cluster.md b/docs/src/gs/create-workload-cluster.md index 625daccd8..53cf44298 100644 --- a/docs/src/gs/create-workload-cluster.md +++ b/docs/src/gs/create-workload-cluster.md @@ -11,17 +11,17 @@ further configured with the parameters below. The following Oracle Cloud Infrastructure (OCI) configuration parameters are available when creating a workload cluster on OCI using one of our predefined templates: -| Parameter | Default Value | Description | -|-------------------------------------------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `OCI_COMPARTMENT_ID` | | The OCID of the compartment in which to create the required compute, storage and network resources. | -| `OCI_IMAGE_ID` | | The OCID of the image for the kubernetes nodes. This same image is used for both the control plane and the worker nodes. | -| `OCI_CONTROL_PLANE_MACHINE_TYPE` | VM.Standard.E4.Flex | The [shape](https://docs.oracle.com/en-us/iaas/Content/Compute/References/computeshapes.htm) of the Kubernetes control plane machine. | -| `OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS` | 1 | The number of OCPUs allocated to the control plane instance. | -| `OCI_NODE_MACHINE_TYPE` | VM.Standard.E4.Flex | The [shape](https://docs.oracle.com/en-us/iaas/Content/Compute/References/computeshapes.htm) of the Kubernetes worker machine. | -| `OCI_NODE_MACHINE_TYPE_OCPUS` | 1 | The number of OCPUs allocated to the worker instance. | -| `OCI_SSH_KEY` | | The public SSH key to be added to the Kubernetes nodes. It can be used to login to the node and troubleshoot failures. | -| `OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION` | true | Enables [in-flight Transport Layer Security (TLS) 1.2 encryption](https://docs.oracle.com/en-us/iaas/Content/File/Tasks/intransitencryption.htm) of data between control plane nodes and their associated block storage devices. | -| `OCI_NODE_PV_TRANSIT_ENCRYPTION` | true | Enables [in-flight Transport Layer Security (TLS) 1.2 encryption](https://docs.oracle.com/en-us/iaas/Content/File/Tasks/intransitencryption.htm) of data between worker nodes and their associated block storage devices. | +| Parameter | Default Value | Description | +|-------------------------------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `OCI_COMPARTMENT_ID` | | The OCID of the compartment in which to create the required compute, storage and network resources. | +| `OCI_IMAGE_ID` | | The OCID of the image for the kubernetes nodes. This same image is used for both the control plane and the worker nodes. | +| `OCI_CONTROL_PLANE_MACHINE_TYPE` | VM.Standard.E4.Flex | The [shape](https://docs.oracle.com/en-us/iaas/Content/Compute/References/computeshapes.htm) of the Kubernetes control plane machine. | +| `OCI_CONTROL_PLANE_MACHINE_TYPE_OCPUS` | 1 | The number of OCPUs allocated to the control plane instance. | +| `OCI_NODE_MACHINE_TYPE` | VM.Standard.E4.Flex | The [shape](https://docs.oracle.com/en-us/iaas/Content/Compute/References/computeshapes.htm) of the Kubernetes worker machine. | +| `OCI_NODE_MACHINE_TYPE_OCPUS` | 1 | The number of OCPUs allocated to the worker instance. | +| `OCI_SSH_KEY` | | The public SSH key to be added to the Kubernetes nodes. It can be used to login to the node and troubleshoot failures. | +| `OCI_CONTROL_PLANE_PV_TRANSIT_ENCRYPTION` | true | Enables [in-flight Transport Layer Security (TLS) 1.2 encryption](https://docs.oracle.com/en-us/iaas/Content/File/Tasks/intransitencryption.htm) of data between control plane nodes and their associated block storage devices. | +| `OCI_NODE_PV_TRANSIT_ENCRYPTION` | true | Enables [in-flight Transport Layer Security (TLS) 1.2 encryption](https://docs.oracle.com/en-us/iaas/Content/File/Tasks/intransitencryption.htm) of data between worker nodes and their associated block storage devices. | *NOTE* Only specific [bare metal shapes](https://docs.oracle.com/en-us/iaas/releasenotes/changes/60d602f5-abb3-4639-aa19-292a5744a808/) support in-transit encryption. If an unsupported shape is specified, the deployment will fail completely. diff --git a/main.go b/main.go index 6e4a771b0..700241cbe 100644 --- a/main.go +++ b/main.go @@ -21,19 +21,19 @@ import ( "os" infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + + _ "k8s.io/client-go/plugin/pkg/client/auth" + "github.com/oracle/cluster-api-provider-oci/cloud/config" + "github.com/oracle/cluster-api-provider-oci/cloud/ociutil" "github.com/oracle/cluster-api-provider-oci/cloud/scope" "github.com/oracle/cluster-api-provider-oci/controllers" - "github.com/oracle/oci-go-sdk/v63/core" - "github.com/oracle/oci-go-sdk/v63/identity" - "github.com/oracle/oci-go-sdk/v63/networkloadbalancer" "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/klog/v2/klogr" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -105,6 +105,10 @@ func main() { setupLog.Error(err, "invalid auth config file") os.Exit(1) } + if err = ociutil.ValidateAuthConfig(authConfig); err != nil { + setupLog.Error(err, "auth config is invalid") + os.Exit(1) + } ociAuthConfigProvider, err := config.NewConfigurationProvider(authConfig) if err != nil { @@ -121,50 +125,29 @@ func main() { os.Exit(1) } - vcnClient, err := core.NewVirtualNetworkClientWithConfigurationProvider(ociAuthConfigProvider) + clientProvider, err := scope.NewClientProvider(authConfig) if err != nil { - setupLog.Error(err, "unable to create OCI VCN Client") - os.Exit(1) - } - - lbClient, err := networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider) - if err != nil { - setupLog.Error(err, "unable to create OCI LB Client") - os.Exit(1) - } - - identityClient, err := identity.NewIdentityClientWithConfigurationProvider(ociAuthConfigProvider) - if err != nil { - setupLog.Error(err, "unable to create OCI Identity Client") + setupLog.Error(err, "unable to create OCI ClientProvider") os.Exit(1) } if err = (&controllers.OCIClusterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - VCNClient: vcnClient, - NetworkLoadBalancerClient: lbClient, - IdentityClient: identityClient, - Region: region, - Recorder: mgr.GetEventRecorderFor("ocicluster-controller"), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Region: region, + ClientProvider: clientProvider, + Recorder: mgr.GetEventRecorderFor("ocicluster-controller"), }).SetupWithManager(ctx, mgr, controller.Options{}); err != nil { setupLog.Error(err, "unable to create controller", "controller", scope.OCIClusterKind) os.Exit(1) } - computeClient, err := core.NewComputeClientWithConfigurationProvider(ociAuthConfigProvider) - if err != nil { - setupLog.Error(err, "unable to create OCI Compute Client") - os.Exit(1) - } - if err = (&controllers.OCIMachineReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ComputeClient: computeClient, - VCNClient: vcnClient, - NetworkLoadBalancerClient: lbClient, - Recorder: mgr.GetEventRecorderFor("ocimachine-controller"), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ClientProvider: clientProvider, + Region: region, + Recorder: mgr.GetEventRecorderFor("ocimachine-controller"), }).SetupWithManager(ctx, mgr, controller.Options{}); err != nil { setupLog.Error(err, "unable to create controller", "controller", scope.OCIMachineKind) os.Exit(1) diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index dec20e27c..52288e8e0 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -23,6 +23,7 @@ source "${REPO_ROOT}/hack/ensure-tags.sh" : "${OCI_ORACLE_LINUX_IMAGE_ID:?Environment variable empty or not defined.}" : "${OCI_UPGRADE_IMAGE_ID:?Environment variable empty or not defined.}" : "${OCI_CCM_TEST_IMAGE_ID:?Environment variable empty or not defined.}" +: "${OCI_ALTERNATIVE_REGION_IMAGE_ID:?Environment variable empty or not defined.}" export LOCAL_ONLY=${LOCAL_ONLY:-"true"} @@ -37,6 +38,8 @@ export OCI_NODE_MACHINE_TYPE="${OCI_NODE_MACHINE_TYPE:-"VM.Standard.E3.Flex"}" export OCI_NODE_MACHINE_TYPE_OCPUS="${OCI_NODE_MACHINE_TYPE_OCPUS:-"1"}" export KIND_EXPERIMENTAL_DOCKER_NETWORK="bridge" +export OCI_ALTERNATIVE_REGION="${OCI_ALTERNATIVE_REGION:-"us-sanjose-1"}" + # Generate SSH key. if [ -z "${OCI_SSH_KEY}" ]; then echo "generating sshkey for e2e" diff --git a/test/e2e/cluster_test.go b/test/e2e/cluster_test.go index 5d13df5fa..51ed34644 100644 --- a/test/e2e/cluster_test.go +++ b/test/e2e/cluster_test.go @@ -107,7 +107,7 @@ var _ = Describe("Workload cluster creation", func() { dumpSpecResourcesAndCleanup(ctx, cleanInput) }) - It("With 1 control-plane nodes and 1 worker nodes", func() { + It("Default CNI - with 1 control-plane nodes and 1 worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "simple") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: bootstrapClusterProxy, @@ -130,7 +130,7 @@ var _ = Describe("Workload cluster creation", func() { }, result) }) - It("With 3 control plane nodes spread across failure domains", func() { + It("Default CNI - With 3 control plane nodes spread across failure domains", func() { clusterName = getClusterName(clusterNamePrefix, "3nodecontrolplane") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: bootstrapClusterProxy, @@ -395,6 +395,31 @@ var _ = Describe("Workload cluster creation", func() { }, result) }) }) + + When("Multi-Region workload cluster creation", func() { + + It("Alternative region - With 1 control-plane nodes and 1 worker nodes", func() { + clusterName = getClusterName(clusterNamePrefix, "alternative-region") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "alternative-region", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) }) func verifyMultipleNsgSubnet(ctx context.Context, namespace string, clusterName string, mcDeployments []*clusterv1.MachineDeployment) { diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 15582a02f..7d91fae99 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -53,6 +53,7 @@ providers: contract: v1beta1 files: - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template.yaml" + - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-alternative-region.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-bare-metal.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-kcp-remediation.yaml" - sourcePath: "../data/infrastructure-oci/v1beta1/cluster-template-md-remediation.yaml" diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region.yaml new file mode 100644 index 000000000..4a4377728 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region.yaml @@ -0,0 +1,790 @@ +apiVersion: v1 +data: + cloud-controller-manager.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + stringData: + cloud-provider.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + loadBalancer: + disabled: true + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager + spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: + serviceAccountName: cloud-controller-manager + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=configmaps + - -v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + labels: + kubernetes.io/cluster-service: "true" + rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + + - apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - patch + - update + + # For leader election + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + + # For the PVL + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: oci-cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + namespace: default +--- +apiVersion: v1 +data: + csi.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: oci-volume-provisioner + namespace: kube-system + stringData: + config.yaml: |- + useInstancePrincipals: true + + # compartment configures Compartment within which the cluster resides. + compartment: ${OCI_COMPARTMENT_ID} + + # Optional rate limit controls for accessing OCI API + rateLimiter: + rateLimitQPSRead: 20.0 + rateLimitBucketRead: 5 + rateLimitQPSWrite: 20.0 + rateLimitBucketWrite: 5 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-controller + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-controller + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-controller + role: csi-oci + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + containers: + - name: csi-volume-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --volume-name-prefix=csi + - --feature-gates=Topology=true + - --timeout=120s + - --enable-leader-election=true + - --leader-election-type=leases + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v2.2.0 + args: + - --csi-address=/var/run/shared-tmpfs/csi.sock + - --timeout=120s + - --leader-election=true + - --leader-election-namespace=kube-system + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + - name: oci-csi-controller-driver + args: + - --v=2 + - --endpoint=unix://var/run/shared-tmpfs/csi.sock + command: + - /usr/local/bin/oci-csi-controller-driver + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/oci/ + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true + - mountPath: /var/run/shared-tmpfs + name: shared-tmpfs + volumes: + - name: config + secret: + secretName: oci-volume-provisioner + - name: kubernetes + hostPath: + path: /etc/kubernetes + - name: shared-tmpfs + emptyDir: {} + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + --- + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: oci-csi-iscsiadm + namespace: kube-system + data: + iscsiadm: | + #!/bin/sh + if [ -x /host/sbin/iscsiadm ]; then + chroot /host /sbin/iscsiadm "$@" + elif [ -x /host/usr/local/sbin/iscsiadm ]; then + chroot /host /usr/local/sbin/iscsiadm "$@" + elif [ -x /host/bin/iscsiadm ]; then + chroot /host /bin/iscsiadm "$@" + elif [ -x /host/usr/local/bin/iscsiadm ]; then + chroot /host /usr/local/bin/iscsiadm "$@" + else + chroot /host iscsiadm "$@" + fi + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + annotations: + deprecated.daemonset.template.generation: "1" + generation: 1 + name: csi-oci-node + namespace: kube-system + spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: csi-oci-node + template: + metadata: + creationTimestamp: null + labels: + app: csi-oci-node + role: csi-oci + spec: + containers: + - name: oci-csi-node-driver + args: + - --v=2 + - --endpoint=unix:///csi/csi.sock + - --nodeid=$(KUBE_NODE_NAME) + - --loglevel=debug + command: + - /usr/local/bin/oci-csi-node-driver + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin + image: iad.ocir.io/oracle/cloud-provider-oci:0.12.0 + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + - mountPath: /host + name: host-root + - mountPath: /sbin/iscsiadm + name: chroot-iscsiadm + subPath: iscsiadm + - name: csi-node-registrar + args: + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + securityContext: + privileged: true + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/blockvolume.csi.oraclecloud.com /registration/blockvolume.csi.oraclecloud.com-reg.sock + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /registration + name: registration-dir + dnsPolicy: ClusterFirst + hostNetwork: true + imagePullSecrets: + - name: image-pull-secret + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: csi-oci-node-sa + serviceAccountName: csi-oci-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + type: "" + name: device-dir + - hostPath: + path: / + type: Directory + name: host-root + - configMap: + name: oci-csi-iscsiadm + defaultMode: 0755 + name: chroot-iscsiadm + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: csi-oci-node-sa + namespace: kube-system + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci + namespace: kube-system + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["volume.oci.oracle.com"] + resources: ["blockscsiinfos"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "csinodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "create", "update"] + --- + + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: csi-oci-binding + subjects: + - kind: ServiceAccount + name: csi-oci-node-sa + namespace: kube-system + roleRef: + kind: ClusterRole + name: csi-oci + apiGroup: rbac.authorization.k8s.io +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: ${CLUSTER_NAME}-oci-csi + namespace: default +--- +apiVersion: v1 +binaryData: null +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-ccm-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-cloud-controller-manager + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-csi-resource-set + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-oci-csi + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 +spec: + clusterSelector: + matchLabels: + cni: calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${POD_CIDR:="192.168.0.0/16"} + serviceDomain: ${SERVICE_DOMAIN:="cluster.local"} + services: + cidrBlocks: + - ${SERVICE_CIDR:="10.128.0.0/12"} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCICluster + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + dns: {} + etcd: {} + kubernetesVersion: ${KUBERNETES_VERSION} + networking: {} + scheduler: {} + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + joinConfiguration: + discovery: {} + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + cloud-provider: external + provider-id: oci://{{ ds["id"] }} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OCIMachineTemplate + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} +spec: + compartmentId: ${OCI_COMPARTMENT_ID} + region: ${OCI_ALTERNATIVE_REGION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_ALTERNATIVE_REGION_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_CONTROL_PLANE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + compartmentId: ${OCI_COMPARTMENT_ID} + imageId: ${OCI_ALTERNATIVE_REGION_IMAGE_ID} + metadata: + ssh_authorized_keys: ${OCI_SSH_KEY} + shape: ${OCI_NODE_MACHINE_TYPE} + shapeConfig: + ocpus: "1" diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/cluster.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/cluster.yaml new file mode 100644 index 000000000..c73417e9b --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/cluster.yaml @@ -0,0 +1,17 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCICluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + name: "${CLUSTER_NAME}" +spec: + region: "${OCI_ALTERNATIVE_REGION}" +--- +kind: OCIMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + imageId: "${OCI_ALTERNATIVE_REGION_IMAGE_ID}" diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/kustomization.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/kustomization.yaml new file mode 100644 index 000000000..e9e16ce0d --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/kustomization.yaml @@ -0,0 +1,9 @@ +bases: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../bases/crs.yaml + - ../bases/ccm.yaml + +patchesStrategicMerge: + - ./cluster.yaml + - ./md.yaml diff --git a/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/md.yaml b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/md.yaml new file mode 100644 index 000000000..58e932bb6 --- /dev/null +++ b/test/e2e/data/infrastructure-oci/v1beta1/cluster-template-alternative-region/md.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OCIMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + imageId: "${OCI_ALTERNATIVE_REGION_IMAGE_ID}" + diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 7b25338c1..6c3fb16f3 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -34,8 +34,11 @@ import ( "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" infrastructurev1beta1 "github.com/oracle/cluster-api-provider-oci/api/v1beta1" + oci_config "github.com/oracle/cluster-api-provider-oci/cloud/config" + "github.com/oracle/cluster-api-provider-oci/cloud/scope" + "github.com/oracle/cluster-api-provider-oci/cloud/services/compute" + "github.com/oracle/cluster-api-provider-oci/cloud/services/vcn" "github.com/oracle/oci-go-sdk/v63/common" - "github.com/oracle/oci-go-sdk/v63/core" "github.com/oracle/oci-go-sdk/v63/identity" "github.com/oracle/oci-go-sdk/v63/networkloadbalancer" "k8s.io/apimachinery/pkg/runtime" @@ -94,11 +97,11 @@ var ( // usePRArtifacts specifies whether or not to use the build from a PR of the Kubernetes repository usePRArtifacts bool - computeClient core.ComputeClient + computeClient compute.ComputeClient - vcnClient core.VirtualNetworkClient + vcnClient vcn.Client - lbClient networkloadbalancer.NetworkLoadBalancerClient + lbClient *networkloadbalancer.NetworkLoadBalancerClient adCount int ) @@ -192,22 +195,28 @@ var _ = SynchronizedBeforeSuite(func() []byte { Expect(err).NotTo(HaveOccurred()) region, err := base64.StdEncoding.DecodeString(os.Getenv("OCI_REGION_B64")) Expect(err).NotTo(HaveOccurred()) - conf := common.NewRawConfigurationProvider( - string(tenancyId), - string(userId), - string(region), - string(fingerprint), - string(key), - common.String(string(passphrase))) - computeClient, err = core.NewComputeClientWithConfigurationProvider(conf) - Expect(err).NotTo(HaveOccurred()) - identityClient, err := identity.NewIdentityClientWithConfigurationProvider(conf) - Expect(err).NotTo(HaveOccurred()) - vcnClient, err = core.NewVirtualNetworkClientWithConfigurationProvider(conf) + + clientProvider, err := scope.NewClientProvider(&oci_config.AuthConfig{ + Region: string(region), + TenancyID: string(tenancyId), + UserID: string(userId), + PrivateKey: string(key), + Fingerprint: string(fingerprint), + Passphrase: string(passphrase), + UseInstancePrincipals: false, + }) Expect(err).NotTo(HaveOccurred()) - lbClient, err = networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(conf) + + ociClients, err := clientProvider.GetOrBuildClient(string(region)) Expect(err).NotTo(HaveOccurred()) + computeClient = ociClients.ComputeClient + identityClient := ociClients.IdentityClient + vcnClient = ociClients.VCNClient + lbClient = ociClients.LoadBalancerClient + Expect(identityClient).NotTo(BeNil()) + Expect(lbClient).NotTo(BeNil()) + req := identity.ListAvailabilityDomainsRequest{CompartmentId: common.String(os.Getenv("OCI_COMPARTMENT_ID"))} resp, err := identityClient.ListAvailabilityDomains(context.Background(), req) adCount = len(resp.Items)