Skip to content

Commit

Permalink
feat: add support for clusters in multiple regions
Browse files Browse the repository at this point in the history
Adding `ClientProvider` allows clusters to use different `OCIClients`
groups to interact with the regional APIs.
  • Loading branch information
joekr committed Mar 31, 2022
1 parent 76da8d3 commit 209b749
Show file tree
Hide file tree
Showing 26 changed files with 1,550 additions and 122 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ bin

# Output of the go coverage tool, specifically when used with LiteIDE
*.out
cover.html
testbin/
out/

Expand Down
8 changes: 5 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ ARTIFACTS ?= $(ROOT_DIR)/_artifacts
KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
KUBETEST_FAST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance-fast.yaml)
GINKGO_FOCUS ?= Workload cluster creation
GINKGO_SKIP ?= "Bare Metal"
GINKGO_SKIP ?= "Bare Metal|Multi-Region"
# Image URL to use all building/pushing image targets
IMG ?= controller:latest
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
Expand Down Expand Up @@ -106,7 +106,8 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
test: manifests generate fmt vet ## Run tests.
mkdir -p ${ENVTEST_ASSETS_DIR}
test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out
source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out; go tool cover -html=cover.out -o cover.html


##@ Build

Expand Down Expand Up @@ -241,6 +242,7 @@ serve-book: build-book ## Build and serve the book with live-reloading enabled
.PHONY: generate-e2e-templates ## Generate OCI infrastructure templates for e2e test suite.
generate-e2e-templates: kustomize
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template.yaml
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region.yaml
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal.yaml
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation.yaml
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation.yaml
Expand All @@ -256,7 +258,7 @@ generate-e2e-templates: kustomize
.PHONY: test-e2e-run
test-e2e-run: generate-e2e-templates ginkgo $(ENVSUBST) ## Run e2e tests
envsubst < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \
$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip=$(GINKGO_SKIP) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
Expand Down
4 changes: 4 additions & 0 deletions api/v1beta1/ocicluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ type OCIClusterSpec struct {
// Compartment to create the cluster network.
CompartmentId string `mandatory:"true" json:"compartmentId"`

// Region the cluster operates in. It must be one of available regions in Region Identifier format.
// See https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm
Region string `json:"region,omitempty"`

// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
// +optional
ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
Expand Down
29 changes: 29 additions & 0 deletions cloud/ociutil/ociutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package ociutil
import (
"context"
"fmt"
"github.com/oracle/cluster-api-provider-oci/cloud/config"
nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
"net/http"
"time"
Expand Down Expand Up @@ -112,3 +113,31 @@ func BuildClusterTags(clusterUUID string) map[string]string {
tags["ClusterUUID"] = clusterUUID
return tags
}

func ValidateAuthConfig(authConfig *config.AuthConfig) error {
if authConfig == nil {
return errors.New("authConfig can't be nil")
}

if len(authConfig.Region) <= 0 {
return errors.New("authConfig.region can not be empty")
}

if len(authConfig.TenancyID) <= 0 {
return errors.New("authConfig.tenancyId can not be empty")
}

if len(authConfig.UserID) <= 0 {
return errors.New("authConfig.userId can not be empty")
}

if len(authConfig.PrivateKey) <= 0 {
return errors.New("authConfig.privateKey can not be empty")
}

if len(authConfig.Fingerprint) <= 0 {
return errors.New("authConfig.fingerprint can not be empty")
}

return nil
}
101 changes: 101 additions & 0 deletions cloud/ociutil/ociutil_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import (
"reflect"
"testing"

"github.com/oracle/cluster-api-provider-oci/cloud/config"
"github.com/oracle/oci-go-sdk/v63/core"
"github.com/pkg/errors"
)

func TestGetCloudProviderConfig(t *testing.T) {
Expand Down Expand Up @@ -83,3 +85,102 @@ func TestAddToDefaultClusterTags(t *testing.T) {
}
}
}

func TestValidateAuthConfig(t *testing.T) {
const valid string = "valid"
const invalid string = "invalid"
testCases := []struct {
name string
authConfigIn *config.AuthConfig
expected string
expectedErr error
}{
{
name: "AuthConfig is nil",
authConfigIn: nil,
expected: invalid,
expectedErr: errors.New("authConfig can't be nil"),
},
{
name: "AuthConfig is missing region",
authConfigIn: &config.AuthConfig{
TenancyID: "test.tenant",
UserID: "test.user",
PrivateKey: "private-key",
Fingerprint: "00:00:00:00",
},
expected: invalid,
expectedErr: errors.New("authConfig.region can not be empty"),
},
{
name: "AuthConfig is missing Tenancy",
authConfigIn: &config.AuthConfig{
Region: "us-bozeman-1",
UserID: "test.user",
PrivateKey: "private-key",
Fingerprint: "00:00:00:00",
},
expected: invalid,
expectedErr: errors.New("authConfig.tenancyId can not be empty"),
},
{
name: "AuthConfig is missing User",
authConfigIn: &config.AuthConfig{
Region: "us-bozeman-1",
TenancyID: "test.tenant",
PrivateKey: "private-key",
Fingerprint: "00:00:00:00",
},
expected: invalid,
expectedErr: errors.New("authConfig.userId can not be empty"),
},
{
name: "AuthConfig is missing Private Key",
authConfigIn: &config.AuthConfig{
Region: "us-bozeman-1",
TenancyID: "test.tenant",
UserID: "test.user",
Fingerprint: "00:00:00:00",
},
expected: invalid,
expectedErr: errors.New("authConfig.privateKey can not be empty"),
},
{
name: "AuthConfig is valid Fingerprint",
authConfigIn: &config.AuthConfig{
Region: "us-bozeman-1",
TenancyID: "test.tenant",
UserID: "test.user",
PrivateKey: "private-key",
},
expected: invalid,
expectedErr: errors.New("authConfig.fingerprint can not be empty"),
},
{
name: "AuthConfig is valid",
authConfigIn: &config.AuthConfig{
Region: "us-bozeman-1",
TenancyID: "test.tenant",
UserID: "test.user",
PrivateKey: "private-key",
Fingerprint: "00:00:00:00",
},
expected: valid,
expectedErr: nil,
},
}
for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
err := ValidateAuthConfig(tt.authConfigIn)
if tt.expectedErr != nil {
if !reflect.DeepEqual(err.Error(), tt.expectedErr.Error()) {
t.Errorf("Test (%s) \n Expected %q, \n Actual %q", tt.name, tt.expectedErr, err)
}
} else {
if tt.expected != valid {
t.Errorf("Test (%s) \n Expected %q, \n", tt.name, tt.expected)
}
}
})
}
}
169 changes: 169 additions & 0 deletions cloud/scope/clients.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/*
Copyright (c) 2021, 2022 Oracle and/or its affiliates.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scope

import (
"sync"

"github.com/go-logr/logr"
"github.com/oracle/cluster-api-provider-oci/cloud/config"
"github.com/oracle/cluster-api-provider-oci/cloud/ociutil"
"github.com/oracle/cluster-api-provider-oci/cloud/services/compute"
"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
"github.com/oracle/oci-go-sdk/v63/common"
"github.com/oracle/oci-go-sdk/v63/core"
"github.com/oracle/oci-go-sdk/v63/identity"
"github.com/oracle/oci-go-sdk/v63/networkloadbalancer"
"github.com/pkg/errors"
"k8s.io/klog/v2/klogr"
)

// OCIClients is the struct of all the needed OCI clients
type OCIClients struct {
ComputeClient compute.ComputeClient
VCNClient vcn.Client
LoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient
IdentityClient *identity.IdentityClient
}

// ClientProvider defines the regional clients
type ClientProvider struct {
*logr.Logger
ociClients map[string]OCIClients
ociClientsLock *sync.RWMutex
ociAuthConfigProvider common.ConfigurationProvider
authConfig *config.AuthConfig
}

// NewClientProvider builds the ClientProvider with a client for the given region
func NewClientProvider(authConfig *config.AuthConfig) (*ClientProvider, error) {
if err := ociutil.ValidateAuthConfig(authConfig); err != nil {
return nil, errors.Wrapf(err, "NewClientProvider authConfig is invalid")
}

log := klogr.New()

ociAuthConfigProvider, err := config.NewConfigurationProvider(authConfig)
if err != nil {
log.Error(err, "authentication provider could not be initialised")
return nil, err
}

provider := ClientProvider{
Logger: &log,
ociAuthConfigProvider: ociAuthConfigProvider,
ociClients: map[string]OCIClients{},
ociClientsLock: new(sync.RWMutex),
authConfig: authConfig,
}
_, err = provider.GetOrBuildClient(authConfig.Region)
if err != nil {
return nil, err
}

return &provider, nil
}

// GetOrBuildClient if the OCIClients exist for the region they are returned, if not clients will build them
func (c *ClientProvider) GetOrBuildClient(region string) (OCIClients, error) {
if len(region) <= 0 {
return OCIClients{}, errors.New("ClientProvider.GetOrBuildClient region can not be empty")
}

c.ociClientsLock.RLock()
clients, regionalClientsExists := c.ociClients[region]
c.ociClientsLock.RUnlock()

if regionalClientsExists {
return clients, nil
}

regionalAuth := c.authConfig
regionalAuth.Region = region
ociAuthConfigProvider, err := config.NewConfigurationProvider(regionalAuth)
if err != nil {
return OCIClients{}, err
}

c.ociClientsLock.Lock()
defer c.ociClientsLock.Unlock()
regionalClient, err := createClients(ociAuthConfigProvider, c.Logger)
if err != nil {
return regionalClient, err
}
c.ociClients[region] = regionalClient

return regionalClient, nil
}

func createClients(oCIAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (OCIClients, error) {
vcnClient, err := createVncClient(oCIAuthConfigProvider, logger)
lbClient, err := createLbClient(oCIAuthConfigProvider, logger)
identityClient, err := createIdentityClient(oCIAuthConfigProvider, logger)
computeClient, err := createComputeClient(oCIAuthConfigProvider, logger)

if err != nil {
return OCIClients{}, err
}

return OCIClients{
VCNClient: vcnClient,
LoadBalancerClient: lbClient,
IdentityClient: identityClient,
ComputeClient: computeClient,
}, err
}

func createVncClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.VirtualNetworkClient, error) {
vcnClient, err := core.NewVirtualNetworkClientWithConfigurationProvider(ociAuthConfigProvider)
if err != nil {
logger.Error(err, "unable to create OCI VCN Client")
return nil, err
}

return &vcnClient, nil
}

func createLbClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*networkloadbalancer.NetworkLoadBalancerClient, error) {
lbClient, err := networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider)
if err != nil {
logger.Error(err, "unable to create OCI LB Client")
return nil, err
}

return &lbClient, nil
}

func createIdentityClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*identity.IdentityClient, error) {
identityClient, err := identity.NewIdentityClientWithConfigurationProvider(ociAuthConfigProvider)
if err != nil {
logger.Error(err, "unable to create OCI Identity Client")
return nil, err
}

return &identityClient, nil
}

func createComputeClient(ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.ComputeClient, error) {
computeClient, err := core.NewComputeClientWithConfigurationProvider(ociAuthConfigProvider)
if err != nil {
logger.Error(err, "unable to create OCI Compute Client")
return nil, err
}

return &computeClient, nil
}
Loading

0 comments on commit 209b749

Please sign in to comment.