rancher · Danil-Grigorev · Oct 9, 2023
@@ -190,7 +190,6 @@ manifests: vendor controller-gen ## Generate WebhookConfiguration, ClusterRole a
 vendor:
 	go mod tidy
 	go mod vendor
-	go mod verify
 
 .PHONY: vendor-clean
 vendor-clean:
@@ -242,7 +241,7 @@ ARTIFACTS ?= ${ROOT_DIR}/_artifacts
 KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))
 
 .PHONY: test
-test: $(SETUP_ENVTEST) manifests ## Run tests.
+test: $(SETUP_ENVTEST) manifests kubectl ## Run tests.
 	KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS)
 
 ##@ Build

@@ -0,0 +1,163 @@
+/*
+Copyright 2023 SUSE.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
+	"github.com/rancher-sandbox/rancher-turtles/internal/rancher/setup"
+	"github.com/rancher-sandbox/rancher-turtles/internal/test"
+	turtlesnaming "github.com/rancher-sandbox/rancher-turtles/util/naming"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util/kubeconfig"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+var (
+	rancherEnv     *envtest.Environment
+	otherEnv       *envtest.Environment
+	rancherCfg     *rest.Config
+	otherCfg       *rest.Config
+	rancherCl      client.Client
+	otherCl        client.Client
+	clusterCtx     context.Context
+	cancel         context.CancelFunc
+	capiCluster    *clusterv1.Cluster
+	rancherCluster *provisioningv1.Cluster
+	r              *CAPIImportReconciler
+)
+
+var _ = Describe("In separate clusters", func() {
+
+	BeforeEach(func() {
+		By("bootstrapping rancher environment")
+		var err error
+		rancherEnv = &envtest.Environment{
+			CRDDirectoryPaths: []string{
+				filepath.Join("..", "..", "hack", "crd", "bases"),
+			},
+			ErrorIfCRDPathMissing: true,
+			Scheme:                test.RancherScheme,
+		}
+		rancherCfg, rancherCl, err = test.StartEnvTest(rancherEnv)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(rancherCfg).NotTo(BeNil())
+		Expect(rancherCl).NotTo(BeNil())
+
+		By("Bootstrapping other cluster environment")
+		otherEnv = &envtest.Environment{
+			CRDDirectoryPaths: []string{
+				filepath.Join("..", "..", "hack", "crd", "bases"),
+			},
+			ErrorIfCRDPathMissing: true,
+			Scheme:                test.PartialScheme,
+		}
+		otherCfg, otherCl, err = test.StartEnvTest(otherEnv)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(otherCfg).NotTo(BeNil())
+		Expect(otherCl).NotTo(BeNil())
+
+		Expect(otherCl.Create(ctx, &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: testNamespace,
+				Labels: map[string]string{
+					importLabelName: "true",
+				},
+			},
+		})).To(Succeed())
+
+		Expect(rancherCl.Create(ctx, &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: testNamespace,
+			},
+		})).To(Succeed())
+
+		clusterCtx, cancel = context.WithCancel(ctx)
+		capiCluster = &clusterv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test-cluster",
+				Namespace: testNamespace,
+			},
+		}
+
+		rancherCluster = &provisioningv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      turtlesnaming.Name(capiCluster.Name).ToRancherName(),
+				Namespace: testNamespace,
+			},
+		}
+	})
+
+	AfterEach(func() {
+		cancel()
+		By("tearing down the 2 clsuter environment")
+		Expect(test.StopEnvTest(rancherEnv)).To(Succeed())
+		Expect(test.StopEnvTest(otherEnv)).To(Succeed())
+	})
+
+	It("minimal controller setup should create a Rancher cluster object from a CAPI cluster object located in a different cluster", func() {
+		mgr, err := ctrl.NewManager(otherCfg, ctrl.Options{
+			Scheme:                 otherEnv.Scheme,
+			MetricsBindAddress:     "0",
+			HealthProbeBindAddress: "0",
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		config := kubeconfig.FromEnvTestConfig(rancherCfg, &clusterv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+		})
+		kubeconfigFile, err := os.CreateTemp("", "kubeconfig")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.Remove(kubeconfigFile.Name())
+		Expect(os.WriteFile(kubeconfigFile.Name(), config, 0600)).To(Succeed())
+
+		rancher, err := setup.RancherCluster(mgr, kubeconfigFile.Name())
+		Expect(err).ToNot(HaveOccurred())
+
+		reconciler := &CAPIImportReconciler{
+			Client:         mgr.GetClient(),
+			RancherCluster: rancher,
+		}
+		Expect(reconciler.SetupWithManager(ctx, mgr, controller.Options{})).To(Succeed())
+
+		go func() {
+			Expect(mgr.Start(clusterCtx)).To(Succeed())
+		}()
+
+		Expect(otherCl.Create(ctx, capiCluster)).To(Succeed())
+		capiCluster.Status.ControlPlaneReady = true
+		Expect(otherCl.Status().Update(ctx, capiCluster)).To(Succeed())
+
+		args := []string{"get", fmt.Sprintf("clusters.%s", provisioningv1.GroupVersion.Group), rancherCluster.Name, "-n", rancherCluster.Namespace, "--kubeconfig", kubeconfigFile.Name()}
+		Eventually(exec.Command("kubectl", args...).Run()).Should(Succeed())
+	})
+
+})
@@ -36,6 +36,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -68,7 +69,7 @@ const (
 // CAPIImportReconciler represents a reconciler for importing CAPI clusters in Rancher.
 type CAPIImportReconciler struct {
 	Client             client.Client
-	RancherClient      client.Client
+	RancherCluster     cluster.Cluster
 	recorder           record.EventRecorder
 	WatchFilterValue   string
 	Scheme             *runtime.Scheme
@@ -106,7 +107,7 @@ func (r *CAPIImportReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma
 	// Watch Rancher provisioningv2 clusters
 	// NOTE: we will import the types from rancher in the future
 	err = c.Watch(
-		source.Kind(mgr.GetCache(), &provisioningv1.Cluster{}),
+		source.Kind(r.RancherCluster.GetCache(), &provisioningv1.Cluster{}),
 		handler.EnqueueRequestsFromMapFunc(r.rancherClusterToCapiCluster(ctx, capiPredicates)),
 		//&handler.EnqueueRequestForOwner{OwnerType: &clusterv1.Cluster{}},
 	)
@@ -200,7 +201,7 @@ func (r *CAPIImportReconciler) reconcile(ctx context.Context, capiCluster *clust
 		Name:      turtlesnaming.Name(capiCluster.Name).ToRancherName(),
 	}}
 
-	err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
+	err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
 	if client.IgnoreNotFound(err) != nil {
 		log.Error(err, fmt.Sprintf("Unable to fetch rancher cluster %s", client.ObjectKeyFromObject(rancherCluster)))
 		return ctrl.Result{Requeue: true}, err
@@ -218,7 +219,7 @@ func (r *CAPIImportReconciler) reconcileNormal(ctx context.Context, capiCluster
 ) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
 
-	err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
+	err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)
 	if apierrors.IsNotFound(err) {
 		shouldImport, err := util.ShouldAutoImport(ctx, log, r.Client, capiCluster, importLabelName)
 		if err != nil {
@@ -230,7 +231,7 @@ func (r *CAPIImportReconciler) reconcileNormal(ctx context.Context, capiCluster
 			return ctrl.Result{}, nil
 		}
 
-		if err := r.RancherClient.Create(ctx, &provisioningv1.Cluster{
+		if err := r.RancherCluster.GetClient().Create(ctx, &provisioningv1.Cluster{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      turtlesnaming.Name(capiCluster.Name).ToRancherName(),
 				Namespace: capiCluster.Namespace,
@@ -323,7 +324,7 @@ func (r *CAPIImportReconciler) getClusterRegistrationManifest(ctx context.Contex
 		Name:      clusterRegistrationTokenName,
 		Namespace: clusterName,
 	}}
-	err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(token), token)
+	err := r.RancherCluster.GetClient().Get(ctx, client.ObjectKeyFromObject(token), token)
 
 	if client.IgnoreNotFound(err) != nil {
 		return "", fmt.Errorf("error getting registration token for cluster %s: %w", clusterName, err)

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"bufio"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -43,6 +44,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util/secret"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
@@ -53,12 +55,24 @@ var _ = Describe("reconcile CAPI Cluster", func() {
 		rancherCluster           *provisioningv1.Cluster
 		clusterRegistrationToken *managementv3.ClusterRegistrationToken
 		capiKubeconfigSecret     *corev1.Secret
+		cancel                   context.CancelFunc
+		clusterCtx               context.Context
 	)
 
 	BeforeEach(func() {
+		// rancher and rancher-turtles deployed in the same cluster
+		clusterCtx, cancel = context.WithCancel(ctx)
+		cluster, err := cluster.New(testEnv.Config, func(clusterOptions *cluster.Options) {
+			clusterOptions.Scheme = testEnv.Scheme
+		})
+		Expect(err).ToNot(HaveOccurred())
+		go func() {
+			Expect(cluster.Start(clusterCtx)).To(Succeed())
+		}()
+
 		r = &CAPIImportReconciler{
 			Client:             cl,
-			RancherClient:      cl, // rancher and rancher-turtles deployed in the same cluster
+			RancherCluster:     cluster,
 			remoteClientGetter: remote.NewClusterClient,
 		}
 
@@ -95,6 +109,7 @@ var _ = Describe("reconcile CAPI Cluster", func() {
 	})
 
 	AfterEach(func() {
+		defer cancel()
 		objs, err := manifestToObjects(strings.NewReader(testdata.ImportManifest))
 		clientObjs := []client.Object{
 			capiCluster,

@@ -59,6 +59,7 @@ var _ = BeforeSuite(func() {
 			filepath.Join("..", "..", "hack", "crd", "bases"),
 		},
 		ErrorIfCRDPathMissing: true,
+		Scheme:                test.FullScheme,
 	}
 	cfg, cl, err = test.StartEnvTest(testEnv)
 	Expect(err).NotTo(HaveOccurred())

@@ -14,7 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package rancher contains the rancher provisioning.cattle.io/v1 and
-// management.cattle.io/v3 API proxy implementations.
+// Package rancher contains rancher provisioning.cattle.io/v1 and
+// management.cattle.io/v3 API proxy implementations and cluster connectivity
+// setup procedures.
 // +kubebuilder:object:generate=true
 package rancher