Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Remote pkg & sync strategy label #1426

Merged
merged 69 commits into from
May 22, 2024
Merged
Changes from 1 commit
Commits
Show all changes
69 commits
Select commit Hold shift + click to select a range
a6241c3
remove sync strategy and local client
lindnerby Mar 22, 2024
f62aba4
fmt
lindnerby Mar 22, 2024
416e853
reduce client lookup
lindnerby Mar 22, 2024
64221bd
revert type removal
lindnerby Mar 26, 2024
db71164
Merge branch 'main' into skr-client-provider
lindnerby Mar 26, 2024
c61a8cd
change client type
lindnerby Mar 26, 2024
912b733
stash
lindnerby Mar 27, 2024
438cd50
add remote namespace as shared const
lindnerby Mar 28, 2024
41655b4
reduce kyma context
lindnerby Apr 3, 2024
c44792d
Merge branch 'main' into skr-client-provider
lindnerby Apr 8, 2024
d010bd8
extract skrcontextfactory
lindnerby Apr 9, 2024
7277bc4
adapt integration tests
lindnerby Apr 9, 2024
64ccc04
Merge branch 'main' into skr-client-provider
lindnerby Apr 9, 2024
17528f1
add test interface impl and explicit kcp client for crd creation
lindnerby Apr 9, 2024
78aec5d
integration tests
lindnerby Apr 10, 2024
636e07c
adapt all integration test dependencies
lindnerby Apr 18, 2024
4de845e
Merge remote-tracking branch 'upstream/main' into skr-client-provider
lindnerby Apr 18, 2024
618ddef
add kcp client
lindnerby Apr 18, 2024
2784c49
fmt
lindnerby Apr 18, 2024
69bc176
some webhook manager client creation
lindnerby Apr 19, 2024
69896a5
Merge branch 'main' into skr-client-provider
lindnerby Apr 19, 2024
6b72fc3
fix swapped clients
lindnerby Apr 19, 2024
444048a
Merge branch 'main' into skr-client-provider
lindnerby Apr 19, 2024
1139549
use correct client for crd update
lindnerby Apr 22, 2024
ccfa0e6
remove debug
lindnerby Apr 22, 2024
82bf9ba
revert to new client usage
lindnerby Apr 22, 2024
542f52c
test
lindnerby Apr 22, 2024
da5113d
delete context and use kyma. use client cache in integration test fac…
lindnerby Apr 23, 2024
b6b717b
fix
lindnerby Apr 23, 2024
ccce36f
inject factory
lindnerby Apr 23, 2024
d41d61a
use mgr.GetClient
lindnerby Apr 23, 2024
63b2a54
remove kcpaddr resolving from constr func
lindnerby Apr 23, 2024
0a41471
fix
lindnerby Apr 23, 2024
9c70d7f
try eventually
lindnerby Apr 26, 2024
ef3d3c9
Merge branch 'main' into skr-client-provider
lindnerby Apr 26, 2024
1531d46
use skrcontextfactory impl in integration tests correctly
lindnerby Apr 26, 2024
1b55639
Merge branch 'main' into skr-client-provider
lindnerby Apr 29, 2024
831a7f7
fmt
lindnerby Apr 30, 2024
a5b90cd
fix purge-controller integration tests
lindnerby Apr 30, 2024
43c38a3
use single cluster for remaining integration tests
lindnerby Apr 30, 2024
d46cfc4
wrap eventually for skr context retrieval
lindnerby Apr 30, 2024
d375b31
Merge remote-tracking branch 'upstream/main' into skr-client-provider
lindnerby May 2, 2024
42cae40
filter namespace creation err
lindnerby May 2, 2024
37a6c71
exlude sync on single cluster mode
lindnerby May 3, 2024
5e65815
Merge remote-tracking branch 'upstream/main' into skr-client-provider
lindnerby May 3, 2024
681ee5a
Merge remote-tracking branch 'upstream/main' into skr-client-provider
lindnerby May 3, 2024
b1e1be0
fmt
lindnerby May 5, 2024
e2cea0f
Merge remote-tracking branch 'upstream/main' into skr-client-provider
lindnerby May 5, 2024
2d8c218
linting
lindnerby May 5, 2024
9757717
move to internal
lindnerby May 5, 2024
8bb7a78
rename
lindnerby May 5, 2024
73d5db2
linting
lindnerby May 6, 2024
0e97561
linting
lindnerby May 6, 2024
951af6d
Merge remote-tracking branch 'upstream/main' into skr-client-provider
lindnerby May 6, 2024
f925a78
fix merge leftover
lindnerby May 6, 2024
a7194d9
fix coverage file
lindnerby May 6, 2024
a4e1762
Merge branch 'main' into skr-client-provider
lindnerby May 8, 2024
cd0122c
Merge remote-tracking branch 'upstream/main' into skr-client-provider
lindnerby May 8, 2024
dad6d3f
Merge branch 'main' into skr-client-provider
lindnerby May 17, 2024
8f343b0
fix merge
lindnerby May 17, 2024
5548290
fix links
lindnerby May 21, 2024
969025e
add invalidate cache func
lindnerby May 21, 2024
13e04c4
move namespace creation to context
lindnerby May 21, 2024
e41df8a
extract deleted skr handling
lindnerby May 21, 2024
4763698
call init in purge controller
lindnerby May 21, 2024
24d443c
Merge branch 'main' into skr-client-provider
lindnerby May 21, 2024
505294a
Merge branch 'main' into skr-client-provider
lindnerby May 22, 2024
733e501
Merge branch 'main' into skr-client-provider
lindnerby May 22, 2024
517d209
Merge branch 'main' into skr-client-provider
lindnerby May 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add remote namespace as shared const
lindnerby committed Mar 28, 2024
commit 438cd50cd8f7bd4bc477d8984ad396eb1320ed6d
11 changes: 6 additions & 5 deletions api/shared/operator_labels.go
Original file line number Diff line number Diff line change
@@ -23,11 +23,12 @@ const (
PurposeLabel = OperatorGroup + Separator + "purpose"
CertManager = "klm-watcher-cert-manager"
// SkipReconcileLabel indicates this specific resource will be skipped during reconciliation.
SkipReconcileLabel = OperatorGroup + Separator + "skip-reconciliation"
UnmanagedKyma = "unmanaged-kyma"
DefaultRemoteKymaName = "default"
InternalLabel = OperatorGroup + Separator + "internal"
BetaLabel = OperatorGroup + Separator + "beta"
SkipReconcileLabel = OperatorGroup + Separator + "skip-reconciliation"
UnmanagedKyma = "unmanaged-kyma"
DefaultRemoteKymaName = "default"
DefaultRemoteNamespace = "kyma-system"
InternalLabel = OperatorGroup + Separator + "internal"
BetaLabel = OperatorGroup + Separator + "beta"

// Controls ModuleTemplate sync logic.
// If put on the Kyma object, allows to disable sync for all ModuleTemplatesByLabel
25 changes: 3 additions & 22 deletions internal/controller/kyma_controller.go
Original file line number Diff line number Diff line change
@@ -69,7 +69,7 @@ type KymaReconciler struct {
client.Client
record.EventRecorder
queue.RequeueIntervals
KymaSyncContext remote.SyncContextFactory
KymaClientFactory remote.KymaClientFactory
DescriptorProvider *provider.CachedDescriptorProvider
SyncRemoteCrds remote.SyncCrdsUseCase
SKRWebhookManager *watcher.SKRWebhookManifestManager
@@ -157,24 +157,6 @@ func (r *KymaReconciler) deleteRemoteClientCache(ctx context.Context, kyma *v1be
r.RemoteClientCache.Del(client.ObjectKeyFromObject(kyma))
}

// getSyncedContext returns either the original context (in case Syncing is disabled) or initiates a sync-context
// with a remote client and returns that context instead.
// In case of failure, original context should be returned.
func (r *KymaReconciler) getSyncedContext(ctx context.Context, kyma *v1beta2.Kyma) (context.Context, error) {
if !r.SyncKymaEnabled(kyma) {
return ctx, nil
}

remoteClient := remote.NewClientWithConfig(r.Client, r.KcpRestConfig)
ctxWithSync, err := remote.InitializeSyncContext(ctx, kyma,
r.RemoteSyncNamespace, remoteClient, r.RemoteClientCache)
if err != nil {
return ctx, err
}

return ctxWithSync, nil
}

func (r *KymaReconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctrl.Result, error) {
if !kyma.DeletionTimestamp.IsZero() && kyma.Status.State != shared.StateDeleting {
if err := r.deleteRemoteKyma(ctx, kyma); err != nil {
@@ -228,7 +210,6 @@ func (r *KymaReconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctr
}

if r.SyncKymaEnabled(kyma) {
// update the remote kyma with the state of the control plane
if err := r.syncStatusToRemote(ctx, kyma); err != nil {
r.Metrics.RecordRequeueReason(metrics.StatusSyncToRemote, queue.UnexpectedRequeue)
return r.requeueWithError(ctx, kyma, fmt.Errorf("could not synchronize remote kyma status: %w", err))
@@ -240,7 +221,7 @@ func (r *KymaReconciler) reconcile(ctx context.Context, kyma *v1beta2.Kyma) (ctr

func (r *KymaReconciler) syncCrdsAndUpdateKymaAnnotations(ctx context.Context, kyma *v1beta2.Kyma) (bool, error) {

r.KymaSyncContext.GetSyncContext(ctx, kyma, na)
r.KymaClientFactory.GetClient(ctx, kyma, na)
syncContext, err := remote.SyncContextFromContext(ctx)
if err != nil {
return false, fmt.Errorf("failed to get syncContext: %w", err)
@@ -284,7 +265,7 @@ func (r *KymaReconciler) enqueueNormalEvent(kyma *v1beta2.Kyma, reason EventReas
}

func (r *KymaReconciler) fetchRemoteKyma(ctx context.Context, controlPlaneKyma *v1beta2.Kyma) (*v1beta2.Kyma, error) {
syncContext, err := remote.SyncContextFromContext(ctx)
syncContext, err := r.KymaClientFactory.GetClient(ctx, controlPlaneKyma, r.RemoteSyncNamespace)
if err != nil {
return nil, fmt.Errorf("failed to get syncContext: %w", err)
}
128 changes: 51 additions & 77 deletions pkg/remote/kyma_synchronization_context.go
Original file line number Diff line number Diff line change
@@ -23,58 +23,53 @@ import (

var ErrNotFoundAndKCPKymaUnderDeleting = errors.New("not found and kcp kyma under deleting")

type SyncContextFactory interface {
GetSyncContext(ctx context.Context, kyma *v1beta2.Kyma, syncNamespace string) (*KymaSynchronizationContext, error)
type KymaClientFactory interface {
GetClient(ctx context.Context, kyma *v1beta2.Kyma) (*KymaClient, error)
}

type KymaSynchronizationContext struct {
RuntimeClient Client
type KymaClient struct {
Client
}

type KymaSyncContextFactory struct {
kcpClient Client
clientCache *ClientCache
clientCache *ClientCache
clientLookup *ClientLookup
}

func (k *KymaSyncContextFactory) GetSyncContext(ctx context.Context, kyma *v1beta2.Kyma, syncNamespace string) (*KymaSynchronizationContext, error) {
skr, err := NewClientLookup(k.kcpClient, k.clientCache).Lookup(ctx, client.ObjectKeyFromObject(kyma))
func (k *KymaSyncContextFactory) GetClient(ctx context.Context, kyma *v1beta2.Kyma) (*KymaClient, error) {
skrClient, err := k.clientLookup.Lookup(ctx, client.ObjectKeyFromObject(kyma))
if err != nil {
return nil, err
}

syncContext := &KymaSynchronizationContext{
RuntimeClient: skr,
syncContext := &KymaClient{
Client: skrClient,
}

if err := syncContext.ensureRemoteNamespaceExists(ctx, syncNamespace); err != nil {
if err := syncContext.ensureNamespaceExists(ctx); err != nil {
return nil, err
}

return syncContext, nil
}

func (c *KymaSynchronizationContext) GetRemotelySyncedKyma(
ctx context.Context, remoteSyncNamespace string,
) (*v1beta2.Kyma, error) {
remoteKyma := &v1beta2.Kyma{}
remoteKyma.Name = shared.DefaultRemoteKymaName
remoteKyma.Namespace = remoteSyncNamespace
if err := c.RuntimeClient.Get(ctx, client.ObjectKeyFromObject(remoteKyma), remoteKyma); err != nil {
return remoteKyma, fmt.Errorf("failed to get remote kyma: %w", err)
func (k *KymaClient) getRemotelySyncedKyma(ctx context.Context) (*v1beta2.Kyma, error) {
skrKyma := &v1beta2.Kyma{
ObjectMeta: apimetav1.ObjectMeta{
Name: shared.DefaultRemoteKymaName,
Namespace: shared.DefaultRemoteNamespace,
},
}

return remoteKyma, nil
}

func RemoveFinalizersFromRemoteKyma(
ctx context.Context, remoteSyncNamespace string,
) error {
syncContext, err := SyncContextFromContext(ctx)
if err != nil {
return fmt.Errorf("failed to get syncContext: %w", err)
if err := k.Client.Get(ctx, client.ObjectKeyFromObject(skrKyma), skrKyma); err != nil {
return skrKyma, fmt.Errorf("failed to get remote kyma: %w", err)
}

remoteKyma, err := syncContext.GetRemotelySyncedKyma(ctx, remoteSyncNamespace)
return skrKyma, nil
}

func (k *KymaClient) RemoveFinalizersFromRemoteKyma(ctx context.Context) error {
remoteKyma, err := k.getRemotelySyncedKyma(ctx)
if err != nil {
return err
}
@@ -83,37 +78,29 @@ func RemoveFinalizersFromRemoteKyma(
controllerutil.RemoveFinalizer(remoteKyma, finalizer)
}

err = syncContext.RuntimeClient.Update(ctx, remoteKyma)
err = k.Client.Update(ctx, remoteKyma)
if err != nil {
return fmt.Errorf("failed to update remote kyma when removing finalizers: %w", err)
}
return nil
}

func DeleteRemotelySyncedKyma(
ctx context.Context, remoteSyncNamespace string,
) error {
syncContext, err := SyncContextFromContext(ctx)
if err != nil {
return fmt.Errorf("failed to get syncContext: %w", err)
}
remoteKyma, err := syncContext.GetRemotelySyncedKyma(ctx, remoteSyncNamespace)
func (k *KymaClient) DeleteRemotelySyncedKyma(ctx context.Context) error {
remoteKyma, err := k.getRemotelySyncedKyma(ctx)
if err != nil {
return err
}
err = syncContext.RuntimeClient.Delete(ctx, remoteKyma)
err = k.Client.Delete(ctx, remoteKyma)
if err != nil {
return fmt.Errorf("failed to delete remote kyma: %w", err)
}
return nil
}

// ensureRemoteNamespaceExists tries to ensure existence of a namespace for synchronization based on
// name of controlPlaneKyma.namespace in this order.
func (c *KymaSynchronizationContext) ensureRemoteNamespaceExists(ctx context.Context, syncNamespace string) error {
func (k *KymaClient) ensureNamespaceExists(ctx context.Context) error {
namespace := &apicorev1.Namespace{
ObjectMeta: apimetav1.ObjectMeta{
Name: syncNamespace,
Name: shared.DefaultRemoteNamespace,
Labels: map[string]string{shared.ManagedBy: shared.OperatorName},
},
// setting explicit type meta is required for SSA on Namespaces
@@ -127,40 +114,36 @@ func (c *KymaSynchronizationContext) ensureRemoteNamespaceExists(ctx context.Con

patch := client.RawPatch(types.ApplyPatchType, buf.Bytes())
force := true
fieldManager := "kyma-sync-context"

if err := c.RuntimeClient.Patch(
ctx, namespace, patch, &client.PatchOptions{Force: &force, FieldManager: fieldManager},
); err != nil {
patchOpts := &client.PatchOptions{Force: &force, FieldManager: "kyma-sync-context"}
if err := k.Client.Patch(ctx, namespace, patch, patchOpts); err != nil {
return fmt.Errorf("failed to ensure remote namespace exists: %w", err)
}

return nil
}

func (c *KymaSynchronizationContext) CreateOrUpdateCRD(ctx context.Context, plural string) error {
func (k *KymaClient) createOrUpdateCRD(ctx context.Context, plural string) error {
crd := &apiextensionsv1.CustomResourceDefinition{}
crdFromRuntime := &apiextensionsv1.CustomResourceDefinition{}
var err error
err = c..Get(
ctx, client.ObjectKey{
// this object name is derived from the plural and is the default kustomize value for crd namings, if the CRD
// name changes, this also has to be adjusted here. We can think of making this configurable later
Name: fmt.Sprintf("%s.%s", plural, v1beta2.GroupVersion.Group),
}, crd,
err = k.Client.Get(ctx, client.ObjectKey{
// this object name is derived from the plural and is the default kustomize value for crd namings, if the CRD
// name changes, this also has to be adjusted here. We can think of making this configurable later
Name: fmt.Sprintf("%s.%s", plural, v1beta2.GroupVersion.Group),
}, crd,
)
if err != nil {
return fmt.Errorf("failed to get kyma CRDs on kcp: %w", err)
}

err = c.RuntimeClient.Get(
err = k.Client.Get(
ctx, client.ObjectKey{
Name: fmt.Sprintf("%s.%s", plural, v1beta2.GroupVersion.Group),
}, crdFromRuntime,
)

if util.IsNotFound(err) || !ContainsLatestVersion(crdFromRuntime, v1beta2.GroupVersion.Version) {
return PatchCRD(ctx, c.RuntimeClient, crd)
return PatchCRD(ctx, k.Client, crd)
}

if err != nil {
@@ -170,16 +153,13 @@ func (c *KymaSynchronizationContext) CreateOrUpdateCRD(ctx context.Context, plur
return nil
}

func (c *KymaSynchronizationContext) CreateOrFetchRemoteKyma(
ctx context.Context, kyma *v1beta2.Kyma, remoteSyncNamespace string,
) (*v1beta2.Kyma, error) {
func (k *KymaClient) CreateOrFetchRemoteKyma(ctx context.Context, kyma *v1beta2.Kyma) (*v1beta2.Kyma, error) {
recorder := adapter.RecorderFromContext(ctx)

remoteKyma, err := c.GetRemotelySyncedKyma(ctx, remoteSyncNamespace)
remoteKyma, err := k.getRemotelySyncedKyma(ctx)
if meta.IsNoMatchError(err) || CRDNotFoundErr(err) {
recorder.Event(kyma, "Normal", err.Error(), "CRDs are missing in SKR and will be installed")

if err := c.CreateOrUpdateCRD(ctx, shared.KymaKind.Plural()); err != nil {
if err := k.createOrUpdateCRD(ctx, shared.KymaKind.Plural()); err != nil {
return nil, err
}

@@ -192,7 +172,7 @@ func (c *KymaSynchronizationContext) CreateOrFetchRemoteKyma(
}
kyma.Spec.DeepCopyInto(&remoteKyma.Spec)

err = c.RuntimeClient.Create(ctx, remoteKyma)
err = k.Client.Create(ctx, remoteKyma)
if err != nil {
recorder.Event(kyma, "Normal", "RemoteInstallation", "Kyma was installed to SKR")
return nil, fmt.Errorf("failed to create remote kyma: %w", err)
@@ -205,43 +185,37 @@ func (c *KymaSynchronizationContext) CreateOrFetchRemoteKyma(
return remoteKyma, nil
}

func (c *KymaSynchronizationContext) SynchronizeRemoteKyma(
ctx context.Context,
controlPlaneKyma, remoteKyma *v1beta2.Kyma,
) error {
func (k *KymaClient) SynchronizeRemoteKyma(ctx context.Context, controlPlaneKyma, remoteKyma *v1beta2.Kyma) error {
if !remoteKyma.GetDeletionTimestamp().IsZero() {
return nil
}
recorder := adapter.RecorderFromContext(ctx)

c.SyncWatcherLabelsAnnotations(controlPlaneKyma, remoteKyma)
if err := c.RuntimeClient.Update(ctx, remoteKyma); err != nil {
k.syncWatcherLabelsAnnotations(controlPlaneKyma, remoteKyma)
if err := k.Client.Update(ctx, remoteKyma); err != nil {
recorder.Event(controlPlaneKyma, "Warning", err.Error(), "could not synchronise runtime kyma "+
"spec, watcher labels and annotations")
return fmt.Errorf("failed to synchronise runtime kyma: %w", err)
}

remoteKyma.Status = controlPlaneKyma.Status
if err := c.RuntimeClient.Status().Update(ctx, remoteKyma); err != nil {
if err := k.Client.Status().Update(ctx, remoteKyma); err != nil {
recorder.Event(controlPlaneKyma, "Warning", err.Error(), "could not update runtime kyma status")
return fmt.Errorf("failed to update runtime kyma status: %w", err)
}
return nil
}

// ReplaceModules replaces modules specification from control plane Kyma with Remote Kyma specifications.
func ReplaceModules(
controlPlaneKyma *v1beta2.Kyma,
remoteKyma *v1beta2.Kyma,
) {
func ReplaceModules(controlPlaneKyma *v1beta2.Kyma, remoteKyma *v1beta2.Kyma) {
controlPlaneKyma.Spec.Modules = []v1beta2.Module{}
controlPlaneKyma.Spec.Modules = append(controlPlaneKyma.Spec.Modules, remoteKyma.Spec.Modules...)
controlPlaneKyma.Spec.Channel = remoteKyma.Spec.Channel
}

// SyncWatcherLabelsAnnotations inserts labels into the given KymaCR, which are needed to ensure
// syncWatcherLabelsAnnotations inserts labels into the given KymaCR, which are needed to ensure
// a working e2e-flow for the runtime-watcher.
func (c *KymaSynchronizationContext) SyncWatcherLabelsAnnotations(controlPlaneKyma, remoteKyma *v1beta2.Kyma) {
func (k *KymaClient) syncWatcherLabelsAnnotations(controlPlaneKyma, remoteKyma *v1beta2.Kyma) {
if remoteKyma.Labels == nil {
remoteKyma.Labels = make(map[string]string)
}
10 changes: 5 additions & 5 deletions pkg/remote/remote_catalog.go
Original file line number Diff line number Diff line change
@@ -87,7 +87,7 @@ var errTemplateCleanup = errors.New("failed to delete obsolete catalog templates
func (c *RemoteCatalog) deleteDiffCatalog(ctx context.Context,
kcpModules []v1beta2.ModuleTemplate,
runtimeModules []v1beta2.ModuleTemplate,
syncContext *KymaSynchronizationContext,
syncContext *KymaClient,
) error {
diffsToDelete := c.diffsToDelete(runtimeModules, kcpModules)
channelLength := len(diffsToDelete)
@@ -116,7 +116,7 @@ var errCatTemplatesApply = errors.New("could not apply catalog templates")

func (c *RemoteCatalog) createOrUpdateCatalog(ctx context.Context,
kcpModules []v1beta2.ModuleTemplate,
syncContext *KymaSynchronizationContext,
syncContext *KymaClient,
) error {
channelLength := len(kcpModules)
results := make(chan error, channelLength)
@@ -159,14 +159,14 @@ func containsCRDNotFoundError(errs []error) bool {
}

func (c *RemoteCatalog) patchDiff(
ctx context.Context, diff *v1beta2.ModuleTemplate, syncContext *KymaSynchronizationContext,
ctx context.Context, diff *v1beta2.ModuleTemplate, syncContext *KymaClient,
deleteInsteadOfPatch bool,
) error {
var err error
if deleteInsteadOfPatch {
err = syncContext.RuntimeClient.Delete(ctx, diff)
err = syncContext.SkrClient.Delete(ctx, diff)
} else {
err = syncContext.RuntimeClient.Patch(
err = syncContext.SkrClient.Patch(
ctx, diff, client.Apply, c.settings.SSAPatchOptions,
)
}
1 change: 0 additions & 1 deletion pkg/testutils/utils.go
Original file line number Diff line number Diff line change
@@ -112,7 +112,6 @@ func NewSKRCluster(scheme *machineryruntime.Scheme) (client.Client, *envtest.Env
//}

skrClient, err := client.New(authUser.Config(), client.Options{Scheme: scheme})

return skrClient, skrEnv, err
}