Merge branch 'main' into refactor_available_module

.github/actions/deploy-lifecycle-manager-e2e/action.yml

@@ -109,7 +109,8 @@ runs:
         matrix.e2e-test == 'module-status-decoupling-with-deployment' ||
         matrix.e2e-test == 'purge-metrics' ||
         matrix.e2e-test == 'self-signed-certificate-rotation' ||
-        matrix.e2e-test == 'mandatory-module-metrics'}}  
+        matrix.e2e-test == 'mandatory-module-metrics' ||
+        matrix.e2e-test == 'mandatory-module-metrics-with-old-naming-pattern'}}
       shell: bash
       run: |
         kubectl patch svc klm-controller-manager-metrics -p '{"spec": {"type": "LoadBalancer"}}' -n kcp-system

.github/actions/deploy-template-operator-with-modulereleasemeta/action.yml

@@ -20,6 +20,9 @@ runs:
         cp ../lifecycle-manager/scripts/tests/deploy_modulereleasemeta.sh .
     - name: Create and apply Template Operator ModuleTemplate from the latest release
       working-directory: template-operator
+      if: ${{ matrix.e2e-test != 'mandatory-module' &&
+        matrix.e2e-test != 'mandatory-module-metrics'
+        }}
       shell: bash
       run: |
         modulectl create --config-file ./module-config.yaml --registry http://localhost:5111 --insecure
@@ -28,13 +31,19 @@ runs:
         kubectl apply -f template.yaml
     - name: Create and apply Template Operator ModuleTemplate with ModuleDeploymentNameInOlderVersion
       working-directory: template-operator
+      if: ${{ matrix.e2e-test != 'mandatory-module' &&
+        matrix.e2e-test != 'mandatory-module-metrics'
+        }}
       shell: bash
       run: |
         make build-manifests
         yq eval '(. | select(.kind == "Deployment") | .metadata.name) = "${{ env.ModuleDeploymentNameInOlderVersion }}"' -i template-operator.yaml
         ./deploy_moduletemplate.sh ${{ env.ModuleName }} ${{ env.OlderVersion }}
     - name: Create and apply Template Operator ModuleTemplate with ModuleDeploymentNameInNewerVersion
       working-directory: template-operator
+      if: ${{ matrix.e2e-test != 'mandatory-module' &&
+        matrix.e2e-test != 'mandatory-module-metrics' 
+        }}
       shell: bash
       run: |
         make build-manifests
@@ -70,14 +79,27 @@ runs:
       shell: bash
       run: |
         ./deploy_modulereleasemeta.sh ${{ env.ModuleName }} fast:${{ env.NewerVersion }} regular:${{ env.OlderVersion }}
-    - name: Create Template Operator Module as Mandatory Module
-      working-directory: lifecycle-manager
+
+    - name: Create and apply Template Operator Module as Mandatory Module
+      working-directory: template-operator
       if: ${{ matrix.e2e-test == 'mandatory-module' ||
         matrix.e2e-test == 'mandatory-module-metrics'
         }}
       shell: bash
       run: |
-        kubectl apply -f tests/e2e/moduletemplate/mandatory_moduletemplate_template_operator_v1.yaml
+        make build-manifests
+        yq eval '(. | select(.kind == "Deployment") | .metadata.name) = "${{ env.ModuleDeploymentNameInOlderVersion }}"' -i template-operator.yaml
+        ./deploy_moduletemplate.sh ${{ env.ModuleName }} ${{ env.OlderVersionForMandatoryModule }} true true
+
+    - name: Create ModuleTemplate in a new version for Mandatory module
+      if: ${{ matrix.e2e-test == 'mandatory-module'}}
+      working-directory: template-operator
+      shell: bash
+      run: |
+        make build-manifests
+        yq eval '(. | select(.kind == "Deployment") | .metadata.name) = "${{ env.ModuleDeploymentNameInNewerVersion }}"' -i template-operator.yaml
+        ./deploy_moduletemplate.sh ${{ env.ModuleName }} ${{ env.NewerVersionForMandatoryModule }} true true false
+        cp template.yaml ../lifecycle-manager/tests/e2e/mandatory_template_v2.yaml
     - name: Create and apply ModuleReleaseMeta Template Operator with newer version in fast channel and older version in regular channel
       working-directory: template-operator
       if: ${{ matrix.e2e-test == 'non-blocking-deletion' }}

.github/actions/deploy-template-operator/action.yml

@@ -47,8 +47,8 @@ runs:
         kubectl apply -f tests/e2e/moduletemplate/moduletemplate_template_operator_v2_direct_version.yaml
     - name: Create Template Operator Module as Mandatory Module
       working-directory: lifecycle-manager
-      if: ${{ matrix.e2e-test == 'mandatory-module' ||
-        matrix.e2e-test == 'mandatory-module-metrics' 
+      if: ${{ matrix.e2e-test == 'mandatory-module-with-old-naming-pattern' ||
+        matrix.e2e-test == 'mandatory-module-metrics-with-old-naming-pattern' 
         }}
       shell: bash
       run: |

.github/workflows/test-e2e-with-modulereleasemeta.yml

@@ -115,6 +115,8 @@ jobs:
           ModuleDeploymentNameInOlderVersion: template-operator-v1-controller-manager
           NewerVersion: 2.4.2-e2e-test
           OlderVersion: 1.1.1-e2e-test
+          OlderVersionForMandatoryModule: 1.1.0-smoke-test
+          NewerVersionForMandatoryModule: 2.4.1-smoke-test
           VersionForStatefulSetInWarning: 1.0.0-warning-statefulset
           VersionForDeploymentInWarning: 1.0.0-warning-deployment
           VersionForMisconfiguredDeploymentImage: 1.0.0-misconfigured-deployment

.github/workflows/test-e2e.yml

@@ -58,8 +58,8 @@ jobs:
           - ca-certificate-rotation
           - istio-gateway-secret-rotation
           - self-signed-certificate-rotation
-          - mandatory-module
-          - mandatory-module-metrics
+          - mandatory-module-with-old-naming-pattern
+          - mandatory-module-metrics-with-old-naming-pattern
           - misconfigured-kyma-secret
           - rbac-privileges
           - ocm-compatible-module-template

docs/contributor/02-controllers.md

@@ -40,7 +40,7 @@ Lifecycle Manager uses two Mandatory Modules Controllers:
 * [Mandatory modules installation controller](../../internal/controller/mandatorymodule/installation_controller.go) deals with the reconciliation of mandatory modules
 * [Mandatory modules deletion controller](../../internal/controller/mandatorymodule/deletion_controller.go) deals with the deletion of mandatory modules
 
-Since the channel concept does not apply to mandatory modules, the Mandatory Modules Installation Controller fetches all the Mandatory ModuleTemplate CRs without any channel filtering. It then translates the ModuleTemplate CR for the mandatory module to a [Manifest CR](../../api/v1beta2/manifest_types.go) with an OwnerReference to the Kyma CR. Similarly to the [Kyma Controller](../../internal/controller/kyma/controller.go),
+Since the channel concept does not apply to mandatory modules, the Mandatory Modules Installation Controller fetches all the Mandatory ModuleTemplate CRs with the 'operator.kyma-project.io/mandatory-module' label. If multiple ModuleTemplates exist for the same mandatory module, the Controller fetches the ModuleTemplate with the highest version. It then translates the ModuleTemplate CR for the mandatory module to a [Manifest CR](../../api/v1beta2/manifest_types.go) with an OwnerReference to the Kyma CR. Similarly to the [Kyma Controller](../../internal/controller/kyma/controller.go),
 it propagates changes from the ModuleTemplate CR to the Manifest CR. The mandatory ModuleTemplate CR is not synchronized to the remote cluster and the module status does not appear in the Kyma CR status. If a mandatory module needs to be removed from all clusters, the corresponding ModuleTemplate CR needs to be deleted. The Mandatory Module Deletion Controller picks this event up and marks all associated Manifest CRs for deletion. To ensure that the ModuleTemplate CR is not removed immediately, the controller adds a finalizer to the ModuleTemplate CR. Once all associated Manifest CRs are deleted, the finalizer is removed and the ModuleTemplate CR is deleted.
 
 ## Manifest Controller

docs/contributor/resources/03-moduletemplate.md

@@ -166,5 +166,3 @@ The list is purely informational and does not introduce functional changes to th
 These are the synchronization labels available on the ModuleTemplate CR:
 
 * `operator.kyma-project.io/sync`: A boolean value. If set to `false`, this ModuleTemplate CR is not synchronized with any remote cluster. The default value is `true`.
-* `operator.kyma-project.io/internal`: A boolean value. If set to `true`, marks the ModuleTemplate CR as an `internal` module. It is then synchronized only for these remote clusters which are managed by the Kyma CR with the same `operator.kyma-project.io/internal` label explicitly set to `true`. The default value is `false`.
-* `operator.kyma-project.io/beta` A boolean value. If set to `true`, marks the ModuleTemplate CR as a `beta` module. It is then synchronized only for these remote clusters which are managed by the Kyma CR with the same `operator.kyma-project.io/beta` label explicitly set to `true`. The default value is `false`.

docs/contributor/resources/05-modulereleasemeta.md

@@ -16,6 +16,18 @@ kubectl get crd modulereleasemetas.operator.kyma-project.io -o yaml
 
 The **moduleName** defines the name of the module for which the channel assignments are listed.
 
+### **.spec.beta**
+
+The **beta** flag defines if the module is a `beta` module. If marked as `beta`, it is only synced to SKRs where the Kyma CR is marked with the `"operator.kyma-project.io/beta": "true"` label. This includes the ModuleTemplates related to this module.
+
+The default value is `false`.
+
+### **.spec.internal**
+
+The **internal** flag defines if the module is an `internal` module. If marked as `internal`, it is only synced to SKRs where the Kyma CR is marked with the `"operator.kyma-project.io/internal": "true"` label. This includes the ModuleTemplates related to this module.
+
+The default value is `false`.
+
 ### **.spec.channels**
 
 The **channels** define each module channel with its corresponding version. Each channel can only have one version assigned.
@@ -24,6 +36,8 @@ See the following example:
 ```yaml
 spec:
   moduleName: keda
+  beta: false
+  internal: false
   channels:
     - channel: regular
       version: 1.0.0

internal/controller/mandatorymodule/deletion_controller.go

@@ -117,7 +117,7 @@ func (r *DeletionReconciler) getCorrespondingManifests(ctx context.Context,
 	if err := r.List(ctx, manifests, &client.ListOptions{
 		Namespace:     template.Namespace,
 		LabelSelector: k8slabels.SelectorFromSet(k8slabels.Set{shared.IsMandatoryModule: "true"}),
-	}); err != nil {
+	}); client.IgnoreNotFound(err) != nil {
 		return nil, fmt.Errorf("not able to list mandatory module manifests: %w", err)
 	}
 
@@ -136,7 +136,9 @@ func (r *DeletionReconciler) removeManifests(ctx context.Context, manifests []v1
 	return nil
 }
 
-func filterManifestsByAnnotation(manifests []v1beta2.Manifest, annotationKey, annotationValue string) []v1beta2.Manifest {
+func filterManifestsByAnnotation(manifests []v1beta2.Manifest,
+	annotationKey, annotationValue string,
+) []v1beta2.Manifest {
 	filteredManifests := make([]v1beta2.Manifest, 0)
 	for _, manifest := range manifests {
 		if manifest.Annotations[annotationKey] == annotationValue {

pkg/templatelookup/mandatory.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/Masterminds/semver/v3"
 	k8slabels "k8s.io/apimachinery/pkg/labels"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -22,14 +23,79 @@ func GetMandatory(ctx context.Context, kymaClient client.Reader) (ModuleTemplate
 		return nil, fmt.Errorf("could not list mandatory ModuleTemplates: %w", err)
 	}
 
+	// maps module name to the module template of the highest version encountered
 	mandatoryModules := make(map[string]*ModuleTemplateInfo)
 	for _, moduleTemplate := range mandatoryModuleTemplateList.Items {
 		if moduleTemplate.DeletionTimestamp.IsZero() {
-			mandatoryModules[moduleTemplate.Name] = &ModuleTemplateInfo{
-				ModuleTemplate: &moduleTemplate,
+			currentModuleTemplate := &moduleTemplate
+			moduleName := GetModuleName(currentModuleTemplate)
+			if mandatoryModules[moduleName] != nil {
+				var err error
+				currentModuleTemplate, err = GetModuleTemplateWithHigherVersion(currentModuleTemplate,
+					mandatoryModules[moduleName].ModuleTemplate)
+				if err != nil {
+					mandatoryModules[moduleName] = &ModuleTemplateInfo{
+						ModuleTemplate: nil,
+						Err:            err,
+					}
+					continue
+				}
+			}
+			mandatoryModules[moduleName] = &ModuleTemplateInfo{
+				ModuleTemplate: currentModuleTemplate,
 				Err:            nil,
 			}
 		}
 	}
 	return mandatoryModules, nil
 }
+
+func GetModuleName(moduleTemplate *v1beta2.ModuleTemplate) string {
+	if moduleTemplate.Spec.ModuleName != "" {
+		return moduleTemplate.Spec.ModuleName
+	}
+
+	// https://github.com/kyma-project/lifecycle-manager/issues/2135
+	// Remove this after warden ModuleTemplate is created using modulectl
+	return moduleTemplate.Labels[shared.ModuleName]
+}
+
+func GetModuleSemverVersion(moduleTemplate *v1beta2.ModuleTemplate) (*semver.Version, error) {
+	if moduleTemplate.Spec.Version != "" {
+		version, err := semver.NewVersion(moduleTemplate.Spec.Version)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse version as a semver: %s: %w",
+				moduleTemplate.Spec.Version, err)
+		}
+		return version, nil
+	}
+
+	// https://github.com/kyma-project/lifecycle-manager/issues/2135
+	// Remove this after warden ModuleTemplate is created using modulectl
+	version, err := semver.NewVersion(moduleTemplate.Annotations[shared.ModuleVersionAnnotation])
+	if err != nil {
+		return nil, fmt.Errorf("could not parse version as a semver %s: %w",
+			moduleTemplate.Annotations[shared.ModuleVersionAnnotation], err)
+	}
+	return version, nil
+}
+
+func GetModuleTemplateWithHigherVersion(firstModuleTemplate, secondModuleTemplate *v1beta2.ModuleTemplate) (*v1beta2.ModuleTemplate,
+	error,
+) {
+	firstVersion, err := GetModuleSemverVersion(firstModuleTemplate)
+	if err != nil {
+		return nil, err
+	}
+
+	secondVersion, err := GetModuleSemverVersion(secondModuleTemplate)
+	if err != nil {
+		return nil, err
+	}
+
+	if firstVersion.GreaterThan(secondVersion) {
+		return firstModuleTemplate, nil
+	}
+
+	return secondModuleTemplate, nil
+}