Add a dry_run validation to ensure no output_secret_name duplications…

… exist. (#4812)

reconcile/external_resources/integration.py

@@ -3,6 +3,7 @@
 from typing import Any
 
 from reconcile.external_resources.manager import (
+    ExternalResourceDryRunsValidator,
     ExternalResourcesInventory,
     ExternalResourcesManager,
     setup_factories,
@@ -89,6 +90,10 @@ def create_er_manager(
     m_inventory = load_module_inventory(get_modules())
     namespaces = [ns for ns in get_namespaces() if ns.external_resources]
     er_inventory = ExternalResourcesInventory(namespaces)
+    state_manager = ExternalResourcesStateDynamoDB(
+        aws_api=aws_api,
+        table_name=er_settings.state_dynamodb_table,
+    )
 
     if not workers_cluster:
         workers_cluster = er_settings.workers_cluster.name
@@ -104,10 +109,7 @@ def create_er_manager(
         ),
         er_inventory=er_inventory,
         module_inventory=m_inventory,
-        state_manager=ExternalResourcesStateDynamoDB(
-            aws_api=aws_api,
-            table_name=er_settings.state_dynamodb_table,
-        ),
+        state_manager=state_manager,
         reconciler=K8sExternalResourcesReconciler(
             controller=build_job_controller(
                 integration=QONTRACT_INTEGRATION,
@@ -128,6 +130,9 @@ def create_er_manager(
             thread_pool_size=thread_pool_size,
             dry_run=dry_run,
         ),
+        dry_runs_validator=ExternalResourceDryRunsValidator(
+            state_manager, er_inventory
+        ),
     )
 
 

reconcile/external_resources/manager.py

@@ -1,4 +1,5 @@
 import logging
+from collections import Counter
 from collections.abc import Iterable
 from datetime import UTC, datetime
 
@@ -19,6 +20,7 @@
     ExternalResourceKey,
     ExternalResourceModuleConfiguration,
     ExternalResourceOrphanedResourcesError,
+    ExternalResourceOutputResourceNameDuplications,
     ExternalResourcesInventory,
     ExternalResourceValidationError,
     ModuleInventory,
@@ -73,6 +75,41 @@ def setup_factories(
     return of
 
 
+class ExternalResourceDryRunsValidator:
+    def __init__(
+        self,
+        state_manager: ExternalResourcesStateDynamoDB,
+        er_inventory: ExternalResourcesInventory,
+    ):
+        self.state_mgr = state_manager
+        self.er_inventory = er_inventory
+
+    def _check_output_resource_name_duplications(
+        self,
+    ) -> None:
+        specs = Counter(
+            (
+                spec.cluster_name,
+                spec.namespace_name,
+                spec.output_resource_name,
+            )
+            for spec in self.er_inventory.values()
+        )
+        if duplicates := [key for key, count in specs.items() if count > 1]:
+            raise ExternalResourceOutputResourceNameDuplications(duplicates)
+
+    def _check_orphaned_objects(self) -> None:
+        state_keys = self.state_mgr.get_all_resource_keys()
+        inventory_keys = set(self.er_inventory.keys())
+        orphans = state_keys - inventory_keys
+        if len(orphans) > 0:
+            raise ExternalResourceOrphanedResourcesError(orphans)
+
+    def validate(self) -> None:
+        self._check_orphaned_objects()
+        self._check_output_resource_name_duplications()
+
+
 class ExternalResourcesManager:
     def __init__(
         self,
@@ -84,6 +121,7 @@ def __init__(
         er_inventory: ExternalResourcesInventory,
         factories: ObjectFactory[ExternalResourceFactory],
         secrets_reconciler: InClusterSecretsReconciler,
+        dry_runs_validator: ExternalResourceDryRunsValidator,
         thread_pool_size: int,
     ) -> None:
         self.state_mgr = state_manager
@@ -96,6 +134,7 @@ def __init__(
         self.secrets_reconciler = secrets_reconciler
         self.errors: dict[ExternalResourceKey, ExternalResourceValidationError] = {}
         self.thread_pool_size = thread_pool_size
+        self.dry_runs_validator = dry_runs_validator
 
     def _get_reconcile_action(
         self, reconciliation: Reconciliation, state: ExternalResourceState
@@ -199,13 +238,6 @@ def _get_deleted_objects_reconciliations(self) -> set[Reconciliation]:
             to_reconcile.add(r)
         return to_reconcile
 
-    def _check_orphaned_objects(self) -> None:
-        state_keys = self.state_mgr.get_all_resource_keys()
-        inventory_keys = set(self.er_inventory.keys())
-        orphans = state_keys - inventory_keys
-        if len(orphans) > 0:
-            raise ExternalResourceOrphanedResourcesError(orphans)
-
     def _get_reconciliation_status(
         self,
         r: Reconciliation,
@@ -374,7 +406,7 @@ def handle_resources(self) -> None:
             self._sync_secrets(to_sync_keys=to_sync_keys | pending_sync_keys)
 
     def handle_dry_run_resources(self) -> None:
-        self._check_orphaned_objects()
+        self.dry_runs_validator.validate()
         desired_r = self._get_desired_objects_reconciliations()
         deleted_r = self._get_deleted_objects_reconciliations()
         reconciliations = desired_r.union(deleted_r)

reconcile/external_resources/model.py

@@ -45,6 +45,17 @@ def __init__(self, orphans: Iterable["ExternalResourceKey"]) -> None:
         super().__init__("".join(msg))
 
 
+class ExternalResourceOutputResourceNameDuplications(Exception):
+    def __init__(self, duplicates: Iterable[tuple[str, str, str]]) -> None:
+        msg = [
+            "There are output_resource_name attribute duplications. ",
+            "output_resource_name must be unique within a cluster/namespace.\n"
+            "Duplications:\n",
+            "\n".join(map(str, duplicates)),
+        ]
+        super().__init__("".join(msg))
+
+
 class ExternalResourceValidationError(Exception):
     errors: list[str] = []
 

reconcile/test/external_resources/test_manager.py

@@ -7,6 +7,7 @@
 from pytest_mock import MockerFixture
 
 from reconcile.external_resources.manager import (
+    ExternalResourceDryRunsValidator,
     ExternalResourcesManager,
     ReconcileStatus,
     ReconciliationStatus,
@@ -36,10 +37,12 @@ def manager(
     module_inventory: ModuleInventory,
 ) -> ExternalResourcesManager:
     er_inventory = ExternalResourcesInventory([])
+    state_mgr = Mock(spec=ExternalResourcesStateDynamoDB)
     secret_reader = Mock()
     reconciler = Mock(spec=ExternalResourcesReconciler)
     secrets_reconciler = Mock()
     factories = setup_factories(settings, module_inventory, er_inventory, secret_reader)
+    dry_runs_validator = ExternalResourceDryRunsValidator(state_mgr, er_inventory)
 
     return ExternalResourcesManager(
         secret_reader=secret_reader,
@@ -51,6 +54,7 @@ def manager(
         er_inventory=er_inventory,
         secrets_reconciler=secrets_reconciler,
         thread_pool_size=1,
+        dry_runs_validator=dry_runs_validator,
     )