diff --git a/reconcile/external_resources/integration.py b/reconcile/external_resources/integration.py index 2c1af81d4..459a5c6ef 100644 --- a/reconcile/external_resources/integration.py +++ b/reconcile/external_resources/integration.py @@ -3,6 +3,7 @@ from typing import Any from reconcile.external_resources.manager import ( + ExternalResourceDryRunsValidator, ExternalResourcesInventory, ExternalResourcesManager, setup_factories, @@ -89,6 +90,10 @@ def create_er_manager( m_inventory = load_module_inventory(get_modules()) namespaces = [ns for ns in get_namespaces() if ns.external_resources] er_inventory = ExternalResourcesInventory(namespaces) + state_manager = ExternalResourcesStateDynamoDB( + aws_api=aws_api, + table_name=er_settings.state_dynamodb_table, + ) if not workers_cluster: workers_cluster = er_settings.workers_cluster.name @@ -104,10 +109,7 @@ def create_er_manager( ), er_inventory=er_inventory, module_inventory=m_inventory, - state_manager=ExternalResourcesStateDynamoDB( - aws_api=aws_api, - table_name=er_settings.state_dynamodb_table, - ), + state_manager=state_manager, reconciler=K8sExternalResourcesReconciler( controller=build_job_controller( integration=QONTRACT_INTEGRATION, @@ -128,6 +130,9 @@ def create_er_manager( thread_pool_size=thread_pool_size, dry_run=dry_run, ), + dry_runs_validator=ExternalResourceDryRunsValidator( + state_manager, er_inventory + ), ) diff --git a/reconcile/external_resources/manager.py b/reconcile/external_resources/manager.py index 70b971603..6723e6253 100644 --- a/reconcile/external_resources/manager.py +++ b/reconcile/external_resources/manager.py @@ -1,4 +1,5 @@ import logging +from collections import Counter from collections.abc import Iterable from datetime import UTC, datetime @@ -19,6 +20,7 @@ ExternalResourceKey, ExternalResourceModuleConfiguration, ExternalResourceOrphanedResourcesError, + ExternalResourceOutputResourceNameDuplications, ExternalResourcesInventory, ExternalResourceValidationError, ModuleInventory, @@ -73,6 +75,41 @@ def setup_factories( return of +class ExternalResourceDryRunsValidator: + def __init__( + self, + state_manager: ExternalResourcesStateDynamoDB, + er_inventory: ExternalResourcesInventory, + ): + self.state_mgr = state_manager + self.er_inventory = er_inventory + + def _check_output_resource_name_duplications( + self, + ) -> None: + specs = Counter( + ( + spec.cluster_name, + spec.namespace_name, + spec.output_resource_name, + ) + for spec in self.er_inventory.values() + ) + if duplicates := [key for key, count in specs.items() if count > 1]: + raise ExternalResourceOutputResourceNameDuplications(duplicates) + + def _check_orphaned_objects(self) -> None: + state_keys = self.state_mgr.get_all_resource_keys() + inventory_keys = set(self.er_inventory.keys()) + orphans = state_keys - inventory_keys + if len(orphans) > 0: + raise ExternalResourceOrphanedResourcesError(orphans) + + def validate(self) -> None: + self._check_orphaned_objects() + self._check_output_resource_name_duplications() + + class ExternalResourcesManager: def __init__( self, @@ -84,6 +121,7 @@ def __init__( er_inventory: ExternalResourcesInventory, factories: ObjectFactory[ExternalResourceFactory], secrets_reconciler: InClusterSecretsReconciler, + dry_runs_validator: ExternalResourceDryRunsValidator, thread_pool_size: int, ) -> None: self.state_mgr = state_manager @@ -96,6 +134,7 @@ def __init__( self.secrets_reconciler = secrets_reconciler self.errors: dict[ExternalResourceKey, ExternalResourceValidationError] = {} self.thread_pool_size = thread_pool_size + self.dry_runs_validator = dry_runs_validator def _get_reconcile_action( self, reconciliation: Reconciliation, state: ExternalResourceState @@ -199,13 +238,6 @@ def _get_deleted_objects_reconciliations(self) -> set[Reconciliation]: to_reconcile.add(r) return to_reconcile - def _check_orphaned_objects(self) -> None: - state_keys = self.state_mgr.get_all_resource_keys() - inventory_keys = set(self.er_inventory.keys()) - orphans = state_keys - inventory_keys - if len(orphans) > 0: - raise ExternalResourceOrphanedResourcesError(orphans) - def _get_reconciliation_status( self, r: Reconciliation, @@ -374,7 +406,7 @@ def handle_resources(self) -> None: self._sync_secrets(to_sync_keys=to_sync_keys | pending_sync_keys) def handle_dry_run_resources(self) -> None: - self._check_orphaned_objects() + self.dry_runs_validator.validate() desired_r = self._get_desired_objects_reconciliations() deleted_r = self._get_deleted_objects_reconciliations() reconciliations = desired_r.union(deleted_r) diff --git a/reconcile/external_resources/model.py b/reconcile/external_resources/model.py index 7afd93e8d..4aa68c8d5 100644 --- a/reconcile/external_resources/model.py +++ b/reconcile/external_resources/model.py @@ -45,6 +45,17 @@ def __init__(self, orphans: Iterable["ExternalResourceKey"]) -> None: super().__init__("".join(msg)) +class ExternalResourceOutputResourceNameDuplications(Exception): + def __init__(self, duplicates: Iterable[tuple[str, str, str]]) -> None: + msg = [ + "There are output_resource_name attribute duplications. ", + "output_resource_name must be unique within a cluster/namespace.\n" + "Duplications:\n", + "\n".join(map(str, duplicates)), + ] + super().__init__("".join(msg)) + + class ExternalResourceValidationError(Exception): errors: list[str] = [] diff --git a/reconcile/test/external_resources/test_manager.py b/reconcile/test/external_resources/test_manager.py index c813e06f3..2adea1bbc 100644 --- a/reconcile/test/external_resources/test_manager.py +++ b/reconcile/test/external_resources/test_manager.py @@ -7,6 +7,7 @@ from pytest_mock import MockerFixture from reconcile.external_resources.manager import ( + ExternalResourceDryRunsValidator, ExternalResourcesManager, ReconcileStatus, ReconciliationStatus, @@ -36,10 +37,12 @@ def manager( module_inventory: ModuleInventory, ) -> ExternalResourcesManager: er_inventory = ExternalResourcesInventory([]) + state_mgr = Mock(spec=ExternalResourcesStateDynamoDB) secret_reader = Mock() reconciler = Mock(spec=ExternalResourcesReconciler) secrets_reconciler = Mock() factories = setup_factories(settings, module_inventory, er_inventory, secret_reader) + dry_runs_validator = ExternalResourceDryRunsValidator(state_mgr, er_inventory) return ExternalResourcesManager( secret_reader=secret_reader, @@ -51,6 +54,7 @@ def manager( er_inventory=er_inventory, secrets_reconciler=secrets_reconciler, thread_pool_size=1, + dry_runs_validator=dry_runs_validator, )