From ab4ad21b13efd14b026f78c2e1cea64b7af68a27 Mon Sep 17 00:00:00 2001 From: Jordi Piriz Date: Tue, 2 Jul 2024 12:42:45 +0200 Subject: [PATCH] Implement resources explicit deletion --- reconcile/external_resources/manager.py | 28 ++++++++++--------- reconcile/external_resources/meta.py | 3 ++ reconcile/external_resources/model.py | 5 ---- .../external_resources_namespaces.gql | 1 + .../external_resources_namespaces.py | 2 ++ reconcile/gql_definitions/introspection.json | 12 ++++++++ reconcile/utils/external_resource_spec.py | 4 +++ 7 files changed, 37 insertions(+), 18 deletions(-) diff --git a/reconcile/external_resources/manager.py b/reconcile/external_resources/manager.py index edd31efa9c..22a041cadb 100644 --- a/reconcile/external_resources/manager.py +++ b/reconcile/external_resources/manager.py @@ -1,4 +1,3 @@ -import json import logging from collections.abc import Iterable from datetime import UTC, datetime @@ -43,6 +42,7 @@ from reconcile.utils.secret_reader import SecretReaderBase FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2" +FLAG_DELETE_RESOURCE = "delete" def setup_factories( @@ -157,13 +157,13 @@ def _resource_needs_reconciliation( def _get_desired_objects_reconciliations(self) -> set[Reconciliation]: r: set[Reconciliation] = set() for key, spec in self.er_inventory.items(): + if spec.marked_to_delete: + continue module = self.module_inventory.get_from_spec(spec) - try: resource = self._build_external_resource(spec, self.er_inventory) except ExternalResourceValidationError as e: - k = ExternalResourceKey.from_spec(spec) - self.errors[k] = e + self.errors[key] = e continue reconciliation = Reconciliation( @@ -179,21 +179,23 @@ def _get_desired_objects_reconciliations(self) -> set[Reconciliation]: return r def _get_deleted_objects_reconciliations(self) -> set[Reconciliation]: - desired_keys = set(self.er_inventory.keys()) - state_resource_keys = self.state_mgr.get_all_resource_keys() - deleted_keys = state_resource_keys - desired_keys - r: set[Reconciliation] = set() + to_reconcile: set[Reconciliation] = set() + deleted_keys = (k for k, v in self.er_inventory.items() if v.marked_to_delete) for key in deleted_keys: state = self.state_mgr.get_external_resource_state(key) - reconciliation = Reconciliation( + if state.resource_status == ResourceStatus.NOT_EXISTS: + logging.debug("Resource has already been removed. key: %s", key) + continue + + r = Reconciliation( key=key, resource_hash=state.reconciliation.resource_hash, module_configuration=state.reconciliation.module_configuration, input=state.reconciliation.input, action=Action.DESTROY, ) - r.add(reconciliation) - return r + to_reconcile.add(r) + return to_reconcile def _update_in_progress_state( self, r: Reconciliation, state: ExternalResourceState @@ -310,8 +312,8 @@ def _build_external_resource( return resource def _serialize_resource_input(self, resource: ExternalResource) -> str: - return json.dumps( - resource.dict(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}}) + return resource.json( + exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2, FLAG_DELETE_RESOURCE}} ) def handle_resources(self) -> None: diff --git a/reconcile/external_resources/meta.py b/reconcile/external_resources/meta.py index 8b563407ca..3efa8b102f 100644 --- a/reconcile/external_resources/meta.py +++ b/reconcile/external_resources/meta.py @@ -11,3 +11,6 @@ SECRET_ANN_IDENTIFIER = SECRET_ANN_PREFIX + "/identifier" SECRET_UPDATED_AT = SECRET_ANN_PREFIX + "/updated_at" SECRET_UPDATED_AT_TIMEFORMAT = "%Y-%m-%dT%H:%M:%SZ" + +FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2" +FLAG_DELETE_RESOURCE = "delete" diff --git a/reconcile/external_resources/model.py b/reconcile/external_resources/model.py index 4f066d8aaf..811d114b20 100644 --- a/reconcile/external_resources/model.py +++ b/reconcile/external_resources/model.py @@ -1,4 +1,3 @@ -import base64 import hashlib import json from abc import ( @@ -88,7 +87,6 @@ def __init__(self, namespaces: Iterable[NamespaceV1]) -> None: ] for spec in desired_specs: - # self.set(ExternalResourceKey.from_spec(spec), spec) self._inventory[ExternalResourceKey.from_spec(spec)] = spec def __getitem__(self, key: ExternalResourceKey) -> ExternalResourceSpec | None: @@ -239,6 +237,3 @@ def hash(self) -> str: return hashlib.md5( json.dumps(self.data, sort_keys=True).encode("utf-8") ).hexdigest() - - def serialize_input(self) -> str: - return base64.b64encode(json.dumps(self.dict()).encode()).decode() diff --git a/reconcile/gql_definitions/external_resources/external_resources_namespaces.gql b/reconcile/gql_definitions/external_resources/external_resources_namespaces.gql index 6e9ccbe1b0..b7f4222ef0 100644 --- a/reconcile/gql_definitions/external_resources/external_resources_namespaces.gql +++ b/reconcile/gql_definitions/external_resources/external_resources_namespaces.gql @@ -51,6 +51,7 @@ query ExternalResourcesNamespaces { loss_impact } managed_by_erv2 + delete } ... on NamespaceTerraformResourceS3_v1 { region diff --git a/reconcile/gql_definitions/external_resources/external_resources_namespaces.py b/reconcile/gql_definitions/external_resources/external_resources_namespaces.py index 8348c55496..724063aa86 100644 --- a/reconcile/gql_definitions/external_resources/external_resources_namespaces.py +++ b/reconcile/gql_definitions/external_resources/external_resources_namespaces.py @@ -108,6 +108,7 @@ loss_impact } managed_by_erv2 + delete } ... on NamespaceTerraformResourceS3_v1 { region @@ -552,6 +553,7 @@ class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1): event_notifications: Optional[list[AWSRDSEventNotificationV1]] = Field(..., alias="event_notifications") data_classification: Optional[AWSRDSDataClassificationV1] = Field(..., alias="data_classification") managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2") + delete: Optional[bool] = Field(..., alias="delete") class AWSS3EventNotificationV1(ConfiguredBaseModel): diff --git a/reconcile/gql_definitions/introspection.json b/reconcile/gql_definitions/introspection.json index 250e3a1b1f..8232f566c2 100644 --- a/reconcile/gql_definitions/introspection.json +++ b/reconcile/gql_definitions/introspection.json @@ -42173,6 +42173,18 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "delete", + "description": null, + "args": [], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, diff --git a/reconcile/utils/external_resource_spec.py b/reconcile/utils/external_resource_spec.py index 34acd77bf8..a3589a2ef7 100644 --- a/reconcile/utils/external_resource_spec.py +++ b/reconcile/utils/external_resource_spec.py @@ -92,6 +92,10 @@ class ExternalResourceSpec: init=False, compare=False, repr=False, hash=False, default_factory=lambda: {} ) + @property + def marked_to_delete(self) -> bool: + return self.resource.get("delete") or False + @property def provider(self) -> str: return self.resource["provider"]