Backport from https://review.openstack.org/#/c/203751/

From dde701306ca4f044591ec4f7848b1881eed024da Mon Sep 17 00:00:00 2001
From: Cale Rath <ctrath@us.ibm.com>
Date: Mon, 20 Jul 2015 13:55:52 +0000
Subject: [PATCH] Purge soft-deleted instances cmd

This commit adds the functionality to purge soft-deleted
instances records from the DB, along with data in tables
directly related to the soft-deleted instance.

Implements: blueprint purge-deleted-instances-cmd

Change-Id: I3434debdf4df488e031e5218d30e49e5f8cb9e40

nova/cmd/manage.py

@@ -997,6 +997,31 @@ def null_instance_uuid_scan(self, delete=False):
             print(_('There were no records found where '
                     'instance_uuid was NULL.'))
 
+    @args('--dry-run', action='store_true', dest='dry_run',
+          default=False, help='Print SQL statements instead of executing')
+    @args('--older-than', dest='older_than',
+          default=90, help='Days from today to begin purging')
+    @args('--max-number', metavar='<number>', dest='max_number',
+          help='Maximum number of instances to consider')
+    def purge_deleted_instances(self, dry_run=False, older_than=90,
+                                max_number=None):
+        """Removes soft deleted instances data"""
+        older_than = int(older_than)
+        if max_number:
+            max_number = int(max_number)
+
+        admin_context = context.get_admin_context(read_deleted='yes')
+        instance_count = db.instances_purge_deleted(admin_context, dry_run,
+                                                    older_than,
+                                                    max_number)
+        if not dry_run:
+            print(_("Purged %d instances from the DB") % instance_count)
+        else:
+            print (_("There are at least %d records in the DB for deleted "
+                     "instances.  Run this command again without the dry-run "
+                     "option to purge these records.")
+                     % instance_count)
+
     def _run_migration(self, ctxt, max_count):
         ran = 0
         for migration_meth in self.online_migrations:

nova/db/api.py

@@ -691,6 +691,21 @@ def instance_destroy(context, instance_uuid, constraint=None):
     return IMPL.instance_destroy(context, instance_uuid, constraint)
 
 
+def instances_purge_deleted(context, dry_run=False, older_than=90,
+                            max_number=None):
+    """Removes soft deleted instance data
+
+    :param dry_run: If true, prints data that will be deleted without
+            performing an actual delete.
+    :param since: Number of days in the past, from today,
+            to purge instance data
+    :param max_number: Maximum number of instances to consider
+    :returns: number of purged instances.
+    """
+    return IMPL.instances_purge_deleted(context, dry_run, older_than,
+                                        max_number)
+
+
 def instance_get_by_uuid(context, uuid, columns_to_join=None):
     """Get an instance or raise if it does not exist."""
     return IMPL.instance_get_by_uuid(context, uuid, columns_to_join)

nova/db/sqlalchemy/api.py

@@ -41,6 +41,7 @@
 import sqlalchemy as sa
 from sqlalchemy import and_
 from sqlalchemy.exc import NoSuchTableError
+from sqlalchemy.inspection import inspect as sqlalchemyinspect
 from sqlalchemy import MetaData
 from sqlalchemy import or_
 from sqlalchemy.orm import aliased
@@ -1965,6 +1966,119 @@ def instance_destroy(context, instance_uuid, constraint=None):
     return instance_ref
 
 
+def _related_fks_get(table_name):
+    """For a given table name, return all the tables that have a foreign
+       key relationship to the given table
+
+    :param table_name: The table in which to find related tables
+    :return: A list of Column sqlalchemy objects that represent a foreign
+             key to the given provided table in table_name
+    """
+    engine = get_engine()
+    engine.connect()
+    metadata = MetaData()
+    metadata.bind = engine
+    metadata.reflect(engine)
+
+    ret = []
+    for table in metadata.sorted_tables:
+        for col in table.columns:
+            fkeys = col.foreign_keys or []
+            for fkey in fkeys:
+                if fkey.column.table.name == table_name:
+                    ret.append(col)
+                    break
+    return ret
+
+
+def _purge_records(query, model, context, dry_run, max_number=None):
+    """Performs a deep delete of table records.  This will find each related
+       table, related by foreign key relationships, to the given table provided
+
+    :param query: The query in which to execute
+    :param model: The model in which the query represents
+    :param context: DB context
+    :param dry_run: If true, don't perform an actual delete
+    :param max_number: The maximum number of records for the current request
+    :return:
+    """
+    fks = _related_fks_get(model.__tablename__)
+
+    offset = 0
+    limit = 50
+    while True:
+        if max_number is not None:
+            if offset >= max_number:
+                break
+            elif limit > (max_number - offset):
+                limit = max_number - offset
+
+        q = query.limit(limit)
+        if dry_run:
+            q = q.offset(offset)
+        results = q.all()
+        results_len = len(results)
+        offset += results_len
+        if results_len <= 0:
+            break
+
+        for fk in fks:
+            model_class = None
+            found_model = false
+
+            for model_class in six.itervalues(models.__dict__):
+                if hasattr(model_class, "__tablename__"):
+                    if model_class.__tablename__ == fk.table.name:
+                        found_model = true
+                        break
+
+            if found_model:
+                for referenced_key in fk.foreign_keys:
+                    if (referenced_key.column.table.name ==
+                            model.__tablename__):
+                        fk_ids = [x[referenced_key.column.name] for x
+                                  in results]
+                        # In order for sqlalchemy to produce kosher SQL,
+                        # the actual fk attr needs to be used from the
+                        # sqlalchemy model class.
+                        method_attr = getattr(model_class, fk.name)
+                        related_model_query = model_query(context,
+                                                          model_class).\
+                                                filter(method_attr.in_(fk_ids))
+                        _purge_records(related_model_query, model_class,
+                                       context, dry_run)
+
+        # The original 'query' can't be used for delete because a delete
+        # is not allowed when a limit is set.  Therefore, here I generate a
+        # new query for deletion using the primary keys from the original
+        # result set
+        delete_query = model_query(context, model)
+        pk_columns = [key.name for key in sqlalchemyinspect(model).primary_key]
+
+        for pk_column in pk_columns:
+            pk_attr = getattr(model, pk_column)
+            delete_query = delete_query.filter(
+                pk_attr.in_([x[pk_column] for x in results]))
+
+        if not dry_run:
+            delete_query.delete(synchronize_session='fetch')
+    return offset
+
+
+@require_context
+@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
+@pick_context_manager_writer
+def instances_purge_deleted(context, dry_run=False, older_than=90,
+                            max_number=None):
+    purge_before = timeutils.utcnow() - datetime.timedelta(days=older_than)
+    instance_query = model_query(context, models.Instance).\
+                         filter(and_(models.Instance.deleted != 0,
+                         models.Instance.deleted_at < purge_before))
+
+    return _purge_records(instance_query, models.Instance, context,
+                          dry_run, max_number)
+
+
 @require_context
 @pick_context_manager_reader_allow_async
 def instance_get_by_uuid(context, uuid, columns_to_join=None):