core: fail unomonitored backups

If the command monitoring a VM backup is no longer present in the
command_entities table, it will be impossible to finalize the backup and
it will be stuck in the finalizing phase.

This patch checks periodically for backups that are active but are no
longer monitored and fails them, notifying the user a cleanup may be
required.

Bug-Url: https://bugzilla.redhat.com/2107590
Signed-off-by: Benny Zlotnik <bzlotnik@redhat.com>

backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/backup/DbEntityCleanupManager.java

@@ -1,19 +1,31 @@
 package org.ovirt.engine.core.bll.storage.backup;
 
 import java.util.Date;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import javax.annotation.PostConstruct;
 import javax.enterprise.concurrent.ManagedScheduledExecutorService;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
 import org.apache.commons.lang.exception.ExceptionUtils;
+import org.ovirt.engine.core.common.AuditLogType;
 import org.ovirt.engine.core.common.BackendService;
+import org.ovirt.engine.core.common.action.ActionType;
+import org.ovirt.engine.core.common.action.VmBackupParameters;
+import org.ovirt.engine.core.common.businessentities.CommandEntity;
+import org.ovirt.engine.core.common.businessentities.VmBackup;
+import org.ovirt.engine.core.common.businessentities.VmBackupPhase;
 import org.ovirt.engine.core.common.config.Config;
 import org.ovirt.engine.core.common.config.ConfigValues;
+import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector;
+import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableImpl;
+import org.ovirt.engine.core.dao.CommandEntityDao;
 import org.ovirt.engine.core.dao.ImageTransferDao;
 import org.ovirt.engine.core.dao.VmBackupDao;
+import org.ovirt.engine.core.dao.VmDao;
 import org.ovirt.engine.core.utils.threadpool.ThreadPools;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,6 +48,12 @@ public class DbEntityCleanupManager implements BackendService {
     @Inject
     private ImageTransferDao imageTransferDao;
     @Inject
+    private CommandEntityDao commandEntityDao;
+    @Inject
+    private VmDao vmDao;
+    @Inject
+    private AuditLogDirector auditLogDirector;
+    @Inject
     @ThreadPools(ThreadPools.ThreadPoolType.EngineScheduledThreadPool)
     private ManagedScheduledExecutorService executor;
 
@@ -88,6 +106,7 @@ private void cleanCompletedDbEntities() {
                 new Date(currentTimeMillis - TimeUnit.MILLISECONDS.convert(failedImageTransferTime, TimeUnit.MINUTES));
 
         try {
+            failUnmonitoredBackups();
             vmBackupDao.deleteCompletedBackups(succeededBackupsDeleteTime, failedBackupsDeleteTime);
         } catch (Throwable t) {
             log.error("Failed to delete completed backups: {}", ExceptionUtils.getRootCauseMessage(t));
@@ -100,4 +119,19 @@ private void cleanCompletedDbEntities() {
             log.debug("Exception", t);
         }
     }
+
+    private void failUnmonitoredBackups() {
+        AuditLogableImpl auditLogable = new AuditLogableImpl();
+
+        vmBackupDao.getUnmonitoredBackups()
+                .stream()
+                .forEach(vmBackup -> {
+                    auditLogable.addCustomValue("BackupId", vmBackup.getId().toString());
+                    auditLogable.addCustomValue("VmName", vmDao.get(vmBackup.getVmId()).getName());
+                    auditLogDirector.log(auditLogable, AuditLogType.VM_BACKUP_FAILED_UNMONITORED);
+
+                   vmBackup.setPhase(VmBackupPhase.FAILED);
+                   vmBackupDao.update(vmBackup);
+                });
+    }
 }

backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java

@@ -1612,6 +1612,8 @@ public enum AuditLogType {
     VM_BACKUP_SCRATCH_DISK_CREATION_SUCCEEDED(10805),
     VM_BACKUP_SCRATCH_DISK_CREATION_FAILED(10806, AuditLogSeverity.ERROR),
     VM_BACKUP_SNAPSHOT_POSSIBLE_LEFTOVER(10813, AuditLogSeverity.WARNING),
+    VM_BACKUP_FAILED_UNMONITORED(10814, AuditLogSeverity.ERROR),
+
 
     // Host Devices
     VM_ADD_HOST_DEVICES(10800),

backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dao/VmBackupDao.java

@@ -81,4 +81,6 @@ public interface VmBackupDao extends GenericDao<VmBackup, Guid> {
      * @param backupId the VM backup ID
      */
     void setBackupStopped(Guid backupId);
+
+    List<VmBackup> getUnmonitoredBackups();
 }

backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dao/VmBackupDaoImpl.java

@@ -149,4 +149,10 @@ public void setBackupStopped(Guid backupId) {
                 getCustomMapSqlParameterSource()
                         .addValue("backup_id", backupId));
     }
+
+    @Override
+    public List<VmBackup> getUnmonitoredBackups() {
+        MapSqlParameterSource parameterSource = getCustomMapSqlParameterSource();
+        return getCallsHandler().executeReadList("GetUnmonitoredBackups", vmBackupRowMapper, parameterSource);
+    }
 }

backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties

@@ -1359,3 +1359,4 @@ VM_BACKUP_SCRATCH_DISK_CREATION_FAILED=Failed to create scratch disk for VM ${Vm
 VNICS_OUT_OF_SYNC_ON_NETWORK_UPDATE=Update to network ${NetworkName} was not applied to virtual network interfaces [${VnicNames}]. The actual configuration on the interfaces may differ from the displayed one.
 VNICS_OUT_OF_SYNC_ON_PROFILE_UPDATE=Update to network interface profile ${ProfileName} was not applied to virtual network interfaces [${VnicNames}]. The actual configuration on the interfaces may differ from the displayed one.
 VM_BACKUP_SNAPSHOT_POSSIBLE_LEFTOVER=Possible failure while removing a snapshot that was generated automatically during backup of VM '${VmName}'. The snapshot may be removed manually
+VM_BACKUP_FAILED_UNMONITORED=Backup ${BackupId} for VM ${VmName} was marked as failed as it is no longer monitored, cleanup may be required.

packaging/dbscripts/vm_backups_sp.sql

@@ -225,3 +225,19 @@ BEGIN
             );
 END;$FUNCTION$
 LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION GetUnmonitoredBackups ()
+RETURNS SETOF vm_backups STABLE AS $FUNCTION$
+BEGIN
+    RETURN QUERY
+
+    SELECT *
+    FROM vm_backups
+    WHERE phase != 'Succeeded' AND phase != 'Failed'
+    AND NOT EXISTS(
+        SELECT command_id
+        FROM command_entities
+        WHERE command_id = backup_id
+    );
+END;$FUNCTION$
+LANGUAGE plpgsql;