core: fail unomonitored backups

If the command monitoring a VM backup is no longer present in the
command_entities table, it will be impossible to finalize the backup and
it will be stuck in the finalizing phase.

This patch checks periodically for backups that are active but are no
longer monitored and fails them, notifying the user a cleanup may be
required.

Bug-Url: https://bugzilla.redhat.com/2107590
Signed-off-by: Benny Zlotnik <bzlotnik@redhat.com>

backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/storage/backup/DbEntityCleanupManager.java

@@ -1,19 +1,31 @@
 package org.ovirt.engine.core.bll.storage.backup;
 
 import java.util.Date;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import javax.annotation.PostConstruct;
 import javax.enterprise.concurrent.ManagedScheduledExecutorService;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
 import org.apache.commons.lang.exception.ExceptionUtils;
+import org.ovirt.engine.core.common.AuditLogType;
 import org.ovirt.engine.core.common.BackendService;
+import org.ovirt.engine.core.common.action.ActionType;
+import org.ovirt.engine.core.common.action.VmBackupParameters;
+import org.ovirt.engine.core.common.businessentities.CommandEntity;
+import org.ovirt.engine.core.common.businessentities.VmBackup;
+import org.ovirt.engine.core.common.businessentities.VmBackupPhase;
 import org.ovirt.engine.core.common.config.Config;
 import org.ovirt.engine.core.common.config.ConfigValues;
+import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector;
+import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableImpl;
+import org.ovirt.engine.core.dao.CommandEntityDao;
 import org.ovirt.engine.core.dao.ImageTransferDao;
 import org.ovirt.engine.core.dao.VmBackupDao;
+import org.ovirt.engine.core.dao.VmDao;
 import org.ovirt.engine.core.utils.threadpool.ThreadPools;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,6 +48,12 @@ public class DbEntityCleanupManager implements BackendService {
     @Inject
     private ImageTransferDao imageTransferDao;
     @Inject
+    private CommandEntityDao commandEntityDao;
+    @Inject
+    private VmDao vmDao;
+    @Inject
+    private AuditLogDirector auditLogDirector;
+    @Inject
     @ThreadPools(ThreadPools.ThreadPoolType.EngineScheduledThreadPool)
     private ManagedScheduledExecutorService executor;
 
@@ -88,6 +106,7 @@ private void cleanCompletedDbEntities() {
                 new Date(currentTimeMillis - TimeUnit.MILLISECONDS.convert(failedImageTransferTime, TimeUnit.MINUTES));
 
         try {
+            failUnmonitoredBackups();
             vmBackupDao.deleteCompletedBackups(succeededBackupsDeleteTime, failedBackupsDeleteTime);
         } catch (Throwable t) {
             log.error("Failed to delete completed backups: {}", ExceptionUtils.getRootCauseMessage(t));
@@ -100,4 +119,19 @@ private void cleanCompletedDbEntities() {
             log.debug("Exception", t);
         }
     }
+
+    private void failUnmonitoredBackups() {
+        AuditLogableImpl auditLogable = new AuditLogableImpl();
+
+        vmBackupDao.getUnmonitoredBackups()
+                .stream()
+                .forEach(vmBackup -> {
+                    auditLogable.addCustomValue("BackupId", vmBackup.getId().toString());
+                    auditLogable.addCustomValue("VmName", vmDao.get(vmBackup.getVmId()).getName());
+                    auditLogDirector.log(auditLogable, AuditLogType.VM_BACKUP_FAILED_UNMONITORED);
+
+                   vmBackup.setPhase(VmBackupPhase.FAILED);
+                   vmBackupDao.update(vmBackup);
+                });
+    }
 }

backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java

@@ -1612,6 +1612,8 @@ public enum AuditLogType {
     VM_BACKUP_SCRATCH_DISK_CREATION_SUCCEEDED(10805),
     VM_BACKUP_SCRATCH_DISK_CREATION_FAILED(10806, AuditLogSeverity.ERROR),
     VM_BACKUP_SNAPSHOT_POSSIBLE_LEFTOVER(10813, AuditLogSeverity.WARNING),
+    VM_BACKUP_FAILED_UNMONITORED(10814, AuditLogSeverity.ERROR),
+
 
     // Host Devices
     VM_ADD_HOST_DEVICES(10800),

backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dao/VmBackupDao.java

@@ -81,4 +81,6 @@ public interface VmBackupDao extends GenericDao<VmBackup, Guid> {
      * @param backupId the VM backup ID
      */
     void setBackupStopped(Guid backupId);
+
+    List<VmBackup> getUnmonitoredBackups();
 }

backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dao/VmBackupDaoImpl.java

@@ -149,4 +149,10 @@ public void setBackupStopped(Guid backupId) {
                 getCustomMapSqlParameterSource()
                         .addValue("backup_id", backupId));
     }
+
+    @Override
+    public List<VmBackup> getUnmonitoredBackups() {
+        MapSqlParameterSource parameterSource = getCustomMapSqlParameterSource();
+        return getCallsHandler().executeReadList("GetUnmonitoredBackups", vmBackupRowMapper, parameterSource);
+    }
 }