Merge remote-tracking branch 'apache/4.19'

api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java

@@ -103,7 +103,7 @@ public String getEventType() {
 
     @Override
     public String getEventDescription() {
-        return "Adding user "+getUsername()+" to Project "+getProjectId();
+        return "Adding user " + getUsername() + " to project: " + getProjectId();
     }
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/account/DeleteUserFromProjectCmd.java

@@ -81,7 +81,6 @@ public String getEventDescription() {
         return "Removing user " + userId + " from project: " + projectId;
     }
 
-
     @Override
     public long getEntityOwnerId() {
         Project project = _projectService.getProject(projectId);

framework/security/src/main/java/org/apache/cloudstack/framework/security/keystore/KeystoreManager.java

@@ -18,6 +18,7 @@
 
 import com.cloud.agent.api.LogLevel;
 import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.utils.Pair;
 import com.cloud.utils.component.Manager;
 
 public interface KeystoreManager extends Manager {
@@ -59,7 +60,7 @@ public String getRootCACert() {
         }
     }
 
-    boolean validateCertificate(String certificate, String key, String domainSuffix);
+    Pair<Boolean, String> validateCertificate(String certificate, String key, String domainSuffix);
 
     void saveCertificate(String name, String certificate, String key, String domainSuffix);
 

framework/security/src/main/java/org/apache/cloudstack/framework/security/keystore/KeystoreManagerImpl.java

@@ -30,6 +30,7 @@
 
 import javax.inject.Inject;
 
+import com.cloud.utils.Pair;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
@@ -45,24 +46,28 @@ public class KeystoreManagerImpl extends ManagerBase implements KeystoreManager
     private KeystoreDao _ksDao;
 
     @Override
-    public boolean validateCertificate(String certificate, String key, String domainSuffix) {
+    public Pair<Boolean, String> validateCertificate(String certificate, String key, String domainSuffix) {
+        String errMsg = null;
         if (StringUtils.isAnyEmpty(certificate, key, domainSuffix)) {
-            logger.error("Invalid parameter found in (certificate, key, domainSuffix) tuple for domain: " + domainSuffix);
-            return false;
+            errMsg = String.format("Invalid parameter found in (certificate, key, domainSuffix) tuple for domain: %s", domainSuffix);
+            logger.error(errMsg);
+            return new Pair<>(false, errMsg);
         }
 
         try {
             String ksPassword = "passwordForValidation";
             byte[] ksBits = CertificateHelper.buildAndSaveKeystore(domainSuffix, certificate, getKeyContent(key), ksPassword);
             KeyStore ks = CertificateHelper.loadKeystore(ksBits, ksPassword);
-            if (ks != null)
-                return true;
-
-            logger.error("Unabled to construct keystore for domain: " + domainSuffix);
+            if (ks != null) {
+                return new Pair<>(true, errMsg);
+            }
+            errMsg = String.format("Unable to construct keystore for domain: %s", domainSuffix);
+            logger.error(errMsg);
         } catch (Exception e) {
-            logger.error("Certificate validation failed due to exception for domain: " + domainSuffix, e);
+            errMsg = String.format("Certificate validation failed due to exception for domain: %s", domainSuffix);
+            logger.error(errMsg, e);
         }
-        return false;
+        return new Pair<>(false, errMsg);
     }
 
     @Override

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -55,6 +55,7 @@
 import com.vmware.vim25.FolderFileInfo;
 import com.vmware.vim25.HostDatastoreBrowserSearchResults;
 import com.vmware.vim25.HostDatastoreBrowserSearchSpec;
+import com.vmware.vim25.VirtualCdromIsoBackingInfo;
 import com.vmware.vim25.VirtualMachineConfigSummary;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.backup.PrepareForBackupRestorationCommand;
@@ -2737,8 +2738,9 @@ private boolean multipleIsosAtached(DiskTO[] sortedDisks) {
 
     private DiskTO[] getDisks(DiskTO[] sortedDisks) {
        return Arrays.stream(sortedDisks).filter(vol -> ((vol.getPath() != null &&
-                vol.getPath().contains("configdrive"))) || (vol.getType() != Volume.Type.ISO)).toArray(DiskTO[]::new);
+                vol.getPath().contains(ConfigDrive.CONFIGDRIVEDIR))) || (vol.getType() != Volume.Type.ISO)).toArray(DiskTO[]::new);
     }
+
     private void configureIso(VmwareHypervisorHost hyperHost, VirtualMachineMO vmMo, DiskTO vol,
                               VirtualDeviceConfigSpec[] deviceConfigSpecArray, int ideUnitNumber, int i) throws Exception {
         TemplateObjectTO iso = (TemplateObjectTO) vol.getData();
@@ -4447,6 +4449,8 @@ protected Answer execute(StopCommand cmd) {
                             msg = "Have problem in powering off VM " + cmd.getVmName() + ", let the process continue";
                             logger.warn(msg);
                         }
+
+                        disconnectConfigDriveIsoIfExists(vmMo);
                         return new StopAnswer(cmd, msg, true);
                     }
 
@@ -4465,6 +4469,30 @@ protected Answer execute(StopCommand cmd) {
         }
     }
 
+    private void disconnectConfigDriveIsoIfExists(VirtualMachineMO vmMo) {
+        try {
+            List<VirtualDevice> isoDevices = vmMo.getIsoDevices();
+            if (CollectionUtils.isEmpty(isoDevices)) {
+                return;
+            }
+
+            for (VirtualDevice isoDevice : isoDevices) {
+                if (!(isoDevice.getBacking() instanceof VirtualCdromIsoBackingInfo)) {
+                    continue;
+                }
+                String isoFilePath = ((VirtualCdromIsoBackingInfo)isoDevice.getBacking()).getFileName();
+                if (!isoFilePath.contains(ConfigDrive.CONFIGDRIVEDIR)) {
+                    continue;
+                }
+                logger.info(String.format("Disconnecting config drive at location: %s", isoFilePath));
+                vmMo.detachIso(isoFilePath, true);
+                return;
+            }
+        } catch (Exception e) {
+            logger.warn(String.format("Couldn't check/disconnect config drive, error: %s", e.getMessage()), e);
+        }
+    }
+
     protected Answer execute(RebootRouterCommand cmd) {
         RebootAnswer answer = (RebootAnswer) execute((RebootCommand) cmd);
 

server/src/main/java/com/cloud/event/ActionEventInterceptor.java

@@ -88,13 +88,13 @@ public void interceptComplete(Method method, Object target, Object event) {
         for (ActionEvent actionEvent : getActionEvents(method)) {
             CallContext ctx = CallContext.current();
             long userId = ctx.getCallingUserId();
-            long accountId = ctx.getProject() != null ? ctx.getProject().getProjectAccountId() : ctx.getCallingAccountId();    //This should be the entity owner id rather than the Calling User Account Id.
             long startEventId = ctx.getStartEventId();
             String eventDescription = getEventDescription(actionEvent, ctx);
             Long eventResourceId = getEventResourceId(actionEvent, ctx);
             String eventResourceType = getEventResourceType(actionEvent, ctx);
             String eventType = getEventType(actionEvent, ctx);
             boolean isEventDisplayEnabled = ctx.isEventDisplayEnabled();
+            long accountId = ActionEventUtils.getOwnerAccountId(ctx, eventType, ctx.getCallingAccountId());
 
             if (eventType.equals(""))
                 return;
@@ -118,13 +118,13 @@ public void interceptException(Method method, Object target, Object event) {
         for (ActionEvent actionEvent : getActionEvents(method)) {
             CallContext ctx = CallContext.current();
             long userId = ctx.getCallingUserId();
-            long accountId = ctx.getCallingAccountId();
             long startEventId = ctx.getStartEventId();
             String eventDescription = getEventDescription(actionEvent, ctx);
             Long eventResourceId = getEventResourceId(actionEvent, ctx);
             String eventResourceType = getEventResourceType(actionEvent, ctx);
             String eventType = getEventType(actionEvent, ctx);
             boolean isEventDisplayEnabled = ctx.isEventDisplayEnabled();
+            long accountId = ActionEventUtils.getOwnerAccountId(ctx, eventType, ctx.getCallingAccountId());
 
             if (eventType.equals(""))
                 return;

server/src/main/java/com/cloud/event/ActionEventUtils.java

@@ -22,6 +22,7 @@
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import javax.annotation.PostConstruct;
@@ -112,6 +113,8 @@ public static Long onActionEvent(Long userId, Long accountId, Long domainId, Str
      */
     public static Long onScheduledActionEvent(Long userId, Long accountId, String type, String description, Long resourceId, String resourceType, boolean eventDisplayEnabled, long startEventId) {
         Ternary<Long, String, String> resourceDetails = getResourceDetails(resourceId, resourceType, type);
+        CallContext ctx = CallContext.current();
+        accountId = getOwnerAccountId(ctx, type, accountId);
         Event event = persistActionEvent(userId, accountId, null, null, type, Event.State.Scheduled,
                 eventDisplayEnabled, description, resourceDetails.first(), resourceDetails.third(), startEventId);
         publishOnEventBus(event, userId, accountId, EventCategory.ACTION_EVENT.getName(), type,
@@ -127,7 +130,7 @@ public static void startNestedActionEvent(String eventType, String eventDescript
     public static void onStartedActionEventFromContext(String eventType, String eventDescription, Long resourceId, String resourceType, boolean eventDisplayEnabled) {
         CallContext ctx = CallContext.current();
         long userId = ctx.getCallingUserId();
-        long accountId = ctx.getProject() != null ? ctx.getProject().getProjectAccountId() : ctx.getCallingAccountId();    //This should be the entity owner id rather than the Calling User Account Id.
+        long accountId = getOwnerAccountId(ctx, eventType, ctx.getCallingAccountId());
         long startEventId = ctx.getStartEventId();
 
         if (!eventType.equals(""))
@@ -413,7 +416,11 @@ private static void populateFirstClassEntities(Map<String, String> eventDescript
                 LOGGER.trace("Caught exception while populating first class entities for event bus, moving on");
             }
         }
-
     }
 
+    public static long getOwnerAccountId(CallContext ctx, String eventType, long callingAccountId) {
+        List<String> mainProjectEvents = List.of(EventTypes.EVENT_PROJECT_CREATE, EventTypes.EVENT_PROJECT_UPDATE, EventTypes.EVENT_PROJECT_DELETE);
+        long accountId = ctx.getProject() != null && !mainProjectEvents.stream().anyMatch(eventType::equalsIgnoreCase) ? ctx.getProject().getProjectAccountId() : callingAccountId;    //This should be the entity owner id rather than the Calling User Account Id.
+        return accountId;
+    }
 }

server/src/main/java/com/cloud/projects/ProjectManagerImpl.java

@@ -293,16 +293,16 @@ public Project doInTransaction(TransactionStatus status) {
                 assignAccountToProject(project, ownerFinal.getId(), ProjectAccount.Role.Admin,
                         Optional.ofNullable(finalUser).map(User::getId).orElse(null),  null);
 
-        if (project != null) {
-            CallContext.current().setEventDetails("Project id=" + project.getId());
-            CallContext.current().putContextParameter(Project.class, project.getUuid());
-        }
+                if (project != null) {
+                    CallContext.current().setEventDetails("Project id=" + project.getId());
+                    CallContext.current().putContextParameter(Project.class, project.getUuid());
+                }
 
-        //Increment resource count
+                //Increment resource count
                 _resourceLimitMgr.incrementResourceCount(ownerFinal.getId(), ResourceType.project);
 
-        return project;
-    }
+                return project;
+            }
         });
 
         messageBus.publish(_name, ProjectManager.MESSAGE_CREATE_TUNGSTEN_PROJECT_EVENT, PublishScope.LOCAL, project);
@@ -1290,7 +1290,7 @@ public List<Long> listPermittedProjectAccounts(long accountId) {
     }
 
     @Override
-    @ActionEvent(eventType = EventTypes.EVENT_PROJECT_ACTIVATE, eventDescription = "activating project")
+    @ActionEvent(eventType = EventTypes.EVENT_PROJECT_ACTIVATE, eventDescription = "activating project", async = true)
     @DB
     public Project activateProject(final long projectId) {
         Account caller = CallContext.current().getCallingAccount();

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -17,6 +17,7 @@
 package com.cloud.server;
 
 import java.lang.reflect.Field;
+import java.security.cert.CertificateException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
@@ -43,6 +44,7 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.security.CertificateHelper;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.SecurityChecker;
 import org.apache.cloudstack.affinity.AffinityGroupProcessor;
@@ -4572,13 +4574,12 @@ public String uploadCertificate(final UploadCustomCertificateCmd cmd) {
 
         final String certificate = cmd.getCertificate();
         final String key = cmd.getPrivateKey();
+        String domainSuffix = cmd.getDomainSuffix();
 
-        if (cmd.getPrivateKey() != null && !_ksMgr.validateCertificate(certificate, key, cmd.getDomainSuffix())) {
-            throw new InvalidParameterValueException("Failed to pass certificate validation check");
-        }
+        validateCertificate(certificate, key, domainSuffix);
 
         if (cmd.getPrivateKey() != null) {
-            _ksMgr.saveCertificate(ConsoleProxyManager.CERTIFICATE_NAME, certificate, key, cmd.getDomainSuffix());
+            _ksMgr.saveCertificate(ConsoleProxyManager.CERTIFICATE_NAME, certificate, key, domainSuffix);
 
             // Reboot ssvm here since private key is present - meaning server cert being passed
             final List<SecondaryStorageVmVO> alreadyRunning = _secStorageVmDao.getSecStorageVmListInStates(null, State.Running, State.Migrating, State.Starting);
@@ -4595,6 +4596,24 @@ public String uploadCertificate(final UploadCustomCertificateCmd cmd) {
         + "please give a few minutes for console access and storage services service to be up and working again";
     }
 
+    private void validateCertificate(String certificate, String key, String domainSuffix) {
+        if (key != null) {
+            Pair<Boolean, String> result = _ksMgr.validateCertificate(certificate, key, domainSuffix);
+            if (!result.first()) {
+                throw new InvalidParameterValueException(String.format("Failed to pass certificate validation check with error: %s", result.second()));
+            }
+        } else {
+            try {
+                logger.debug(String.format("Trying to validate the root certificate format"));
+                CertificateHelper.buildCertificate(certificate);
+            } catch (CertificateException e) {
+                String errorMsg = String.format("Failed to pass certificate validation check with error: Certificate validation failed due to exception: %s", e.getMessage());
+                logger.error(errorMsg);
+                throw new InvalidParameterValueException(errorMsg);
+            }
+        }
+    }
+
     @Override
     public List<String> getHypervisors(final Long zoneId) {
         final List<String> result = new ArrayList<>();

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -4776,17 +4776,24 @@ private void updateVMDiskController(UserVmVO vm, Map<String, String> customParam
                 vm.setDetail(VmDetailConstants.DATA_DISK_CONTROLLER, dataDiskControllerSetting);
             }
 
-            String controllerSetting = StringUtils.defaultIfEmpty(_configDao.getValue(Config.VmwareRootDiskControllerType.key()),
-                    Config.VmwareRootDiskControllerType.getDefaultValue());
-
             // Don't override if VM already has root/data disk controller detail
             if (vm.getDetail(VmDetailConstants.ROOT_DISK_CONTROLLER) == null) {
-                vm.setDetail(VmDetailConstants.ROOT_DISK_CONTROLLER, controllerSetting);
+                String vmwareRootDiskControllerTypeFromSetting = StringUtils.defaultIfEmpty(_configDao.getValue(Config.VmwareRootDiskControllerType.key()),
+                        Config.VmwareRootDiskControllerType.getDefaultValue());
+                vm.setDetail(VmDetailConstants.ROOT_DISK_CONTROLLER, vmwareRootDiskControllerTypeFromSetting);
             }
+
             if (vm.getDetail(VmDetailConstants.DATA_DISK_CONTROLLER) == null) {
-                if (controllerSetting.equalsIgnoreCase("scsi")) {
-                    vm.setDetail(VmDetailConstants.DATA_DISK_CONTROLLER, "scsi");
+                String finalRootDiskController = vm.getDetail(VmDetailConstants.ROOT_DISK_CONTROLLER);
+                // Set the data disk controller detail same as the final scsi root disk controller if VM doesn't have data disk controller detail
+                // This is to ensure the disk controller is available for the data disks, as all the SCSI controllers are created with same controller type
+                String scsiControllerPattern = "(?i)\\b(scsi|lsilogic|lsilogicsas|lsisas1068|buslogic|pvscsi)\\b";
+                if (finalRootDiskController.matches(scsiControllerPattern)) {
+                    logger.info(String.format("Data disk controller was not defined, but root disk is using SCSI controller [%s]." +
+                            "To ensure disk controllers are available for the data disks, the data disk controller is updated to match the root disk controller.", finalRootDiskController));
+                    vm.setDetail(VmDetailConstants.DATA_DISK_CONTROLLER, finalRootDiskController);
                 } else {
+                    logger.info("Data disk controller was not defined; defaulting to 'osdefault'.");
                     vm.setDetail(VmDetailConstants.DATA_DISK_CONTROLLER, "osdefault");
                 }
             }

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java

@@ -3201,6 +3201,14 @@ public VirtualDevice getIsoDevice() throws Exception {
         return null;
     }
 
+    public List<VirtualDevice> getIsoDevices() throws Exception {
+        List<VirtualDevice> devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device");
+        if (CollectionUtils.isEmpty(devices)) {
+            return new ArrayList<>();
+        }
+        return devices.stream().filter(device -> device instanceof VirtualCdrom).collect(Collectors.toList());
+    }
+
     public VirtualDevice getIsoDevice(int key) throws Exception {
         List<VirtualDevice> devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device");
         if (devices != null && devices.size() > 0) {