From bfc1395edb1510165d2fadc50f0941be609757ba Mon Sep 17 00:00:00 2001 From: Lucia Jelinkova Date: Fri, 22 Apr 2022 12:37:33 +0200 Subject: [PATCH] core: Change the CPUPolicyUnit for dedicated pinning Before the introduction of the exclusively pinned CPUs, the logic of CPUPolicyUnit allowed to check the filtering constraints for each VM in the vm group individually. The constraint was that the number of VM's CPUs had to be >= host CPUs. With the introduction of the exclusively pinned CPUs, that is no more possible - if the group contained VMs with both, shared and exclusively pinned CPUs, we need to calculate the CPU count constraints for the whole group (similar to huge pages in HugePagesFilterPolicyUnit). Now the algorithm for calculating if the vm group fits into the host is as follows: 1. Calculate the host CPU count 2. Calculate the currently exclusively pinned CPUs (including pending) 3. Pin the vms that are being schedulled and count how many CPUs will be taken 4. Calculate how many shared CPUs are required to be left on the host as the maximum of required shared CPUs for vmGroup, pending VMs and running VMs. The host can fit the VMs if: hostCpuCount - exclusiveCpus - maxSharedCpuCount >= 0 Note that the calculation of the previous values may differ based on the cluster setting of "Count threads as cores". --- .../bll/scheduling/InternalPolicyUnits.java | 2 + .../scheduling/pending/PendingCpuCores.java | 10 ++ .../scheduling/policyunits/CPUPolicyUnit.java | 165 +++++++++++++----- .../policyunits/CpuTopologyPolicyUnit.java | 74 ++++++++ .../utils/VdsCpuUnitPinningHelper.java | 44 ++++- .../policyunits/CPUPolicyUnitTest.java | 33 ++-- .../core/common/errors/EngineMessage.java | 2 + .../engine/core/vdsbroker/VdsManager.java | 9 + .../engine/core/vdsbroker/VmManager.java | 7 + .../vdsbroker/monitoring/HostMonitoring.java | 6 + .../ovirt/engine/ui/frontend/AppErrors.java | 4 + .../engine/ui/frontend/AppErrors.properties | 2 + 12 files changed, 297 insertions(+), 61 deletions(-) create mode 100644 backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CpuTopologyPolicyUnit.java diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/InternalPolicyUnits.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/InternalPolicyUnits.java index fcda64acd1a..2d03b8ea027 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/InternalPolicyUnits.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/InternalPolicyUnits.java @@ -15,6 +15,7 @@ import org.ovirt.engine.core.bll.scheduling.policyunits.CpuLevelFilterPolicyUnit; import org.ovirt.engine.core.bll.scheduling.policyunits.CpuOverloadPolicyUnit; import org.ovirt.engine.core.bll.scheduling.policyunits.CpuPinningPolicyUnit; +import org.ovirt.engine.core.bll.scheduling.policyunits.CpuTopologyPolicyUnit; import org.ovirt.engine.core.bll.scheduling.policyunits.EmulatedMachineFilterPolicyUnit; import org.ovirt.engine.core.bll.scheduling.policyunits.EvenDistributionBalancePolicyUnit; import org.ovirt.engine.core.bll.scheduling.policyunits.EvenDistributionCPUWeightPolicyUnit; @@ -101,6 +102,7 @@ public class InternalPolicyUnits { mandatoryUnits.add(CompatibilityVersionFilterPolicyUnit.class); mandatoryUnits.add(CpuLevelFilterPolicyUnit.class); + mandatoryUnits.add(CpuTopologyPolicyUnit.class); mandatoryUnits.add(CpuPinningPolicyUnit.class); mandatoryUnits.add(HostDeviceFilterPolicyUnit.class); mandatoryUnits.add(PinToHostPolicyUnit.class); diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/pending/PendingCpuCores.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/pending/PendingCpuCores.java index faa3a86af9c..856482f9353 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/pending/PendingCpuCores.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/pending/PendingCpuCores.java @@ -1,6 +1,7 @@ package org.ovirt.engine.core.bll.scheduling.pending; import org.ovirt.engine.core.bll.scheduling.utils.VmSpecificPendingResourceEqualizer; +import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy; import org.ovirt.engine.core.common.businessentities.VDS; import org.ovirt.engine.core.common.businessentities.VM; import org.ovirt.engine.core.compat.Guid; @@ -10,15 +11,20 @@ * by not yet started VM on a specified host */ public class PendingCpuCores extends PendingResource { + + private CpuPinningPolicy cpuPinningPolicy; + private int coreCount; public PendingCpuCores(VDS host, VM vm, int coreCount) { super(host, vm); + this.cpuPinningPolicy = vm.getCpuPinningPolicy(); this.coreCount = coreCount; } public PendingCpuCores(Guid host, VM vm, int coreCount) { super(host, vm); + this.cpuPinningPolicy = vm.getCpuPinningPolicy(); this.coreCount = coreCount; } @@ -30,6 +36,10 @@ public void setCoreCount(int coreCount) { this.coreCount = coreCount; } + public CpuPinningPolicy getCpuPinningPolicy() { + return cpuPinningPolicy; + } + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override public boolean equals(Object other) { diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnit.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnit.java index a3a6287a06a..6b977847ed0 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnit.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnit.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.inject.Inject; @@ -9,15 +10,20 @@ import org.ovirt.engine.core.bll.scheduling.SchedulingContext; import org.ovirt.engine.core.bll.scheduling.SchedulingUnit; import org.ovirt.engine.core.bll.scheduling.SlaValidator; +import org.ovirt.engine.core.bll.scheduling.pending.PendingCpuCores; +import org.ovirt.engine.core.bll.scheduling.pending.PendingCpuPinning; import org.ovirt.engine.core.bll.scheduling.pending.PendingResourceManager; import org.ovirt.engine.core.bll.scheduling.utils.VdsCpuUnitPinningHelper; import org.ovirt.engine.core.common.businessentities.VDS; import org.ovirt.engine.core.common.businessentities.VM; +import org.ovirt.engine.core.common.businessentities.VdsCpuUnit; import org.ovirt.engine.core.common.errors.EngineMessage; import org.ovirt.engine.core.common.scheduling.PerHostMessages; import org.ovirt.engine.core.common.scheduling.PolicyUnit; import org.ovirt.engine.core.common.scheduling.PolicyUnitType; import org.ovirt.engine.core.common.utils.VmCpuCountHelper; +import org.ovirt.engine.core.compat.Guid; +import org.ovirt.engine.core.vdsbroker.ResourceManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,69 +31,142 @@ guid = "6d636bf6-a35c-4f9d-b68d-0731f720cddc", name = "CPU", type = PolicyUnitType.FILTER, - description = "Filters out hosts with less CPUs than VM's CPUs" -) + description = "Filters out hosts with less CPUs than VM's CPUs") public class CPUPolicyUnit extends PolicyUnitImpl { private static final Logger log = LoggerFactory.getLogger(CPUPolicyUnit.class); @Inject private VdsCpuUnitPinningHelper vdsCpuUnitPinningHelper; + @Inject + private ResourceManager resourceManager; + public CPUPolicyUnit(PolicyUnit policyUnit, PendingResourceManager pendingResourceManager) { super(policyUnit, pendingResourceManager); } + /** + * Filters out the hosts that do not have enough free CPUs to accommodate the shared and exclusively pinned CPUs + * required by the vmGroup. + */ @Override - public List filter(SchedulingContext context, List hosts, VM vm, PerHostMessages messages) { - List list = new ArrayList<>(); - - for (VDS vds : hosts) { - if (VmCpuCountHelper.isResizeAndPinPolicy(vm) && !VmCpuCountHelper.isDynamicCpuTopologySet(vm)) { - if (vds.getCpuCores() / vds.getCpuSockets() > 1) { - list.add(vds); - } else { - messages.addMessage(vds.getId(), EngineMessage.VAR__DETAIL__NOT_ENOUGH_CORES.toString()); - log.debug("Host '{}' has only one core per socket. Resize and pin requires more than one core per socket", vds.getName()); - } + public List filter(SchedulingContext context, List hosts, List vmGroup, PerHostMessages messages) { + List candidates = new ArrayList<>(); + + int maxVmGroupSharedCpuCount = countMaxVmGroupSharedCpuCount(vmGroup); + + for (VDS host : hosts) { + + if (host.getCpuSockets() == null || host.getCpuCores() == null || host.getCpuThreads() == null) { + log.warn("Unknown number of cores for host {}.", host.getName()); continue; } - Integer cores = SlaValidator.getEffectiveCpuCores(vds, context.getCluster().getCountThreadsAsCores()); - if (cores != null) { - int numOfCpus = VmCpuCountHelper.isDynamicCpuTopologySet(vm) ? - vm.getCurrentNumOfCpus(false) : vm.getNumOfCpus(false); - if (vm.getCpuPinningPolicy().isExclusive()) { - int futureCpus = context.getCluster().getCountThreadsAsCores() ? cores - vds.getVmsCoresCount() : - cores - (int) Math.ceil(vds.getVmsCoresCount() / (vds.getCpuThreads() / vds.getCpuCores())); - if (numOfCpus > futureCpus) { - messageNotEnoughCores(vds, cores, vm, messages); - continue; - } - } else { - int takenCpus = 0; - // takenCpus are CPUs (threads), we should consider switching it to cores when necessary. - takenCpus = context.getCluster().getCountThreadsAsCores() ? - vdsCpuUnitPinningHelper.getDedicatedCount(vds.getId()) : - vdsCpuUnitPinningHelper.countTakenCores(vds); - cores = cores - takenCpus; - if (numOfCpus > cores) { - messageNotEnoughCores(vds, cores, vm, messages); - continue; - } - } + // total number of host CPUs (either core or threads based on countThreadsAsCores) + Integer hostCpuCount = + SlaValidator.getEffectiveCpuCores(host, context.getCluster().getCountThreadsAsCores()); + + // all CPUs (either core or threads based on countThreadsAsCores) + // unavailable due to the exclusive pinning (counted for running VMs, pending, vmGroup) + int exclusiveCpus = countExclusiveCPUs(host, vmGroup, context.getCluster().getCountThreadsAsCores()); + + // the maximal number of CPU count for all VMs (running, pending, vmGroup) + // it determines how many shared CPUs needs to be available on the host + long maxSharedCpuCount = Math.max(countMaxRunningVmsSharedCpuCount(host), maxVmGroupSharedCpuCount); + + if (hostCpuCount - exclusiveCpus - maxSharedCpuCount < 0) { + messageNotEnoughCores(host, messages); + continue; } - list.add(vds); + candidates.add(host); + } + return candidates; + } + + /** + * Counts how many CPUs will be unavailable due to the exclusive pinning. The method + * takes a copy of the current CPU topology, applies pending resources and pins all of the exclusively + * pinned VMs. Then counts either the unavailable threads or the whole cores based on the countThreadsAsCores + * parameter. + * @param host Host candidate + * @param vmGroup VMs to be scheduled + * @param countThreadsAsCores If the threads should be counted as cores + * @return Number of CPUs that are exclusively pinned or blocked by an exclusive pinning or Integer.MAX_VALUE + * if the pinning of the exclusive CPUs is not possible (shared pinnings should be checked in {@link CpuPinningPolicyUnit} + */ + private int countExclusiveCPUs(VDS host, List vmGroup, boolean countThreadsAsCores) { + + List cpuTopology = getEffectiveCpuTopology(host); + + if (cpuTopology.isEmpty()) { + return 0; } - return list; + + for (VM vm : vmGroup) { + if (!host.getId().equals(vm.getRunOnVds())) { + List allocatedCpus = + vdsCpuUnitPinningHelper.updatePhysicalCpuAllocations(vm, cpuTopology, host.getId()); + if (vm.getCpuPinningPolicy().isExclusive() && (allocatedCpus == null || allocatedCpus.isEmpty())) { + return Integer.MAX_VALUE; + } + } + } + + return vdsCpuUnitPinningHelper.countUnavailableCpus( + cpuTopology, + countThreadsAsCores); + } + + private List getEffectiveCpuTopology(VDS host) { + List cpuTopology = resourceManager.getVdsManager(host.getId()).getCpuTopology(); + + Map> vmToPendingExclusiveCpuPinnings = + PendingCpuPinning.collectForHost(getPendingResourceManager(), host.getId()); + + vdsCpuUnitPinningHelper.previewPinOfPendingExclusiveCpus(cpuTopology, vmToPendingExclusiveCpuPinnings); + return cpuTopology; + } + + /** + * Finds the VM with the maximum number of shared CPUs and returns the CPU count + * @param vmGroup VMs to be scheduled + * @return Max count of shared CPUs in the vm group + */ + private int countMaxVmGroupSharedCpuCount(List vmGroup) { + // the number of host's shared CPUs has to >= as the number of CPUs of any VM + int maxVmGroupSharedCpuCount = vmGroup.stream() + .filter(vm -> !vm.getCpuPinningPolicy().isExclusive()) + .mapToInt(vm -> VmCpuCountHelper.getDynamicNumOfCpu(vm)) + .max() + .orElse(0); + return maxVmGroupSharedCpuCount; + } + + /** + * Checks running VMs and pending resources and finds the VM with the maximum number + * of shared CPU and returns the CPU count + * @param host Host candidate + * @return Max count of shared CPUs in running VMs and pending resources + */ + private long countMaxRunningVmsSharedCpuCount(VDS host) { + + long maxPendingSharedCount = pendingResourceManager.pendingHostResources(host.getId(), PendingCpuCores.class) + .stream() + .filter(pending -> !pending.getCpuPinningPolicy().isExclusive()) + .mapToLong(pending -> pending.getCoreCount()) + .max() + .orElse(0); + + long maxRunningSharedCpuCount = resourceManager.getVdsManager(host.getId()).getMinRequiredSharedCpusCount(); + + return Math.max(maxPendingSharedCount, maxRunningSharedCpuCount); } - private void messageNotEnoughCores(VDS vds, int cores, VM vm, PerHostMessages messages) { + private void messageNotEnoughCores(VDS vds, PerHostMessages messages) { messages.addMessage(vds.getId(), EngineMessage.VAR__DETAIL__NOT_ENOUGH_CORES.toString()); - log.debug("Host '{}' has less cores ({}) than vm cores ({})", - vds.getName(), - cores, - VmCpuCountHelper.getDynamicNumOfCpu(vm)); + log.debug("Host '{}' has not enough available cores to schedule vms)", + vds.getName()); } } diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CpuTopologyPolicyUnit.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CpuTopologyPolicyUnit.java new file mode 100644 index 00000000000..a4be90f34c5 --- /dev/null +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CpuTopologyPolicyUnit.java @@ -0,0 +1,74 @@ +package org.ovirt.engine.core.bll.scheduling.policyunits; + +import java.util.ArrayList; +import java.util.List; + +import org.ovirt.engine.core.bll.scheduling.PolicyUnitImpl; +import org.ovirt.engine.core.bll.scheduling.SchedulingContext; +import org.ovirt.engine.core.bll.scheduling.SchedulingUnit; +import org.ovirt.engine.core.bll.scheduling.pending.PendingResourceManager; +import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy; +import org.ovirt.engine.core.common.businessentities.VDS; +import org.ovirt.engine.core.common.businessentities.VM; +import org.ovirt.engine.core.common.errors.EngineMessage; +import org.ovirt.engine.core.common.scheduling.PerHostMessages; +import org.ovirt.engine.core.common.scheduling.PolicyUnit; +import org.ovirt.engine.core.common.scheduling.PolicyUnitType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SchedulingUnit( + guid = "35c2f1a5-8928-48e9-81ac-4c49eb49d60e", + name = "CPUTopology", + type = PolicyUnitType.FILTER, + description = "Runs VMs only on hosts with a proper CPU topology") +public class CpuTopologyPolicyUnit extends PolicyUnitImpl { + + private static final Logger log = LoggerFactory.getLogger(CpuTopologyPolicyUnit.class); + + public CpuTopologyPolicyUnit(PolicyUnit policyUnit, PendingResourceManager pendingResourceManager) { + super(policyUnit, pendingResourceManager); + } + + @Override + public List filter(SchedulingContext context, List hosts, VM vm, PerHostMessages messages) { + List candidates = new ArrayList<>(); + + for (VDS host : hosts) { + + if (host.getCpuSockets() == null || host.getCpuCores() == null || host.getCpuThreads() == null) { + log.warn("Unknown number of cores for host {}.", host.getName()); + continue; + } + + // when the VM uses Resize and PIN CPU pinning policy the host needs to have + // more than one core per socket + if (vm.getCpuPinningPolicy() == CpuPinningPolicy.RESIZE_AND_PIN_NUMA) { + int coresPerSocket = host.getCpuCores() / host.getCpuSockets(); + + if (coresPerSocket <= 1) { + messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__NOT_ENOUGH_CORES_PER_SOCKET_FOR_RESIZE_AND_PIN.toString()); + log.debug( + "Host '{}' has only one core per socket. Resize and pin requires more than one core per socket", + host.getName()); + continue; + } + } else if (vm.getCpuPinningPolicy() == CpuPinningPolicy.DEDICATED) { + // the dedicated pinning requires that all vThreads of the same vCore are pinned + // to the same physical core. That is why the number of threads per core for the VM + // cannot exceed the number of threads per core on the host + int threadsPerCore = host.getCpuThreads() / host.getCpuCores(); + + if (threadsPerCore < vm.getThreadsPerCpu()) { + messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__NOT_ENOUGH_THREADS_PER_CORE_FOR_DEDICATED.toString()); + log.debug( + "Host '{}' has not enough CPU threads per core to run the VM with Dedicated CPU pinning policy", + host.getName()); + continue; + } + } + candidates.add(host); + } + return candidates; + } +} diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/utils/VdsCpuUnitPinningHelper.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/utils/VdsCpuUnitPinningHelper.java index 8db650752e9..16540d89b1c 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/utils/VdsCpuUnitPinningHelper.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/utils/VdsCpuUnitPinningHelper.java @@ -11,7 +11,6 @@ import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy; import org.ovirt.engine.core.common.businessentities.NumaNodeStatistics; -import org.ovirt.engine.core.common.businessentities.VDS; import org.ovirt.engine.core.common.businessentities.VM; import org.ovirt.engine.core.common.businessentities.VdsCpuUnit; import org.ovirt.engine.core.common.businessentities.VdsNumaNode; @@ -106,6 +105,25 @@ public List updatePhysicalCpuAllocations(VM vm, Map. The list of VdsCpuUnit we are going to use. If not possible, return null. + */ + public List updatePhysicalCpuAllocations(VM vm, List cpuTopology, Guid hostId) { + if (cpuTopology.isEmpty()) { + return new ArrayList<>(); + } // 'Resize and pin NUMA' policy also acts as manual pinning. if (!vm.getCpuPinningPolicy().isExclusive()) { @@ -210,8 +228,7 @@ private List allocateDedicatedCpus(List cpuTopology, VM return cpusToBeAllocated; } - public int countTakenCores(VDS host) { - List cpuTopology = resourceManager.getVdsManager(host.getId()).getCpuTopology(); + private int countTakenCores(List cpuTopology) { if (cpuTopology.isEmpty()) { return 0; } @@ -254,9 +271,24 @@ private List getNonExclusiveCpusInCore(List cpuTopology, return getCpusInCore(getCoresInSocket(cpuTopology, socketId), coreId).stream().filter(cpu -> !cpu.isExclusive()).collect(Collectors.toList()); } - public int getDedicatedCount(Guid vdsId) { - return (int) resourceManager.getVdsManager(vdsId).getCpuTopology().stream() - .filter(VdsCpuUnit::isExclusive).count(); + private int getDedicatedCount(List cpuTopology) { + return (int) cpuTopology.stream().filter(VdsCpuUnit::isExclusive).count(); + } + + /** + * Counts how many CPUs (cores or threads, depending on the countThreadsAsCores) will be unavailable + * due to the exclusive pinning. + * + * @param cpuTopology CPU topology we want to process + * @param countThreadsAsCores If the threads should be counted as cores + * @return Number of CPUs that are exclusively pinned or blocked by an exclusive pinning + */ + public int countUnavailableCpus(List cpuTopology, boolean countThreadsAsCores) { + if (countThreadsAsCores) { + return getDedicatedCount(cpuTopology); + } else { + return countTakenCores(cpuTopology); + } } private List getOnlineSockets(List cpuTopology) { diff --git a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnitTest.java b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnitTest.java index 3a7036e66c7..e4732f5be35 100644 --- a/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnitTest.java +++ b/backend/manager/modules/bll/src/test/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnitTest.java @@ -1,7 +1,9 @@ package org.ovirt.engine.core.bll.scheduling.policyunits; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Collections; @@ -16,12 +18,15 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.ovirt.engine.core.bll.scheduling.SchedulingContext; +import org.ovirt.engine.core.bll.scheduling.pending.PendingResourceManager; import org.ovirt.engine.core.bll.scheduling.utils.VdsCpuUnitPinningHelper; import org.ovirt.engine.core.common.businessentities.Cluster; import org.ovirt.engine.core.common.businessentities.VDS; import org.ovirt.engine.core.common.businessentities.VM; import org.ovirt.engine.core.common.scheduling.PerHostMessages; import org.ovirt.engine.core.compat.Guid; +import org.ovirt.engine.core.vdsbroker.ResourceManager; +import org.ovirt.engine.core.vdsbroker.VdsManager; @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) @@ -29,8 +34,17 @@ public class CPUPolicyUnitTest { @Mock private VdsCpuUnitPinningHelper vdsCpuUnitPinningHelper; + @Mock + private PendingResourceManager pendingHostResources; + + @Mock + private VdsManager vdsManager; + + @Mock + private ResourceManager resourceManager; + @InjectMocks - private final CPUPolicyUnit cpuPolicyUnit = new CPUPolicyUnit(null, null); + private final CPUPolicyUnit cpuPolicyUnit = new CPUPolicyUnit(null, pendingHostResources); private VDS vdsWithInvalidCpuInfo; @@ -53,11 +67,15 @@ public void setUp() { vdsWithCores = new VDS(); vdsWithCores.setId(Guid.newGuid()); + vdsWithCores.setCpuSockets(1); vdsWithCores.setCpuCores(2); vdsWithCores.setCpuThreads(4); cluster = new Cluster(); cluster.setId(Guid.newGuid()); + + when(pendingHostResources.pendingHostResources(any(), any())).thenReturn(Collections.emptyList()); + when(resourceManager.getVdsManager(any())).thenReturn(vdsManager); } /** @@ -70,16 +88,6 @@ public void shouldFilterInvalidHosts() { assertThat(candidates).containsOnly(vdsWithCores); } - /** - * When no cpu information at all is available, consider the host - */ - @Test - public void shouldKeepHostsWithNoCpuInformation() { - vdsWithInvalidCpuInfo.setCpuCores(null); - final List candidates = filter(); - assertThat(candidates).contains(vdsWithInvalidCpuInfo); - } - @Test public void shouldFilterHostWithNotEnoughThreads() { cluster.setCountThreadsAsCores(true); @@ -113,7 +121,8 @@ public void shouldKeepHostsWithEnoughThreads() { private List filter() { return cpuPolicyUnit.filter(new SchedulingContext(cluster, Collections.emptyMap()), Arrays.asList(vdsWithInvalidCpuInfo, vdsWithCores), - vm, mock(PerHostMessages.class)); + Collections.singletonList(vm), + mock(PerHostMessages.class)); } } diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/errors/EngineMessage.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/errors/EngineMessage.java index 73e9c21251a..d2459188dbb 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/errors/EngineMessage.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/errors/EngineMessage.java @@ -1550,6 +1550,8 @@ public enum EngineMessage { VAR__DETAIL__NUMA_NOT_SUPPORTED, VAR__DETAIL__NOT_MEMORY_PINNED_NUMA, VAR__DETAIL__NOT_ENOUGH_CORES, + VAR__DETAIL__NOT_ENOUGH_CORES_PER_SOCKET_FOR_RESIZE_AND_PIN, + VAR__DETAIL__NOT_ENOUGH_THREADS_PER_CORE_FOR_DEDICATED, VAR__DETAIL__VM_PINNING_PCPU_DOES_NOT_EXIST, VAR__DETAIL__VM_PINNING_DEDICATED_NOT_FIT, VAR__DETAIL__VM_PINNING_CANT_RESIZE_WITH_DEDICATED, diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java index 4bdcf49ba04..6a69d4c8b2c 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VdsManager.java @@ -182,6 +182,7 @@ public class VdsManager { private HostConnectionRefresherInterface hostRefresher; private volatile boolean inServerRebootTimeout; private List cpuTopology; + private int minRequiredSharedCpusCount; VdsManager(VDS vds, ResourceManager resourceManager) { this.resourceManager = resourceManager; @@ -1350,4 +1351,12 @@ public void unpinVmCpus(Guid vmId) { cpuTopology.stream().filter(cpu -> cpu.getVmIds().contains(vmId)) .forEach(cpu -> cpu.unPinVm(vmId)); } + + public void setMinRequiredSharedCpusCount(int minSharedCpusCount) { + this.minRequiredSharedCpusCount = minSharedCpusCount; + } + + public int getMinRequiredSharedCpusCount() { + return minRequiredSharedCpusCount < 1 ? 1 : minRequiredSharedCpusCount; + } } diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VmManager.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VmManager.java index f99af3f5488..ae6192de685 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VmManager.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/VmManager.java @@ -15,6 +15,7 @@ import org.ovirt.engine.core.common.businessentities.ArchitectureType; import org.ovirt.engine.core.common.businessentities.BiosType; import org.ovirt.engine.core.common.businessentities.Cluster; +import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy; import org.ovirt.engine.core.common.businessentities.OriginType; import org.ovirt.engine.core.common.businessentities.VM; import org.ovirt.engine.core.common.businessentities.VMStatus; @@ -47,6 +48,7 @@ public class VmManager { private int memSizeMb; private int minAllocatedMem; private int numOfCpus; + private CpuPinningPolicy cpuPinningPolicy; private Version clusterCompatibilityVersion; private ArchitectureType clusterArchitecture; private BiosType clusterBiosType; @@ -126,6 +128,7 @@ private void updateStaticFields(VmStatic vmStatic) { memSizeMb = vmStatic.getMemSizeMb(); minAllocatedMem = vmStatic.getMinAllocatedMem(); numOfCpus = vmStatic.getNumOfCpus(); + cpuPinningPolicy = vmStatic.getCpuPinningPolicy(); final Cluster cluster = clusterDao.get(vmStatic.getClusterId()); clusterCompatibilityVersion = cluster.getCompatibilityVersion(); clusterArchitecture = cluster.getArchitecture(); @@ -293,6 +296,10 @@ public int getNumOfCpus() { return numOfCpus; } + public CpuPinningPolicy getCpuPinningPolicy() { + return cpuPinningPolicy; + } + public void setClusterCompatibilityVersion(Version version) { this.clusterCompatibilityVersion = version; } diff --git a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/monitoring/HostMonitoring.java b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/monitoring/HostMonitoring.java index a8519bf7125..5902a7d5e5b 100644 --- a/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/monitoring/HostMonitoring.java +++ b/backend/manager/modules/vdsbroker/src/main/java/org/ovirt/engine/core/vdsbroker/monitoring/HostMonitoring.java @@ -887,6 +887,7 @@ public static boolean refreshCommitedMemory(VDS host, Map vmIdTo int memCommited = host.getGuestOverhead(); int vmsCoresCount = 0; + int maxSharedCpus = 0; for (Map.Entry entry : vmIdToStatus.entrySet()) { VMStatus status = entry.getValue(); @@ -897,10 +898,15 @@ public static boolean refreshCommitedMemory(VDS host, Map vmIdTo if (vmManager != null) { memCommited += vmManager.getVmMemoryWithOverheadInMB(); vmsCoresCount += vmManager.getNumOfCpus(); + int sharedCpus = vmManager.getCpuPinningPolicy() != null && !vmManager.getCpuPinningPolicy().isExclusive() + ? vmManager.getNumOfCpus() : 0; + maxSharedCpus = Math.max(maxSharedCpus, sharedCpus); } } } + resourceManager.getVdsManager(host.getId()).setMinRequiredSharedCpusCount(maxSharedCpus); + if (memCommited != host.getMemCommited()) { host.setMemCommited(memCommited); memoryUpdated = true; diff --git a/frontend/webadmin/modules/frontend/src/main/java/org/ovirt/engine/ui/frontend/AppErrors.java b/frontend/webadmin/modules/frontend/src/main/java/org/ovirt/engine/ui/frontend/AppErrors.java index 29f0cfb02ce..79bd1315e47 100644 --- a/frontend/webadmin/modules/frontend/src/main/java/org/ovirt/engine/ui/frontend/AppErrors.java +++ b/frontend/webadmin/modules/frontend/src/main/java/org/ovirt/engine/ui/frontend/AppErrors.java @@ -2703,6 +2703,10 @@ public interface AppErrors extends ConstantsWithLookup { String VAR__DETAIL__NOT_ENOUGH_CORES(); + String VAR__DETAIL__NOT_ENOUGH_CORES_PER_SOCKET_FOR_RESIZE_AND_PIN(); + + String VAR__DETAIL__NOT_ENOUGH_THREADS_PER_CORE_FOR_DEDICATED(); + String VAR__DETAIL__VM_PINNING_PCPU_DOES_NOT_EXIST(); String VAR__DETAIL__VM_PINNING_DEDICATED_NOT_FIT(); diff --git a/frontend/webadmin/modules/frontend/src/main/resources/org/ovirt/engine/ui/frontend/AppErrors.properties b/frontend/webadmin/modules/frontend/src/main/resources/org/ovirt/engine/ui/frontend/AppErrors.properties index 4a5f4db5055..1d16f47e4a2 100644 --- a/frontend/webadmin/modules/frontend/src/main/resources/org/ovirt/engine/ui/frontend/AppErrors.properties +++ b/frontend/webadmin/modules/frontend/src/main/resources/org/ovirt/engine/ui/frontend/AppErrors.properties @@ -1282,6 +1282,8 @@ VAR__DETAIL__LOW_CPU_LEVEL=$detailMessage its CPU flags are missing VM's require VAR__DETAIL__CPU_FLAGS_NOT_IDENTICAL=$detailMessage its CPU flags are not identical to VM's required CPU flags. It is missing flags: "${missingFlags}", and has additional flags: "${additionalFlags}". VAR__DETAIL__NETWORK_MISSING=$detailMessage network(s) ${networkNames} are missing VAR__DETAIL__NOT_ENOUGH_CORES=$detailMessage it has an insufficient amount of CPU cores to run the VM +VAR__DETAIL__NOT_ENOUGH_CORES_PER_SOCKET_FOR_RESIZE_AND_PIN=$detailMessage it has to have more than one core per socket for VMs with Resize and Pin CPU pinning policy +VAR__DETAIL__NOT_ENOUGH_THREADS_PER_CORE_FOR_DEDICATED=$detailMessage it has not enough CPU threads per core to run the VM with Dedicated CPU pinning policy VAR__DETAIL__NOT_ENOUGH_HUGE_PAGES=$detailMessage there are not enough free huge pages to run the VM VAR__DETAIL__NOT_ENOUGH_MEMORY=$detailMessage its available memory is too low (${availableMem} MB) to run the VM VAR__DETAIL__NOT_HE_HOST=$detailMessage it is not a Hosted Engine host.