Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduce isolated threads CPUs #260

Merged
merged 2 commits into from
May 8, 2022
Merged

Introduce isolated threads CPUs #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b79f880
core: introduce isolated threads cpus
liranr23 Apr 12, 2022
c8fa057
frontend: introduce isolate threads
liranr23 Apr 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...anager/modules/bll/src/main/java/org/ovirt/engine/core/bll/HotSetNumberOfCpusCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ protected boolean validate() {
valid = failValidation(EngineMessage.HOT_PLUG_CPU_CONFLICT,
String.format("%1$s", getVm().getCpuPinningPolicy().name()));
}
if (getVm().getCpuPinningPolicy() == CpuPinningPolicy.DEDICATED) {
if (getVm().getCpuPinningPolicy().isExclusive()) {
valid = failValidation(EngineMessage.HOT_PLUG_CPU_IS_NOT_SUPPORTED_DEDICATED,
String.format("$policy %1$s", getVm().getCpuPinningPolicy().name()));
}
Expand Down
18 changes: 12 additions & 6 deletions backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/MigrateVmCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import org.ovirt.engine.core.common.action.LockProperties.Scope;
import org.ovirt.engine.core.common.action.MigrateVmParameters;
import org.ovirt.engine.core.common.action.PlugAction;
import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy;
import org.ovirt.engine.core.common.businessentities.MigrationMethod;
import org.ovirt.engine.core.common.businessentities.OriginType;
import org.ovirt.engine.core.common.businessentities.VDS;
Expand Down Expand Up @@ -429,10 +428,17 @@ private MigrateVDSCommandParameters createMigrateVDSCommandParameters() {
if (parallelMigrations == null) {
maxIncomingMigrations = maxOutgoingMigrations = effectiveMigrationPolicy.getMaxMigrations();
}
if (getVm().getCpuPinningPolicy() == CpuPinningPolicy.DEDICATED) {
cpuSets = CpuPinningHelper.getAllPinnedPCpus(getDedicatedCpuPinning(getDestinationVdsManager())).stream()
.map(Object::toString).collect(Collectors.toList());
String numaPinningString = vmHandler.createNumaPinningForDedicated(getVm(), getDestinationVdsId());
if (getVm().getCpuPinningPolicy().isExclusive()) {
String cpuPinning = getExclusiveCpuPinning(getDestinationVdsManager());
List<CpuPinningHelper.PinnedCpu> pinnedCpus = CpuPinningHelper.parseCpuPinning(cpuPinning);
cpuSets = new LinkedList<>();
for (CpuPinningHelper.PinnedCpu pin : pinnedCpus) {
for (Integer pCpu : pin.getpCpus()) {
cpuSets.add(pCpu.toString());
break;
}
liranr23 marked this conversation as resolved.
Show resolved Hide resolved
}
String numaPinningString = vmHandler.createNumaPinningForExclusiveCpuPinning(getVm(), getDestinationVdsId());
numaNodeSets = NumaPinningHelper.parseNumaSets(numaPinningString);
}

Expand Down Expand Up @@ -624,7 +630,7 @@ public void runningSucceded() {
try {
queryDowntime();
addNumaPinningForDedicated(getDestinationVdsId());
setDedicatedCpus(getDestinationVdsManager());
setExclusiveCpuPinning(getDestinationVdsManager());
vmDynamicDao.clearMigratingToVdsAndSetDynamicPinning(getVmId(), getVm().getCurrentCpuPinning(), getVm().getCurrentNumaPinning());
updateVmAfterMigrationToDifferentCluster();
plugPassthroughNics();
Expand Down
2 changes: 1 addition & 1 deletion backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/RunVmCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -958,7 +958,7 @@ protected boolean getVdsToRunOn() {

addNumaPinningForDedicated(getVdsId());
addNumaPinningForResizeAndPin();
setDedicatedCpus(getVdsManager());
setExclusiveCpuPinning(getVdsManager());

return true;
}
Expand Down
13 changes: 6 additions & 7 deletions backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/RunVmCommandBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.ovirt.engine.core.common.action.ActionType;
import org.ovirt.engine.core.common.action.ProcessDownVmParameters;
import org.ovirt.engine.core.common.action.VmOperationParameterBase;
import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy;
import org.ovirt.engine.core.common.businessentities.IVdsAsyncCommand;
import org.ovirt.engine.core.common.businessentities.OpenstackNetworkProviderProperties;
import org.ovirt.engine.core.common.businessentities.Provider;
Expand Down Expand Up @@ -404,21 +403,21 @@ protected void initParametersForExternalNetworks(VDS vds, boolean isMigration) {
}

protected void addNumaPinningForDedicated(Guid vdsId) {
String numaPinningString = vmHandler.createNumaPinningForDedicated(getVm(), vdsId);
String numaPinningString = vmHandler.createNumaPinningForExclusiveCpuPinning(getVm(), vdsId);
getVm().setCurrentNumaPinning(numaPinningString);
}

protected void setDedicatedCpus(VdsManager vdsManager) {
if (getVm().getCpuPinningPolicy() != CpuPinningPolicy.DEDICATED) {
protected void setExclusiveCpuPinning(VdsManager vdsManager) {
if (!getVm().getCpuPinningPolicy().isExclusive()) {
return;
}
getVm().setCurrentCpuPinning(getDedicatedCpuPinning(vdsManager));
getVm().setCurrentCpuPinning(getExclusiveCpuPinning(vdsManager));
}

protected String getDedicatedCpuPinning(VdsManager vdsManager) {
protected String getExclusiveCpuPinning(VdsManager vdsManager) {
List<VdsCpuUnit> vdsCpuUnits = vdsManager.getCpuTopology().stream()
.filter(cpu -> cpu.getVmIds().contains(getVmId())).sorted().collect(Collectors.toList());
return CpuPinningHelper.createCpuPinningString(vdsCpuUnits);
return CpuPinningHelper.createCpuPinningString(vdsCpuUnits, getVm().getCpuPinningPolicy());
}

protected VdsManager getVdsManager() {
Expand Down
20 changes: 6 additions & 14 deletions backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VmHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1339,27 +1339,18 @@ public void setCpuPinningByNumaPinning(VM vm, Guid vdsId) {
vm.setCurrentCpuPinning(res);
}

public String createNumaPinningForDedicated(VM vm, Guid vdsId) {
if (vm.getCpuPinningPolicy() != CpuPinningPolicy.DEDICATED
|| vm.getvNumaNodeList() == null
|| vm.getvNumaNodeList().isEmpty()
|| vdsId == null) {
return null;
}

public String createNumaPinningForExclusiveCpuPinning(VM vm, Guid vdsId) {
List<VdsCpuUnit> cpuUnits = resourceManager.getVdsManager(vdsId)
.getCpuTopology()
.stream()
.filter(cpuUnit -> cpuUnit.getVmIds().contains(vm.getId()))
.collect(Collectors.toList());

return NumaPinningHelper.createNumaPinningAccordingToCpuPinning(
vm.getvNumaNodeList(),
cpuUnits);
return createNumaPinningForExclusiveCpuPinning(vm, cpuUnits);
}

public String createNumaPinningForDedicated(VM vm, List<VdsCpuUnit> cpuUnits) {
if (vm.getCpuPinningPolicy() != CpuPinningPolicy.DEDICATED
public String createNumaPinningForExclusiveCpuPinning(VM vm, List<VdsCpuUnit> cpuUnits) {
if (!vm.getCpuPinningPolicy().isExclusive()
|| vm.getvNumaNodeList() == null
|| vm.getvNumaNodeList().isEmpty()
|| cpuUnits == null) {
Expand All @@ -1368,7 +1359,8 @@ public String createNumaPinningForDedicated(VM vm, List<VdsCpuUnit> cpuUnits) {

return NumaPinningHelper.createNumaPinningAccordingToCpuPinning(
vm.getvNumaNodeList(),
cpuUnits);
cpuUnits,
vm.getCpuPinningPolicy());
}

public void autoSelectResumeBehavior(VmBase vmBase) {
Expand Down
5 changes: 2 additions & 3 deletions ...ger/modules/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/SchedulingManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
import org.ovirt.engine.core.common.AuditLogType;
import org.ovirt.engine.core.common.BackendService;
import org.ovirt.engine.core.common.businessentities.Cluster;
import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy;
import org.ovirt.engine.core.common.businessentities.HugePage;
import org.ovirt.engine.core.common.businessentities.NumaNodeStatistics;
import org.ovirt.engine.core.common.businessentities.NumaTuneMode;
Expand Down Expand Up @@ -438,7 +437,7 @@ private Map<Guid, Guid> schedule(Cluster cluster,
vmHandler.setCpuPinningByNumaPinning(vm, host.getId());
List<VdsCpuUnit> dedicatedCpuPinning = vdsCpuUnitPinningHelper.updatePhysicalCpuAllocations(vm,
PendingCpuPinning.collectForHost(getPendingResourceManager(), host.getId()), host.getId());
String numaPinningString = vmHandler.createNumaPinningForDedicated(vm, dedicatedCpuPinning);
String numaPinningString = vmHandler.createNumaPinningForExclusiveCpuPinning(vm, dedicatedCpuPinning);
updateDedicatedNumaMemoryConsumption(vm, host, numaPinningString, numaConsumptionPerVm);
addPendingResources(vm, host, numaConsumptionPerVm.getOrDefault(vm.getId(), Collections.emptyMap()), dedicatedCpuPinning);
hostsToNotifyPending.add(bestHostId);
Expand All @@ -463,7 +462,7 @@ private Map<Guid, Guid> schedule(Cluster cluster,

private void updateDedicatedNumaMemoryConsumption(VM vm, VDS host, String numaPinningString,
Map<Guid, Map<Integer, NumaNodeMemoryConsumption>> numaConsumptionPerVm) {
if (vm.getCpuPinningPolicy() == CpuPinningPolicy.DEDICATED) {
if (vm.getCpuPinningPolicy().isExclusive()) {
var currentNumaPinning = NumaPinningHelper.parseNumaMapping(numaPinningString);
for (VmNumaNode vmNumaNode : vm.getvNumaNodeList()) {
var vdsNumaNodesPinned = currentNumaPinning.get(vmNumaNode.getIndex());
Expand Down
3 changes: 1 addition & 2 deletions ...les/bll/src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CPUPolicyUnit.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.ovirt.engine.core.bll.scheduling.SlaValidator;
import org.ovirt.engine.core.bll.scheduling.pending.PendingResourceManager;
import org.ovirt.engine.core.bll.scheduling.utils.VdsCpuUnitPinningHelper;
import org.ovirt.engine.core.common.businessentities.CpuPinningPolicy;
import org.ovirt.engine.core.common.businessentities.VDS;
import org.ovirt.engine.core.common.businessentities.VM;
import org.ovirt.engine.core.common.errors.EngineMessage;
Expand Down Expand Up @@ -57,7 +56,7 @@ public List<VDS> filter(SchedulingContext context, List<VDS> hosts, VM vm, PerHo
if (cores != null) {
int numOfCpus = VmCpuCountHelper.isDynamicCpuTopologySet(vm) ?
vm.getCurrentNumOfCpus(false) : vm.getNumOfCpus(false);
if (vm.getCpuPinningPolicy() == CpuPinningPolicy.DEDICATED) {
if (vm.getCpuPinningPolicy().isExclusive()) {
int futureCpus = context.getCluster().getCountThreadsAsCores() ? cores - vds.getVmsCoresCount() :
cores - (int) Math.ceil(vds.getVmsCoresCount() / (vds.getCpuThreads() / vds.getCpuCores()));
if (numOfCpus > futureCpus) {
Expand Down
117 changes: 57 additions & 60 deletions .../src/main/java/org/ovirt/engine/core/bll/scheduling/policyunits/CpuPinningPolicyUnit.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,73 +67,70 @@ public List<VDS> filter(final SchedulingContext context,
return hosts;
}


// only add hosts as candidates which have all required CPUs up and running
final List<VDS> candidates = new ArrayList<>();

switch (vm.getCpuPinningPolicy()) {
case MANUAL:
if (StringUtils.isEmpty(cpuPinning)) {
return hosts;
}
case RESIZE_AND_PIN_NUMA:
// collect all pinned host cpus and merge them into one set
final Set<Integer> pinnedCpus = CpuPinningHelper.getAllPinnedPCpus(cpuPinning);
for (final VDS host : hosts) {
final Collection<Integer> onlineHostCpus = SlaValidator.getOnlineCpus(host);

Map<Guid, List<VdsCpuUnit>> vmToPendingDedicatedCpuPinnings =
PendingCpuPinning.collectForHost(getPendingResourceManager(), host.getId());
var cpuTopology = resourceManager.getVdsManager(host.getId()).getCpuTopology();
vdsCpuUnitPinningHelper.previewPinOfPendingExclusiveCpus(cpuTopology, vmToPendingDedicatedCpuPinnings);
final Collection<Integer> dedicatedCpus = cpuTopology.stream().filter(VdsCpuUnit::isDedicated).map(VdsCpuUnit::getCpu).collect(Collectors.toList());

if (!dedicatedCpus.isEmpty() && vm.getCpuPinningPolicy() == CpuPinningPolicy.RESIZE_AND_PIN_NUMA && pinnedCpus.isEmpty()) {
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__VM_PINNING_CANT_RESIZE_WITH_DEDICATED.name());
log.debug("Host {} does not satisfy CPU pinning constraints, cannot match virtual topology " +
"with available CPUs due to exclusively pinned cpus on the host .", host.getId());
continue;
}
final Collection availableCpus = CollectionUtils.subtract(onlineHostCpus, dedicatedCpus);
final Collection difference = CollectionUtils.subtract(pinnedCpus, availableCpus);
if (difference.isEmpty()) {
candidates.add(host);
} else {
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__VM_PINNING_PCPU_DOES_NOT_EXIST.name());
messages.addMessage(host.getId(), String.format("$missingCores %1$s", StringUtils.join(difference, ", ")));
log.debug("Host {} does not satisfy the cpu pinning constraints because of missing, exclusively pinned or offline cpus {}.",
host.getId(),
StringUtils.join(difference, ", "));
}
if (vm.getCpuPinningPolicy().isExclusive()) {
for (final VDS host : hosts) {
if (host.getCpuTopology() == null || host.getCpuTopology().isEmpty()) {
// means CPU topology not reported for this host, lets ignore it
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__NO_HOST_CPU_DATA.name());
log.debug("Host {} does not have the cpu topology data.", host.getId());
continue;
}
break;
case DEDICATED:
for (final VDS host : hosts) {
if (host.getCpuTopology() == null || host.getCpuTopology().isEmpty()) {
// means CPU topology not reported for this host, lets ignore it
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__NO_HOST_CPU_DATA.name());
log.debug("Host {} does not have the cpu topology data.", host.getId());
continue;
}

Map<Guid, List<VdsCpuUnit>> vmToPendingDedicatedCpuPinnings =
PendingCpuPinning.collectForHost(getPendingResourceManager(), host.getId());
boolean isDedicatedCpuPinningPossibleAtHost =
vdsCpuUnitPinningHelper.isDedicatedCpuPinningPossibleAtHost(
vmToPendingDedicatedCpuPinnings,
vm,
host.getId());
if (isDedicatedCpuPinningPossibleAtHost) {
candidates.add(host);
} else {
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__VM_PINNING_DEDICATED_NOT_FIT.name());
log.debug("Host {} does not satisfy CPU pinning constraints, cannot match virtual topology " +
"with available CPUs.", host.getId());
}
Map<Guid, List<VdsCpuUnit>> vmToPendingDedicatedCpuPinnings =
PendingCpuPinning.collectForHost(getPendingResourceManager(), host.getId());
boolean isDedicatedCpuPinningPossibleAtHost =
vdsCpuUnitPinningHelper.isExclusiveCpuPinningPossibleOnHost(
vmToPendingDedicatedCpuPinnings,
vm,
host.getId());
if (isDedicatedCpuPinningPossibleAtHost) {
candidates.add(host);
} else {
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__VM_PINNING_DEDICATED_NOT_FIT.name());
log.debug("Host {} does not satisfy CPU pinning constraints, cannot match virtual topology " +
"with available CPUs.", host.getId());
}
break;
}
return candidates;
}
// shared CPUs
if (StringUtils.isEmpty(cpuPinning)) {
// For 'Resize and Pin NUMA', on RunVm, we get here before determining the CPU pinning for the VM
// so the dynamic CPU pinning is not set and hosts should not be filtered by the logic below.
return hosts;
}
// collect all pinned host cpus and merge them into one set
final Set<Integer> pinnedCpus = CpuPinningHelper.getAllPinnedPCpus(cpuPinning);
for (final VDS host : hosts) {
final Collection<Integer> onlineHostCpus = SlaValidator.getOnlineCpus(host);

Map<Guid, List<VdsCpuUnit>> vmToPendingDedicatedCpuPinnings =
PendingCpuPinning.collectForHost(getPendingResourceManager(), host.getId());
var cpuTopology = resourceManager.getVdsManager(host.getId()).getCpuTopology();
vdsCpuUnitPinningHelper.previewPinOfPendingExclusiveCpus(cpuTopology, vmToPendingDedicatedCpuPinnings);
final Collection<Integer> dedicatedCpus = cpuTopology.stream().filter(VdsCpuUnit::isExclusive).map(VdsCpuUnit::getCpu).collect(Collectors.toList());

if (!dedicatedCpus.isEmpty() && vm.getCpuPinningPolicy() == CpuPinningPolicy.RESIZE_AND_PIN_NUMA && pinnedCpus.isEmpty()) {
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__VM_PINNING_CANT_RESIZE_WITH_DEDICATED.name());
log.debug("Host {} does not satisfy CPU pinning constraints, cannot match virtual topology " +
"with available CPUs due to exclusively pinned cpus on the host .", host.getId());
continue;
}
final Collection availableCpus = CollectionUtils.subtract(onlineHostCpus, dedicatedCpus);
final Collection difference = CollectionUtils.subtract(pinnedCpus, availableCpus);
if (difference.isEmpty()) {
candidates.add(host);
} else {
messages.addMessage(host.getId(), EngineMessage.VAR__DETAIL__VM_PINNING_PCPU_DOES_NOT_EXIST.name());
messages.addMessage(host.getId(), String.format("$missingCores %1$s", StringUtils.join(difference, ", ")));
log.debug("Host {} does not satisfy the cpu pinning constraints because of missing, exclusively pinned or offline cpus {}.",
host.getId(),
StringUtils.join(difference, ", "));
}
}

return candidates;
}
}
Loading