Skip to content

Commit

Permalink
core: release core reminders
Browse files Browse the repository at this point in the history 
We now release using unPin the VdsCpuUnit we need. This is good for
`dedicated` policy. When we are handling `isolate-threads` policy we
need to release the whole core. Otherwise the engine thinks it's taken.

Change-Id: I0b028e0dc5db72f5ef3099059f355d975a81fc90
Bug-Url: https://bugzilla.redhat.com/2103620
Signed-off-by: Liran Rotenberg <lrotenbe@redhat.com>
  • Loading branch information
@liranr23
liranr23 committed Jul 12, 2022
1 parent af4ac85 commit 56c2646
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 35 deletions.
126 changes: 91 additions & 35 deletions ...bll/src/main/java/org/ovirt/engine/core/bll/scheduling/utils/VdsCpuUnitPinningHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ public List<VdsCpuUnit> updatePhysicalCpuAllocations(VM vm, List<VdsCpuUnit> cpu
}

filterSocketsWithInsufficientMemoryForNumaNode(cpuTopology, vm, hostId);
List<VdsCpuUnit> cpusToBeAllocated = allocateDedicatedCpus(cpuTopology, vm, hostId);
List<VdsCpuUnit> cpusToBeAllocated = allocateExclusiveCpus(cpuTopology, vm, hostId);
if (vm.getCpuPinningPolicy() == CpuPinningPolicy.ISOLATE_THREADS) {
List<Integer> socketsUsed = cpusToBeAllocated.stream().map(VdsCpuUnit::getSocket).distinct().collect(Collectors.toList());
int pCores = 0;
Expand Down Expand Up @@ -140,11 +140,21 @@ public List<VdsCpuUnit> allocateManualCpus(List<VdsCpuUnit> cpuTopology, VM vm)
return cpusToBeAllocated;
}

private List<VdsCpuUnit> allocateExclusiveCpus(List<VdsCpuUnit> cpuTopology, VM vm, Guid hostId) {
switch (vm.getCpuPinningPolicy()) {
case DEDICATED:
return allocateDedicatedCpus(cpuTopology, vm, hostId);
case ISOLATE_THREADS:
return allocateIsolatedThreadsCpus(cpuTopology, vm, hostId);
default:
return Collections.emptyList();
}
}

private List<VdsCpuUnit> allocateDedicatedCpus(List<VdsCpuUnit> cpuTopology, VM vm, Guid hostId) {
List<VdsCpuUnit> cpusToBeAllocated = new ArrayList<>();
int socketsLeft = vm.getNumOfSockets();
int onlineSockets = getOnlineSockets(cpuTopology).size();
int numOfAllocatedCPUs = 0;
while (onlineSockets > 0 && socketsLeft > 0) {
List<VdsCpuUnit> cpusInChosenSocket = getCoresInSocket(cpuTopology, getMaxFreedSocket(cpuTopology));
if (cpusInChosenSocket.isEmpty()) {
Expand All @@ -159,48 +169,78 @@ private List<VdsCpuUnit> allocateDedicatedCpus(List<VdsCpuUnit> cpuTopology, VM
// coreCount is based on the VM topology
int coreCount = 0;
for (int core : getOnlineCores(cpusInChosenSocket)) {
switch (vm.getCpuPinningPolicy()) {
case DEDICATED:
List<VdsCpuUnit> freeCpusInCore = getFreeCpusInCore(cpusInChosenSocket, core);
int coreThreads = freeCpusInCore.size();
while (coreThreads >= vm.getThreadsPerCpu() &&
cpusToBeAllocated.size() < vm.getNumOfCpus() &&
coreCount / vm.getCpuPerSocket() < amountOfVSocketsInPSockets) {
for (int thread = 0; thread < vm.getThreadsPerCpu() && cpusToBeAllocated.size() < vm.getNumOfCpus(); thread++) {
VdsCpuUnit cpuUnit = freeCpusInCore.remove(0);
cpuUnit.pinVm(vm.getId(), vm.getCpuPinningPolicy());
cpusToBeAllocated.add(cpuUnit);
}
coreCount++;
coreThreads -= vm.getThreadsPerCpu();
}
break;
case ISOLATE_THREADS:
List<VdsCpuUnit> cpusInCore = getCpusInCore(cpusInChosenSocket, core);
if (cpusInCore.stream().anyMatch(VdsCpuUnit::isPinned)) {
continue;
}
if (numOfAllocatedCPUs < vm.getNumOfCpus() && coreCount / vm.getCpuPerSocket() < amountOfVSocketsInPSockets) {
cpusInCore.forEach(cpu -> {
cpu.pinVm(vm.getId(), vm.getCpuPinningPolicy());
cpusToBeAllocated.add(cpu);
});
numOfAllocatedCPUs++;
if (numOfAllocatedCPUs % vm.getThreadsPerCpu() == 0) {
coreCount++;
}
}
List<VdsCpuUnit> freeCpusInCore = getFreeCpusInCore(cpusInChosenSocket, core);
int coreThreads = freeCpusInCore.size();
while (coreThreads >= vm.getThreadsPerCpu() &&
cpusToBeAllocated.size() < vm.getNumOfCpus() &&
coreCount / vm.getCpuPerSocket() < amountOfVSocketsInPSockets) {
for (int thread = 0; thread < vm.getThreadsPerCpu() && cpusToBeAllocated.size() < vm.getNumOfCpus(); thread++) {
VdsCpuUnit cpuUnit = freeCpusInCore.remove(0);
cpuUnit.pinVm(vm.getId(), vm.getCpuPinningPolicy());
cpusToBeAllocated.add(cpuUnit);
}
coreCount++;
coreThreads -= vm.getThreadsPerCpu();
}
}
socketsLeft -= coreCount / vm.getCpuPerSocket();
int coresReminder = coreCount % vm.getCpuPerSocket();
for (int i = 0; i < coresReminder * vm.getThreadsPerCpu(); i++) {
if (!cpusToBeAllocated.isEmpty()) {
VdsCpuUnit releasedCpu = cpusToBeAllocated.remove(cpusToBeAllocated.size() - 1);
releasedCpu.unPinVm(vm.getId());
releaseDedicatedCpu(vm.getId(), cpusToBeAllocated);
}
}
onlineSockets--;
}
if (socketsLeft > 0) {
// We didn't manage to allocate the required sockets. We should never get here.
return Collections.emptyList();
}
return cpusToBeAllocated;
}

private List<VdsCpuUnit> allocateIsolatedThreadsCpus(List<VdsCpuUnit> cpuTopology, VM vm, Guid hostId) {
List<VdsCpuUnit> cpusToBeAllocated = new ArrayList<>();
int socketsLeft = vm.getNumOfSockets();
int onlineSockets = getOnlineSockets(cpuTopology).size();
int numOfAllocatedCPUs = 0;
while (onlineSockets > 0 && socketsLeft > 0) {
List<VdsCpuUnit> cpusInChosenSocket = getCoresInSocket(cpuTopology, getMaxFreedSocket(cpuTopology));
if (cpusInChosenSocket.isEmpty()) {
break;
}
int amountOfVSocketsInPSockets = Integer.MAX_VALUE;
if (!vm.getvNumaNodeList().isEmpty()) {
int highestAmountOfvNumaNodesInSocket = getVirtualNumaNodesInSocket(cpuTopology, vm, hostId, cpusInChosenSocket.get(0).getSocket());
int vNumaNodesInVirtualSocket = (int) Math.ceil((float)vm.getvNumaNodeList().size() / (float)vm.getNumOfSockets());
amountOfVSocketsInPSockets = highestAmountOfvNumaNodesInSocket / vNumaNodesInVirtualSocket;
}
// coreCount is based on the VM topology
int coreCount = 0;
for (int core : getOnlineCores(cpusInChosenSocket)) {
List<VdsCpuUnit> cpusInCore = getCpusInCore(cpusInChosenSocket, core);
if (cpusInCore.stream().anyMatch(VdsCpuUnit::isPinned)) {
continue;
}
if (numOfAllocatedCPUs < vm.getNumOfCpus() && coreCount / vm.getCpuPerSocket() < amountOfVSocketsInPSockets) {
cpusInCore.forEach(cpu -> {
cpu.pinVm(vm.getId(), vm.getCpuPinningPolicy());
cpusToBeAllocated.add(cpu);
});
numOfAllocatedCPUs++;
if (numOfAllocatedCPUs % vm.getThreadsPerCpu() == 0) {
coreCount++;
}
}
}
socketsLeft -= coreCount / vm.getCpuPerSocket();
int coresReminder = coreCount % vm.getCpuPerSocket();
for (int i = 0; i < coresReminder; i++) {
if (!cpusToBeAllocated.isEmpty()) {
releaseIsolateThreadsCpu(vm.getId(), cpusToBeAllocated);
numOfAllocatedCPUs--;
}
}
onlineSockets--;
}
if (socketsLeft > 0) {
Expand All @@ -210,6 +250,22 @@ private List<VdsCpuUnit> allocateDedicatedCpus(List<VdsCpuUnit> cpuTopology, VM
return cpusToBeAllocated;
}

private void releaseDedicatedCpu(Guid vmId, List<VdsCpuUnit> cpusToBeAllocated) {
VdsCpuUnit releasedCpu = cpusToBeAllocated.remove(cpusToBeAllocated.size() - 1);
releasedCpu.unPinVm(vmId);
}

private void releaseIsolateThreadsCpu(Guid vmId, List<VdsCpuUnit> cpusToBeAllocated) {
VdsCpuUnit cpuUnit = cpusToBeAllocated.get(cpusToBeAllocated.size() - 1);
int coreId = cpuUnit.getCore();
int socketId = cpuUnit.getSocket();
List<VdsCpuUnit> cpusToRemove = getCpusInCore(getCoresInSocket(cpusToBeAllocated, socketId), coreId);
cpusToRemove.forEach(vdsCpuUnit -> {
vdsCpuUnit.unPinVm(vmId);
cpusToBeAllocated.remove(vdsCpuUnit);
});
}

private int countTakenCores(List<VdsCpuUnit> cpuTopology) {
if (cpuTopology.isEmpty()) {
return 0;
Expand Down
36 changes: 36 additions & 0 deletions ...src/test/java/org/ovirt/engine/core/bll/scheduling/utils/VdsCpuUnitPinningHelperTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,4 +416,40 @@ public void shouldSucceedToDedicateEqualSockets() {
assertEquals(6, cpus.stream().filter(cpu -> cpu.getSocket() == 1).count());
}

@Test
public void releaseReminderOnIsolateThreads() {
VM vm = new VM();
vm.setId(Guid.newGuid());
vm.setNumOfSockets(2);
vm.setCpuPerSocket(3);
vm.setThreadsPerCpu(1);
vm.setCpuPinningPolicy(CpuPinningPolicy.ISOLATE_THREADS);

List<VdsCpuUnit> cpuTopology = new ArrayList<>();
cpuTopology.add(new VdsCpuUnit(0, 0, 0, 0));
cpuTopology.add(new VdsCpuUnit(0, 0, 0, 1));
cpuTopology.add(new VdsCpuUnit(0, 0, 1, 0));
VdsCpuUnit vdsmTaken = new VdsCpuUnit(0, 0, 1, 1);
vdsmTaken.pinVm(Guid.SYSTEM, CpuPinningPolicy.MANUAL);
cpuTopology.add(vdsmTaken);
cpuTopology.add(new VdsCpuUnit(0, 0, 2, 0));
cpuTopology.add(new VdsCpuUnit(0, 0, 2, 1));
cpuTopology.add(new VdsCpuUnit(0, 0, 3, 0));
cpuTopology.add(new VdsCpuUnit(0, 0, 3, 1));

cpuTopology.add(new VdsCpuUnit(1, 1, 0, 0));
cpuTopology.add(new VdsCpuUnit(1, 1, 0, 1));
cpuTopology.add(new VdsCpuUnit(1, 1, 1, 0));
cpuTopology.add(new VdsCpuUnit(1, 1, 1, 0));
cpuTopology.add(new VdsCpuUnit(1, 1, 2, 0));
cpuTopology.add(new VdsCpuUnit(1, 1, 2, 1));
cpuTopology.add(new VdsCpuUnit(1, 1, 3, 0));
cpuTopology.add(new VdsCpuUnit(1, 1, 3, 1));

when(vdsManager.getCpuTopology()).thenReturn(cpuTopology);
List<VdsCpuUnit> cpus = vdsCpuUnitPinningHelper.updatePhysicalCpuAllocations(vm, new HashMap<>(), host.getId());
assertEquals(12, cpus.size());
assertEquals(12, cpuTopology.stream().filter(VdsCpuUnit::isExclusive).count());
}

}

0 comments on commit 56c2646

Please sign in to comment.