Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Runtime#availableProcessors on Linux #16512

Closed
wants to merge 1 commit into from
+3 −225
Closed

Use Runtime#availableProcessors on Linux #16512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 3 additions & 37 deletions src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,27 +14,21 @@

package com.google.devtools.build.lib.actions;

import com.google.common.base.Splitter;
import com.google.common.io.Files;
import com.google.devtools.build.lib.unix.ProcMeminfoParser;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import javax.annotation.Nullable;

/**
* This class estimates the local host's resource capacity for Linux.
*/
public class LocalHostResourceManagerLinux {
private static String cpuInfoContent = null;

private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
private static final String CPU_INFO_FILE = "/proc/cpuinfo";
private static final String MEM_INFO_FILE = "/proc/meminfo";

private static int getLogicalCpuCount() throws IOException {
String content = getCpuInfoContent();
return getLogicalCpuCountHelper(content);
// As of JDK 11, availableProcessors is aware of cgroups as commonly used by containers.
// https://hg.openjdk.java.net/jdk/hs/rev/7f22774a5f42#l6.178
return Runtime.getRuntime().availableProcessors();
}

private static double getMemoryInMb() throws IOException {
Expand All @@ -56,34 +50,6 @@ public static ResourceSet getLocalHostResources() {
}
}

private static String getCpuInfoContent() throws IOException {
if (cpuInfoContent == null) {
cpuInfoContent = readContent(CPU_INFO_FILE);
}
return cpuInfoContent;
}

private static String readContent(String filename) throws IOException {
return Files.asCharSource(new File(filename), Charset.defaultCharset()).read();
}

/**
* For testing purposes only. Do not use it.
*/
public static int getLogicalCpuCountHelper(String content) throws IOException {
int count = 0;
Iterable<String> lines = NEWLINE_SPLITTER.split(content);
for (String line : lines) {
if (line.startsWith("processor")) {
count++;
}
}
if (count == 0) {
throw new IllegalArgumentException("Can't get logical CPU count");
}
return count;
}

public static double getMemoryInMbHelper(String memInfoFileName) throws IOException {
ProcMeminfoParser memInfo = new ProcMeminfoParser(memInfoFileName);
double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());
Expand Down
188 changes: 0 additions & 188 deletions src/test/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinuxTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,29 +31,6 @@ public class LocalHostResourceManagerLinuxTest {

@Test
public void testNonHyperthreadedMachine() throws Exception {
String cpuinfoContent =
StringUtilities.joinLines(
"processor\t: 0",
"vendor_id\t: GenuineIntel",
"cpu family\t: 15",
"model\t\t: 4",
"model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
"stepping\t: 10",
"cpu MHz\t\t: 3400.000",
"cache size\t: 2048 KB",
"fpu\t\t: yes",
"fpu_exception\t: yes",
"cpuid level\t: 5",
"wp\t\t: yes",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca "
+ "cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
+ "syscall nx lm constant_tsc up pni monitor ds_cpl est cid cx16 "
+ "xtpr lahf_lm",
"bogomips\t: 6803.83",
"clflush size\t: 64",
"cache_alignment\t: 128",
"address sizes\t: 36 bits physical, 48 bits virtual",
"power management:");
String meminfoContent =
StringUtilities.joinLines(
"MemTotal: 3091732 kB",
Expand Down Expand Up @@ -87,7 +64,6 @@ public void testNonHyperthreadedMachine() throws Exception {
"HugePages_Rsvd: 0",
"Hugepagesize: 2048 kB");
String meminfoFile = scratch.file("test_meminfo_nonht", meminfoContent).getPathString();
assertThat(LocalHostResourceManagerLinux.getLogicalCpuCountHelper(cpuinfoContent)).isEqualTo(1);
// +/- 0.1MB
assertThat(LocalHostResourceManagerLinux.getMemoryInMbHelper(meminfoFile))
.isWithin(0.1)
Expand All @@ -96,58 +72,6 @@ public void testNonHyperthreadedMachine() throws Exception {

@Test
public void testHyperthreadedMachine() throws Exception {
String cpuinfoContent =
StringUtilities.joinLines(
"processor\t: 0",
"vendor_id\t: GenuineIntel",
"cpu family\t: 15",
"model\t\t: 4",
"model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
"stepping\t: 1",
"cpu MHz\t\t: 3400.245",
"cache size\t: 1024 KB",
"physical id\t: 0",
"siblings\t: 2",
"core id\t\t: 0",
"cpu cores\t: 1",
"fpu\t\t: yes",
"fpu_exception\t: yes",
"cpuid level\t: 5",
"wp\t\t: yes",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge "
+ "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
+ "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr",
"bogomips\t: 6806.31",
"clflush size\t: 64",
"cache_alignment\t: 128",
"address sizes\t: 36 bits physical, 48 bits virtual",
"power management:",
"",
"processor\t: 1",
"vendor_id\t: GenuineIntel",
"cpu family\t: 15",
"model\t\t: 4",
"model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz",
"stepping\t: 1",
"cpu MHz\t\t: 3400.245",
"cache size\t: 1024 KB",
"physical id\t: 0",
"siblings\t: 2",
"core id\t\t: 0",
"cpu cores\t: 1",
"fpu\t\t: yes",
"fpu_exception\t: yes",
"cpuid level\t: 5",
"wp\t\t: yes",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge "
+ "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm "
+ "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr",
"bogomips\t: 6800.76",
"clflush size\t: 64",
"cache_alignment\t: 128",
"address sizes\t: 36 bits physical, 48 bits virtual",
"power management:",
"");
String meminfoContent =
StringUtilities.joinLines(
"MemTotal: 3092004 kB",
Expand Down Expand Up @@ -181,7 +105,6 @@ public void testHyperthreadedMachine() throws Exception {
"HugePages_Rsvd: 0",
"Hugepagesize: 2048 kB");
String meminfoFile = scratch.file("test_meminfo_ht", meminfoContent).getPathString();
assertThat(LocalHostResourceManagerLinux.getLogicalCpuCountHelper(cpuinfoContent)).isEqualTo(2);
// +/- 0.1MB
assertThat(LocalHostResourceManagerLinux.getMemoryInMbHelper(meminfoFile))
.isWithin(0.1)
Expand All @@ -190,116 +113,6 @@ public void testHyperthreadedMachine() throws Exception {

@Test
public void testAMDMachine() throws Exception {
String cpuinfoContent =
StringUtilities.joinLines(
"processor\t: 0",
"vendor_id\t: AuthenticAMD",
"cpu family\t: 15",
"model\t\t: 65",
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
"stepping\t: 2",
"cpu MHz\t\t: 2200.000",
"cache size\t: 1024 KB",
"physical id\t: 0",
"siblings\t: 2",
"core id\t\t: 0",
"cpu cores\t: 2",
"fpu\t\t: yes",
"fpu_exception\t: yes",
"cpuid level\t: 1",
"wp\t\t: yes",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ "cmp_legacy svm cr8_legacy",
"bogomips\t: 4425.84",
"TLB size\t: 1024 4K pages",
"clflush size\t: 64",
"cache_alignment\t: 64",
"address sizes\t: 40 bits physical, 48 bits virtual",
"power management: ts fid vid ttp tm stc",
"",
"processor\t: 1",
"vendor_id\t: AuthenticAMD",
"cpu family\t: 15",
"model\t\t: 65",
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
"stepping\t: 2",
"cpu MHz\t\t: 2200.000",
"cache size\t: 1024 KB",
"physical id\t: 0",
"siblings\t: 2",
"core id\t\t: 1",
"cpu cores\t: 2",
"fpu\t\t: yes",
"fpu_exception\t: yes",
"cpuid level\t: 1",
"wp\t\t: yes",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ "cmp_legacy svm cr8_legacy",
"bogomips\t: 4460.61",
"TLB size\t: 1024 4K pages",
"clflush size\t: 64",
"cache_alignment\t: 64",
"address sizes\t: 40 bits physical, 48 bits virtual",
"power management: ts fid vid ttp tm stc",
"",
"processor\t: 2",
"vendor_id\t: AuthenticAMD",
"cpu family\t: 15",
"model\t\t: 65",
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
"stepping\t: 2",
"cpu MHz\t\t: 2200.000",
"cache size\t: 1024 KB",
"physical id\t: 1",
"siblings\t: 2",
"core id\t\t: 0",
"cpu cores\t: 2",
"fpu\t\t: yes",
"fpu_exception\t: yes",
"cpuid level\t: 1",
"wp\t\t: yes",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ "cmp_legacy svm cr8_legacy",
"bogomips\t: 4420.45",
"TLB size\t: 1024 4K pages",
"clflush size\t: 64",
"cache_alignment\t: 64",
"address sizes\t: 40 bits physical, 48 bits virtual",
"power management: ts fid vid ttp tm stc",
"",
"processor\t: 3",
"vendor_id\t: AuthenticAMD",
"cpu family\t: 15",
"model\t\t: 65",
"model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE",
"stepping\t: 2",
"cpu MHz\t\t: 2200.000",
"cache size\t: 1024 KB",
"physical id\t: 1",
"siblings\t: 2",
"core id\t\t: 1",
"cpu cores\t: 2",
"fpu\t\t: yes",
"fpu_exception\t: yes",
"cpuid level\t: 1",
"wp\t\t: yes",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr "
+ "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall "
+ "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm "
+ "cmp_legacy svm cr8_legacy",
"bogomips\t: 4460.39",
"TLB size\t: 1024 4K pages",
"clflush size\t: 64",
"cache_alignment\t: 64",
"address sizes\t: 40 bits physical, 48 bits virtual",
"power management: ts fid vid ttp tm stc",
"");
String meminfoContent =
StringUtilities.joinLines(
"MemTotal: 8223956 kB",
Expand Down Expand Up @@ -333,7 +146,6 @@ public void testAMDMachine() throws Exception {
"HugePages_Rsvd: 0",
"Hugepagesize: 2048 kB");
String meminfoFile = scratch.file("test_meminfo_amd", meminfoContent).getPathString();
assertThat(LocalHostResourceManagerLinux.getLogicalCpuCountHelper(cpuinfoContent)).isEqualTo(4);
// +/- 0.1MB
assertThat(LocalHostResourceManagerLinux.getMemoryInMbHelper(meminfoFile))
.isWithin(0.1)
Expand Down