diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD index 925dfcc790291a..823dffa6253c46 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/BUILD +++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD @@ -128,6 +128,7 @@ java_library( "//third_party:flogger", "//third_party:guava", "//third_party:jsr305", + "//third_party:rxjava3", "//third_party/protobuf:protobuf_java", ], ) @@ -311,6 +312,7 @@ java_library( "//third_party:flogger", "//third_party:guava", "//third_party:jsr305", + "//third_party:rxjava3", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java index 84336e6a55f267..5303f863622deb 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java @@ -154,6 +154,35 @@ public String parseIfMatches(String tag) throws ValidationException { return null; }); + /** How many extra resources an action requires for execution. */ + public static final ParseableRequirement RESOURCES = + ParseableRequirement.create( + "resources::", + Pattern.compile("resources:(.+:.+)"), + s -> { + Preconditions.checkNotNull(s); + + int splitIndex = s.indexOf(":"); + String resourceCount = s.substring(splitIndex+1); + int value; + try { + value = Integer.parseInt(resourceCount); + } catch (NumberFormatException e) { + return "can't be parsed as an integer"; + } + + // De-and-reserialize & compare to only allow canonical integer formats. + if (!Integer.toString(value).equals(resourceCount)) { + return "must be in canonical format (e.g. '4' instead of '+04')"; + } + + if (value < 1) { + return "can't be zero or negative"; + } + + return null; + }); + /** If an action supports running in persistent worker mode. */ public static final String SUPPORTS_WORKERS = "supports-workers"; diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java index 772e604e757786..475c136552200e 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java @@ -27,8 +27,12 @@ import com.google.devtools.build.lib.worker.WorkerPool; import java.io.IOException; import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import javax.annotation.Nullable; @@ -171,13 +175,19 @@ public static ResourceManager instance() { // definition in the ResourceSet class. private double usedRam; + // Used amount of extra resources. Corresponds to the extra resource + // definition in the ResourceSet class. + private Map usedExtraResources; + // Used local test count. Corresponds to the local test count definition in the ResourceSet class. private int usedLocalTestCount; /** If set, local-only actions are given priority over dynamically run actions. */ private boolean prioritizeLocalActions; - private ResourceManager() {} + private ResourceManager() { + usedExtraResources = new HashMap<>(); + } @VisibleForTesting public static ResourceManager instanceForTestingOnly() { @@ -192,6 +202,7 @@ public static ResourceManager instanceForTestingOnly() { public synchronized void resetResourceUsage() { usedCpu = 0; usedRam = 0; + usedExtraResources = new HashMap<>(); usedLocalTestCount = 0; for (Pair request : localRequests) { request.second.latch.countDown(); @@ -286,6 +297,17 @@ private Worker incrementResources(ResourceSet resources) throws IOException, InterruptedException { usedCpu += resources.getCpuUsage(); usedRam += resources.getMemoryMb(); + + resources.getExtraResourceUsage().entrySet().forEach( + resource -> { + String key = (String)resource.getKey(); + float value = resource.getValue(); + if (usedExtraResources.containsKey(key)) { + value += (float)usedExtraResources.get(key); + } + usedExtraResources.put(key, value); + }); + usedLocalTestCount += resources.getLocalTestCount(); if (resources.getWorkerKey() != null) { @@ -298,6 +320,7 @@ private Worker incrementResources(ResourceSet resources) public synchronized boolean inUse() { return usedCpu != 0.0 || usedRam != 0.0 + || !usedExtraResources.isEmpty() || usedLocalTestCount != 0 || !localRequests.isEmpty() || !dynamicWorkerRequests.isEmpty() @@ -405,6 +428,13 @@ private boolean release(ResourceSet resources, @Nullable Worker worker) private synchronized void releaseResourcesOnly(ResourceSet resources) { usedCpu -= resources.getCpuUsage(); usedRam -= resources.getMemoryMb(); + + for (Map.Entry resource : resources.getExtraResourceUsage().entrySet()) { + String key = (String)resource.getKey(); + float value = (float)usedExtraResources.get(key) - resource.getValue(); + usedExtraResources.put(key, value); + } + usedLocalTestCount -= resources.getLocalTestCount(); // TODO(bazel-team): (2010) rounding error can accumulate and value below can end up being @@ -416,6 +446,19 @@ private synchronized void releaseResourcesOnly(ResourceSet resources) { if (usedRam < epsilon) { usedRam = 0; } + + Set toRemove = new HashSet<>(); + usedExtraResources.entrySet().forEach( + resource -> { + String key = (String)resource.getKey(); + float value = (float)usedExtraResources.get(key); + if (value < epsilon) { + toRemove.add(key); + } + }); + for (String key : toRemove) { + usedExtraResources.remove(key); + } } private synchronized boolean processAllWaitingThreads() throws IOException, InterruptedException { @@ -454,6 +497,23 @@ private synchronized void processWaitingThreads(Deque resource : resources.getExtraResourceUsage().entrySet()) { + String key = (String)resource.getKey(); + float used = (float)usedExtraResources.getOrDefault(key, 0f); + float requested = resource.getValue(); + float available = (float)availableResources.getExtraResourceUsage().getOrDefault(key, 0f); + float epsilon = 0.0001f; // Account for possible rounding errors. + if (requested != 0.0 && used != 0.0 && requested + used > available + epsilon) { + return false; + } + } + return true; + } + // Method will return true if all requested resources are considered to be available. @VisibleForTesting boolean areResourcesAvailable(ResourceSet resources) { @@ -472,7 +532,7 @@ boolean areResourcesAvailable(ResourceSet resources) { workerKey == null || (activeWorkers < availableWorkers && workerPool.couldBeBorrowed(workerKey)); - if (usedCpu == 0.0 && usedRam == 0.0 && usedLocalTestCount == 0 && workerIsAvailable) { + if (usedCpu == 0.0 && usedRam == 0.0 && usedExtraResources.isEmpty() && usedLocalTestCount == 0 && workerIsAvailable) { return true; } // Use only MIN_NECESSARY_???_RATIO of the resource value to check for @@ -503,7 +563,8 @@ boolean areResourcesAvailable(ResourceSet resources) { localTestCount == 0 || usedLocalTestCount == 0 || usedLocalTestCount + localTestCount <= availableLocalTestCount; - return cpuIsAvailable && ramIsAvailable && localTestCountIsAvailable && workerIsAvailable; + boolean extraResourcesIsAvailable = areExtraResourcesAvailable(resources); + return cpuIsAvailable && ramIsAvailable && extraResourcesIsAvailable && localTestCountIsAvailable && workerIsAvailable; } @VisibleForTesting diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java index a0b60461e9541f..3453fb2d8c26ab 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java @@ -14,13 +14,16 @@ package com.google.devtools.build.lib.actions; +import com.google.common.base.Joiner; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Doubles; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.worker.WorkerKey; import com.google.devtools.common.options.Converter; import com.google.devtools.common.options.OptionsParsingException; +import io.reactivex.rxjava3.annotations.NonNull; import java.util.Iterator; import java.util.NoSuchElementException; import javax.annotation.Nullable; @@ -43,16 +46,26 @@ public class ResourceSet implements ResourceSetOrBuilder { /** The number of CPUs, or fractions thereof. */ private final double cpuUsage; + /** + * Map of extra resources (for example: GPUs, embedded boards, ...) mapping + * name of the resource to a value. + */ + private final ImmutableMap extraResourceUsage; + /** The number of local tests. */ private final int localTestCount; /** The workerKey of used worker. Null if no worker is used. */ @Nullable private final WorkerKey workerKey; - private ResourceSet( - double memoryMb, double cpuUsage, int localTestCount, @Nullable WorkerKey workerKey) { + private ResourceSet(double memoryMb, double cpuUsage, int localTestCount, @Nullable WorkerKey workerKey) { + this(memoryMb, cpuUsage, ImmutableMap.of(), localTestCount, workerKey); + } + + private ResourceSet(double memoryMb, double cpuUsage, @NonNull ImmutableMap extraResourceUsage, int localTestCount, @Nullable WorkerKey workerKey) { this.memoryMb = memoryMb; this.cpuUsage = cpuUsage; + this.extraResourceUsage = extraResourceUsage; this.localTestCount = localTestCount; this.workerKey = workerKey; } @@ -83,21 +96,31 @@ public static ResourceSet createWithLocalTestCount(int localTestCount) { } /** - * Returns a new ResourceSet with the provided values for memoryMb, cpuUsage, ioUsage, and + * Returns a new ResourceSet with the provided values for memoryMb, cpuUsage, and * localTestCount. Most action resource definitions should use {@link #createWithRamCpu} or {@link * #createWithLocalTestCount(int)}. Use this method primarily when constructing ResourceSets that * represent available resources. */ public static ResourceSet create(double memoryMb, double cpuUsage, int localTestCount) { - return createWithWorkerKey(memoryMb, cpuUsage, localTestCount, /* workerKey= */ null); + return ResourceSet.create(memoryMb, cpuUsage, ImmutableMap.of(), localTestCount, /* wolkerKey= */ null); + } + + /** + * Returns a new ResourceSet with the provided values for memoryMb, cpuUsage, extraResources, and + * localTestCount. Most action resource definitions should use {@link #createWithRamCpu} or + * {@link #createWithLocalTestCount(int)}. Use this method primarily when constructing + * ResourceSets that represent available resources. + */ + public static ResourceSet create(double memoryMb, double cpuUsage, ImmutableMap extraResourceUsage, int localTestCount) { + return createWithWorkerKey(memoryMb, cpuUsage, extraResourceUseage, localTestCount, /* workerKey= */ null); } public static ResourceSet createWithWorkerKey( - double memoryMb, double cpuUsage, int localTestCount, WorkerKey workerKey) { - if (memoryMb == 0 && cpuUsage == 0 && localTestCount == 0 && workerKey == null) { + double memoryMb, double cpuUsage, ImmutableMap extraResourceUsage, int localTestCount, WorkerKey workerKey) { + if (memoryMb == 0 && cpuUsage == 0 && extraResourceUsage.size() == 0 && localTestCount == 0 && workerKey == null) { return ZERO; } - return new ResourceSet(memoryMb, cpuUsage, localTestCount, workerKey); + return new ResourceSet(memoryMb, cpuUsage, extraResourceUsage, localTestCount, workerKey); } /** Returns the amount of real memory (resident set size) used in MB. */ @@ -124,6 +147,10 @@ public double getCpuUsage() { return cpuUsage; } + public ImmutableMap getExtraResourceUsage() { + return extraResourceUsage; + } + /** Returns the local test count used. */ public int getLocalTestCount() { return localTestCount; @@ -138,6 +165,7 @@ public String toString() { + "CPU: " + cpuUsage + "\n" + + Joiner.on("\n").withKeyValueSeparator(": ").join(extraResourceUsage.entrySet()) + "Local tests: " + localTestCount + "\n"; diff --git a/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java b/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java index 82bc1a94ebfedb..5041c7a0129685 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/test/TestTargetProperties.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.TestAction; import com.google.devtools.build.lib.server.FailureDetails.TestAction.Code; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -160,25 +161,29 @@ public ResourceSet getLocalResourceUsage(Label label, boolean usingLocalTestJobs } ResourceSet testResourcesFromSize = TestTargetProperties.getResourceSetFromSize(size); + return ResourceSet.create( + testResourcesFromSize.getMemoryMb(), + getLocalCpuResourceUsage(label), + getLocalExtraResourceUsage(label), + testResourcesFromSize.getLocalTestCount()); + } + private double getLocalCpuResourceUsage(Label label) throws UserExecException { + ResourceSet testResourcesFromSize = TestTargetProperties.getResourceSetFromSize(size); // Tests can override their CPU reservation with a "cpu:" tag. - ResourceSet testResourcesFromTag = null; + double cpuCount = -1.0; for (String tag : executionInfo.keySet()) { try { String cpus = ExecutionRequirements.CPU.parseIfMatches(tag); if (cpus != null) { - if (testResourcesFromTag != null) { + if (cpuCount != -1.0) { String message = String.format( "%s has more than one '%s' tag, but duplicate tags aren't allowed", label, ExecutionRequirements.CPU.userFriendlyName()); throw new UserExecException(createFailureDetail(message, Code.DUPLICATE_CPU_TAGS)); } - testResourcesFromTag = - ResourceSet.create( - testResourcesFromSize.getMemoryMb(), - Float.parseFloat(cpus), - testResourcesFromSize.getLocalTestCount()); + cpuCount = Float.parseFloat(cpus); } } catch (ValidationException e) { String message = @@ -191,10 +196,44 @@ public ResourceSet getLocalResourceUsage(Label label, boolean usingLocalTestJobs throw new UserExecException(createFailureDetail(message, Code.INVALID_CPU_TAG)); } } + return cpuCount != -1.0 ? cpuCount : testResourcesFromSize.getCpuUsage(); + } - return testResourcesFromTag != null ? testResourcesFromTag : testResourcesFromSize; + private ImmutableMap getLocalExtraResourceUsage(Label label) throws UserExecException { + // Tests can specify requirements for extra resources using "resources::" tag. + Map extraResources = new HashMap<>(); + for (String tag : executionInfo.keySet()) { + try { + String extras = ExecutionRequirements.RESOURCES.parseIfMatches(tag); + if (extras != null) { + int splitIndex = extras.indexOf(":"); + String resourceName = extras.substring(0, splitIndex); + String resourceCount = extras.substring(splitIndex+1); + if (extraResources.get(resourceName) != null) { + String message = + String.format( + "%s has more than one '%s' tag, but duplicate tags aren't allowed", + label, ExecutionRequirements.RESOURCES.userFriendlyName()); + throw new UserExecException(createFailureDetail(message, Code.DUPLICATE_CPU_TAGS)); + } + extraResources.put(resourceName, Float.parseFloat(resourceCount)); + } + } catch (ValidationException e) { + String message = + String.format( + "%s has a '%s' tag, but its value '%s' didn't pass validation: %s", + label, + ExecutionRequirements.RESOURCES.userFriendlyName(), + e.getTagValue(), + e.getMessage()); + throw new UserExecException(createFailureDetail(message, Code.INVALID_CPU_TAG)); + } + } + return ImmutableMap.copyOf(extraResources); } + + private static FailureDetail createFailureDetail(String message, Code detailedCode) { return FailureDetail.newBuilder() .setMessage(message) diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index a11962834709f5..38d89bbb84991c 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -110,6 +110,7 @@ import java.time.Duration; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -937,10 +938,14 @@ private Builder createBuilder( public static void configureResourceManager(ResourceManager resourceMgr, BuildRequest request) { ExecutionOptions options = request.getOptions(ExecutionOptions.class); resourceMgr.setPrioritizeLocalActions(options.prioritizeLocalActions); + ImmutableMap extraResources = options.localExtraResources.stream().collect( + ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue, (v1,v2) -> v1)); + resourceMgr.setAvailableResources( ResourceSet.create( options.localRamResources, options.localCpuResources, + extraResources, options.usingLocalTestJobs() ? options.localTestJobs : Integer.MAX_VALUE)); } diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java index 334f47af1877f9..a1cc160c17583c 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java +++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java @@ -325,6 +325,23 @@ public boolean shouldMaterializeParamFiles() { converter = RamResourceConverter.class) public float localRamResources; + @Option( + name = "local_extra_resources", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + allowMultiple = true, + help = + "Set the number of extra resources available to Bazel. " + + "Takes in a string-float pair. Can be used multiple times to specify multiple " + + "types of extra resources. Bazel will limit concurrently running test actions " + + "based on the available extra resources and the extra resources required " + + "by the test actions. Tests can declare the amount of extra resources they need " + + "by using a tag of the \"resources::\" format. " + + "Available CPU, RAM and test job resources cannot be set with this flag.", + converter = Converters.StringToFloatAssignmentConverter.class) + public List> localExtraResources; + @Option( name = "local_test_jobs", defaultValue = "auto", diff --git a/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java b/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java index 744cefce6363bb..6f148c1e75f790 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java +++ b/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java @@ -57,7 +57,8 @@ private static boolean legalExecInfoKeys(String tag) { || tag.startsWith("disable-") || tag.startsWith("cpu:") || tag.equals(ExecutionRequirements.LOCAL) - || tag.equals(ExecutionRequirements.WORKER_KEY_MNEMONIC); + || tag.equals(ExecutionRequirements.WORKER_KEY_MNEMONIC) + || tag.startsWith("resources:"); } private TargetUtils() {} // Uninstantiable. diff --git a/src/main/java/com/google/devtools/common/options/Converters.java b/src/main/java/com/google/devtools/common/options/Converters.java index 0a8a9499dcc5f7..29c5f823fe250c 100644 --- a/src/main/java/com/google/devtools/common/options/Converters.java +++ b/src/main/java/com/google/devtools/common/options/Converters.java @@ -472,6 +472,24 @@ public String getTypeDescription() { } } + /** + * A converter for for assignments from a string value to a float value. + */ + public static class StringToFloatAssignmentConverter implements Converter> { + private static final AssignmentConverter baseConverter = new AssignmentConverter(); + + @Override + public Map.Entry convert(String input) throws OptionsParsingException, NumberFormatException { + Map.Entry stringEntry = baseConverter.convert(input); + return Maps.immutableEntry(stringEntry.getKey(), Float.parseFloat(stringEntry.getValue())); + } + + @Override + public String getTypeDescription() { + return "a named float, 'name=value'"; + } + } + /** * A converter for command line flag aliases. It does additional validation on the name and value * of the assignment to ensure they conform to the naming limitations. diff --git a/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java index 05c817f085cf8b..b5242917a15119 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java +++ b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java @@ -68,7 +68,14 @@ public final class ResourceManagerTest { @Before public void configureResourceManager() throws Exception { rm.setAvailableResources( - ResourceSet.create(/*memoryMb=*/ 1000, /*cpuUsage=*/ 1, /* localTestCount= */ 2)); + ResourceSet.create( + /*memoryMb=*/ 1000, + /*cpuUsage=*/ 1, + /*extraResourceUsage*/ ImmutableMap.of( + "gpu", 2.0f, + "fancyresource", 1.5f + ), + /* localTestCount= */ 2)); counter = new AtomicInteger(0); sync = new CyclicBarrier(2); sync2 = new CyclicBarrier(2); @@ -107,12 +114,12 @@ private ResourceHandle acquire(double ram, double cpu, int tests) return acquire(ram, cpu, tests, ResourcePriority.LOCAL); } - private ResourceHandle acquire(double ram, double cpu, int tests, String mnemonic) + private ResourceHandle acquire(double ram, double cpu, int tests, ImmutableMap extraResources, String mnemonic) throws InterruptedException, IOException { return rm.acquireResources( resourceOwner, - ResourceSet.createWithWorkerKey(ram, cpu, tests, createWorkerKey(mnemonic)), + ResourceSet.createWithWorkerKey(ram, cpu, tests, extraResources, createWorkerKey(mnemonic)), ResourcePriority.LOCAL); } @@ -120,6 +127,10 @@ private void release(double ram, double cpu, int tests) throws IOException, Inte rm.releaseResources(resourceOwner, ResourceSet.create(ram, cpu, tests), /* worker=*/ null); } + private void release(double ram, double cpu, int tests, ImmutableMap extraResources) { + rm.releaseResources(resourceOwner, ResourceSet.create(ram, cpu, extraResources, tests)); + } + private void validate(int count) { assertThat(counter.incrementAndGet()).isEqualTo(count); } @@ -167,6 +178,13 @@ public void testOverBudgetRequests() throws Exception { assertThat(rm.inUse()).isTrue(); release(0, 0, bigTests); assertThat(rm.inUse()).isFalse(); + + // Ditto, for extra resources (even if they don't exist in the available resource set): + ImmutableMap bigExtraResources = ImmutableMap.of("gpu", 10.0f, "fancyresource", 10.0f, "nonexisting", 10.0f); + acquire(0, 0, 0, bigExtraResources); + assertThat(rm.inUse()).isTrue(); + release(0, 0, 0, bigExtraResources); + assertThat(rm.inUse()).isFalse(); } @Test @@ -248,11 +266,25 @@ public void testThatTestsCannotBeOverallocated() throws Exception { assertThat(e).hasCauseThat().hasMessageThat().contains("is still alive"); } + @Test + public void testThatExtraResourcesCannotBeOverallocated() throws Exception { + assertThat(rm.inUse()).isFalse(); + + // Given a partially acquired extra resources: + acquire(0, 0, 1, ImmutableMap.of("gpu", 1.0f)); + + // When a request for extra resources is made that would overallocate, + // Then the request fails: + TestThread thread1 = new TestThread(() -> assertThat(acquireNonblocking(0, 0, 0, ImmutableMap.of("gpu", 1.1f))).isNull()); + thread1.start(); + thread1.joinAndAssertState(10000); + } + @Test public void testHasResources() throws Exception { assertThat(rm.inUse()).isFalse(); assertThat(rm.threadHasResources()).isFalse(); - acquire(1, 0.1, 1); + acquire(1, 0.1, 1, ImmutableMap.of("gpu", 1.0f)); assertThat(rm.threadHasResources()).isTrue(); // We have resources in this thread - make sure other threads @@ -273,11 +305,15 @@ public void testHasResources() throws Exception { assertThat(rm.threadHasResources()).isTrue(); release(0, 0, 1); assertThat(rm.threadHasResources()).isFalse(); + acquire(0, 0, 0, ImmutableMap.of("gpu", 1.0f)); + assertThat(rm.threadHasResources()).isTrue(); + release(0, 0, 0, ImmutableMap.of("gpu", 1.0f)); + assertThat(rm.threadHasResources()).isFalse(); }); thread1.start(); thread1.joinAndAssertState(10000); - release(1, 0.1, 1); + release(1, 0.1, 1, ImmutableMap.of("gpu", 1.0f)); assertThat(rm.threadHasResources()).isFalse(); assertThat(rm.inUse()).isFalse(); }