Add logging when worker pool is shrunk.

RELNOTES: None. PiperOrigin-RevId: 573777234 Change-Id: I33bb1e44c959caa6e035a9be3ce218ffcaf05363
bazel-io · Oct 16, 2023 · 22ff6ae · 22ff6ae
1 parent ff4b1d5
commit 22ff6ae
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 12 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/worker/BUILD b/src/main/java/com/google/devtools/build/lib/worker/BUILD
@@ -233,6 +233,7 @@ java_library(
         ":worker_key",
         ":worker_pool",
         "//third_party:apache_commons_pool2",
+        "//third_party:flogger",
         "//third_party:guava",
         "//third_party:jsr305",
     ],

diff --git a/src/main/java/com/google/devtools/build/lib/worker/SimpleWorkerPool.java b/src/main/java/com/google/devtools/build/lib/worker/SimpleWorkerPool.java
@@ -15,6 +15,7 @@
 
 import com.google.common.base.Throwables;
 import com.google.common.eventbus.EventBus;
+import com.google.common.flogger.GoogleLogger;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
@@ -32,6 +33,8 @@
 @ThreadSafe
 final class SimpleWorkerPool extends GenericKeyedObjectPool<WorkerKey, Worker> {
 
+  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+
   /**
    * The subtrahend for maximal toal number of object per key. Unfortunately
    * GenericKeyedObjectPoolConfig doesn't support different number of objects per key, so we need to
@@ -101,7 +104,7 @@ public void invalidateObject(WorkerKey key, Worker obj) throws InterruptedExcept
         if (eventBus != null) {
           eventBus.post(new WorkerEvictedEvent(key.hashCode(), key.getMnemonic()));
         }
-        updateShrunkBy(key);
+        updateShrunkBy(key, obj.getWorkerId());
       }
     } catch (Throwable t) {
       Throwables.propagateIfPossible(t, InterruptedException.class);
@@ -116,18 +119,21 @@ public void returnObject(WorkerKey key, Worker obj) {
       if (eventBus != null) {
         eventBus.post(new WorkerEvictedEvent(key.hashCode(), key.getMnemonic()));
       }
-      updateShrunkBy(key);
+      updateShrunkBy(key, obj.getWorkerId());
     }
   }
 
   public int getMaxTotalPerKey(WorkerKey key) {
     return getMaxTotalPerKey() - shrunkBy.getOrDefault(key, 0);
   }
 
-  private synchronized void updateShrunkBy(WorkerKey workerKey) {
+  private synchronized void updateShrunkBy(WorkerKey workerKey, int workerId) {
     int currentValue = shrunkBy.getOrDefault(workerKey, 0);
     if (getMaxTotalPerKey() - currentValue > 1) {
-      shrunkBy.put(workerKey, currentValue + 1);
+      int newValue = currentValue + 1;
+      logger.atInfo().log(
+          "shrinking %s (worker-%d) by %d.", workerKey.getMnemonic(), workerId, newValue);
+      shrunkBy.put(workerKey, newValue);
     }
   }
 

diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerPoolImpl.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerPoolImpl.java
@@ -18,6 +18,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.eventbus.EventBus;
+import com.google.common.flogger.GoogleLogger;
 import java.io.IOException;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -32,14 +33,20 @@
 /** Implementation of WorkerPool. */
 @ThreadSafe
 public class WorkerPoolImpl implements WorkerPool {
+
+  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+
   /** Unless otherwise specified, the max number of workers per WorkerKey. */
   private static final int DEFAULT_MAX_WORKERS = 4;
+
   /** Unless otherwise specified, the max number of multiplex workers per WorkerKey. */
   private static final int DEFAULT_MAX_MULTIPLEX_WORKERS = 8;
 
   private final WorkerPoolConfig workerPoolConfig;
+
   /** Map of singleplex worker pools, one per mnemonic. */
   private final ImmutableMap<String, SimpleWorkerPool> workerPools;
+
   /** Map of multiplex worker pools, one per mnemonic. */
   private final ImmutableMap<String, SimpleWorkerPool> multiplexPools;
 
@@ -130,10 +137,10 @@ public void evictWithPolicy(EvictionPolicy<Worker> evictionPolicy) throws Interr
   private void evictWithPolicy(EvictionPolicy<Worker> evictionPolicy, SimpleWorkerPool pool)
       throws InterruptedException {
     try {
-        pool.setEvictionPolicy(evictionPolicy);
-        pool.evict();
-      } catch (Throwable t) {
-        Throwables.propagateIfPossible(t, InterruptedException.class);
+      pool.setEvictionPolicy(evictionPolicy);
+      pool.evict();
+    } catch (Throwable t) {
+      Throwables.propagateIfPossible(t, InterruptedException.class);
       throw new VerifyException("unexpected", t);
     }
   }
@@ -186,11 +193,17 @@ public synchronized void setDoomedWorkers(ImmutableSet<Integer> workerIds) {
   @Override
   public synchronized void clearDoomedWorkers() {
     this.doomedWorkers = ImmutableSet.of();
-    for (SimpleWorkerPool pool : workerPools.values()) {
-      pool.clearShrunkBy();
+    for (Entry<String, SimpleWorkerPool> entry : workerPools.entrySet()) {
+      logger.atInfo().log(
+          "clearing shrunk by values for %s worker pool",
+          entry.getKey().isEmpty() ? "shared" : entry.getKey());
+      entry.getValue().clearShrunkBy();
     }
-    for (SimpleWorkerPool pool : multiplexPools.values()) {
-      pool.clearShrunkBy();
+    for (Entry<String, SimpleWorkerPool> entry : multiplexPools.entrySet()) {
+      logger.atInfo().log(
+          "clearing shrunk by values for %s multiplex worker pool",
+          entry.getKey().isEmpty() ? "shared" : entry.getKey());
+      entry.getValue().clearShrunkBy();
     }
   }