Define the FingerprintValueService, bundling fingerprinting dependenc…

…ies.

Injects the FingerprintValueService where it is needed, but it doesn't do
anything yet.

PiperOrigin-RevId: 614753186
Change-Id: I06dca45500fd93f0e143016fca212497ce2d7c4b

src/main/java/com/google/devtools/build/lib/skyframe/serialization/FingerprintValueService.java

@@ -0,0 +1,141 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe.serialization;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.protobuf.ByteString;
+import java.io.IOException;
+import java.util.concurrent.Executor;
+import javax.annotation.Nullable;
+
+/**
+ * Bundles the components needed to store serialized values by fingerprint, the storage interface,
+ * the cache and the hash function for computing fingerprints.
+ */
+public final class FingerprintValueService {
+  private final Executor executor;
+  private final FingerprintValueStore store;
+  private final FingerprintValueCache cache;
+
+  /**
+   * The function used to generate fingerprints.
+   *
+   * <p>Used to derive {@link #fingerprintPlaceholder} and {@link #fingerprintLength}.
+   */
+  private final HashFunction hashFunction;
+
+  private final ByteString fingerprintPlaceholder;
+  private final int fingerprintLength;
+
+  @VisibleForTesting
+  public static FingerprintValueService createForTesting() {
+    return createForTesting(
+        FingerprintValueStore.inMemoryStore(), /* exerciseDeserializationForTesting= */ true);
+  }
+
+  @VisibleForTesting
+  public static FingerprintValueService createForTesting(FingerprintValueStore store) {
+    return createForTesting(store, /* exerciseDeserializationForTesting= */ true);
+  }
+
+  @VisibleForTesting
+  public static FingerprintValueService createForTesting(
+      boolean exerciseDeserializationForTesting) {
+    return createForTesting(
+        FingerprintValueStore.inMemoryStore(), exerciseDeserializationForTesting);
+  }
+
+  private static FingerprintValueService createForTesting(
+      FingerprintValueStore store, boolean exerciseDeserializationForTesting) {
+    return new FingerprintValueService(
+        newSingleThreadExecutor(),
+        store,
+        new FingerprintValueCache(exerciseDeserializationForTesting),
+        Hashing.murmur3_128());
+  }
+
+  public FingerprintValueService(
+      Executor executor,
+      FingerprintValueStore store,
+      FingerprintValueCache cache,
+      HashFunction hashFunction) {
+    this.executor = executor;
+    this.store = store;
+    this.cache = cache;
+    this.hashFunction = hashFunction;
+
+    this.fingerprintPlaceholder = fingerprint(new byte[] {});
+    this.fingerprintLength = fingerprintPlaceholder.size();
+  }
+
+  /** Delegates to {@link FingerprintValueStore#put}. */
+  ListenableFuture<Void> put(ByteString fingerprint, byte[] serializedBytes) {
+    return store.put(fingerprint, serializedBytes);
+  }
+
+  /** Delegates to {@link FingerprintValueStore#get}. */
+  ListenableFuture<byte[]> get(ByteString fingerprint) throws IOException {
+    return store.get(fingerprint);
+  }
+
+  /** Delegates to {@link FingerprintValueCache#getOrClaimPutOperation}. */
+  @Nullable
+  Object getOrClaimPutOperation(
+      Object obj, @Nullable Object distinguisher, ListenableFuture<PutOperation> putOperation) {
+    return cache.getOrClaimPutOperation(obj, distinguisher, putOperation);
+  }
+
+  /** Delegates to {@link FingerprintValueCache#getOrClaimGetOperation}. */
+  @Nullable
+  Object getOrClaimGetOperation(
+      ByteString fingerprint,
+      @Nullable Object distinguisher,
+      ListenableFuture<Object> getOperation) {
+    return cache.getOrClaimGetOperation(fingerprint, distinguisher, getOperation);
+  }
+
+  /** Computes the fingerprint of {@code bytes}. */
+  ByteString fingerprint(byte[] bytes) {
+    return ByteString.copyFrom(hashFunction.hashBytes(bytes).asBytes());
+  }
+
+  /**
+   * A placeholder fingerprint to use when the actual fingerprint is not yet available.
+   *
+   * <p>The placeholder has the same length as the real fingerprint so the real fingerprint can
+   * overwrite the placeholder when it becomes available.
+   */
+  ByteString fingerprintPlaceholder() {
+    return fingerprintPlaceholder;
+  }
+
+  /** The fixed length of fingerprints. */
+  int fingerprintLength() {
+    return fingerprintLength;
+  }
+
+  /**
+   * Executor for chaining work on top of futures returned by {@link #put} or {@link #get}.
+   *
+   * <p>Those callbacks may be executing on RPC threads that should not be blocked.
+   */
+  Executor getExecutor() {
+    return executor;
+  }
+}

src/main/java/com/google/devtools/build/lib/skyframe/serialization/FutureHelpers.java

@@ -14,14 +14,36 @@
 package com.google.devtools.build.lib.skyframe.serialization;
 
 import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
+import static com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly;
 
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.io.IOException;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 
 /** Helpers for serialization futures. */
 public final class FutureHelpers {
 
+  /**
+   * Waits for {@code future} and returns the result.
+   *
+   * <p>Handles exceptions by converting them into {@link SerializationException}.
+   */
+  @CanIgnoreReturnValue // may be called for side effects
+  public static <T> T waitForSerializationFuture(ListenableFuture<T> future)
+      throws SerializationException {
+    try {
+      // TODO: b/297857068 - revisit whether this should handle to interrupts. As of 02/09/24,
+      // serialization doesn't handle interrupts so introducing them here could lead to unforseen
+      // problems.
+      return getUninterruptibly(future);
+    } catch (ExecutionException e) {
+      throw asSerializationException(e.getCause());
+    }
+  }
+
   /** Combines a list of {@code Void} futures into a single future. */
   static ListenableFuture<Void> aggregateStatusFutures(List<ListenableFuture<Void>> futures) {
     if (futures.size() == 1) {
@@ -30,5 +52,15 @@ static ListenableFuture<Void> aggregateStatusFutures(List<ListenableFuture<Void>
     return Futures.whenAllSucceed(futures).call(() -> null, directExecutor());
   }
 
+  private static SerializationException asSerializationException(Throwable cause) {
+    if (cause instanceof SerializationException) {
+      return new SerializationException(cause);
+    }
+    if (cause instanceof IOException) {
+      return new SerializationException("serialization I/O error", cause);
+    }
+    return new SerializationException("unexpected serialization error", cause);
+  }
+
   private FutureHelpers() {}
 }

src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecs.java

@@ -77,8 +77,10 @@ public MemoizingSerializationContext getMemoizingSerializationContextForTesting(
   }
 
   @VisibleForTesting // private
-  public SharedValueSerializationContext getSharedValueSerializationContextForTesting() {
-    return SharedValueSerializationContext.createForTesting(getCodecRegistry(), getDependencies());
+  public SharedValueSerializationContext getSharedValueSerializationContextForTesting(
+      FingerprintValueService fingerprintValueService) {
+    return SharedValueSerializationContext.createForTesting(
+        getCodecRegistry(), getDependencies(), fingerprintValueService);
   }
 
   @VisibleForTesting // private
@@ -98,9 +100,10 @@ public MemoizingDeserializationContext getMemoizingDeserializationContextForTest
   }
 
   @VisibleForTesting // private
-  public SharedValueDeserializationContext getSharedValueDeserializationContextForTesting() {
+  public SharedValueDeserializationContext getSharedValueDeserializationContextForTesting(
+      FingerprintValueService fingerprintValueService) {
     return SharedValueDeserializationContext.createForTesting(
-        getCodecRegistry(), getDependencies());
+        getCodecRegistry(), getDependencies(), fingerprintValueService);
   }
 
   /**
@@ -152,10 +155,11 @@ public byte[] serializeMemoizedToBytes(
   }
 
   /** Serializes {@code subject} using a {@link SharedValueSerializationContext}. */
-  public SerializationResult<ByteString> serializeMemoizedAndBlocking(Object subject)
+  public SerializationResult<ByteString> serializeMemoizedAndBlocking(
+      FingerprintValueService fingerprintValueService, Object subject)
       throws SerializationException {
     return SharedValueSerializationContext.serializeToResult(
-        getCodecRegistry(), getDependencies(), subject);
+        getCodecRegistry(), getDependencies(), fingerprintValueService, subject);
   }
 
   /**
@@ -164,9 +168,15 @@ public SerializationResult<ByteString> serializeMemoizedAndBlocking(Object subje
    * @param dependencyOverrides dependencies to override, see {@link #overrideDependencies}
    */
   public SerializationResult<ByteString> serializeMemoizedAndBlocking(
-      Object subject, ClassToInstanceMap<?> dependencyOverrides) throws SerializationException {
+      FingerprintValueService fingerprintValueService,
+      Object subject,
+      ImmutableClassToInstanceMap<?> dependencyOverrides)
+      throws SerializationException {
     return SharedValueSerializationContext.serializeToResult(
-        getCodecRegistry(), overrideDependencies(getDependencies(), dependencyOverrides), subject);
+        getCodecRegistry(),
+        overrideDependencies(getDependencies(), dependencyOverrides),
+        fingerprintValueService,
+        subject);
   }
 
   public Object deserialize(byte[] data) throws SerializationException {
@@ -191,16 +201,23 @@ public Object deserializeMemoized(byte[] data) throws SerializationException {
         getCodecRegistry(), getDependencies(), data);
   }
 
-  public Object deserializeMemoizedAndBlocking(ByteString data) throws SerializationException {
+  public Object deserializeMemoizedAndBlocking(
+      FingerprintValueService fingerprintValueService, ByteString data)
+      throws SerializationException {
     return SharedValueDeserializationContext.deserializeWithSharedValues(
-        getCodecRegistry(), getDependencies(), data);
+        getCodecRegistry(), getDependencies(), fingerprintValueService, data);
   }
 
   public Object deserializeMemoizedAndBlocking(
-      ByteString data, ImmutableClassToInstanceMap<?> dependencyOverrides)
+      FingerprintValueService fingerprintValueService,
+      ByteString data,
+      ImmutableClassToInstanceMap<?> dependencyOverrides)
       throws SerializationException {
     return SharedValueDeserializationContext.deserializeWithSharedValues(
-        getCodecRegistry(), overrideDependencies(getDependencies(), dependencyOverrides), data);
+        getCodecRegistry(),
+        overrideDependencies(getDependencies(), dependencyOverrides),
+        fingerprintValueService,
+        data);
   }
 
   static Object deserializeStreamFully(CodedInputStream codedIn, DeserializationContext context)

src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationException.java

@@ -25,6 +25,10 @@ public SerializationException(String msg) {
     super(msg);
   }
 
+  public SerializationException(Throwable cause) {
+    super(cause);
+  }
+
   public SerializationException(String msg, Throwable cause) {
     super(msg, cause);
   }

src/main/java/com/google/devtools/build/lib/skyframe/serialization/SharedValueDeserializationContext.java

@@ -26,29 +26,41 @@
  */
 // TODO: b/297857068 - complete this implementation
 final class SharedValueDeserializationContext extends MemoizingDeserializationContext {
+  private final FingerprintValueService fingerprintValueService;
+
   @VisibleForTesting // private
   static SharedValueDeserializationContext createForTesting(
-      ObjectCodecRegistry codecRegistry, ImmutableClassToInstanceMap<Object> dependencies) {
-    return new SharedValueDeserializationContext(codecRegistry, dependencies);
+      ObjectCodecRegistry codecRegistry,
+      ImmutableClassToInstanceMap<Object> dependencies,
+      FingerprintValueService fingerprintValueService) {
+    return new SharedValueDeserializationContext(
+        codecRegistry, dependencies, fingerprintValueService);
   }
 
   private SharedValueDeserializationContext(
-      ObjectCodecRegistry codecRegistry, ImmutableClassToInstanceMap<Object> dependencies) {
+      ObjectCodecRegistry codecRegistry,
+      ImmutableClassToInstanceMap<Object> dependencies,
+      FingerprintValueService fingerprintValueService) {
     super(codecRegistry, dependencies);
+    this.fingerprintValueService = fingerprintValueService;
   }
 
   static Object deserializeWithSharedValues(
       ObjectCodecRegistry codecRegistry,
       ImmutableClassToInstanceMap<Object> dependencies,
+      FingerprintValueService fingerprintValueService,
       ByteString bytes)
       throws SerializationException {
     return ObjectCodecs.deserializeStreamFully(
-        bytes.newCodedInput(), new SharedValueDeserializationContext(codecRegistry, dependencies));
+        bytes.newCodedInput(),
+        new SharedValueDeserializationContext(
+            codecRegistry, dependencies, fingerprintValueService));
   }
 
   @Override
   public SharedValueDeserializationContext getFreshContext() {
-    return new SharedValueDeserializationContext(getRegistry(), getDependencies());
+    return new SharedValueDeserializationContext(
+        getRegistry(), getDependencies(), fingerprintValueService);
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/skyframe/serialization/SharedValueSerializationContext.java

@@ -33,6 +33,8 @@
  * SerializationResult#getFutureToBlockWritesOn}.
  */
 final class SharedValueSerializationContext extends MemoizingSerializationContext {
+  private final FingerprintValueService fingerprintValueService;
+
   /**
    * Futures that represent writes to remote storage.
    *
@@ -44,13 +46,19 @@ final class SharedValueSerializationContext extends MemoizingSerializationContex
 
   @VisibleForTesting // private
   static SharedValueSerializationContext createForTesting(
-      ObjectCodecRegistry codecRegistry, ImmutableClassToInstanceMap<Object> dependencies) {
-    return new SharedValueSerializationContext(codecRegistry, dependencies);
+      ObjectCodecRegistry codecRegistry,
+      ImmutableClassToInstanceMap<Object> dependencies,
+      FingerprintValueService fingerprintValueService) {
+    return new SharedValueSerializationContext(
+        codecRegistry, dependencies, fingerprintValueService);
   }
 
   private SharedValueSerializationContext(
-      ObjectCodecRegistry codecRegistry, ImmutableClassToInstanceMap<Object> dependencies) {
+      ObjectCodecRegistry codecRegistry,
+      ImmutableClassToInstanceMap<Object> dependencies,
+      FingerprintValueService fingerprintValueService) {
     super(codecRegistry, dependencies);
+    this.fingerprintValueService = fingerprintValueService;
   }
 
   /**
@@ -62,12 +70,13 @@ private SharedValueSerializationContext(
   static SerializationResult<ByteString> serializeToResult(
       ObjectCodecRegistry codecRegistry,
       ImmutableClassToInstanceMap<Object> dependencies,
+      FingerprintValueService fingerprintValueService,
       @Nullable Object subject)
       throws SerializationException {
-    SharedValueSerializationContext context =
-        new SharedValueSerializationContext(codecRegistry, dependencies);
     ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
     CodedOutputStream codedOut = CodedOutputStream.newInstance(bytesOut);
+    SharedValueSerializationContext context =
+        new SharedValueSerializationContext(codecRegistry, dependencies, fingerprintValueService);
     try {
       context.serialize(subject, codedOut);
       codedOut.flush();
@@ -79,7 +88,8 @@ static SerializationResult<ByteString> serializeToResult(
 
   @Override
   public SharedValueSerializationContext getFreshContext() {
-    return new SharedValueSerializationContext(getCodecRegistry(), getDependencies());
+    return new SharedValueSerializationContext(
+        getCodecRegistry(), getDependencies(), fingerprintValueService);
   }
 
   /**