Add rctx.watch_tree() to watch a directory tree

- Added `rctx.watch_tree()` to watch a directory tree, which includes all transitive descendants' names, and if they're files, their contents. - Added a new SkyFunction DirectoryTreeDigestFunction to do the heavy lifting. - In the future, for performance, we could try to get this skyfunction to have a mode where it only digests stat(), to use as heuristics (similar to #21044) Work towards #20952.
bazelbuild · Feb 15, 2024 · 1cf3caa · 1cf3caa
1 parent bae3435
commit 1cf3caa
Show file tree

Hide file tree

Showing 10 changed files with 359 additions and 4 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD
@@ -43,6 +43,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/shell",
         "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:ignored_package_prefixes_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository",

diff --git a/...va/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java b/...va/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java
@@ -34,17 +34,16 @@
 import com.google.devtools.build.lib.repository.RepositoryFetchProgress;
 import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException;
 import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
-import com.google.devtools.build.lib.rules.repository.RepoRecordedInput.Dirents;
+import com.google.devtools.build.lib.rules.repository.RepoRecordedInput.RepoCacheFriendlyPath;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
 import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper;
 import com.google.devtools.build.lib.runtime.ProcessWrapper;
 import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
-import com.google.devtools.build.lib.skyframe.DirectoryListingValue;
+import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue;
 import com.google.devtools.build.lib.util.StringUtilities;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
 import com.google.devtools.build.lib.vfs.RootedPath;
 import com.google.devtools.build.lib.vfs.SyscallCache;
 import com.google.devtools.build.skyframe.SkyFunction.Environment;
@@ -84,6 +83,7 @@ public class StarlarkRepositoryContext extends StarlarkBaseExternalContext {
   private final StructImpl attrObject;
   private final ImmutableSet<PathFragment> ignoredPatterns;
   private final SyscallCache syscallCache;
+  private final HashMap<RepoRecordedInput.DirTree, String> recordedDirTreeInputs = new HashMap<>();
 
   /**
    * Create a new context (repository_ctx) object for a Starlark repository rule ({@code rule}
@@ -137,6 +137,10 @@ protected String getIdentifyingStringForLogging() {
     return RepositoryFetchProgress.repositoryFetchContextString(repoName);
   }
 
+  public ImmutableMap<RepoRecordedInput.DirTree, String> getRecordedDirTreeInputs() {
+    return ImmutableMap.copyOf(recordedDirTreeInputs);
+  }
+
   @StarlarkMethod(
       name = "name",
       structField = true,
@@ -572,6 +576,49 @@ public void extract(
     env.getListener().post(new ExtractProgress(outputPath.getPath().toString()));
   }
 
+  @StarlarkMethod(
+      name = "watch_tree",
+      doc = "watches a tree!",
+      parameters = {
+        @Param(
+            name = "path",
+            allowedTypes = {
+              @ParamType(type = String.class),
+              @ParamType(type = Label.class),
+              @ParamType(type = StarlarkPath.class)
+            },
+            doc = "path of the directory tree to watch."),
+      })
+  public void watchTree(Object path)
+      throws EvalException, InterruptedException, RepositoryFunctionException {
+    StarlarkPath p = getPath("watch_tree()", path);
+    if (!p.isDir()) {
+      throw Starlark.errorf("can't call watch_tree() on non-directory %s", p);
+    }
+    RepoCacheFriendlyPath repoCacheFriendlyPath =
+        toRepoCacheFriendlyPath(p.getPath(), ShouldWatch.YES);
+    if (repoCacheFriendlyPath == null) {
+      return;
+    }
+    RootedPath rootedPath = repoCacheFriendlyPath.getRootedPath(env, directories);
+    if (rootedPath == null) {
+      throw new NeedsSkyframeRestartException();
+    }
+    try {
+      DirectoryTreeDigestValue digestValue =
+          (DirectoryTreeDigestValue)
+              env.getValueOrThrow(DirectoryTreeDigestValue.key(rootedPath), IOException.class);
+      if (digestValue == null) {
+        throw new NeedsSkyframeRestartException();
+      }
+
+      recordedDirTreeInputs.put(
+          new RepoRecordedInput.DirTree(repoCacheFriendlyPath), digestValue.hexDigest());
+    } catch (IOException e) {
+      throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+    }
+  }
+
   @Override
   public String toString() {
     return "repository_ctx[" + rule.getLabel() + "]";

diff --git a/...a/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java b/...a/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java
@@ -312,6 +312,7 @@ private RepositoryDirectoryValue.Builder fetchInternal(
       // Modify marker data to include the files/dirents used by the rule's implementation function.
       recordedInputValues.putAll(starlarkRepositoryContext.getRecordedFileInputs());
       recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirentsInputs());
+      recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirTreeInputs());
 
       // Ditto for environment variables accessed via `getenv`.
       for (String envKey : starlarkRepositoryContext.getAccumulatedEnvKeys()) {

diff --git a/src/main/java/com/google/devtools/build/lib/rules/BUILD b/src/main/java/com/google/devtools/build/lib/rules/BUILD
@@ -374,6 +374,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",
         "//src/main/java/com/google/devtools/build/lib/util",

diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java
@@ -28,6 +28,7 @@
 import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction;
 import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue;
 import com.google.devtools.build.lib.skyframe.DirectoryListingValue;
+import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue;
 import com.google.devtools.build.lib.skyframe.PrecomputedValue;
 import com.google.devtools.build.lib.skyframe.RepositoryMappingValue;
 import com.google.devtools.build.lib.util.Fingerprint;
@@ -92,7 +93,9 @@ public abstract static class Parser {
   public static RepoRecordedInput parse(String s) {
     List<String> parts = Splitter.on(':').limit(2).splitToList(s);
     for (Parser parser :
-        new Parser[] {File.PARSER, Dirents.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER}) {
+        new Parser[] {
+          File.PARSER, Dirents.PARSER, DirTree.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER
+        }) {
       if (parts.get(0).equals(parser.getPrefix())) {
         return parser.parse(parts.get(1));
       }
@@ -438,6 +441,76 @@ public static String getDirentsMarkerValue(Path path) throws IOException {
     }
   }
 
+  public static final class DirTree extends RepoRecordedInput {
+    public static final Parser PARSER =
+        new Parser() {
+          @Override
+          public String getPrefix() {
+            return "DIRTREE";
+          }
+
+          @Override
+          public RepoRecordedInput parse(String s) {
+            return new DirTree(RepoCacheFriendlyPath.parse(s));
+          }
+        };
+
+    private final RepoCacheFriendlyPath path;
+
+    public DirTree(RepoCacheFriendlyPath path) {
+      this.path = path;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof DirTree)) {
+        return false;
+      }
+      DirTree that = (DirTree) o;
+      return Objects.equals(path, that.path);
+    }
+
+    @Override
+    public int hashCode() {
+      return path.hashCode();
+    }
+
+    @Override
+    public String toStringInternal() {
+      return path.toString();
+    }
+
+    @Override
+    public Parser getParser() {
+      return PARSER;
+    }
+
+    @Nullable
+    @Override
+    public SkyKey getSkyKey(BlazeDirectories directories) {
+      return path.getRepoDirSkyKeyOrNull();
+    }
+
+    @Override
+    public boolean isUpToDate(
+        Environment env, BlazeDirectories directories, @Nullable String oldValue)
+        throws InterruptedException {
+      RootedPath rootedPath = path.getRootedPath(env, directories);
+      if (rootedPath == null) {
+        return false;
+      }
+      DirectoryTreeDigestValue value =
+          (DirectoryTreeDigestValue) env.getValue(DirectoryTreeDigestValue.key(rootedPath));
+      if (value == null) {
+        return false;
+      }
+      return oldValue.equals(value.hexDigest());
+    }
+  }
+
   /** Represents an environment variable accessed during the repo fetch. */
   public static final class EnvVar extends RepoRecordedInput {
     static final Parser PARSER =

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD
@@ -125,6 +125,7 @@ java_library(
         ":diff_awareness_manager",
         ":directory_listing_function",
         ":directory_listing_state_value",
+        ":directory_tree_digest_function",
         ":ephemeral_check_if_output_consumed",
         ":exclusive_test_build_driver_value",
         ":execution_finished_event",
@@ -1248,6 +1249,34 @@ java_library(
     ],
 )
 
+java_library(
+    name = "directory_tree_digest_function",
+    srcs = ["DirectoryTreeDigestFunction.java"],
+    deps = [
+        ":directory_listing_value",
+        ":directory_tree_digest_value",
+        ":dirents",
+        "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
+        "//src/main/java/com/google/devtools/build/lib/util",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//third_party:guava",
+        "//third_party:jsr305",
+    ],
+)
+
+java_library(
+    name = "directory_tree_digest_value",
+    srcs = ["DirectoryTreeDigestValue.java"],
+    deps = [
+        ":sky_functions",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//third_party:auto_value",
+    ],
+)
+
 java_library(
     name = "dirents",
     srcs = ["Dirents.java"],