Add rctx.watch_tree() to watch a directory tree

- Added `rctx.watch_tree()` to watch a directory tree, which includes all transitive descendants' names, and if they're files, their contents.
  -  In the future we could add glob patterns to this method.
- Added a new SkyFunction DirectoryTreeDigestFunction to do the heavy lifting.
  - In the future, for performance, we could try to get this skyfunction to have a mode where it only digests stat(), to use as heuristics (similar to #21044)

Work towards #20952.

Closes #21362.

PiperOrigin-RevId: 608667062
Change-Id: Ibacbb7af4cf4d7628fe8fcf06e2c4fa50e811e4e

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD

@@ -43,6 +43,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/shell",
         "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:ignored_package_prefixes_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository",

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java

@@ -33,21 +33,26 @@
 import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
 import com.google.devtools.build.lib.repository.RepositoryFetchProgress;
 import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException;
+import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
+import com.google.devtools.build.lib.rules.repository.RepoRecordedInput.RepoCacheFriendlyPath;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
 import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper;
 import com.google.devtools.build.lib.runtime.ProcessWrapper;
 import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
+import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue;
 import com.google.devtools.build.lib.util.StringUtilities;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
 import com.google.devtools.build.lib.vfs.SyscallCache;
 import com.google.devtools.build.skyframe.SkyFunction.Environment;
 import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.InvalidPathException;
+import java.util.HashMap;
 import java.util.Map;
 import javax.annotation.Nullable;
 import net.starlark.java.annot.Param;
@@ -78,6 +83,7 @@ public class StarlarkRepositoryContext extends StarlarkBaseExternalContext {
   private final StructImpl attrObject;
   private final ImmutableSet<PathFragment> ignoredPatterns;
   private final SyscallCache syscallCache;
+  private final HashMap<RepoRecordedInput.DirTree, String> recordedDirTreeInputs = new HashMap<>();
 
   /**
    * Create a new context (repository_ctx) object for a Starlark repository rule ({@code rule}
@@ -131,6 +137,10 @@ protected String getIdentifyingStringForLogging() {
     return RepositoryFetchProgress.repositoryFetchContextString(repoName);
   }
 
+  public ImmutableMap<RepoRecordedInput.DirTree, String> getRecordedDirTreeInputs() {
+    return ImmutableMap.copyOf(recordedDirTreeInputs);
+  }
+
   @StarlarkMethod(
       name = "name",
       structField = true,
@@ -566,6 +576,54 @@ public void extract(
     env.getListener().post(new ExtractProgress(outputPath.getPath().toString()));
   }
 
+  @StarlarkMethod(
+      name = "watch_tree",
+      doc =
+          "Tells Bazel to watch for changes to any files or directories transitively under the "
+              + "given path. Any changes to the contents of files, the existence of files or "
+              + "directories, file names or directory names, will cause this repo to be "
+              + "refetched.<p>Note that attempting to watch paths inside the repo currently being "
+              + "fetched will result in an error. ",
+      parameters = {
+        @Param(
+            name = "path",
+            allowedTypes = {
+              @ParamType(type = String.class),
+              @ParamType(type = Label.class),
+              @ParamType(type = StarlarkPath.class)
+            },
+            doc = "path of the directory tree to watch."),
+      })
+  public void watchTree(Object path)
+      throws EvalException, InterruptedException, RepositoryFunctionException {
+    StarlarkPath p = getPath("watch_tree()", path);
+    if (!p.isDir()) {
+      throw Starlark.errorf("can't call watch_tree() on non-directory %s", p);
+    }
+    RepoCacheFriendlyPath repoCacheFriendlyPath =
+        toRepoCacheFriendlyPath(p.getPath(), ShouldWatch.YES);
+    if (repoCacheFriendlyPath == null) {
+      return;
+    }
+    RootedPath rootedPath = repoCacheFriendlyPath.getRootedPath(env, directories);
+    if (rootedPath == null) {
+      throw new NeedsSkyframeRestartException();
+    }
+    try {
+      DirectoryTreeDigestValue digestValue =
+          (DirectoryTreeDigestValue)
+              env.getValueOrThrow(DirectoryTreeDigestValue.key(rootedPath), IOException.class);
+      if (digestValue == null) {
+        throw new NeedsSkyframeRestartException();
+      }
+
+      recordedDirTreeInputs.put(
+          new RepoRecordedInput.DirTree(repoCacheFriendlyPath), digestValue.hexDigest());
+    } catch (IOException e) {
+      throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+    }
+  }
+
   @Override
   public String toString() {
     return "repository_ctx[" + rule.getLabel() + "]";

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java

@@ -312,6 +312,7 @@ private RepositoryDirectoryValue.Builder fetchInternal(
       // Modify marker data to include the files/dirents used by the rule's implementation function.
       recordedInputValues.putAll(starlarkRepositoryContext.getRecordedFileInputs());
       recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirentsInputs());
+      recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirTreeInputs());
 
       // Ditto for environment variables accessed via `getenv`.
       for (String envKey : starlarkRepositoryContext.getAccumulatedEnvKeys()) {

src/main/java/com/google/devtools/build/lib/rules/BUILD

@@ -374,6 +374,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",
         "//src/main/java/com/google/devtools/build/lib/util",

src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java

@@ -28,6 +28,7 @@
 import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction;
 import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue;
 import com.google.devtools.build.lib.skyframe.DirectoryListingValue;
+import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue;
 import com.google.devtools.build.lib.skyframe.PrecomputedValue;
 import com.google.devtools.build.lib.skyframe.RepositoryMappingValue;
 import com.google.devtools.build.lib.util.Fingerprint;
@@ -92,7 +93,9 @@ public abstract static class Parser {
   public static RepoRecordedInput parse(String s) {
     List<String> parts = Splitter.on(':').limit(2).splitToList(s);
     for (Parser parser :
-        new Parser[] {File.PARSER, Dirents.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER}) {
+        new Parser[] {
+          File.PARSER, Dirents.PARSER, DirTree.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER
+        }) {
       if (parts.get(0).equals(parser.getPrefix())) {
         return parser.parse(parts.get(1));
       }
@@ -112,7 +115,7 @@ public static boolean areAllValuesUpToDate(
       throws InterruptedException {
     env.getValuesAndExceptions(
         recordedInputValues.keySet().stream()
-            .map(rri -> rri.getSkyKey(directories))
+            .map(RepoRecordedInput::getSkyKey)
             .filter(Objects::nonNull)
             .collect(toImmutableSet()));
     if (env.valuesMissing()) {
@@ -159,13 +162,13 @@ public int compareTo(RepoRecordedInput o) {
    * no SkyKey is needed.
    */
   @Nullable
-  public abstract SkyKey getSkyKey(BlazeDirectories directories);
+  public abstract SkyKey getSkyKey();
 
   /**
    * Returns whether the given {@code oldValue} is still up-to-date for this recorded input. This
-   * method can assume that {@link #getSkyKey(BlazeDirectories)} is already evaluated; it can
-   * request further Skyframe evaluations, and if any values are missing, this method can return any
-   * value (doesn't matter what) and will be reinvoked after a Skyframe restart.
+   * method can assume that {@link #getSkyKey()} is already evaluated; it can request further
+   * Skyframe evaluations, and if any values are missing, this method can return any value (doesn't
+   * matter what) and will be reinvoked after a Skyframe restart.
    */
   public abstract boolean isUpToDate(
       Environment env, BlazeDirectories directories, @Nullable String oldValue)
@@ -330,7 +333,7 @@ public static String fileValueToMarkerValue(FileValue fileValue) throws IOExcept
 
     @Override
     @Nullable
-    public SkyKey getSkyKey(BlazeDirectories directories) {
+    public SkyKey getSkyKey() {
       return path.getRepoDirSkyKeyOrNull();
     }
 
@@ -405,7 +408,7 @@ public Parser getParser() {
 
     @Nullable
     @Override
-    public SkyKey getSkyKey(BlazeDirectories directories) {
+    public SkyKey getSkyKey() {
       return path.getRepoDirSkyKeyOrNull();
     }
 
@@ -438,6 +441,81 @@ public static String getDirentsMarkerValue(Path path) throws IOException {
     }
   }
 
+  /**
+   * Represents an entire directory tree accessed during the fetch. Anything under the tree changing
+   * (including adding/removing/renaming files or directories and changing file contents) will cause
+   * it to go out of date.
+   */
+  public static final class DirTree extends RepoRecordedInput {
+    public static final Parser PARSER =
+        new Parser() {
+          @Override
+          public String getPrefix() {
+            return "DIRTREE";
+          }
+
+          @Override
+          public RepoRecordedInput parse(String s) {
+            return new DirTree(RepoCacheFriendlyPath.parse(s));
+          }
+        };
+
+    private final RepoCacheFriendlyPath path;
+
+    public DirTree(RepoCacheFriendlyPath path) {
+      this.path = path;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof DirTree)) {
+        return false;
+      }
+      DirTree that = (DirTree) o;
+      return Objects.equals(path, that.path);
+    }
+
+    @Override
+    public int hashCode() {
+      return path.hashCode();
+    }
+
+    @Override
+    public String toStringInternal() {
+      return path.toString();
+    }
+
+    @Override
+    public Parser getParser() {
+      return PARSER;
+    }
+
+    @Nullable
+    @Override
+    public SkyKey getSkyKey() {
+      return path.getRepoDirSkyKeyOrNull();
+    }
+
+    @Override
+    public boolean isUpToDate(
+        Environment env, BlazeDirectories directories, @Nullable String oldValue)
+        throws InterruptedException {
+      RootedPath rootedPath = path.getRootedPath(env, directories);
+      if (rootedPath == null) {
+        return false;
+      }
+      DirectoryTreeDigestValue value =
+          (DirectoryTreeDigestValue) env.getValue(DirectoryTreeDigestValue.key(rootedPath));
+      if (value == null) {
+        return false;
+      }
+      return oldValue.equals(value.hexDigest());
+    }
+  }
+
   /** Represents an environment variable accessed during the repo fetch. */
   public static final class EnvVar extends RepoRecordedInput {
     static final Parser PARSER =
@@ -487,7 +565,7 @@ public String toStringInternal() {
     }
 
     @Override
-    public SkyKey getSkyKey(BlazeDirectories directories) {
+    public SkyKey getSkyKey() {
       return ActionEnvironmentFunction.key(name);
     }
 
@@ -497,7 +575,7 @@ public boolean isUpToDate(
         throws InterruptedException {
       String v = PrecomputedValue.REPO_ENV.get(env).get(name);
       if (v == null) {
-        v = ((ClientEnvironmentValue) env.getValue(getSkyKey(directories))).getValue();
+        v = ((ClientEnvironmentValue) env.getValue(getSkyKey())).getValue();
       }
       // Note that `oldValue` can be null if the env var was not set.
       return Objects.equals(oldValue, v);
@@ -558,7 +636,7 @@ public String toStringInternal() {
     }
 
     @Override
-    public SkyKey getSkyKey(BlazeDirectories directories) {
+    public SkyKey getSkyKey() {
       // Since we only record repo mapping entries for repos defined in Bzlmod, we can request the
       // WORKSPACE-less version of the main repo mapping (as no repos defined in Bzlmod can see
       // stuff from WORKSPACE).
@@ -571,8 +649,7 @@ public SkyKey getSkyKey(BlazeDirectories directories) {
     public boolean isUpToDate(
         Environment env, BlazeDirectories directories, @Nullable String oldValue)
         throws InterruptedException {
-      RepositoryMappingValue repoMappingValue =
-          (RepositoryMappingValue) env.getValue(getSkyKey(directories));
+      RepositoryMappingValue repoMappingValue = (RepositoryMappingValue) env.getValue(getSkyKey());
       return repoMappingValue != RepositoryMappingValue.NOT_FOUND_VALUE
           && RepositoryName.createUnvalidated(oldValue)
               .equals(repoMappingValue.getRepositoryMapping().get(apparentName));

src/main/java/com/google/devtools/build/lib/skyframe/BUILD

@@ -125,6 +125,7 @@ java_library(
         ":diff_awareness_manager",
         ":directory_listing_function",
         ":directory_listing_state_value",
+        ":directory_tree_digest_function",
         ":ephemeral_check_if_output_consumed",
         ":exclusive_test_build_driver_value",
         ":execution_finished_event",
@@ -1248,6 +1249,33 @@ java_library(
     ],
 )
 
+java_library(
+    name = "directory_tree_digest_function",
+    srcs = ["DirectoryTreeDigestFunction.java"],
+    deps = [
+        ":directory_listing_value",
+        ":directory_tree_digest_value",
+        "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
+        "//src/main/java/com/google/devtools/build/lib/util",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//third_party:guava",
+        "//third_party:jsr305",
+    ],
+)
+
+java_library(
+    name = "directory_tree_digest_value",
+    srcs = ["DirectoryTreeDigestValue.java"],
+    deps = [
+        ":sky_functions",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//third_party:auto_value",
+    ],
+)
+
 java_library(
     name = "dirents",
     srcs = ["Dirents.java"],