From 89c2a268a706118b5b779dee2ccf9c704fa6379a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?X=C3=B9d=C5=8Dng=20Y=C3=A1ng?= Date: Wed, 14 Feb 2024 23:02:19 -0500 Subject: [PATCH] Add `rctx.watch_tree()` to watch a directory tree - Added `rctx.watch_tree()` to watch a directory tree, which includes all transitive descendants' names, and if they're files, their contents. - Added a new SkyFunction DirectoryTreeDigestFunction to do the heavy lifting. - In the future, for performance, we could try to get this skyfunction to have a mode where it only digests stat(), to use as heuristics (similar to #21044) Work towards #20952. --- .../build/lib/bazel/repository/starlark/BUILD | 1 + .../starlark/StarlarkRepositoryContext.java | 53 ++++++- .../starlark/StarlarkRepositoryFunction.java | 1 + .../com/google/devtools/build/lib/rules/BUILD | 1 + .../rules/repository/RepoRecordedInput.java | 75 ++++++++- .../google/devtools/build/lib/skyframe/BUILD | 29 ++++ .../skyframe/DirectoryTreeDigestFunction.java | 149 ++++++++++++++++++ .../skyframe/DirectoryTreeDigestValue.java | 51 ++++++ .../build/lib/skyframe/SkyFunctions.java | 2 + .../build/lib/skyframe/SkyframeExecutor.java | 1 + 10 files changed, 359 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunction.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestValue.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD index b0f34b7bac8755..bbc23ef10fe95f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD @@ -43,6 +43,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/shell", "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function", "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value", + "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value", "//src/main/java/com/google/devtools/build/lib/skyframe:ignored_package_prefixes_value", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java index f6a32cdef62b5e..88083781ba112d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java @@ -34,17 +34,16 @@ import com.google.devtools.build.lib.repository.RepositoryFetchProgress; import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException; import com.google.devtools.build.lib.rules.repository.RepoRecordedInput; -import com.google.devtools.build.lib.rules.repository.RepoRecordedInput.Dirents; +import com.google.devtools.build.lib.rules.repository.RepoRecordedInput.RepoCacheFriendlyPath; import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; import com.google.devtools.build.lib.runtime.ProcessWrapper; import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor; -import com.google.devtools.build.lib.skyframe.DirectoryListingValue; +import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue; import com.google.devtools.build.lib.util.StringUtilities; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.SkyFunction.Environment; @@ -84,6 +83,7 @@ public class StarlarkRepositoryContext extends StarlarkBaseExternalContext { private final StructImpl attrObject; private final ImmutableSet ignoredPatterns; private final SyscallCache syscallCache; + private final HashMap recordedDirTreeInputs = new HashMap<>(); /** * Create a new context (repository_ctx) object for a Starlark repository rule ({@code rule} @@ -137,6 +137,10 @@ protected String getIdentifyingStringForLogging() { return RepositoryFetchProgress.repositoryFetchContextString(repoName); } + public ImmutableMap getRecordedDirTreeInputs() { + return ImmutableMap.copyOf(recordedDirTreeInputs); + } + @StarlarkMethod( name = "name", structField = true, @@ -572,6 +576,49 @@ public void extract( env.getListener().post(new ExtractProgress(outputPath.getPath().toString())); } + @StarlarkMethod( + name = "watch_tree", + doc = "watches a tree!", + parameters = { + @Param( + name = "path", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = Label.class), + @ParamType(type = StarlarkPath.class) + }, + doc = "path of the directory tree to watch."), + }) + public void watchTree(Object path) + throws EvalException, InterruptedException, RepositoryFunctionException { + StarlarkPath p = getPath("watch_tree()", path); + if (!p.isDir()) { + throw Starlark.errorf("can't call watch_tree() on non-directory %s", p); + } + RepoCacheFriendlyPath repoCacheFriendlyPath = + toRepoCacheFriendlyPath(p.getPath(), ShouldWatch.YES); + if (repoCacheFriendlyPath == null) { + return; + } + RootedPath rootedPath = repoCacheFriendlyPath.getRootedPath(env, directories); + if (rootedPath == null) { + throw new NeedsSkyframeRestartException(); + } + try { + DirectoryTreeDigestValue digestValue = + (DirectoryTreeDigestValue) + env.getValueOrThrow(DirectoryTreeDigestValue.key(rootedPath), IOException.class); + if (digestValue == null) { + throw new NeedsSkyframeRestartException(); + } + + recordedDirTreeInputs.put( + new RepoRecordedInput.DirTree(repoCacheFriendlyPath), digestValue.hexDigest()); + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + } + @Override public String toString() { return "repository_ctx[" + rule.getLabel() + "]"; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java index d350edef1cff03..fd7d66aaa3f562 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java @@ -312,6 +312,7 @@ private RepositoryDirectoryValue.Builder fetchInternal( // Modify marker data to include the files/dirents used by the rule's implementation function. recordedInputValues.putAll(starlarkRepositoryContext.getRecordedFileInputs()); recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirentsInputs()); + recordedInputValues.putAll(starlarkRepositoryContext.getRecordedDirTreeInputs()); // Ditto for environment variables accessed via `getenv`. for (String envKey : starlarkRepositoryContext.getAccumulatedEnvKeys()) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/BUILD b/src/main/java/com/google/devtools/build/lib/rules/BUILD index c3c100c5ea4a68..cfb7c415690ea9 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/BUILD @@ -374,6 +374,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/skyframe:action_environment_function", "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value", "//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value", + "//src/main/java/com/google/devtools/build/lib/skyframe:directory_tree_digest_value", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value", "//src/main/java/com/google/devtools/build/lib/util", diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java index 9c8a6369109119..a6c733bc94e912 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; import com.google.devtools.build.lib.skyframe.DirectoryListingValue; +import com.google.devtools.build.lib.skyframe.DirectoryTreeDigestValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.RepositoryMappingValue; import com.google.devtools.build.lib.util.Fingerprint; @@ -92,7 +93,9 @@ public abstract static class Parser { public static RepoRecordedInput parse(String s) { List parts = Splitter.on(':').limit(2).splitToList(s); for (Parser parser : - new Parser[] {File.PARSER, Dirents.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER}) { + new Parser[] { + File.PARSER, Dirents.PARSER, DirTree.PARSER, EnvVar.PARSER, RecordedRepoMapping.PARSER + }) { if (parts.get(0).equals(parser.getPrefix())) { return parser.parse(parts.get(1)); } @@ -438,6 +441,76 @@ public static String getDirentsMarkerValue(Path path) throws IOException { } } + public static final class DirTree extends RepoRecordedInput { + public static final Parser PARSER = + new Parser() { + @Override + public String getPrefix() { + return "DIRTREE"; + } + + @Override + public RepoRecordedInput parse(String s) { + return new DirTree(RepoCacheFriendlyPath.parse(s)); + } + }; + + private final RepoCacheFriendlyPath path; + + public DirTree(RepoCacheFriendlyPath path) { + this.path = path; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DirTree)) { + return false; + } + DirTree that = (DirTree) o; + return Objects.equals(path, that.path); + } + + @Override + public int hashCode() { + return path.hashCode(); + } + + @Override + public String toStringInternal() { + return path.toString(); + } + + @Override + public Parser getParser() { + return PARSER; + } + + @Nullable + @Override + public SkyKey getSkyKey(BlazeDirectories directories) { + return path.getRepoDirSkyKeyOrNull(); + } + + @Override + public boolean isUpToDate( + Environment env, BlazeDirectories directories, @Nullable String oldValue) + throws InterruptedException { + RootedPath rootedPath = path.getRootedPath(env, directories); + if (rootedPath == null) { + return false; + } + DirectoryTreeDigestValue value = + (DirectoryTreeDigestValue) env.getValue(DirectoryTreeDigestValue.key(rootedPath)); + if (value == null) { + return false; + } + return oldValue.equals(value.hexDigest()); + } + } + /** Represents an environment variable accessed during the repo fetch. */ public static final class EnvVar extends RepoRecordedInput { static final Parser PARSER = diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index dab78bd1eccbea..dc21c789d9295f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD @@ -125,6 +125,7 @@ java_library( ":diff_awareness_manager", ":directory_listing_function", ":directory_listing_state_value", + ":directory_tree_digest_function", ":ephemeral_check_if_output_consumed", ":exclusive_test_build_driver_value", ":execution_finished_event", @@ -1248,6 +1249,34 @@ java_library( ], ) +java_library( + name = "directory_tree_digest_function", + srcs = ["DirectoryTreeDigestFunction.java"], + deps = [ + ":directory_listing_value", + ":directory_tree_digest_value", + ":dirents", + "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", + "//src/main/java/com/google/devtools/build/lib/util", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/skyframe", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:guava", + "//third_party:jsr305", + ], +) + +java_library( + name = "directory_tree_digest_value", + srcs = ["DirectoryTreeDigestValue.java"], + deps = [ + ":sky_functions", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:auto_value", + ], +) + java_library( name = "dirents", srcs = ["Dirents.java"], diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunction.java new file mode 100644 index 00000000000000..95a22c0db428f4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestFunction.java @@ -0,0 +1,149 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.SkyframeLookupResult; +import java.io.IOException; +import java.util.stream.StreamSupport; +import javax.annotation.Nullable; + +/** A {@link SkyFunction} for {@link DirectoryTreeDigestValue}s. */ +public final class DirectoryTreeDigestFunction implements SkyFunction { + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws InterruptedException, DirectoryTreeDigestFunctionException { + RootedPath rootedPath = (RootedPath) skyKey.argument(); + DirectoryListingValue dirListingValue = + (DirectoryListingValue) env.getValue(DirectoryListingValue.key(rootedPath)); + if (dirListingValue == null) { + return null; + } + + // Get the names of entries directly in this directory, and sort them. This sets the basis for + // subsequent digests. + ImmutableSet sortedDirents = + StreamSupport.stream(dirListingValue.getDirents().spliterator(), /* parallel= */ false) + .sorted() + .map(Dirent::getName) + .sorted() + .collect(toImmutableSet()); + + // Turn each entry into a FileValue. + ImmutableList fileValues = getFileValues(env, sortedDirents, rootedPath); + if (fileValues == null) { + return null; + } + + // For each entry that is a directory (or a symlink to a directory), find its own + // DirectoryTreeDigestValue. + ImmutableList subDirTreeDigests = getSubDirTreeDigests(env, fileValues); + if (subDirTreeDigests == null) { + return null; + } + + // Finally, we're ready to digest everything together! + Fingerprint fp = new Fingerprint(); + fp.addStrings(sortedDirents); + fp.addStrings(subDirTreeDigests); + try { + for (FileValue fileValue : fileValues) { + fp.addBoolean(fileValue.isDirectory()); + if (!fileValue.isDirectory()) { + byte[] digest = fileValue.realFileStateValue().getDigest(); + if (digest == null) { + // Fast digest not available, or it would have been in the FileValue. + digest = fileValue.realRootedPath().asPath().getDigest(); + } + fp.addBytes(digest); + } + } + } catch (IOException e) { + throw new DirectoryTreeDigestFunctionException(e); + } + + return DirectoryTreeDigestValue.of(fp.hexDigestAndReset()); + } + + @Nullable + private static ImmutableList getFileValues( + Environment env, ImmutableSet sortedDirents, RootedPath rootedPath) + throws InterruptedException { + ImmutableSet fileValueKeys = + sortedDirents.stream() + .map( + dirent -> + FileValue.key( + RootedPath.toRootedPath( + rootedPath.getRoot(), + rootedPath.getRootRelativePath().getRelative(dirent)))) + .collect(toImmutableSet()); + SkyframeLookupResult result = env.getValuesAndExceptions(fileValueKeys); + if (env.valuesMissing()) { + return null; + } + ImmutableList fileValues = + fileValueKeys.stream() + .map(result::get) + .map(FileValue.class::cast) + .collect(toImmutableList()); + if (env.valuesMissing()) { + return null; + } + return fileValues; + } + + @Nullable + private static ImmutableList getSubDirTreeDigests( + Environment env, ImmutableList fileValues) throws InterruptedException { + ImmutableSet dirTreeDigestValueKeys = + fileValues.stream() + .filter(FileValue::isDirectory) + .map(fv -> DirectoryTreeDigestValue.key(fv.realRootedPath())) + .collect(toImmutableSet()); + SkyframeLookupResult result = env.getValuesAndExceptions(dirTreeDigestValueKeys); + if (env.valuesMissing()) { + return null; + } + ImmutableList dirTreeDigests = + dirTreeDigestValueKeys.stream() + .map(result::get) + .map(DirectoryTreeDigestValue.class::cast) + .map(DirectoryTreeDigestValue::hexDigest) + .collect(toImmutableList()); + if (env.valuesMissing()) { + return null; + } + return dirTreeDigests; + } + + private static final class DirectoryTreeDigestFunctionException extends SkyFunctionException { + public DirectoryTreeDigestFunctionException(IOException e) { + super(e, Transience.TRANSIENT); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestValue.java new file mode 100644 index 00000000000000..246abf09872020 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryTreeDigestValue.java @@ -0,0 +1,51 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.AbstractSkyKey; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * Contains information about the recursive digest of a directory tree, including all transitive + * descendant files and their contents. + */ +@AutoValue +public abstract class DirectoryTreeDigestValue implements SkyValue { + public abstract String hexDigest(); + + public static DirectoryTreeDigestValue of(String hexDigest) { + return new AutoValue_DirectoryTreeDigestValue(hexDigest); + } + + public static Key key(RootedPath path) { + return new Key(path); + } + + /** Key type for {@link DirectoryTreeDigestValue}. */ + public static class Key extends AbstractSkyKey { + + private Key(RootedPath rootedPath) { + super(rootedPath); + } + + @Override + public SkyFunctionName functionName() { + return SkyFunctions.DIRECTORY_TREE_DIGEST; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index 0cf31c94232ff4..ec27b8fcec3318 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -30,6 +30,8 @@ public final class SkyFunctions { SkyFunctionName.createNonHermetic("DIRECTORY_LISTING_STATE"); public static final SkyFunctionName DIRECTORY_LISTING = SkyFunctionName.createHermetic("DIRECTORY_LISTING"); + public static final SkyFunctionName DIRECTORY_TREE_DIGEST = + SkyFunctionName.createHermetic("DIRECTORY_TREE_DIGEST"); // Hermetic even though package lookups secretly access the set of deleted packages, because // SequencedSkyframeExecutor deletes any affected PACKAGE_LOOKUP nodes when that set changes. public static final SkyFunctionName PACKAGE_LOOKUP = diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index d4286034031398..ec3e8e2b31f1d9 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -589,6 +589,7 @@ private ImmutableMap skyFunctions() { new FileSymlinkInfiniteExpansionUniquenessFunction()); map.put(FileValue.FILE, new FileFunction(pkgLocator, directories)); map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction()); + map.put(SkyFunctions.DIRECTORY_TREE_DIGEST, new DirectoryTreeDigestFunction()); map.put( SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(