diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java index b8568292e9913e..0d069fbf2d878e 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; import com.google.devtools.build.lib.actions.cache.ActionCache; -import com.google.devtools.build.lib.actions.cache.DigestUtils; +import com.google.devtools.build.lib.actions.cache.MetadataDigestUtils; import com.google.devtools.build.lib.actions.cache.MetadataHandler; import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; import com.google.devtools.build.lib.collect.nestedset.NestedSet; @@ -159,7 +159,7 @@ private static boolean validateArtifacts( for (Artifact artifact : actionInputs.toList()) { mdMap.put(artifact.getExecPathString(), getMetadataMaybe(metadataHandler, artifact)); } - return !Arrays.equals(DigestUtils.fromMetadata(mdMap), entry.getFileDigest()); + return !Arrays.equals(MetadataDigestUtils.fromMetadata(mdMap), entry.getFileDigest()); } private void reportCommand(EventHandler handler, Action action) { @@ -347,7 +347,8 @@ private boolean mustExecute( } Map usedEnvironment = computeUsedEnv(action, clientEnv, remoteDefaultPlatformProperties); - if (!Arrays.equals(entry.getUsedClientEnvDigest(), DigestUtils.fromEnv(usedEnvironment))) { + if (!Arrays.equals( + entry.getUsedClientEnvDigest(), MetadataDigestUtils.fromEnv(usedEnvironment))) { reportClientEnv(handler, action, usedEnvironment); actionCache.accountMiss(MissReason.DIFFERENT_ENVIRONMENT); return true; diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD index e9299eff4111d3..2887382e5b2d64 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/BUILD +++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD @@ -51,7 +51,7 @@ java_library( "LocalHostResourceFallback.java", "MiddlemanType.java", "ResourceSet.java", - "cache/DigestUtils.java", + "cache/MetadataDigestUtils.java", ], ), deps = [ @@ -213,14 +213,12 @@ java_library( "FileStateValue.java", "FileValue.java", "InconsistentFilesystemException.java", - "cache/DigestUtils.java", + "cache/MetadataDigestUtils.java", ], deps = [ ":artifacts", ":has_digest", - "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/concurrent", - "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec", "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/util:var_int", diff --git a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java index 5d3369a61837a5..aae6b76413bcec 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java @@ -20,12 +20,12 @@ import com.google.common.base.Preconditions; import com.google.common.hash.HashFunction; import com.google.common.io.BaseEncoding; -import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.util.BigIntegerFingerprint; import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java index 2de29690253828..f7b7b0a8d513f6 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java @@ -87,7 +87,7 @@ final class Entry { public Entry(String key, Map usedClientEnv, boolean discoversInputs) { actionKey = key; - this.usedClientEnvDigest = DigestUtils.fromEnv(usedClientEnv); + this.usedClientEnvDigest = MetadataDigestUtils.fromEnv(usedClientEnv); files = discoversInputs ? new ArrayList() : null; mdMap = new HashMap<>(); } @@ -141,7 +141,7 @@ public byte[] getUsedClientEnvDigest() { */ public byte[] getFileDigest() { if (digest == null) { - digest = DigestUtils.fromMetadata(mdMap); + digest = MetadataDigestUtils.fromMetadata(mdMap); mdMap = null; } return digest; @@ -182,7 +182,9 @@ public String toString() { .append("\n"); builder.append(" digestKey = "); if (digest == null) { - builder.append(formatDigest(DigestUtils.fromMetadata(mdMap))).append(" (from mdMap)\n"); + builder + .append(formatDigest(MetadataDigestUtils.fromMetadata(mdMap))) + .append(" (from mdMap)\n"); } else { builder.append(formatDigest(digest)).append("\n"); } diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java index f9fec407e092bf..87bacdc872baab 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java @@ -26,6 +26,7 @@ import com.google.devtools.build.lib.util.PersistentMap; import com.google.devtools.build.lib.util.StringIndexer; import com.google.devtools.build.lib.util.VarInt; +import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.UnixGlob; import java.io.ByteArrayOutputStream; @@ -381,14 +382,14 @@ private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) { VarInt.putVarInt(actionKeyBytes.length, sink); sink.write(actionKeyBytes); - DigestUtils.write(entry.getFileDigest(), sink); + MetadataDigestUtils.write(entry.getFileDigest(), sink); VarInt.putVarInt(entry.discoversInputs() ? files.size() : NO_INPUT_DISCOVERY_COUNT, sink); for (String file : files) { VarInt.putVarInt(indexer.getOrCreateIndex(file), sink); } - DigestUtils.write(entry.getUsedClientEnvDigest(), sink); + MetadataDigestUtils.write(entry.getUsedClientEnvDigest(), sink); return sink.toByteArray(); } catch (IOException e) { @@ -410,7 +411,7 @@ private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) thro source.get(actionKeyBytes); String actionKey = new String(actionKeyBytes, ISO_8859_1); - byte[] digest = DigestUtils.read(source); + byte[] digest = MetadataDigestUtils.read(source); int count = VarInt.getVarInt(source); if (count != NO_INPUT_DISCOVERY_COUNT && count < 0) { @@ -430,7 +431,7 @@ private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) thro files = builder.build(); } - byte[] usedClientEnvDigest = DigestUtils.read(source); + byte[] usedClientEnvDigest = MetadataDigestUtils.read(source); if (source.remaining() > 0) { throw new IOException("serialized entry data has not been fully decoded"); diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataDigestUtils.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataDigestUtils.java new file mode 100644 index 00000000000000..4071b63a227645 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataDigestUtils.java @@ -0,0 +1,85 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions.cache; + +import com.google.devtools.build.lib.actions.FileArtifactValue; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.VarInt; +import com.google.devtools.build.lib.vfs.DigestUtils; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Map; +import javax.annotation.Nullable; + +/** Utility class for digests/metadata relating to the action cache. */ +public final class MetadataDigestUtils { + private MetadataDigestUtils() {} + + /** + * @param source the byte buffer source. + * @return the digest from the given buffer. + */ + public static byte[] read(ByteBuffer source) { + int size = VarInt.getVarInt(source); + byte[] bytes = new byte[size]; + source.get(bytes); + return bytes; + } + + /** Write the digest to the output stream. */ + public static void write(byte[] digest, OutputStream sink) throws IOException { + VarInt.putVarInt(digest.length, sink); + sink.write(digest); + } + + /** + * @param mdMap A collection of (execPath, FileArtifactValue) pairs. Values may be null. + * @return an order-independent digest from the given "set" of (path, metadata) pairs. + */ + public static byte[] fromMetadata(Map mdMap) { + byte[] result = new byte[1]; // reserve the empty string + // Profiling showed that MessageDigest engine instantiation was a hotspot, so create one + // instance for this computation to amortize its cost. + Fingerprint fp = new Fingerprint(); + for (Map.Entry entry : mdMap.entrySet()) { + result = DigestUtils.xor(result, getDigest(fp, entry.getKey(), entry.getValue())); + } + return result; + } + + /** + * @param env A collection of (String, String) pairs. + * @return an order-independent digest of the given set of pairs. + */ + public static byte[] fromEnv(Map env) { + byte[] result = new byte[0]; + Fingerprint fp = new Fingerprint(); + for (Map.Entry entry : env.entrySet()) { + fp.addString(entry.getKey()); + fp.addString(entry.getValue()); + result = DigestUtils.xor(result, fp.digestAndReset()); + } + return result; + } + + private static byte[] getDigest(Fingerprint fp, String execPath, @Nullable FileArtifactValue md) { + fp.addString(execPath); + if (md != null) { + md.addTo(fp); + } + return fp.digestAndReset(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java b/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java index 188bc2bb04c0d2..3316af9f9352c7 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java +++ b/src/main/java/com/google/devtools/build/lib/exec/RunfilesTreeUpdater.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; @@ -80,7 +81,9 @@ private static void updateRunfilesTree( // symbolic link, it is likely a symbolic link to the input manifest, so we cannot trust it as // an up-to-date check. if (!outputManifest.isSymbolicLink() - && Arrays.equals(outputManifest.getDigest(), inputManifest.getDigest())) { + && Arrays.equals( + DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(outputManifest), + DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(inputManifest))) { return; } } catch (IOException e) { diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java index 7223b866e76e62..e5116f09406805 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java @@ -33,6 +33,7 @@ import com.google.devtools.build.lib.remote.options.RemoteOptions; import com.google.devtools.build.lib.util.io.MessageOutputStream; import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.Dirent; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -92,7 +93,7 @@ public void logSpawn( ActionInput input = e.getValue(); Path inputPath = execRoot.getRelative(input.getExecPathString()); if (inputPath.isDirectory()) { - listDirectoryContents(inputPath, (file) -> builder.addInputs(file), metadataProvider); + listDirectoryContents(inputPath, builder::addInputs, metadataProvider); } else { Digest digest = computeDigest(input, null, metadataProvider); builder.addInputsBuilder().setPath(input.getExecPathString()).setDigest(digest); @@ -110,7 +111,7 @@ public void logSpawn( for (Map.Entry e : existingOutputs.entrySet()) { Path path = e.getKey(); if (path.isDirectory()) { - listDirectoryContents(path, (file) -> builder.addActualOutputs(file), metadataProvider); + listDirectoryContents(path, builder::addActualOutputs, metadataProvider); } else { File.Builder outputBuilder = builder.addActualOutputsBuilder(); outputBuilder.setPath(path.relativeTo(execRoot).toString()); @@ -233,9 +234,11 @@ private Digest computeDigest( path = execRoot.getRelative(input.getExecPath()); } // Compute digest manually. + long fileSize = path.getFileSize(); return digest - .setHash(HashCode.fromBytes(path.getDigest()).toString()) - .setSizeBytes(path.getFileSize()) + .setHash( + HashCode.fromBytes(DigestUtils.getDigestWithManualFallback(path, fileSize)).toString()) + .setSizeBytes(fileSize) .build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java index 1b294af8be8994..5066f3c380602f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java @@ -21,10 +21,10 @@ import com.google.common.hash.HashCode; import com.google.common.hash.HashingOutputStream; import com.google.common.io.BaseEncoding; -import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.actions.cache.VirtualActionInput; import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey; import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.protobuf.Message; import java.io.ByteArrayOutputStream; diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java index c819b1f1753191..beb05537ee160e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; -import com.google.devtools.build.lib.actions.FileStateValue.RegularFileStateValue; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.RuleDefinition; @@ -251,8 +250,9 @@ private static boolean verifyLabelMarkerData(Rule rule, String key, String value public static String fileValueToMarkerValue(FileValue fileValue) throws IOException { Preconditions.checkArgument(fileValue.isFile() && !fileValue.isSpecialFile()); // Return the file content digest in hex. fileValue may or may not have the digest available. - byte[] digest = ((RegularFileStateValue) fileValue.realFileStateValue()).getDigest(); + byte[] digest = fileValue.realFileStateValue().getDigest(); if (digest == null) { + // Fast digest not available, or it would have been in the FileValue. digest = fileValue.realRootedPath().asPath().getDigest(); } return BaseEncoding.base16().lowerCase().encode(digest); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CacheFileDigestsModule.java b/src/main/java/com/google/devtools/build/lib/runtime/CacheFileDigestsModule.java index 520e02c6bec11e..ca38d5fcf6c415 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/CacheFileDigestsModule.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/CacheFileDigestsModule.java @@ -17,10 +17,10 @@ import com.google.common.base.Preconditions; import com.google.common.cache.CacheStats; import com.google.common.flogger.GoogleLogger; -import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.ExecutorBuilder; +import com.google.devtools.build.lib.vfs.DigestUtils; /** Enables the caching of file digests in {@link DigestUtils}. */ public class CacheFileDigestsModule extends BlazeModule { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java index 13c78e1ef3ec06..5499dceefa7d12 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionMetadataHandler.java @@ -36,9 +36,9 @@ import com.google.devtools.build.lib.actions.FilesetManifest; import com.google.devtools.build.lib.actions.FilesetManifest.RelativeSymlinkBehavior; import com.google.devtools.build.lib.actions.FilesetOutputSymlink; -import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.actions.cache.MetadataHandler; import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.build.lib.vfs.Dirent; import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.FileStatusWithDigest; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java index 78d2bc949026fd..81a05fe9db20df 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TreeArtifactValue.java @@ -29,7 +29,7 @@ import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.actions.HasDigest; -import com.google.devtools.build.lib.actions.cache.DigestUtils; +import com.google.devtools.build.lib.actions.cache.MetadataDigestUtils; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.lib.util.Fingerprint; @@ -131,7 +131,7 @@ static ArchivedRepresentation create( @AutoCodec.VisibleForSerialization static final TreeArtifactValue EMPTY = new TreeArtifactValue( - DigestUtils.fromMetadata(ImmutableMap.of()), + MetadataDigestUtils.fromMetadata(ImmutableMap.of()), ImmutableSortedMap.of(), /*archivedRepresentation=*/ null, /*entirelyRemote=*/ false); diff --git a/src/main/java/com/google/devtools/build/lib/ssd/BUILD b/src/main/java/com/google/devtools/build/lib/ssd/BUILD index 9069ec7bd620aa..85fb315f63e4b5 100644 --- a/src/main/java/com/google/devtools/build/lib/ssd/BUILD +++ b/src/main/java/com/google/devtools/build/lib/ssd/BUILD @@ -13,7 +13,7 @@ java_library( srcs = glob(["*.java"]), deps = [ "//src/main/java/com/google/devtools/build/lib:runtime", - "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/common/options", "//third_party:guava", ], diff --git a/src/main/java/com/google/devtools/build/lib/ssd/SsdModule.java b/src/main/java/com/google/devtools/build/lib/ssd/SsdModule.java index 2e73896516d005..219329e6dcbccf 100644 --- a/src/main/java/com/google/devtools/build/lib/ssd/SsdModule.java +++ b/src/main/java/com/google/devtools/build/lib/ssd/SsdModule.java @@ -14,9 +14,9 @@ package com.google.devtools.build.lib.ssd; import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.vfs.DigestUtils; import com.google.devtools.common.options.OptionsBase; /** diff --git a/src/main/java/com/google/devtools/build/lib/vfs/BUILD b/src/main/java/com/google/devtools/build/lib/vfs/BUILD index be8dc58ab3a707..70a2769101a16e 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/BUILD +++ b/src/main/java/com/google/devtools/build/lib/vfs/BUILD @@ -51,7 +51,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/concurrent", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec", - "//src/main/java/com/google/devtools/build/lib/util:TestType", "//src/main/java/com/google/devtools/build/lib/util:filetype", "//src/main/java/com/google/devtools/common/options", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java b/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java similarity index 79% rename from src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java rename to src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java index addd5f0ead7b82..24edd82bbef8d4 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.lib.actions.cache; +package com.google.devtools.build.lib.vfs; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -19,21 +19,10 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheStats; import com.google.common.primitives.Longs; -import com.google.devtools.build.lib.actions.FileArtifactValue; -import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; -import com.google.devtools.build.lib.util.Fingerprint; -import com.google.devtools.build.lib.util.VarInt; -import com.google.devtools.build.lib.vfs.FileStatus; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import javax.annotation.Nullable; /** * Utility class for getting digests of files. @@ -152,7 +141,7 @@ private DigestUtils() {} * provide it via extended attribute. */ private static byte[] getDigestInExclusiveMode(Path path) throws IOException { - long startTime = BlazeClock.nanoTime(); + long startTime = Profiler.nanoTimeMaybe(); synchronized (DIGEST_LOCK) { Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, path.getPathString()); return getDigestInternal(path); @@ -160,14 +149,14 @@ private static byte[] getDigestInExclusiveMode(Path path) throws IOException { } private static byte[] getDigestInternal(Path path) throws IOException { - long startTime = BlazeClock.nanoTime(); + long startTime = System.nanoTime(); byte[] digest = path.getDigest(); // When using multi-threaded digesting, it makes no sense to use the throughput of a single // digest operation to determine whether a read was abnormally slow (as the scheduler might just // have preferred other reads). if (!MULTI_THREADED_DIGEST.get()) { - long millis = (BlazeClock.nanoTime() - startTime) / 1000000; + long millis = (System.nanoTime() - startTime) / 1000000; if (millis > SLOW_READ_MILLIS && (path.getFileSize() / millis) < SLOW_READ_THROUGHPUT) { System.err.printf( "Slow read: a %d-byte read from %s took %d ms.%n", path.getFileSize(), path, millis); @@ -230,6 +219,20 @@ public static byte[] getDigestWithManualFallback(Path path, long fileSize) throw return digest != null ? digest : manuallyComputeDigest(path, fileSize); } + /** + * Gets the digest of {@code path}, using a constant-time xattr call if the filesystem supports + * it, and calculating the digest manually otherwise. + * + *

Unlike {@link #getDigestWithManualFallback}, will not rate-limit manual digesting of files, + * so only use this method if the file size is truly unknown and you don't expect many concurrent + * manual digests of large files. + * + * @param path Path of the file. + */ + public static byte[] getDigestWithManualFallbackWhenSizeUnknown(Path path) throws IOException { + return getDigestWithManualFallback(path, -1); + } + /** * Calculates the digest manually. * @@ -271,64 +274,8 @@ public static byte[] manuallyComputeDigest(Path path, long fileSize) throws IOEx return digest; } - /** - * @param source the byte buffer source. - * @return the digest from the given buffer. - * @throws IOException if the byte buffer is incorrectly formatted. - */ - public static byte[] read(ByteBuffer source) throws IOException { - int size = VarInt.getVarInt(source); - byte[] bytes = new byte[size]; - source.get(bytes); - return bytes; - } - - /** Write the digest to the output stream. */ - public static void write(byte[] digest, OutputStream sink) throws IOException { - VarInt.putVarInt(digest.length, sink); - sink.write(digest); - } - - /** - * @param mdMap A collection of (execPath, FileArtifactValue) pairs. Values may be null. - * @return an order-independent digest from the given "set" of (path, metadata) pairs. - */ - public static byte[] fromMetadata(Map mdMap) { - byte[] result = new byte[1]; // reserve the empty string - // Profiling showed that MessageDigest engine instantiation was a hotspot, so create one - // instance for this computation to amortize its cost. - Fingerprint fp = new Fingerprint(); - for (Map.Entry entry : mdMap.entrySet()) { - result = xor(result, getDigest(fp, entry.getKey(), entry.getValue())); - } - return result; - } - - /** - * @param env A collection of (String, String) pairs. - * @return an order-independent digest of the given set of pairs. - */ - public static byte[] fromEnv(Map env) { - byte[] result = new byte[0]; - Fingerprint fp = new Fingerprint(); - for (Map.Entry entry : env.entrySet()) { - fp.addString(entry.getKey()); - fp.addString(entry.getValue()); - result = xor(result, fp.digestAndReset()); - } - return result; - } - - private static byte[] getDigest(Fingerprint fp, String execPath, @Nullable FileArtifactValue md) { - fp.addString(execPath); - if (md != null) { - md.addTo(fp); - } - return fp.digestAndReset(); - } - /** Compute lhs ^= rhs bitwise operation of the arrays. May clobber either argument. */ - private static byte[] xor(byte[] lhs, byte[] rhs) { + public static byte[] xor(byte[] lhs, byte[] rhs) { int n = rhs.length; if (lhs.length >= n) { for (int i = 0; i < n; i++) { diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Path.java b/src/main/java/com/google/devtools/build/lib/vfs/Path.java index 58bb000e5d4692..91e2722a478396 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Path.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/Path.java @@ -766,7 +766,11 @@ public byte[] getFastDigest() throws IOException { } /** - * Returns the digest of the file denoted by the current path, following symbolic links. + * Returns the digest of the file denoted by the current path, following symbolic links. Is not + * guaranteed to call {@link #getFastDigest} internally, even if a fast digest is likely + * available. Callers should prefer {@link DigestUtils#getDigestWithManualFallback} to this method + * unless they know that a fast digest is unavailable and do not need the other features + * (disk-read rate-limiting, global cache) that {@link DigestUtils} provides. * * @return a new byte array containing the file's digest * @throws IOException if the digest could not be computed for any reason @@ -802,7 +806,7 @@ public String getDirectoryDigest() throws IOException { } else { hasher.putChar('-'); } - hasher.putBytes(path.getDigest()); + hasher.putBytes(DigestUtils.getDigestWithManualFallback(path, stat.getSize())); } else if (stat.isDirectory()) { hasher.putChar('d').putUnencodedChars(path.getDirectoryDigest()); } else if (stat.isSymbolicLink()) { @@ -816,7 +820,7 @@ public String getDirectoryDigest() throws IOException { } else { hasher.putChar('-'); } - hasher.putBytes(resolved.getDigest()); + hasher.putBytes(DigestUtils.getDigestWithManualFallbackWhenSizeUnknown(resolved)); } else { // link to a non-file: include the link itself in the hash hasher.putChar('l').putUnencodedChars(link.toString()); diff --git a/src/test/java/com/google/devtools/build/lib/vfs/BUILD b/src/test/java/com/google/devtools/build/lib/vfs/BUILD index 6a6c08aa2d6e53..b74760eaf37482 100644 --- a/src/test/java/com/google/devtools/build/lib/vfs/BUILD +++ b/src/test/java/com/google/devtools/build/lib/vfs/BUILD @@ -68,6 +68,7 @@ java_library( "//src/test/java/com/google/devtools/build/lib/vfs/util", "//third_party:guava", "//third_party:guava-testlib", + "//third_party:jsr305", "//third_party:junit4", "//third_party:truth", "//third_party/protobuf:protobuf_java", diff --git a/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/DigestUtilsTest.java similarity index 96% rename from src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java rename to src/test/java/com/google/devtools/build/lib/vfs/DigestUtilsTest.java index f8f0a5fa1c5aed..26e85301b69784 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java +++ b/src/test/java/com/google/devtools/build/lib/vfs/DigestUtilsTest.java @@ -11,19 +11,14 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.lib.actions; +package com.google.devtools.build.lib.vfs; import static com.google.common.truth.Truth.assertThat; import com.google.common.base.Strings; import com.google.common.cache.CacheStats; -import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.testutil.TestThread; import com.google.devtools.build.lib.testutil.TestUtils; -import com.google.devtools.build.lib.vfs.DigestHashFunction; -import com.google.devtools.build.lib.vfs.FileSystem; -import com.google.devtools.build.lib.vfs.FileSystemUtils; -import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import java.io.IOException; import java.util.Arrays; @@ -36,9 +31,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Tests for DigestUtils. - */ +/** Tests for {@link DigestUtils}. */ @RunWith(JUnit4.class) public class DigestUtilsTest {