diff --git a/src/main/java/com/google/devtools/build/lib/remote/ActionResultDownloader.java b/src/main/java/com/google/devtools/build/lib/remote/ActionResultDownloader.java new file mode 100644 index 00000000000000..f29808e4e7e262 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/ActionResultDownloader.java @@ -0,0 +1,9 @@ +package com.google.devtools.build.lib.remote; + +import build.bazel.remote.execution.v2.ActionResult; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.devtools.build.lib.remote.common.RemotePathResolver; + +public interface ActionResultDownloader { + public ListenableFuture downloadActionResult(ActionResult actionResult, RemotePathResolver remotePathResolver); +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index 890f3e5dc24443..9748cea712b5b0 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -129,6 +129,8 @@ java_library( "//src/main/protobuf:cache_salt_java_proto", "//src/main/protobuf:failure_details_java_proto", "//src/main/protobuf:remote_execution_log_java_proto", + "//src/main/protobuf:remote_output_service_java_grpc", + "//src/main/protobuf:remote_output_service_java_proto", "//third_party:auth", "//third_party:caffeine", "//third_party:flogger", diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteOutputService.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteOutputService.java new file mode 100644 index 00000000000000..0e8168c14183ee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteOutputService.java @@ -0,0 +1,674 @@ +package com.google.devtools.build.lib.remote; + +import static com.google.common.collect.Iterators.filter; +import static com.google.common.collect.Iterators.partition; +import static java.lang.String.format; + +import build.bazel.remote.execution.v2.ActionResult; +import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.DigestFunction; +import build.bazel.remote.execution.v2.OutputDirectory; +import build.bazel.remote.execution.v2.OutputFile; +import build.bazel.remote.execution.v2.OutputSymlink; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.UnmodifiableIterator; +import com.google.common.flogger.GoogleLogger; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionInputMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactPathResolver; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.FilesetOutputSymlink; +import com.google.devtools.build.lib.actions.LostInputsActionExecutionException; +import com.google.devtools.build.lib.actions.cache.OutputMetadataStore; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.remote.RemoteOutputServiceGrpc.RemoteOutputServiceBlockingStub; +import com.google.devtools.build.lib.remote.RemoteOutputServiceGrpc.RemoteOutputServiceFutureStub; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.BatchCreateRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.BatchStatRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.BatchStatResponse; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.CleanRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.FileStatus; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.FinalizeBuildRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.InitialOutputPathContents; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.StartBuildRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.StartBuildResponse; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.StatResponse; +import com.google.devtools.build.lib.remote.common.RemotePathResolver; +import com.google.devtools.build.lib.remote.util.DigestUtil; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.vfs.BatchStat; +import com.google.devtools.build.lib.vfs.DelegateFileSystem; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.FileStatusWithDigest; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.OutputService; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import io.grpc.Status.Code; +import io.grpc.StatusRuntimeException; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.Collection; +import java.util.concurrent.ExecutionException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.annotation.Nullable; + +public class GrpcRemoteOutputService implements OutputService, ActionResultDownloader { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + // State that needs to be retained across invocations of Bazel. The + // current build ID is placed inside, because it ensures that cached + // instances of Path that use GrpcFileSystem always generate RPCs to + // the Remote Output Service using the current build ID; not the one + // under which they were created. + public static class StateAcrossBuilds { + @Nullable private UUID currentBuildId; + @Nullable private UUID previousBuildId; + } + + private final StateAcrossBuilds stateAcrossBuilds; + private final ReferenceCountedChannel channel; + private final String outputBaseId; + private final PathFragment outputPathPrefix; + private final String instanceName; + private final DigestFunction.Value digestFunction; + private final Digest emptyDigest; + private final RemoteRetrier retrier; + private final CommandEnvironment env; + + private PathFragment currentRelativeOutputPath; + private boolean currentBuildSuccessful; + + public GrpcRemoteOutputService(StateAcrossBuilds stateAcrossBuilds, ReferenceCountedChannel channel, String outputBaseId, PathFragment outputPathPrefix, String instanceName, DigestUtil digestUtil, RemoteRetrier retrier, CommandEnvironment env) { + this.stateAcrossBuilds = stateAcrossBuilds; + this.channel = channel; + this.outputBaseId = outputBaseId; + this.outputPathPrefix = outputPathPrefix; + this.instanceName = instanceName; + this.digestFunction = digestUtil.getDigestFunction(); + this.emptyDigest = digestUtil.compute(new byte[] {}); + this.retrier = retrier; + this.env = env; + } + + + @Override + public String getFilesSystemName() { + return "GrpcRemoteOutputService"; + } + + @Override + public ModifiedFileSet startBuild( + EventHandler eventHandler, UUID buildId, boolean finalizeActions) + throws BuildFailedException, AbruptExitException, InterruptedException { + // Notify the remote output service that the build is about to + // start. The remote output service will return the directory in + // which it wants us to let the build take place. + // + // Make the output service aware of the location of the output path + // from our perspective and any symbolic links that will point to + // it. This allows the service to properly resolve symbolic links + // containing absolute paths as part of BatchStat() calls. + StartBuildRequest.Builder builder = StartBuildRequest.newBuilder(); + builder.setOutputBaseId(outputBaseId); + builder.setBuildId(buildId.toString()); + builder.setInstanceName(instanceName); + builder.setDigestFunction(digestFunction); + builder.setOutputPathPrefix(outputPathPrefix.toString()); + String relativeOutputPath = env.getDirectories().getRelativeOutputPath(); + Path originalOutputPath = env.getExecRoot().getRelative(relativeOutputPath); + builder.putOutputPathAliases(originalOutputPath.toString(), "."); + StartBuildRequest request = builder.build(); + StartBuildResponse response; + try { + response = retrier.execute( + () -> + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .startBuild(request))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + // Replace the output path with a symbolic link pointing to the + // directory managed by the remote output service. + PathFragment outputPath = outputPathPrefix.getRelative(response.getOutputPathSuffix()); + try { + try { + originalOutputPath.deleteTree(); + } catch (FileNotFoundException e) {} + originalOutputPath.createSymbolicLink(outputPath); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + stateAcrossBuilds.currentBuildId = buildId; + currentRelativeOutputPath = PathFragment.create(relativeOutputPath); + + if (stateAcrossBuilds.previousBuildId == null || !response.hasInitialOutputPathContents()) { + // Either Bazel or the remote output service has performed no + // build before. + return ModifiedFileSet.EVERYTHING_MODIFIED; + } + InitialOutputPathContents initialContents = response.getInitialOutputPathContents(); + if (!initialContents.getBuildId().equals(stateAcrossBuilds.previousBuildId.toString())) { + // Bazel and the remote output service disagree on the build ID of + // the previous build. + return ModifiedFileSet.EVERYTHING_MODIFIED; + } + // Bazel and the remote output service agree on the build ID of the + // previous build. Return the set of paths that have been modified. + // + // TODO: Do these paths need to be relative to the exec root or the + // output path? Assume the exec root for now. + return ModifiedFileSet.builder() + .modifyAll( + Iterables.transform( + initialContents.getModifiedPathsList(), + (p) -> currentRelativeOutputPath.getRelative(p))) + .build(); + } + + @Override + public void finalizeBuild(boolean buildSuccessful) + throws BuildFailedException, AbruptExitException, InterruptedException { + currentBuildSuccessful = buildSuccessful; + } + + public void sendFinalizeBuild() { + if (stateAcrossBuilds.currentBuildId != null) { + FinalizeBuildRequest.Builder builder = FinalizeBuildRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setBuildSuccessful(currentBuildSuccessful); + + try { + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .finalizeBuild(builder.build())); + stateAcrossBuilds.previousBuildId = stateAcrossBuilds.currentBuildId; + } catch (InterruptedException e) { + stateAcrossBuilds.previousBuildId = null; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof InterruptedException) { + stateAcrossBuilds.previousBuildId = null; + } else { + Throwables.throwIfUnchecked(cause); + throw new UncheckedIOException(new IOException(e)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + stateAcrossBuilds.currentBuildId = null; + currentRelativeOutputPath = null; + } + } + } + + @Override + public void finalizeAction(Action action, OutputMetadataStore outputMetadataStore) + throws IOException, EnvironmentalExecException, InterruptedException { + // TODO: Would this be the right place to call into the remote + // output service to check whether any I/O errors occurred? If so, + // we should likely let createActionFileSystem() call into the + // remote output service to start capturing I/O errors. + } + + private String fixupExecRootPath(PathFragment path) { + return path.relativeTo(currentRelativeOutputPath).toString(); + } + + private static abstract class DumbFileStatus implements FileStatusWithDigest { + @Override + public long getSize() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastModifiedTime() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastChangeTime() throws IOException { + return -1; + } + + @Override + public long getNodeId() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getDigest() throws IOException { + throw new UnsupportedOperationException(); + } + } + + private static class RegularFileStatus extends DumbFileStatus { + private final long size; + private final byte[] digest; + + RegularFileStatus(long size, byte[] digest) { + this.size = size; + this.digest = digest; + } + + @Override + public boolean isFile() { + return true; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isSpecialFile() { + return false; + } + + @Override + public long getSize() throws IOException { + return size; + } + + @Override + public byte[] getDigest() throws IOException { + return digest; + } + } + + private static class DirectoryFileStatus extends DumbFileStatus { + private final long lastModifiedTime; + + DirectoryFileStatus(long lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } + + @Override + public boolean isFile() { + return false; + } + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isSpecialFile() { + return false; + } + + @Override + public long getLastModifiedTime() throws IOException { + return this.lastModifiedTime; + } + + } + + private static class SymlinkFileStatus extends DumbFileStatus { + @Override + public boolean isFile() { + return false; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return true; + } + + @Override + public boolean isSpecialFile() { + return false; + } + } + + private class GrpcBatchStat implements BatchStat { + + public List batchStat(Iterable paths) + throws IOException, InterruptedException { + // TODO: Do we need to partition the input, just like in + // createSymlinkTree(), or is input already guaranteed to be + // bounded in size? + BatchStatRequest.Builder builder = BatchStatRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setIncludeFileDigest(true); + for (PathFragment path : paths) { + builder.addPaths(fixupExecRootPath(path)); + } + + BatchStatResponse responses; + try { + responses = channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .batchStat(builder.build())); + } catch (StatusRuntimeException e) { + throw new IOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, IOException.class, InterruptedException.class); + throw new RuntimeException(e); + } + return Lists.newArrayList( + Iterables.transform( + responses.getResponsesList(), + (response) -> { + if (!response.hasFileStatus()) { + // File not found. + return null; + } + FileStatus fileStatus = response.getFileStatus(); + if (fileStatus.hasFile()) { + FileStatus.File regularFileStatus = fileStatus.getFile(); + if (!regularFileStatus.hasDigest()) { + // The remote output service knows the file exists, but it + // is incapable of returning the digest. This may be caused + // by the kernel holding on to dirty pages in its write-back + // cache. + // + // By returning null, ActionMetadataHandler's + // fileArtifactValueFromArtifact() will compute the file's + // digest separately. + return null; + } + Digest digest = regularFileStatus.getDigest(); + return new RegularFileStatus(digest.getSizeBytes(), DigestUtil.toBinaryDigest(digest)); + } + if (fileStatus.hasDirectory()) { + return new DirectoryFileStatus(fileStatus.getDirectory().getLastModifiedTime().getSeconds()); + } + if (fileStatus.hasSymlink()) { + return new SymlinkFileStatus(); + } + throw new UnsupportedOperationException(); + })); + } + } + + @Override + public BatchStat getBatchStatter() { + return new GrpcBatchStat(); + } + + @Override + public boolean canCreateSymlinkTree() { + return true; + } + + @Override + public void createSymlinkTree(Map symlinks, PathFragment symlinkTreeRoot) + throws ExecException, InterruptedException { + // The provided set of symbolic links may be too large to provide to + // the remote output service at once. Partition the symbolic links + // in groups of 1000, so that BatchCreateRequest messages remain + // small enough. + UnmodifiableIterator>> symlinkBatchIterator = + partition(symlinks.entrySet().iterator(), 1000); + boolean cleanPathPrefix = true; + while (symlinkBatchIterator.hasNext()) { + List> symlinksBatch = symlinkBatchIterator.next(); + BatchCreateRequest.Builder builder = BatchCreateRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setPathPrefix(symlinkTreeRoot.relativeTo(currentRelativeOutputPath).toString()); + // During the first iteration, we should ensure that any existing + // contents of the symlink tree directory are removed. This may be + // triggered by setting the 'clean_path_prefix' option. + builder.setCleanPathPrefix(cleanPathPrefix); + for (Map.Entry symlink : symlinksBatch) { + PathFragment target = symlink.getValue(); + if (target == null) { + // No target indicates an empty file needs to be created. + OutputFile.Builder fileBuilder = builder.addFilesBuilder(); + fileBuilder.setPath(symlink.getKey().toString()); + fileBuilder.setDigest(emptyDigest); + fileBuilder.setIsExecutable(true); + } else { + OutputSymlink.Builder symlinkBuilder = builder.addSymlinksBuilder(); + symlinkBuilder.setPath(symlink.getKey().toString()); + symlinkBuilder.setTarget(symlink.getValue().toString()); + } + } + try { + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .batchCreate(builder.build())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, InterruptedException.class); + throw new RuntimeException(e); + } + cleanPathPrefix = false; + } + } + + @Override + public void clean() throws ExecException, InterruptedException { + CleanRequest.Builder builder = CleanRequest.newBuilder(); + builder.setOutputBaseId(outputBaseId); + try { + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .clean(builder.build())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, InterruptedException.class); + throw new RuntimeException(e); + } + } + + @Override + public ActionFileSystemType actionFileSystemType() { + return ActionFileSystemType.REMOTE_FILE_SYSTEM; + } + + private class GrpcFileSystem extends DelegateFileSystem { + private final PathFragment originalOutputPath; + + public GrpcFileSystem(FileSystem sourceDelegate, PathFragment originalOutputPath) { + super(sourceDelegate); + this.originalOutputPath = originalOutputPath; + } + + @Override + protected byte[] getFastDigest(PathFragment path) throws IOException { + // Don't attempt to compute digests ourselves. Call into the + // remote output service to request the digest. The service may be + // able to return the digest from its bookkeeping, as opposed to + // reading the actual file contents. + BatchStatRequest.Builder builder = BatchStatRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setIncludeFileDigest(true); + builder.setFollowSymlinks(true); + PathFragment relativePath; + try { + relativePath = path.relativeTo(originalOutputPath); + } catch (IllegalArgumentException e) { + // Path is outside the output path. Send it to the regular file system. + return super.getFastDigest(path); + } + builder.addPaths(relativePath.toString()); + + BatchStatResponse responses; + try { + responses = channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .batchStat(builder.build())); + } catch (InterruptedException e) { + return null; + } catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Code.CANCELLED && Thread.currentThread().isInterrupted()) { + // The current thread is interrupted, meaning that gRPC calls + // into the remote output service are cancelled automatically. + // Return null, indicating that a fast digest is not + // available. + // + // Throwing an IOException in this case may cause other parts + // of Bazel to throw InconsistentFilesystemExceptions, which + // leads to spurious build error messages. + return null; + } + throw new IOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof InterruptedException) { + return null; + } + Throwables.propagateIfPossible(cause, IOException.class); + throw new RuntimeException(e); + } + if (responses.getResponsesCount() != 1) { + throw new IOException("Remote output service returned an unexpected number of stat responses"); + } + StatResponse response = responses.getResponses(0); + if (!response.hasFileStatus()) { + // Output service reports that the file does not exist. + throw new FileNotFoundException(); + } + FileStatus fileStatus = response.getFileStatus(); + if (fileStatus.hasFile()) { + FileStatus.File regularFileStatus = fileStatus.getFile(); + if (!regularFileStatus.hasDigest()) { + // The remote output service knows the file exists, but it is + // incapable of returning the digest. This may be caused by + // the kernel holding on to dirty pages in its write-back + // cache. We must compute the digest ourselves. + return super.getFastDigest(path); + } + return DigestUtil.toBinaryDigest(regularFileStatus.getDigest()); + } + if (fileStatus.hasExternal()) { + // Path is a symbolic link that points to a location outside of + // the output path. The output service is unable to give a + // digest. We must compute it ourselves. + return super.getFastDigest( + originalOutputPath.getRelative(fileStatus.getExternal().getNextPath())); + } + throw new IOException(format("Remote output service did not return the status of file %s", path)); + } + } + + @Override + @Nullable + public FileSystem createActionFileSystem( + FileSystem sourceDelegate, + PathFragment execRootFragment, + String relativeOutputPath, + ImmutableList sourceRoots, + ActionInputMap inputArtifactData, + Iterable outputArtifacts, + boolean trackFailedRemoteReads) { + return new GrpcFileSystem(sourceDelegate, execRootFragment.getRelative(relativeOutputPath)); + } + + @Override + public void checkActionFileSystemForLostInputs(FileSystem actionFileSystem, Action action) + throws LostInputsActionExecutionException { + // There is no need to provide an implementation for this function, + // as StartBuild() on the server is supposed to call + // FindMissingBlobs() for all remotely tracked files. This ensures + // that they cannot disappear during the build. + } + + public ListenableFuture downloadActionResult(ActionResult actionResult, RemotePathResolver remotePathResolver) { + Path execRoot = env.getExecRoot(); + // Request that all outputs of the action are created. Do make sure + // to remove the "bazel-out/" prefix from all paths, as that part of + // the path is not managed by the remote output service. + BatchCreateRequest.Builder builder = BatchCreateRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + for (OutputFile file : actionResult.getOutputFilesList()) { + // As there is no guarantee that permissions on files created by + // the remote output service can be modified, make sure that all + // downloaded files are marked executable. + // TODO: Do we only want to do this selectively? + builder.addFilesBuilder() + .mergeFrom(file) + .setPath(fixupExecRootPath(remotePathResolver.outputPathToLocalPath(file.getPath()).relativeTo(execRoot))) + .setIsExecutable(true); + } + for (OutputDirectory directory : actionResult.getOutputDirectoriesList()) { + builder.addDirectoriesBuilder() + .mergeFrom(directory) + .setPath(fixupExecRootPath(remotePathResolver.outputPathToLocalPath(directory.getPath()).relativeTo(execRoot))); + } + for (OutputSymlink symlink : Iterables.concat(actionResult.getOutputFileSymlinksList(), actionResult.getOutputDirectorySymlinksList())) { + builder.addSymlinksBuilder() + .mergeFrom(symlink) + .setPath(fixupExecRootPath(remotePathResolver.outputPathToLocalPath(symlink.getPath()).relativeTo(execRoot))); + } + + return Futures.catchingAsync( + Futures.transform( + channel.withChannelFuture( + channel -> + RemoteOutputServiceGrpc.newFutureStub(channel) + .batchCreate(builder.build())), + (result) -> null, + MoreExecutors.directExecutor()), + StatusRuntimeException.class, + (sre) -> Futures.immediateFailedFuture(new IOException(sre)), + MoreExecutors.directExecutor()); + } + + @Override + public boolean supportsPathResolverForArtifactValues() { + return true; + } + + @Override + public ArtifactPathResolver createPathResolverForArtifactValues( + PathFragment execRoot, + String relativeOutputPath, + FileSystem fileSystem, + ImmutableList pathEntries, + ActionInputMap actionInputMap, + Map> expandedArtifacts, + Map> filesets) { + FileSystem remoteFileSystem = + new GrpcFileSystem(fileSystem, execRoot.getRelative(relativeOutputPath)); + return ArtifactPathResolver.createPathResolver(remoteFileSystem, fileSystem.getPath(execRoot)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index 4892160c702e30..bf12571c848074 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java @@ -45,6 +45,7 @@ final class RemoteActionContextProvider { @Nullable private final ListeningScheduledExecutorService retryScheduler; private final DigestUtil digestUtil; @Nullable private final Path logDir; + @Nullable private final ActionResultDownloader actionResultDownloader; private TempPathGenerator tempPathGenerator; private RemoteExecutionService remoteExecutionService; @Nullable private RemoteActionInputFetcher actionInputFetcher; @@ -58,7 +59,8 @@ private RemoteActionContextProvider( @Nullable ListeningScheduledExecutorService retryScheduler, DigestUtil digestUtil, @Nullable Path logDir, - @Nullable RemoteOutputChecker remoteOutputChecker) { + @Nullable RemoteOutputChecker remoteOutputChecker, + @Nullable ActionResultDownloader actionResultDownloader) { this.executor = executor; this.env = Preconditions.checkNotNull(env, "env"); this.remoteCache = remoteCache; @@ -67,6 +69,7 @@ private RemoteActionContextProvider( this.digestUtil = digestUtil; this.logDir = logDir; this.remoteOutputChecker = remoteOutputChecker; + this.actionResultDownloader = actionResultDownloader; } public static RemoteActionContextProvider createForPlaceholder( @@ -81,7 +84,8 @@ public static RemoteActionContextProvider createForPlaceholder( retryScheduler, digestUtil, /* logDir= */ null, - /* remoteOutputChecker= */ null); + /* remoteOutputChecker= */ null, + /*actionResultDownloader=*/ null); } public static RemoteActionContextProvider createForRemoteCaching( @@ -90,7 +94,8 @@ public static RemoteActionContextProvider createForRemoteCaching( RemoteCache remoteCache, ListeningScheduledExecutorService retryScheduler, DigestUtil digestUtil, - @Nullable RemoteOutputChecker remoteOutputChecker) { + @Nullable RemoteOutputChecker remoteOutputChecker, + ActionResultDownloader actionResultDownloader) { return new RemoteActionContextProvider( executor, env, @@ -99,7 +104,8 @@ public static RemoteActionContextProvider createForRemoteCaching( retryScheduler, digestUtil, /* logDir= */ null, - remoteOutputChecker); + remoteOutputChecker, + actionResultDownloader); } public static RemoteActionContextProvider createForRemoteExecution( @@ -110,7 +116,8 @@ public static RemoteActionContextProvider createForRemoteExecution( ListeningScheduledExecutorService retryScheduler, DigestUtil digestUtil, Path logDir, - @Nullable RemoteOutputChecker remoteOutputChecker) { + @Nullable RemoteOutputChecker remoteOutputChecker, + ActionResultDownloader actionResultDownloader) { return new RemoteActionContextProvider( executor, env, @@ -119,7 +126,8 @@ public static RemoteActionContextProvider createForRemoteExecution( retryScheduler, digestUtil, logDir, - remoteOutputChecker); + remoteOutputChecker, + actionResultDownloader); } private RemotePathResolver createRemotePathResolver() { @@ -170,7 +178,8 @@ private RemoteExecutionService getRemoteExecutionService() { remoteExecutor, tempPathGenerator, captureCorruptedOutputsDir, - remoteOutputChecker); + remoteOutputChecker, + actionResultDownloader); env.getEventBus().register(remoteExecutionService); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 314b9f6fb56e25..7f78c4a39efb8a 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -173,6 +173,7 @@ public class RemoteExecutionService { private final RemoteOptions remoteOptions; @Nullable private final RemoteCache remoteCache; @Nullable private final RemoteExecutionClient remoteExecutor; + @Nullable private final ActionResultDownloader actionResultDownloader; private final TempPathGenerator tempPathGenerator; @Nullable private final Path captureCorruptedOutputsDir; private final Cache> merkleTreeCache; @@ -202,7 +203,8 @@ public RemoteExecutionService( @Nullable RemoteExecutionClient remoteExecutor, TempPathGenerator tempPathGenerator, @Nullable Path captureCorruptedOutputsDir, - @Nullable RemoteOutputChecker remoteOutputChecker) { + @Nullable RemoteOutputChecker remoteOutputChecker, + @Nullable ActionResultDownloader actionResultDownloader) { this.reporter = reporter; this.verboseFailures = verboseFailures; this.execRoot = execRoot; @@ -213,6 +215,7 @@ public RemoteExecutionService( this.remoteOptions = remoteOptions; this.remoteCache = remoteCache; this.remoteExecutor = remoteExecutor; + this.actionResultDownloader = actionResultDownloader; Caffeine merkleTreeCacheBuilder = Caffeine.newBuilder().softValues(); // remoteMerkleTreesCacheSize = 0 means limitless. @@ -1099,6 +1102,33 @@ public InMemoryOutput downloadOutputs(RemoteAction action, RemoteActionResult re checkState(!shutdown.get(), "shutdown"); checkNotNull(remoteCache, "remoteCache can't be null"); + if (actionResultDownloader != null) { + // We have a remote output service process that can do the + // downloading for us. + ActionResult actionResult = result.actionResult; + List> downloads = new ArrayList<>(); + downloads.add(actionResultDownloader.downloadActionResult(actionResult, action.getRemotePathResolver())); + + FileOutErr outErr = action.getSpawnExecutionContext().getFileOutErr(); + FileOutErr tmpOutErr = outErr.childOutErr(); + downloads.addAll( + remoteCache.downloadOutErr( + action.getRemoteActionExecutionContext(), actionResult, tmpOutErr)); + + try { + waitForBulkTransfer(downloads, /* cancelRemainingOnInterrupt=*/ true); + if (tmpOutErr != null) { + FileOutErr.dump(tmpOutErr, outErr); + } + } finally { + if (tmpOutErr != null) { + tmpOutErr.clearOut(); + tmpOutErr.clearErr(); + } + } + return null; + } + FileSystem actionFileSystem = action.getSpawnExecutionContext().getActionFileSystem(); checkState( actionFileSystem instanceof RemoteActionFileSystem, diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index e58bc28010ef4b..eaa107373793bb 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -14,6 +14,8 @@ package com.google.devtools.build.lib.remote; +import static com.google.common.hash.Hashing.md5; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.SECONDS; import build.bazel.remote.execution.v2.DigestFunction; @@ -97,6 +99,7 @@ import com.google.devtools.build.lib.vfs.OutputPermissions; import com.google.devtools.build.lib.vfs.OutputService; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.common.options.OptionsBase; import com.google.devtools.common.options.OptionsParsingResult; import io.grpc.CallCredentials; @@ -131,7 +134,9 @@ public final class RemoteModule extends BlazeModule { @Nullable private RemoteActionInputFetcher actionInputFetcher; @Nullable private RemoteOptions remoteOptions; @Nullable private CommandEnvironment env; - @Nullable private RemoteOutputService remoteOutputService; + @Nullable private OutputService remoteOutputService; + @Nullable private GrpcRemoteOutputService.StateAcrossBuilds remoteOutputServiceStateAcrossBuilds = + new GrpcRemoteOutputService.StateAcrossBuilds(); @Nullable private TempPathGenerator tempPathGenerator; @Nullable private BlockWaitingModule blockWaitingModule; @Nullable private RemoteOutputChecker remoteOutputChecker; @@ -241,7 +246,8 @@ private void initHttpAndDiskCache( remoteCache, /* retryScheduler= */ null, digestUtil, - remoteOutputChecker); + remoteOutputChecker, + /* actionResultDownloader= */ null); } @Override @@ -515,6 +521,55 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { cacheChannel.retain(), callCredentialsProvider, remoteOptions, retrier, digestUtil); cacheChannel.release(); + ActionResultDownloader actionResultDownloader = null; + Preconditions.checkState(remoteOutputService == null, "remoteOutputService must be null"); + if (remoteOptions.remoteOutputService != null) { + if (remoteOptions.remoteOutputsMode != RemoteOutputsMode.ALL) { + throw createOptionsExitException( + "If --remote_output_service is specified, --remote_download_outputs must be set to \"all\"", + FailureDetails.RemoteOptions.Code.OUTPUT_SERVICE_WITH_INCOMPATIBLE_REMOTE_OUTPUTS_MODE); + } + String outputBaseId = remoteOptions.remoteOutputServiceOutputBaseId; + if (Strings.isNullOrEmpty(outputBaseId)) { + outputBaseId = DigestUtil.hashCodeToString(md5().hashString(env.getWorkspace().toString(), UTF_8)); + } + String outputPathPrefix = remoteOptions.remoteOutputServiceOutputPathPrefix; + if (Strings.isNullOrEmpty(outputPathPrefix)) { + throw createOptionsExitException( + "If --remote_output_service is specified, --remote_output_service_output_path_prefix must be set as well", + FailureDetails.RemoteOptions.Code.OUTPUT_SERVICE_WITHOUT_OUTPUT_PATH_REFIX); + } + + ReferenceCountedChannel channel = + createChannel( + executorService, + remoteOptions, + authAndTlsOptions, + /* headersInterceptor= */ null, + loggingInterceptor, + channelFactory, + remoteOptions.remoteOutputService, + remoteOptions.remoteProxy, + maxConcurrencyPerConnection, + maxConnections, + verboseFailures, + env.getReporter(), + rsc, + digestUtil.getDigestFunction(), + ServerCapabilitiesRequirement.NONE); + GrpcRemoteOutputService grpcRemoteOutputService = new GrpcRemoteOutputService( + remoteOutputServiceStateAcrossBuilds, + channel, + outputBaseId, + PathFragment.create(outputPathPrefix), + remoteOptions.remoteInstanceName, + digestUtil, + retrier, + env); + remoteOutputService = grpcRemoteOutputService; + actionResultDownloader = grpcRemoteOutputService; + } + if (enableRemoteExecution) { if (enableDiskCache) { try { @@ -564,7 +619,8 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { retryScheduler, digestUtil, logDir, - remoteOutputChecker); + remoteOutputChecker, + actionResultDownloader); repositoryRemoteExecutorFactoryDelegate.init( new RemoteRepositoryRemoteExecutorFactory( remoteCache, @@ -593,7 +649,13 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { RemoteCache remoteCache = new RemoteCache(cacheClient, remoteOptions, digestUtil); actionContextProvider = RemoteActionContextProvider.createForRemoteCaching( - executorService, env, remoteCache, retryScheduler, digestUtil, remoteOutputChecker); + executorService, + env, + remoteCache, + retryScheduler, + digestUtil, + remoteOutputChecker, + actionResultDownloader); } buildEventArtifactUploaderFactoryDelegate.init( @@ -841,7 +903,6 @@ public void afterCommand() { actionInputFetcher = null; remoteOptions = null; env = null; - remoteOutputService = null; tempPathGenerator = null; rpcLogFile = null; remoteOutputChecker = null; @@ -877,6 +938,16 @@ private static void afterCommandTask( } } + @Override + public void commandComplete() { + if (remoteOutputService != null) { + if (remoteOutputService instanceof GrpcRemoteOutputService) { + ((GrpcRemoteOutputService) remoteOutputService).sendFinalizeBuild(); + } + remoteOutputService = null; + } + } + @Override public void registerSpawnStrategies( SpawnStrategyRegistry.Builder registryBuilder, CommandEnvironment env) { @@ -943,12 +1014,17 @@ public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorB actionContextProvider.setTempPathGenerator(tempPathGenerator); CoreOptions coreOptions = env.getOptions().getOptions(CoreOptions.class); + RemoteOptions remoteOptions = + Preconditions.checkNotNull( + env.getOptions().getOptions(RemoteOptions.class), "RemoteOptions"); OutputPermissions outputPermissions = coreOptions.experimentalWritableOutputs ? OutputPermissions.WRITABLE : OutputPermissions.READONLY; - if (actionContextProvider.getRemoteCache() != null) { + if ( + !(remoteOutputService instanceof GrpcRemoteOutputService) + && actionContextProvider.getRemoteCache() != null) { Preconditions.checkNotNull(remoteOutputChecker, "remoteOutputChecker must not be null"); actionInputFetcher = @@ -983,18 +1059,17 @@ public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorB env.getBlazeWorkspace().getPersistentActionCache(), leaseExtension); - remoteOutputService.setRemoteOutputChecker(remoteOutputChecker); - remoteOutputService.setActionInputFetcher(actionInputFetcher); - remoteOutputService.setLeaseService(leaseService); - remoteOutputService.setFileCacheSupplier(env::getFileCache); + ((RemoteOutputService) remoteOutputService).setRemoteOutputChecker(remoteOutputChecker); + ((RemoteOutputService) remoteOutputService).setActionInputFetcher(actionInputFetcher); + ((RemoteOutputService) remoteOutputService).setLeaseService(leaseService); + ((RemoteOutputService) remoteOutputService).setFileCacheSupplier(env::getFileCache); env.getEventBus().register(remoteOutputService); } } @Override public OutputService getOutputService() { - Preconditions.checkState(remoteOutputService == null, "remoteOutputService must be null"); - if (actionContextProvider.getRemoteCache() != null) { + if (remoteOutputService == null && actionContextProvider.getRemoteCache() != null) { remoteOutputService = new RemoteOutputService(env); } return remoteOutputService; diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java index 1449a0e53a7a49..4fa0f5e7059a01 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java @@ -701,6 +701,42 @@ public RemoteOutputsStrategyConverter() { + " frequency is based on the value of `--experimental_remote_cache_ttl`.") public boolean remoteCacheLeaseExtension; + @Option( + name = "remote_output_service", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "A URI of a remote output service. A remote output service is a daemon that runs next " + + "to Bazel, managing the contents of the bazel-out/ directory. It may use systems " + + "like FUSE to add features such as snapshotting and lazy loading of objects stored " + + "in a remote Content Addressable Storage.") + public String remoteOutputService; + + @Option( + name = "remote_output_service_output_path_prefix", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "The path at which all output path directories are visible that are managed by the " + + "remote output service.") + public String remoteOutputServiceOutputPathPrefix; + + @Option( + name = "remote_output_service_output_base_id", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "The identifier of the workspace in which the build takes place. This value must be " + + "unique for every workspace built through the same remote output service, as it " + + "allows the remote output service to track state for multiple workspaces in " + + "parallel. By default, it is set to the MD5 sum of the path of the workspace. " + + "It may be necessary to override this value if the file system is virtualized " + + "and multiple workspaces are placed at the same path.") + public String remoteOutputServiceOutputBaseId; + @Option( name = "experimental_remote_scrubbing_config", converter = ScrubberConverter.class, diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD index abb19a3d6ef848..ad24a02190a12b 100644 --- a/src/main/protobuf/BUILD +++ b/src/main/protobuf/BUILD @@ -236,6 +236,32 @@ java_library_srcs( deps = [":cache_salt_java_proto"], ) +proto_library( + name = "remote_output_service_proto", + srcs = ["remote_output_service.proto"], + deps = [ + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:timestamp_proto", + "@remoteapis//:build_bazel_remote_execution_v2_remote_execution_proto", + ], +) + +java_proto_library( + name = "remote_output_service_java_proto", + deps = [":remote_output_service_proto"], +) + +java_grpc_library( + name = "remote_output_service_java_grpc", + srcs = [":remote_output_service_proto"], + deps = [":remote_output_service_java_proto"], +) + +java_library_srcs( + name = "remote_output_service_java_proto_srcs", + deps = [":remote_output_service_java_proto"], +) + proto_library( name = "remote_scrubbing_proto", srcs = ["remote_scrubbing.proto"], @@ -307,6 +333,8 @@ filegroup( ":option_filters_java_proto_srcs", ":profile_java_proto_srcs", ":remote_execution_log_java_proto_srcs", + ":remote_output_service_java_grpc_srcs", + ":remote_output_service_java_proto_srcs", ":remote_scrubbing_java_proto_srcs", ":spawn_java_proto_srcs", ":xcode_java_proto_srcs", diff --git a/src/main/protobuf/failure_details.proto b/src/main/protobuf/failure_details.proto index ed165063702a7c..8a841c316678fb 100644 --- a/src/main/protobuf/failure_details.proto +++ b/src/main/protobuf/failure_details.proto @@ -296,6 +296,8 @@ message RemoteOptions { DOWNLOADER_WITHOUT_GRPC_CACHE = 4 [(metadata) = { exit_code: 2 }]; EXECUTION_WITH_INVALID_CACHE = 5 [(metadata) = { exit_code: 2 }]; EXECUTION_WITH_SCRUBBING = 6 [(metadata) = { exit_code: 2 }]; + OUTPUT_SERVICE_WITHOUT_OUTPUT_PATH_REFIX = 7 [(metadata) = { exit_code: 2 }]; + OUTPUT_SERVICE_WITH_INCOMPATIBLE_REMOTE_OUTPUTS_MODE = 8 [(metadata) = { exit_code: 2 }]; } Code code = 1; diff --git a/src/main/protobuf/remote_output_service.proto b/src/main/protobuf/remote_output_service.proto new file mode 100644 index 00000000000000..e8dce0ae7bd7c2 --- /dev/null +++ b/src/main/protobuf/remote_output_service.proto @@ -0,0 +1,320 @@ +// Copyright 2021 The Bazel Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package remote_output_service; + +import "build/bazel/remote/execution/v2/remote_execution.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option java_package = "com.google.devtools.build.lib.remote"; +option java_outer_classname = "RemoteOutputServiceProto"; +option go_package = "remoteoutputservice"; + +// The Remote Output Service may be used by users of the Remote +// Execution API to construct a directory on the local system that +// contains all output files of a build. +// +// Primitive implementations of this API may simply download files from +// the Content Addressable Storage (CAS) and store them at their +// designated location. Complex implementations may use a pseudo file +// system (e.g., FUSE) to support deduplication, lazy loading and +// snapshotting. +// +// Details: +// https://github.com/bazelbuild/proposals/blob/master/designs/2021-02-09-remote-output-service.md +// https://groups.google.com/g/remote-execution-apis/c/qOSWWwBLPzo +// https://groups.google.com/g/bazel-dev/c/lKzENsNd1Do +service RemoteOutputService { + // Methods that can be invoked at any point in time. + + // Clean all data associated with a single output path, so that the + // next invocation of StartBuild() yields an empty output path. This + // may be implemented in a way that's faster than removing all of the + // files from the file system manually. + rpc Clean(CleanRequest) returns (google.protobuf.Empty); + + // Signal that a new build is about to start. + // + // The client uses this call to obtain a directory where outputs of + // the build may be stored, called the output path. Based on the + // parameters provided, the remote output service may provide an empty + // output path, or one that has contents from a previous build of the + // same workspace. + // + // In case the output path contains data from a previous build, the + // remote output service is responsible for calling + // ContentAddressableStorage.FindMissingBlobs() for all of the objects + // that are stored remotely. This ensures that these objects don't + // disappear from the Content Addressable Storage while the build is + // running. Any files that are absent must be removed from the output + // path and reported through InitialOutputPathContents.modified_paths. + rpc StartBuild(StartBuildRequest) returns (StartBuildResponse); + + // Methods that can only be invoked during a build. + + // Create one or more files, directories or symbolic links in the + // output path. + rpc BatchCreate(BatchCreateRequest) returns (google.protobuf.Empty); + + // Obtain the status of one or more files, directories or symbolic + // links that are stored in the input path. + rpc BatchStat(BatchStatRequest) returns (BatchStatResponse); + + // Signal that a build has been completed. + rpc FinalizeBuild(FinalizeBuildRequest) returns (google.protobuf.Empty); +} + +message CleanRequest { + // The output base identifier that was provided to + // StartBuildRequest.output_base_id whose data needs to be removed. + string output_base_id = 1; +} + +message StartBuildRequest { + // A client-chosen value that uniquely identifies the workspace for + // which the build is being started. This value must be set to ensure + // that the remote output service is capable of managing builds for + // distinct workspaces concurrently. + // + // This value must be a valid filename for the operating system on + // which the remote output service and client are being executed. This + // allows the remote output service to create one subdirectory per + // project that needs to be built. + // + // By default, Bazel sets this value to the MD5 sum of the absolute + // path of the workspace directory. This is generally sufficient, + // though a more complex scheme may necessary in case the file system + // namespace is virtualized. + // + // Starting a build finalizes any previous build with the same + // output_base_id that has not been finalized yet. + string output_base_id = 1; + + // A client-chosen value that uniquely identifies this build. This + // value must be provided to most other methods to ensure that + // operations are targeted against the right output path. + string build_id = 2; + + // The instance name that the client uses when communicating with the + // remote execution system. The remote output service uses this value + // when loading objects from the Content Addressable Storage. + string instance_name = 3; + + // The digest function that the client uses when communicating with + // the remote execution system. The remote output service uses this + // value to ensure that FileStatus responses contain digests that were + // computed with right digest function. + build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 4; + + // The absolute path at which the remote output service exposes its + // output paths, as seen from the perspective of the client. + // + // This value needs to be provided by the client, because file system + // namespace virtualization may cause this directory to appear at a + // location that differs from the one used by the service. + // + // The purpose of this field is to ensure that the remote output + // service is capable of expanding symbolic links containing absolute + // paths. + string output_path_prefix = 5; + + // A map of paths on the system that will become symbolic links + // pointing to locations inside the output path. Similar to + // output_path_prefix, this option is used to ensure the remote output + // service is capable of expanding symbolic links. + // + // Map keys are absolute paths, while map values are paths that are + // relative to the output path. + map output_path_aliases = 6; +} + +message InitialOutputPathContents { + // The identifier of a previously finalized build whose results are + // stored in the output path. + string build_id = 1; + + // Paths that have been modified or removed since the build finalized. + // + // If the remote output service freezes the contents of the output + // path between builds, this field can be left empty. + repeated string modified_paths = 2; +} + +message StartBuildResponse { + // If set, the contents of the output path are almost entirely + // identical on the results of a previous build. This information may + // be used by the client to prevent unnecessary scanning of the file + // system. + // + // Servers can leave this field unset in case the contents of the + // output path are empty, not based on a previous build, if no + // tracking of this information is performed, or if the number of + // changes made to the output path is too large to be expressed. + InitialOutputPathContents initial_output_path_contents = 1; + + // A relative path that the client must append to + // StartBuildRequest.output_path_prefix to obtain the full path at + // which outputs of the build are stored. + // + // If the remote output service is incapable of storing the output of + // multiple builds, this string may be left empty. + string output_path_suffix = 2; +} + +message BatchCreateRequest { + // The identifier of the build. The remote output service uses this to + // determine which output path needs to be modified. + string build_id = 1; + + // A path relative to the root of the output path where files, + // symbolic links and directories need to be created. + string path_prefix = 2; + + // Whether the contents of the path prefix should be removed prior to + // creating the specified files. + bool clean_path_prefix = 3; + + // Files that need to be downloaded from the Content Addressable + // Storage. + // + // Any missing parent directories, including those in path_prefix, are + // created as well. If any of the parents refer to a non-directory + // file, they are replaced by an empty directory. If a file or + // directory already exists at the provided path, it is replaced. + // + // This means that symbolic links are not followed when evaluating + // path_prefix and OutputFile.path. + repeated build.bazel.remote.execution.v2.OutputFile files = 4; + + // Symbolic links that need to be created. + // + // Any missing parent directories, including those in path_prefix, are + // created as well. If any of the parents refer to a non-directory + // file, they are replaced by an empty directory. If a file or + // directory already exists at the provided path, it is replaced. + // + // This means that symbolic links are not followed when evaluating + // path_prefix and OutputSymlink.path. + repeated build.bazel.remote.execution.v2.OutputSymlink symlinks = 5; + + // Directories that need to be downloaded from the Content Addressable + // Storage. + // + // Any missing parent directories, including those in path_prefix, are + // created as well. If any of the parents refer to a non-directory + // file, they are replaced by an empty directory. Any file or + // directory that already exists at the provided path is replaced. + // + // This means that symbolic links are not followed when evaluating + // path_prefix and OutputDirectory.path. + repeated build.bazel.remote.execution.v2.OutputDirectory directories = 6; +} + +message BatchStatRequest { + // The identifier of the build. The remote output service uses this to + // determine which output path needs to be inspected. + string build_id = 1; + + // In case the path corresponds to a regular file, include the hash + // and size of the file in the response. + bool include_file_digest = 2; + + // In case the path corresponds to a symbolic link, include the target + // of the symbolic link in the response. + bool include_symlink_target = 3; + + // If the last component of the path corresponds to a symbolic link, + // return the status of the file at the target location. + // + // Symbolic links encountered before the last component of the path + // are always expanded, regardless of the value of this option. + bool follow_symlinks = 4; + + // Paths whose status needs to be obtained. + repeated string paths = 5; +} + +message BatchStatResponse { + // The status response for each of the requested paths, using the same + // order as requested. This means that this list has the same length + // as BatchStatRequest.paths. + repeated StatResponse responses = 1; +} + +message StatResponse { + // The status of the file. If the file corresponding with the + // requested path does not exist, this field will be null. + FileStatus file_status = 1; +} + +message FileStatus { + message File { + // The hash and size of the file. This field is only set when + // BatchStatRequest.include_file_digest is set. + build.bazel.remote.execution.v2.Digest digest = 1; + } + + message Symlink { + // The target of the symbolic link. This field is only set when + // BatchStatRequest.include_symlink_target is set. + string target = 1; + } + + message Directory { + // The time at which the directory contents were last modified. + google.protobuf.Timestamp last_modified_time = 1; + } + + message External { + // The path relative to the root of the output path where the file + // is located. This path is absolute, or it is relative, starting + // with "../". + // + // The client can use this field to obtain the file status manually. + string next_path = 1; + } + + oneof file_type { + // The path resolves to a regular file. + File file = 1; + + // The path resolves to a symbolic link. + // + // This field may not be set if BatchStatRequest.follow_symlinks is + // set to true. + Symlink symlink = 2; + + // The path resolves to a directory. + Directory directory = 3; + + // The path resolves to a location outside the output path. The + // remote output service is unable to determine whether any file + // exists at the resulting path, and can therefore not obtain its + // status. + External external = 4; + } +} + +message FinalizeBuildRequest { + // The identifier of the build that should be finalized. + string build_id = 1; + + // Whether the build completed successfully. The remote output service + // may, for example, use this option to apply different retention + // policies that take the outcome of the build into account. + bool build_successful = 2; +} diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index 81a81e71d3bdc1..b98a21ec039b59 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java @@ -2466,7 +2466,8 @@ private RemoteExecutionService newRemoteExecutionService(RemoteOptions remoteOpt executor, tempPathGenerator, null, - remoteOutputChecker); + remoteOutputChecker, + /*actionResultDownloader=*/ null); } private RunfilesSupplier createRunfilesSupplier(String root, Collection artifacts) { diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java index e1d8d2187162bc..2ef26a4560115f 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java @@ -260,7 +260,8 @@ private RemoteSpawnCache remoteSpawnCacheWithOptions(RemoteOptions options) { null, tempPathGenerator, /* captureCorruptedOutputsDir= */ null, - DUMMY_REMOTE_OUTPUT_CHECKER)); + DUMMY_REMOTE_OUTPUT_CHECKER, + /* actionResultDownloader= */ null)); return new RemoteSpawnCache( execRoot, options, /* verboseFailures= */ true, service, digestUtil); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java index 7884ad37e85d74..ef6edbd7a31338 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java @@ -1147,7 +1147,8 @@ private void testParamFilesAreMaterializedForFlag(String flag) throws Exception executor, tempPathGenerator, /* captureCorruptedOutputsDir= */ null, - remoteOutputChecker); + remoteOutputChecker, + /*actionResultDownloader=*/ null); RemoteSpawnRunner runner = new RemoteSpawnRunner( execRoot, @@ -1682,7 +1683,8 @@ private RemoteSpawnRunner newSpawnRunner( executor, tempPathGenerator, /* captureCorruptedOutputsDir= */ null, - remoteOutputChecker)); + remoteOutputChecker, + /*actionResultDownloader=*/ null)); return new RemoteSpawnRunner( execRoot, diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java index 865ee7894266f7..bfe83f4e031855 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java @@ -346,7 +346,8 @@ public int maxConcurrency() { executor, tempPathGenerator, /* captureCorruptedOutputsDir= */ null, - remoteOutputChecker); + remoteOutputChecker, + /* actionResultDownloader= */ null); client = new RemoteSpawnRunner( execRoot,