From 3d08acbcea8318b677c61b9cba4beaf221c8ce94 Mon Sep 17 00:00:00 2001
From: Ed Schouten <eschouten@apple.com>
Date: Thu, 18 Jun 2020 18:55:05 +0200
Subject: [PATCH] REv2: Allow downloads from being prevented by emitting
 symlinks

By default, Bazel's REv2 client downloads all output files of a build
action, which is slow. This can be disabled by using flags like
--remote_download_minimal, but this causes bazel-out/ to be mostly
empty, making the user experience suboptimal.

This change introduces a middle ground: it allows people to develop
virtual file systems (FUSE, NFS, 9p, etc. etc. etc.) that lazily expose
the CAS onto people's systems. It does this by adding a new flag,
--remote_download_symlink_template, that causes Bazel to generate
symlinks that conform to a given template.
---
 .../build/lib/remote/RemoteCache.java         | 12 +++++++++
 .../lib/remote/options/RemoteOptions.java     | 15 +++++++++++
 .../build/lib/remote/RemoteCacheTests.java    | 27 +++++++++++++++++++
 3 files changed, 54 insertions(+)

diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
index 7c5bdfd130bddc..73b34132930dda 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
@@ -445,6 +445,18 @@ public ListenableFuture<Void> downloadFile(Path path, Digest digest) throws IOEx
       return COMPLETED_SUCCESS;
     }
 
+    if (!options.remoteDownloadSymlinkTemplate.isEmpty()) {
+      // Don't actually download files from the CAS. Instead, create a
+      // symbolic link that points to a location where CAS objects may
+      // be found. This could, for example, be a FUSE file system.
+      path.createSymbolicLink(
+          path.getRelative(
+              options.remoteDownloadSymlinkTemplate
+                  .replace("{hash}", digest.getHash())
+                  .replace("{size_bytes}", String.valueOf(digest.getSizeBytes()))));
+      return COMPLETED_SUCCESS;
+    }
+
     OutputStream out = new LazyFileOutputStream(path);
     SettableFuture<Void> outerF = SettableFuture.create();
     ListenableFuture<Void> f = cacheProtocol.downloadBlob(digest, out);
diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
index fbabade665fdd2..ec4feb60755b35 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -460,6 +460,21 @@ public RemoteOutputsStrategyConverter() {
               + " discard the remotely cached values if they don't match the expected value.")
   public boolean remoteVerifyDownloads;
 
+  @Option(
+      name = "remote_download_symlink_template",
+      defaultValue = "",
+      category = "remote",
+      documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
+      effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+      help =
+          "Instead of downloading remote build outputs to the local machine, create symbolic "
+              + "links. The target of the symbolic links can be specified in the form of a "
+              + "template string. This template string may contain {hash} and {size_bytes} that "
+              + "expand to the hash of the object and the size in bytes, respectively. "
+              + "These symbolic links may, for example, point to a FUSE file system "
+              + "that loads objects from the CAS on demand.")
+  public String remoteDownloadSymlinkTemplate;
+
   // The below options are not configurable by users, only tests.
   // This is part of the effort to reduce the overall number of flags.
 
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java
index 96039baaa95f7c..0578b52a29db08 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTests.java
@@ -1206,6 +1206,33 @@ public void testDownloadEmptyBlobAndFile() throws Exception {
     assertThat(file.getFileSize()).isEqualTo(0);
   }
 
+  @Test
+  public void testDownloadFileWithSymlinkTemplate() throws Exception {
+    // Test that when a symlink template is provided, we don't actually download files to disk.
+    // Instead, a symbolic link should be created that points to a location where the file may
+    // actually be found. That location could, for example, be backed by a FUSE file system that
+    // exposes the Content Addressable Storage.
+
+    // arrange
+    final ConcurrentMap<Digest, byte[]> cas = new ConcurrentHashMap<>();
+
+    Digest helloDigest = digestUtil.computeAsUtf8("hello-contents");
+    cas.put(helloDigest, "hello-contents".getBytes(Charsets.UTF_8));
+
+    Path file = fs.getPath("/execroot/symlink-to-file");
+    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+    options.remoteDownloadSymlinkTemplate = "/home/alice/cas/{hash}-{size_bytes}";
+    RemoteCache remoteCache = new InMemoryRemoteCache(cas, options, digestUtil);
+
+    // act
+    Utils.getFromFuture(remoteCache.downloadFile(file, helloDigest));
+
+    // assert
+    assertThat(file.isSymbolicLink()).isTrue();
+    assertThat(file.readSymbolicLink()).isEqualTo(
+        PathFragment.create("/home/alice/cas/a378b939ad2e1d470a9a28b34b0e256b189e85cb236766edc1d46ec3b6ca82e5-14"));
+  }
+
   @Test
   public void testDownloadDirectory() throws Exception {
     // Test that downloading an output directory works.