Intrinsic to turn FilesContent into Snapshot

src/python/pants/engine/fs.py

@@ -185,6 +185,7 @@ def create_fs_rules():
   return [
     RootRule(Digest),
     RootRule(DirectoriesToMerge),
+    RootRule(FilesContent),
     RootRule(PathGlobs),
     RootRule(DirectoryWithPrefixToStrip),
     RootRule(UrlToFetch),

src/rust/engine/fs/src/lib.rs

@@ -87,7 +87,7 @@ pub struct Link(pub PathBuf);
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub struct Dir(pub PathBuf);
 
-#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
 pub struct File {
   pub path: PathBuf,
   pub is_executable: bool,

src/rust/engine/fs/src/snapshot.rs

@@ -2,15 +2,17 @@
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 use crate::glob_matching::GlobMatching;
-use crate::{Dir, File, PathGlobs, PathStat, PosixFS, Store};
+use crate::{Dir, File, FileContent, PathGlobs, PathStat, PosixFS, Store};
 use bazel_protos;
 use boxfuture::{try_future, BoxFuture, Boxable};
+use bytes::Bytes;
 use futures::future::{self, join_all};
 use futures::Future;
 use hashing::{Digest, Fingerprint};
 use indexmap::{self, IndexMap};
 use itertools::Itertools;
 use protobuf;
+use std::collections::BTreeMap;
 use std::ffi::OsString;
 use std::fmt;
 use std::iter::Iterator;
@@ -496,6 +498,36 @@ impl Snapshot {
       })
       .to_boxed()
   }
+
+  pub fn from_files_content(
+    store: Store,
+    files_content: Vec<FileContent>,
+  ) -> impl Future<Item = Snapshot, Error = String> {
+    let files_content: BTreeMap<File, Bytes> = files_content
+      .into_iter()
+      .map(|fc| {
+        let file = File {
+          path: fc.path,
+          // TODO: Configurable?
+          is_executable: true,
+        };
+        (file, fc.content)
+      })
+      .collect();
+    // TODO: Should there be a way to represent empty directories?
+    let path_stats: Vec<_> = files_content
+      .keys()
+      .map(|file| PathStat::File {
+        path: file.path.clone(),
+        stat: file.clone(),
+      })
+      .collect();
+    let digester = FromMemoryStoreFileByDigest {
+      store: store.clone(),
+      files_content,
+    };
+    Snapshot::from_path_stats(store, &digester, path_stats)
+  }
 }
 
 impl fmt::Debug for Snapshot {
@@ -571,6 +603,30 @@ impl StoreFileByDigest<String> for OneOffStoreFileByDigest {
   }
 }
 
+///
+/// A StoreFileByDigest which reads in-memory file contents and writes to a Store, with no caching.
+///
+#[derive(Clone)]
+struct FromMemoryStoreFileByDigest {
+  store: Store,
+  files_content: BTreeMap<File, Bytes>,
+}
+
+impl StoreFileByDigest<String> for FromMemoryStoreFileByDigest {
+  fn store_by_digest(&self, file: File) -> BoxFuture<Digest, String> {
+    let store = self.store.clone();
+    futures::future::done(
+      self
+        .files_content
+        .get(&file)
+        .cloned()
+        .ok_or_else(|| format!("File {:?} not known", file)),
+    )
+    .and_then(move |content| store.store_file_bytes(content, true))
+    .to_boxed()
+  }
+}
+
 #[cfg(test)]
 mod tests {
   use futures::future::Future;

src/rust/engine/src/nodes.rs

@@ -21,7 +21,7 @@ use crate::rule_graph;
 use crate::selectors;
 use crate::tasks::{self, Intrinsic};
 use boxfuture::{try_future, BoxFuture, Boxable};
-use bytes::{self, BufMut};
+use bytes::{self, BufMut, Bytes};
 use fs::{
   self, Dir, DirectoryListing, File, FileContent, GlobExpansionConjunction, GlobMatching, Link,
   PathGlobs, PathStat, StoreFileByDigest, StrictGlobMatching, VFS,
@@ -271,6 +271,24 @@ impl WrappedNode for Select {
             })
             .to_boxed()
         }
+        &rule_graph::Rule::Intrinsic(Intrinsic { product, input })
+          if product == context.core.types.snapshot
+            && input == context.core.types.files_content =>
+        {
+          let core = context.core.clone();
+          self
+            .select_product(&context, context.core.types.files_content, "intrinsic")
+            .and_then(move |files_content| {
+              let files_content: Vec<_> = externs::project_multi(&files_content, "dependencies")
+                .iter()
+                .map(lift_file_content)
+                .collect();
+              fs::Snapshot::from_files_content(context.core.store(), files_content)
+                .map_err(|str| throw(&str))
+            })
+            .map(move |snapshot| Snapshot::store_snapshot(&core, &snapshot))
+            .to_boxed()
+        }
         &rule_graph::Rule::Intrinsic(Intrinsic { product, input })
           if product == context.core.types.process_result
             && input == context.core.types.process_request =>
@@ -334,6 +352,12 @@ pub fn lift_digest(digest: &Value) -> Result<hashing::Digest, String> {
   ))
 }
 
+pub fn lift_file_content(file_content: &Value) -> fs::FileContent {
+  let path = PathBuf::from(externs::project_str(&file_content, "path"));
+  let content = Bytes::from(externs::project_bytes(&file_content, "content"));
+  fs::FileContent { path, content }
+}
+
 ///
 /// A Node that represents executing a process.
 ///

src/rust/engine/src/tasks.rs

@@ -85,6 +85,10 @@ impl Tasks {
         product: types.files_content,
         input: types.directory_digest,
       },
+      Intrinsic {
+        product: types.snapshot,
+        input: types.files_content,
+      },
       Intrinsic {
         product: types.directory_digest,
         input: types.directories_to_merge,

tests/python/pants_test/engine/test_fs.py

@@ -15,8 +15,9 @@
 from future.utils import PY2, text_type
 
 from pants.engine.fs import (EMPTY_DIRECTORY_DIGEST, Digest, DirectoriesToMerge,
-                             DirectoryToMaterialize, DirectoryWithPrefixToStrip, FilesContent,
-                             PathGlobs, PathGlobsAndRoot, Snapshot, UrlToFetch, create_fs_rules)
+                             DirectoryToMaterialize, DirectoryWithPrefixToStrip, FileContent,
+                             FilesContent, PathGlobs, PathGlobsAndRoot, Snapshot, UrlToFetch,
+                             create_fs_rules)
 from pants.engine.scheduler import ExecutionError
 from pants.option.global_options import GlobMatchErrorBehavior
 from pants.util.collections import assert_single_element
@@ -508,6 +509,31 @@ def test_glob_match_warn_logging(self):
       single_warning = all_warnings[0]
       self.assertEqual("???", str(single_warning))
 
+  def test_files_content_from_bytes(self):
+    # TODO: Should you be allowed to set executable? (currently we always set executable=true)
+    # TODO: Should there be a way to represent empty directories?
+    # TODO: Test with some unicode, some non-unicode byte, and a directory with multiple files
+    files_content = FilesContent((
+      FileContent(
+        path=text_type("cats/roland"),
+        content=b"European Burmese",
+      ),
+      FileContent(
+        path=text_type("treats"),
+        content=b"catnip",
+      )
+    ))
+
+    snapshot = assert_single_element(self.scheduler.product_request(Snapshot, [files_content]))
+    self.assertEquals(("cats/roland", "treats"), snapshot.files)
+    self.assertEquals(
+      snapshot.directory_digest,
+      Digest(
+        fingerprint="1249edd274db4350ce7d295dc2f40768581197f0e13e74d0206a3c408f1c236f",
+        serialized_bytes_length=160,
+      ),
+    )
+
   def prime_store_with_roland_digest(self):
     """This method primes the store with a directory of a file named 'roland' and contents 'European Burmese'."""
     with temporary_dir() as temp_dir: