From cdcfe63bf0b98582e2eafe5bf48b5f3aa7b35460 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 16 May 2019 10:50:00 +0100 Subject: [PATCH] Intrinsic to turn FilesContent into Snapshot --- src/python/pants/engine/fs.py | 1 + src/rust/engine/fs/src/lib.rs | 2 +- src/rust/engine/fs/src/snapshot.rs | 58 ++++++++++++++++++++++- src/rust/engine/src/nodes.rs | 26 +++++++++- src/rust/engine/src/tasks.rs | 4 ++ tests/python/pants_test/engine/test_fs.py | 30 +++++++++++- 6 files changed, 116 insertions(+), 5 deletions(-) diff --git a/src/python/pants/engine/fs.py b/src/python/pants/engine/fs.py index bc07e129911..5d5998c0bda 100644 --- a/src/python/pants/engine/fs.py +++ b/src/python/pants/engine/fs.py @@ -185,6 +185,7 @@ def create_fs_rules(): return [ RootRule(Digest), RootRule(DirectoriesToMerge), + RootRule(FilesContent), RootRule(PathGlobs), RootRule(DirectoryWithPrefixToStrip), RootRule(UrlToFetch), diff --git a/src/rust/engine/fs/src/lib.rs b/src/rust/engine/fs/src/lib.rs index f9bc0ec3fc6..32cf0b56479 100644 --- a/src/rust/engine/fs/src/lib.rs +++ b/src/rust/engine/fs/src/lib.rs @@ -87,7 +87,7 @@ pub struct Link(pub PathBuf); #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct Dir(pub PathBuf); -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct File { pub path: PathBuf, pub is_executable: bool, diff --git a/src/rust/engine/fs/src/snapshot.rs b/src/rust/engine/fs/src/snapshot.rs index 11d6e5fba8b..d58d7540525 100644 --- a/src/rust/engine/fs/src/snapshot.rs +++ b/src/rust/engine/fs/src/snapshot.rs @@ -2,15 +2,17 @@ // Licensed under the Apache License, Version 2.0 (see LICENSE). use crate::glob_matching::GlobMatching; -use crate::{Dir, File, PathGlobs, PathStat, PosixFS, Store}; +use crate::{Dir, File, FileContent, PathGlobs, PathStat, PosixFS, Store}; use bazel_protos; use boxfuture::{try_future, BoxFuture, Boxable}; +use bytes::Bytes; use futures::future::{self, join_all}; use futures::Future; use hashing::{Digest, Fingerprint}; use indexmap::{self, IndexMap}; use itertools::Itertools; use protobuf; +use std::collections::BTreeMap; use std::ffi::OsString; use std::fmt; use std::iter::Iterator; @@ -496,6 +498,36 @@ impl Snapshot { }) .to_boxed() } + + pub fn from_files_content( + store: Store, + files_content: Vec, + ) -> impl Future { + let files_content: BTreeMap = files_content + .into_iter() + .map(|fc| { + let file = File { + path: fc.path, + // TODO: Configurable? + is_executable: true, + }; + (file, fc.content) + }) + .collect(); + // TODO: Should there be a way to represent empty directories? + let path_stats: Vec<_> = files_content + .keys() + .map(|file| PathStat::File { + path: file.path.clone(), + stat: file.clone(), + }) + .collect(); + let digester = FromMemoryStoreFileByDigest { + store: store.clone(), + files_content, + }; + Snapshot::from_path_stats(store, &digester, path_stats) + } } impl fmt::Debug for Snapshot { @@ -571,6 +603,30 @@ impl StoreFileByDigest for OneOffStoreFileByDigest { } } +/// +/// A StoreFileByDigest which reads in-memory file contents and writes to a Store, with no caching. +/// +#[derive(Clone)] +struct FromMemoryStoreFileByDigest { + store: Store, + files_content: BTreeMap, +} + +impl StoreFileByDigest for FromMemoryStoreFileByDigest { + fn store_by_digest(&self, file: File) -> BoxFuture { + let store = self.store.clone(); + futures::future::done( + self + .files_content + .get(&file) + .cloned() + .ok_or_else(|| format!("File {:?} not known", file)), + ) + .and_then(move |content| store.store_file_bytes(content, true)) + .to_boxed() + } +} + #[cfg(test)] mod tests { use futures::future::Future; diff --git a/src/rust/engine/src/nodes.rs b/src/rust/engine/src/nodes.rs index 71dfae528ba..c6d8b0efaa0 100644 --- a/src/rust/engine/src/nodes.rs +++ b/src/rust/engine/src/nodes.rs @@ -21,7 +21,7 @@ use crate::rule_graph; use crate::selectors; use crate::tasks::{self, Intrinsic}; use boxfuture::{try_future, BoxFuture, Boxable}; -use bytes::{self, BufMut}; +use bytes::{self, BufMut, Bytes}; use fs::{ self, Dir, DirectoryListing, File, FileContent, GlobExpansionConjunction, GlobMatching, Link, PathGlobs, PathStat, StoreFileByDigest, StrictGlobMatching, VFS, @@ -271,6 +271,24 @@ impl WrappedNode for Select { }) .to_boxed() } + &rule_graph::Rule::Intrinsic(Intrinsic { product, input }) + if product == context.core.types.snapshot + && input == context.core.types.files_content => + { + let core = context.core.clone(); + self + .select_product(&context, context.core.types.files_content, "intrinsic") + .and_then(move |files_content| { + let files_content: Vec<_> = externs::project_multi(&files_content, "dependencies") + .iter() + .map(lift_file_content) + .collect(); + fs::Snapshot::from_files_content(context.core.store(), files_content) + .map_err(|str| throw(&str)) + }) + .map(move |snapshot| Snapshot::store_snapshot(&core, &snapshot)) + .to_boxed() + } &rule_graph::Rule::Intrinsic(Intrinsic { product, input }) if product == context.core.types.process_result && input == context.core.types.process_request => @@ -334,6 +352,12 @@ pub fn lift_digest(digest: &Value) -> Result { )) } +pub fn lift_file_content(file_content: &Value) -> fs::FileContent { + let path = PathBuf::from(externs::project_str(&file_content, "path")); + let content = Bytes::from(externs::project_bytes(&file_content, "content")); + fs::FileContent { path, content } +} + /// /// A Node that represents executing a process. /// diff --git a/src/rust/engine/src/tasks.rs b/src/rust/engine/src/tasks.rs index 72ab6bc120f..0d3d28103dd 100644 --- a/src/rust/engine/src/tasks.rs +++ b/src/rust/engine/src/tasks.rs @@ -85,6 +85,10 @@ impl Tasks { product: types.files_content, input: types.directory_digest, }, + Intrinsic { + product: types.snapshot, + input: types.files_content, + }, Intrinsic { product: types.directory_digest, input: types.directories_to_merge, diff --git a/tests/python/pants_test/engine/test_fs.py b/tests/python/pants_test/engine/test_fs.py index 33fc1bdc330..847b7663d99 100644 --- a/tests/python/pants_test/engine/test_fs.py +++ b/tests/python/pants_test/engine/test_fs.py @@ -15,8 +15,9 @@ from future.utils import PY2, text_type from pants.engine.fs import (EMPTY_DIRECTORY_DIGEST, Digest, DirectoriesToMerge, - DirectoryToMaterialize, DirectoryWithPrefixToStrip, FilesContent, - PathGlobs, PathGlobsAndRoot, Snapshot, UrlToFetch, create_fs_rules) + DirectoryToMaterialize, DirectoryWithPrefixToStrip, FileContent, + FilesContent, PathGlobs, PathGlobsAndRoot, Snapshot, UrlToFetch, + create_fs_rules) from pants.engine.scheduler import ExecutionError from pants.option.global_options import GlobMatchErrorBehavior from pants.util.collections import assert_single_element @@ -508,6 +509,31 @@ def test_glob_match_warn_logging(self): single_warning = all_warnings[0] self.assertEqual("???", str(single_warning)) + def test_files_content_from_bytes(self): + # TODO: Should you be allowed to set executable? (currently we always set executable=true) + # TODO: Should there be a way to represent empty directories? + # TODO: Test with some unicode, some non-unicode byte, and a directory with multiple files + files_content = FilesContent(( + FileContent( + path=text_type("cats/roland"), + content=b"European Burmese", + ), + FileContent( + path=text_type("treats"), + content=b"catnip", + ) + )) + + snapshot = assert_single_element(self.scheduler.product_request(Snapshot, [files_content])) + self.assertEquals(("cats/roland", "treats"), snapshot.files) + self.assertEquals( + snapshot.directory_digest, + Digest( + fingerprint="1249edd274db4350ce7d295dc2f40768581197f0e13e74d0206a3c408f1c236f", + serialized_bytes_length=160, + ), + ) + def prime_store_with_roland_digest(self): """This method primes the store with a directory of a file named 'roland' and contents 'European Burmese'.""" with temporary_dir() as temp_dir: