Skip to content

Commit

Permalink
Snapshots are stored in the LMDB store not tar files
Browse files Browse the repository at this point in the history
This has a few advantages:
 1. It makes the Bazel remote execution API much easier to implement.
 2. We store each unique file by content exactly one time, rather than
    once per snapshot it belongs to.

It also allows us to delete a lot of code which handles awkward
specifics of tar files.
  • Loading branch information
illicitonion committed Jan 21, 2018
1 parent 5bca8c5 commit c3de15b
Show file tree
Hide file tree
Showing 10 changed files with 305 additions and 481 deletions.
39 changes: 0 additions & 39 deletions src/rust/engine/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion src/rust/engine/fs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ lmdb = "0.7.2"
ordermap = "0.2.8"
protobuf = "1.4.1"
sha2 = "0.6.0"
tar = "0.4.13"
tempdir = "0.3.5"

[dev-dependencies]
Expand Down
33 changes: 0 additions & 33 deletions src/rust/engine/fs/fs_util/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 8 additions & 7 deletions src/rust/engine/fs/fs_util/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ extern crate protobuf;

use boxfuture::{Boxable, BoxFuture};
use clap::{App, Arg, SubCommand};
use fs::{GetFileDigest, ResettablePool, Snapshot, Store, VFS};
use fs::{ResettablePool, Snapshot, Store, StoreFileByDigest, VFS};
use futures::future::{self, Future, join_all};
use hashing::{Digest, Fingerprint};
use protobuf::Message;
Expand Down Expand Up @@ -206,7 +206,7 @@ fn execute(top_match: clap::ArgMatches) -> Result<(), ExitError> {
let digest = FileSaver {
store: store,
posix_fs: Arc::new(posix_fs),
}.digest(&f)
}.store_by_digest(&f)
.wait()
.unwrap();
Ok(println!("{} {}", digest.0, digest.1))
Expand Down Expand Up @@ -246,14 +246,14 @@ fn execute(top_match: clap::ArgMatches) -> Result<(), ExitError> {
.and_then(move |paths| {
Snapshot::from_path_stats(
store.clone(),
Arc::new(FileSaver {
FileSaver {
store: store.clone(),
posix_fs: posix_fs,
}),
},
paths,
)
})
.map(|snapshot| snapshot.digest.unwrap())
.map(|snapshot| snapshot.digest)
.wait()?;
Ok(println!("{} {}", digest.0, digest.1))
}
Expand Down Expand Up @@ -331,13 +331,14 @@ fn make_posix_fs<P: AsRef<Path>>(root: P, pool: Arc<ResettablePool>) -> fs::Posi
fs::PosixFS::new(&root, pool, vec![]).unwrap()
}

#[derive(Clone)]
struct FileSaver {
store: Arc<Store>,
posix_fs: Arc<fs::PosixFS>,
}

impl GetFileDigest<String> for FileSaver {
fn digest(&self, file: &fs::File) -> BoxFuture<Digest, String> {
impl StoreFileByDigest<String> for FileSaver {
fn store_by_digest(&self, file: &fs::File) -> BoxFuture<Digest, String> {
let file_copy = file.clone();
let store = self.store.clone();
self
Expand Down
Loading

0 comments on commit c3de15b

Please sign in to comment.