Skip to content

Commit

Permalink
Apply suggestions from code review
Browse files Browse the repository at this point in the history
Add wjones127 suggestions

Co-authored-by: Will Jones <willjones127@gmail.com>
  • Loading branch information
Blajda and wjones127 authored Jan 30, 2023
1 parent 48a460a commit 85886a5
Showing 1 changed file with 16 additions and 12 deletions.
28 changes: 16 additions & 12 deletions rust/src/operations/filesystem_check.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
//! Audit the Delta Table for active files that do not exist in the underlying filesystem and remove them
//! Audit the Delta Table for active files that do not exist in the underlying filesystem and remove them.
//!
//! Active files are ones that have an add action in the log, but no corresponding remove action.
//! This operation creates a new transaction containing a remove action for each of the missing files.
//!
//! This can be used to repair tables where a data file has been deleted accidentally or
//! purposefully, if the file was corrupted.
//! # Example
//! ```rust ignore
//! let mut table = open_table("../path/to/table")?;
Expand Down Expand Up @@ -69,23 +74,22 @@ impl FileSystemCheckBuilder {

async fn create_fsck_plan(&self) -> DeltaResult<FileSystemCheckPlan> {
let mut files_to_remove = Vec::new();
let mut files_to_check: HashMap<String, &Add> = HashMap::new();
let mut files_to_check: HashMap<String, &Add> = self.state.files().iter()
.map(|active| (active.path.to_owned(), active))
.collect();
let version = self.state.version();
let store = self.store.clone();

self.state.files().iter().for_each(|active| {
files_to_check.insert(active.path.to_owned(), active);
});

let mut files = self.store.list(None).await?;
while let Some(result) = files.next().await {
let file = result?;
files_to_check.remove(file.location.as_ref());
}

files_to_check
let files_to_remove: Vec<Add> = files_to_check
.into_iter()
.for_each(|(_, file)| files_to_remove.push(file.to_owned()));
.map(|(_, file)| file.to_owned())
.collect();

Ok(FileSystemCheckPlan {
files_to_remove,
Expand All @@ -104,8 +108,8 @@ impl FileSystemCheckPlan {
});
}

let mut actions = Vec::new();
let mut removed_file_paths = Vec::new();
let mut actions = Vec::with_capacity(self.files_to_remove.len());
let mut removed_file_paths = Vec::with_capacity(self.files_to_remove.len());
let version = self.version;
let store = &self.store;

Expand Down Expand Up @@ -157,8 +161,8 @@ impl std::future::IntoFuture for FileSystemCheckBuilder {
FileSystemCheckMetrics {
files_removed: plan
.files_to_remove
.iter()
.map(|f| f.path.clone())
.into_iter()
.map(|f| f.path)
.collect(),
dry_run: true,
},
Expand Down

0 comments on commit 85886a5

Please sign in to comment.