-
Notifications
You must be signed in to change notification settings - Fork 242
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2151 from subspace/auditing-multiple-open-files
Implement an abstraction that opens file for auditing multiple times, once for every auditing rayon thread
- Loading branch information
Showing
6 changed files
with
99 additions
and
36 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
53 changes: 53 additions & 0 deletions
53
crates/subspace-farmer/src/single_disk_farm/farming/rayon_files.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
use std::fs::{File, OpenOptions}; | ||
use std::io; | ||
use std::path::Path; | ||
use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt}; | ||
use subspace_farmer_components::ReadAtSync; | ||
|
||
/// Wrapper data structure for multiple files to be used with [`rayon`] thread pool, where the same | ||
/// file is opened multiple times, once for each thread. | ||
pub struct RayonFiles { | ||
files: Vec<File>, | ||
} | ||
|
||
impl ReadAtSync for RayonFiles { | ||
fn read_at(&self, buf: &mut [u8], offset: usize) -> io::Result<()> { | ||
let thread_index = rayon::current_thread_index().ok_or_else(|| { | ||
io::Error::new( | ||
io::ErrorKind::Other, | ||
"Reads must be called from rayon worker thread", | ||
) | ||
})?; | ||
let file = self.files.get(thread_index).ok_or_else(|| { | ||
io::Error::new(io::ErrorKind::Other, "No files entry for this rayon thread") | ||
})?; | ||
|
||
file.read_at(buf, offset) | ||
} | ||
} | ||
|
||
impl ReadAtSync for &RayonFiles { | ||
fn read_at(&self, buf: &mut [u8], offset: usize) -> io::Result<()> { | ||
(*self).read_at(buf, offset) | ||
} | ||
} | ||
|
||
impl RayonFiles { | ||
/// Open file at specified as many times as there is number of threads in current [`rayon`] | ||
/// thread pool. | ||
pub fn open(path: &Path) -> io::Result<Self> { | ||
let files = (0..rayon::current_num_threads()) | ||
.map(|_| { | ||
let file = OpenOptions::new() | ||
.read(true) | ||
.advise_random_access() | ||
.open(path)?; | ||
file.advise_random_access()?; | ||
|
||
Ok::<_, io::Error>(file) | ||
}) | ||
.collect::<Result<Vec<_>, _>>()?; | ||
|
||
Ok(Self { files }) | ||
} | ||
} |