Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Reduce memory usage during plotting/replotting by deferring memory allocation #2051

Merged
merged 1 commit into from
Oct 4, 2023
Merged

Reduce memory usage during plotting/replotting by deferring memory allocation #2051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions crates/pallet-subspace/src/mock.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ use subspace_core_primitives::{
use subspace_erasure_coding::ErasureCoding;
use subspace_farmer_components::auditing::audit_sector;
use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
use subspace_farmer_components::FarmerProtocolInfo;
use subspace_proof_of_space::shim::ShimTable;
use subspace_proof_of_space::{Table, TableGenerator};
Expand Down Expand Up @@ -445,13 +444,12 @@ pub fn create_signed_vote(
min_sector_lifetime: HistorySize::from(NonZeroU64::new(4).unwrap()),
};
let pieces_in_sector = farmer_protocol_info.max_pieces_in_sector;
let sector_size = sector_size(pieces_in_sector);

let mut table_generator = PosTable::generator();

for sector_index in iter::from_fn(|| Some(rand::random())) {
let mut plotted_sector_bytes = vec![0; sector_size];
let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
let mut plotted_sector_bytes = Vec::new();
let mut plotted_sector_metadata_bytes = Vec::new();

let plotted_sector = block_on(plot_sector::<_, PosTable>(
&public_key,
Expand Down
4 changes: 2 additions & 2 deletions crates/sp-lightclient/src/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,8 @@ fn valid_header(
let mut table_generator = PosTable::generator();

for sector_index in iter::from_fn(|| Some(rand::random())) {
let mut plotted_sector_bytes = vec![0; sector_size];
let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
let mut plotted_sector_bytes = Vec::new();
let mut plotted_sector_metadata_bytes = Vec::new();

let plotted_sector = block_on(plot_sector::<_, PosTable>(
&public_key,
Expand Down
4 changes: 2 additions & 2 deletions crates/subspace-farmer-components/benches/auditing.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
} else {
println!("Plotting one sector...");

let mut plotted_sector_bytes = vec![0; sector_size];
let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
let mut plotted_sector_bytes = Vec::new();
let mut plotted_sector_metadata_bytes = Vec::new();

let plotted_sector = block_on(plot_sector::<_, PosTable>(
&public_key,
Expand Down
6 changes: 3 additions & 3 deletions crates/subspace-farmer-components/benches/plotting.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use subspace_core_primitives::crypto::kzg::Kzg;
use subspace_core_primitives::{HistorySize, PublicKey, Record, RecordedHistorySegment};
use subspace_erasure_coding::ErasureCoding;
use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
use subspace_farmer_components::sector::sector_size;
use subspace_farmer_components::FarmerProtocolInfo;
use subspace_proof_of_space::chia::ChiaTable;
use subspace_proof_of_space::Table;
Expand Down Expand Up @@ -58,8 +58,8 @@ fn criterion_benchmark(c: &mut Criterion) {
};

let sector_size = sector_size(pieces_in_sector);
let mut sector_bytes = vec![0; sector_size];
let mut sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
let mut sector_bytes = Vec::new();
let mut sector_metadata_bytes = Vec::new();

let mut group = c.benchmark_group("plotting");
group.throughput(Throughput::Bytes(sector_size as u64));
Expand Down
4 changes: 2 additions & 2 deletions crates/subspace-farmer-components/benches/proving.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
} else {
println!("Plotting one sector...");

let mut plotted_sector_bytes = vec![0; sector_size];
let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
let mut plotted_sector_bytes = Vec::new();
let mut plotted_sector_metadata_bytes = Vec::new();

let plotted_sector = block_on(plot_sector::<_, PosTable>(
&public_key,
Expand Down
4 changes: 2 additions & 2 deletions crates/subspace-farmer-components/benches/reading.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
} else {
println!("Plotting one sector...");

let mut plotted_sector_bytes = vec![0; sector_size];
let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
let mut plotted_sector_bytes = Vec::new();
let mut plotted_sector_metadata_bytes = Vec::new();

let plotted_sector = block_on(plot_sector::<_, PosTable>(
&public_key,
Expand Down
17 changes: 13 additions & 4 deletions crates/subspace-farmer-components/src/plotting.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ pub enum PlottingError {
/// beginning of the sector (seek to desired offset before calling this function and seek back
/// afterwards if necessary).
///
/// Sector output and sector metadata output should be either empty (in which case they'll be
/// resized to correct size automatically) or correctly sized from the beginning or else error will
/// be returned.
///
/// NOTE: Even though this function is async, it has blocking code inside and must be running in a
/// separate thread in order to prevent blocking an executor.
#[allow(clippy::too_many_arguments)]
Expand All @@ -167,8 +171,8 @@ pub async fn plot_sector<PG, PosTable>(
kzg: &Kzg,
erasure_coding: &ErasureCoding,
pieces_in_sector: u16,
sector_output: &mut [u8],
sector_metadata_output: &mut [u8],
sector_output: &mut Vec<u8>,
sector_metadata_output: &mut Vec<u8>,
table_generator: &mut PosTable::Generator,
) -> Result<PlottedSector, PlottingError>
where
Expand All @@ -181,14 +185,16 @@ where

let sector_size = sector_size(pieces_in_sector);

if sector_output.len() != sector_size {
if !sector_output.is_empty() && sector_output.len() != sector_size {
return Err(PlottingError::BadSectorOutputSize {
provided: sector_output.len(),
expected: sector_size,
});
}

if sector_metadata_output.len() < SectorMetadataChecksummed::encoded_size() {
if !sector_metadata_output.is_empty()
&& sector_metadata_output.len() != SectorMetadataChecksummed::encoded_size()
{
return Err(PlottingError::BadSectorMetadataOutputSize {
provided: sector_metadata_output.len(),
expected: SectorMetadataChecksummed::encoded_size(),
Expand Down Expand Up @@ -341,6 +347,9 @@ where
});
});

sector_output.resize(sector_size, 0);
ParthDesai marked this conversation as resolved.
Show resolved Hide resolved
sector_metadata_output.resize(SectorMetadataChecksummed::encoded_size(), 0);

// Write sector to disk in form of following regions:
// * sector contents map
// * record chunks as s-buckets
Expand Down
4 changes: 2 additions & 2 deletions crates/subspace-farmer/src/single_disk_farm/plotting.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,8 @@ where
let plotted_sector;

(sector, sector_metadata, table_generator, plotted_sector) = {
let mut sector = vec![0; sector_size];
let mut sector_metadata = vec![0; sector_metadata_size];
let mut sector = Vec::new();
let mut sector_metadata = Vec::new();

let piece_getter = piece_getter.clone();
let kzg = kzg.clone();
Expand Down
5 changes: 2 additions & 3 deletions test/subspace-test-client/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ use subspace_core_primitives::{
use subspace_erasure_coding::ErasureCoding;
use subspace_farmer_components::auditing::audit_sector;
use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy, PlottedSector};
use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
use subspace_farmer_components::FarmerProtocolInfo;
use subspace_proof_of_space::{Table, TableGenerator};
use subspace_runtime_primitives::opaque::Block;
Expand Down Expand Up @@ -238,8 +237,8 @@ where
.next()
.expect("First block is always producing one segment; qed");
let history_size = HistorySize::from(SegmentIndex::ZERO);
let mut sector = vec![0u8; sector_size(pieces_in_sector)];
let mut sector_metadata = vec![0u8; SectorMetadataChecksummed::encoded_size()];
let mut sector = Vec::new();
let mut sector_metadata = Vec::new();
let sector_index = 0;
let public_key = PublicKey::from(keypair.public.to_bytes());
let farmer_protocol_info = FarmerProtocolInfo {
Expand Down
Loading