Reduce memory usage during plotting/replotting by deferring memory al…

…location

crates/pallet-subspace/src/mock.rs

@@ -50,7 +50,6 @@ use subspace_core_primitives::{
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
-use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::shim::ShimTable;
 use subspace_proof_of_space::{Table, TableGenerator};
@@ -416,13 +415,12 @@ pub fn create_signed_vote(
         min_sector_lifetime: HistorySize::from(NonZeroU64::new(4).unwrap()),
     };
     let pieces_in_sector = farmer_protocol_info.max_pieces_in_sector;
-    let sector_size = sector_size(pieces_in_sector);
 
     let mut table_generator = PosTable::generator();
 
     for sector_index in iter::from_fn(|| Some(rand::random())) {
-        let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
+        let mut plotted_sector_bytes = Vec::new();
+        let mut plotted_sector_metadata_bytes = Vec::new();
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/sp-lightclient/src/tests.rs

@@ -158,8 +158,8 @@ fn valid_header(
     let mut table_generator = PosTable::generator();
 
     for sector_index in iter::from_fn(|| Some(rand::random())) {
-        let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
+        let mut plotted_sector_bytes = Vec::new();
+        let mut plotted_sector_metadata_bytes = Vec::new();
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/subspace-farmer-components/benches/auditing.rs

@@ -113,8 +113,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     } else {
         println!("Plotting one sector...");
 
-        let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
+        let mut plotted_sector_bytes = Vec::new();
+        let mut plotted_sector_metadata_bytes = Vec::new();
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/subspace-farmer-components/benches/plotting.rs

@@ -9,7 +9,7 @@ use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::{HistorySize, PublicKey, Record, RecordedHistorySegment};
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
-use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
+use subspace_farmer_components::sector::sector_size;
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::chia::ChiaTable;
 use subspace_proof_of_space::Table;
@@ -58,8 +58,8 @@ fn criterion_benchmark(c: &mut Criterion) {
     };
 
     let sector_size = sector_size(pieces_in_sector);
-    let mut sector_bytes = vec![0; sector_size];
-    let mut sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
+    let mut sector_bytes = Vec::new();
+    let mut sector_metadata_bytes = Vec::new();
 
     let mut group = c.benchmark_group("plotting");
     group.throughput(Throughput::Bytes(sector_size as u64));

crates/subspace-farmer-components/benches/proving.rs

@@ -116,8 +116,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     } else {
         println!("Plotting one sector...");
 
-        let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
+        let mut plotted_sector_bytes = Vec::new();
+        let mut plotted_sector_metadata_bytes = Vec::new();
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/subspace-farmer-components/benches/reading.rs

@@ -110,8 +110,8 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     } else {
         println!("Plotting one sector...");
 
-        let mut plotted_sector_bytes = vec![0; sector_size];
-        let mut plotted_sector_metadata_bytes = vec![0; SectorMetadataChecksummed::encoded_size()];
+        let mut plotted_sector_bytes = Vec::new();
+        let mut plotted_sector_metadata_bytes = Vec::new();
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,

crates/subspace-farmer-components/src/plotting.rs

@@ -155,6 +155,10 @@ pub enum PlottingError {
 /// beginning of the sector (seek to desired offset before calling this function and seek back
 /// afterwards if necessary).
 ///
+/// Sector output and sector metadata output should be either empty (in which case they'll be
+/// resized to correct size automatically) or correctly sized from the beginning or else error will
+/// be returned.
+///
 /// NOTE: Even though this function is async, it has blocking code inside and must be running in a
 /// separate thread in order to prevent blocking an executor.
 #[allow(clippy::too_many_arguments)]
@@ -167,8 +171,8 @@ pub async fn plot_sector<PG, PosTable>(
     kzg: &Kzg,
     erasure_coding: &ErasureCoding,
     pieces_in_sector: u16,
-    sector_output: &mut [u8],
-    sector_metadata_output: &mut [u8],
+    sector_output: &mut Vec<u8>,
+    sector_metadata_output: &mut Vec<u8>,
     table_generator: &mut PosTable::Generator,
 ) -> Result<PlottedSector, PlottingError>
 where
@@ -181,14 +185,16 @@ where
 
     let sector_size = sector_size(pieces_in_sector);
 
-    if sector_output.len() != sector_size {
+    if !sector_output.is_empty() && sector_output.len() != sector_size {
         return Err(PlottingError::BadSectorOutputSize {
             provided: sector_output.len(),
             expected: sector_size,
         });
     }
 
-    if sector_metadata_output.len() < SectorMetadataChecksummed::encoded_size() {
+    if !sector_metadata_output.is_empty()
+        && sector_metadata_output.len() < SectorMetadataChecksummed::encoded_size()
+    {
         return Err(PlottingError::BadSectorMetadataOutputSize {
             provided: sector_metadata_output.len(),
             expected: SectorMetadataChecksummed::encoded_size(),
@@ -341,6 +347,9 @@ where
                 });
         });
 
+    sector_output.resize(sector_size, 0);
+    sector_metadata_output.resize(SectorMetadataChecksummed::encoded_size(), 0);
+
     // Write sector to disk in form of following regions:
     // * sector contents map
     // * record chunks as s-buckets

crates/subspace-farmer/src/single_disk_farm/plotting.rs

@@ -179,8 +179,8 @@ where
         let plotted_sector;
 
         (sector, sector_metadata, table_generator, plotted_sector) = {
-            let mut sector = vec![0; sector_size];
-            let mut sector_metadata = vec![0; sector_metadata_size];
+            let mut sector = Vec::new();
+            let mut sector_metadata = Vec::new();
 
             let piece_getter = piece_getter.clone();
             let kzg = kzg.clone();

test/subspace-test-client/src/lib.rs

@@ -37,7 +37,6 @@ use subspace_core_primitives::{HistorySize, PosSeed, PublicKey, Record, SegmentI
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy, PlottedSector};
-use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::{Table, TableGenerator};
 use subspace_runtime_primitives::opaque::Block;
@@ -240,8 +239,8 @@ where
         .next()
         .expect("First block is always producing one segment; qed");
     let history_size = HistorySize::from(SegmentIndex::ZERO);
-    let mut sector = vec![0u8; sector_size(pieces_in_sector)];
-    let mut sector_metadata = vec![0u8; SectorMetadataChecksummed::encoded_size()];
+    let mut sector = Vec::new();
+    let mut sector_metadata = Vec::new();
     let sector_index = 0;
     let public_key = PublicKey::from(keypair.public.to_bytes());
     let farmer_protocol_info = FarmerProtocolInfo {