Skip to content

Commit

Permalink
Update tests to new Archiver API
Browse files Browse the repository at this point in the history
  • Loading branch information
@teor2345
teor2345 committed Oct 4, 2024
1 parent cc03fb8 commit b23a4a0
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 117 deletions.
7 changes: 2 additions & 5 deletions crates/pallet-subspace/src/mock.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,11 +287,8 @@ pub fn create_archived_segment() -> &'static NewArchivedSegment {

let mut block = vec![0u8; RecordedHistorySegment::SIZE];
rand::thread_rng().fill(block.as_mut_slice());
archiver
.add_block(block, Default::default(), true)
.into_iter()
.next()
.unwrap()
let block_outcome = archiver.add_block(block, Default::default(), true);
block_outcome.archived_segments.into_iter().next().unwrap()
})
}

Expand Down
5 changes: 3 additions & 2 deletions crates/sc-consensus-subspace/src/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,8 +452,9 @@
// let mut archiver = Archiver::new(kzg).expect("Incorrect parameters for archiver");
//
// let genesis_block = client.block(client.info().genesis_hash).unwrap().unwrap();
// archiver
// .add_block(genesis_block.encode(), BlockObjectMapping::default())
// let block_outcome = archiver.add_block(genesis_block.encode(), BlockObjectMapping::default(), true);
// block_outcome
// .archived_segments
// .into_iter()
// .map(|archived_segment| archived_segment.pieces)
// .collect()
Expand Down
214 changes: 104 additions & 110 deletions crates/subspace-archiving/tests/integration/archiver.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,10 @@ fn archiver() {

(block, object_mapping)
};
let block0_outcome = archiver.add_block(block_0.clone(), block_0_object_mapping.clone(), true);
let archived_segments = block0_outcome.archived_segments;
// There is not enough data to produce archived segment yet
assert!(archiver
.add_block(block_0.clone(), block_0_object_mapping.clone(), true)
.is_empty());
assert!(archived_segments.is_empty());

let (block_1, block_1_object_mapping) = {
let mut block = vec![0u8; RecordedHistorySegment::SIZE / 3 * 2];
Expand Down Expand Up @@ -138,8 +138,8 @@ fn archiver() {
(block, object_mapping)
};
// This should produce 1 archived segment
let archived_segments =
archiver.add_block(block_1.clone(), block_1_object_mapping.clone(), true);
let block1_outcome = archiver.add_block(block_1.clone(), block_1_object_mapping.clone(), true);
let archived_segments = block1_outcome.archived_segments;
assert_eq!(archived_segments.len(), 1);

let first_archived_segment = archived_segments.into_iter().next().unwrap();
Expand All @@ -163,29 +163,46 @@ fn archiver() {
assert_eq!(last_archived_block.partial_archived(), Some(65011701));
}

assert_eq!(
first_archived_segment.object_mapping.len(),
RecordedHistorySegment::NUM_RAW_RECORDS
);
// 4 objects fit into the first segment
assert_eq!(
first_archived_segment
.object_mapping
.iter()
.filter(|object_mapping| !object_mapping.objects().is_empty())
.count(),
4
);
// 4 objects fit into the first segment, 2 from block0, and 2 from block1
let object_mapping = block0_outcome.object_mapping;
assert_eq!(object_mapping.len(), 2);
{
let block_objects = iter::repeat(block_0.as_ref())
.zip(block_0_object_mapping.objects())
.chain(iter::repeat(block_1.as_ref()).zip(block_1_object_mapping.objects()));
let piece_objects = first_archived_segment
.pieces
.source_pieces()
.zip(&first_archived_segment.object_mapping)
.flat_map(|(piece, object_mapping)| iter::repeat(piece).zip(object_mapping.objects()));

let block_objects = iter::repeat(block_0.as_ref()).zip(block_0_object_mapping.objects());
let piece_objects = object_mapping.iter().map(|object_mapping| {
let piece = first_archived_segment
.pieces
.source_pieces()
.nth(object_mapping.piece_index.position() as usize)
.unwrap();
let piece_object = PieceObject {
hash: object_mapping.hash,
offset: object_mapping.offset,
};
(piece, piece_object)
});
compare_block_objects_to_piece_objects(block_objects, piece_objects);
}
let object_mapping = block1_outcome.object_mapping;
assert_eq!(object_mapping.len(), 2);
{
let block_objects = iter::repeat(block_1.as_ref())
.zip(block_1_object_mapping.objects())
.take(2);
let piece_objects = object_mapping
.iter()
.map(|object_mapping| {
let piece = first_archived_segment
.pieces
.source_pieces()
.nth(object_mapping.piece_index.position() as usize)
.unwrap();
let piece_object = PieceObject {
hash: object_mapping.hash,
offset: object_mapping.offset,
};
(piece, piece_object)
})
.take(2);
compare_block_objects_to_piece_objects(block_objects, piece_objects);
}

Expand Down Expand Up @@ -216,12 +233,12 @@ fn archiver() {
block
};
// This should be big enough to produce two archived segments in one go
let archived_segments =
archiver.add_block(block_2.clone(), BlockObjectMapping::default(), true);
let block2_outcome = archiver.add_block(block_2.clone(), BlockObjectMapping::default(), true);
let archived_segments = block2_outcome.archived_segments;
assert_eq!(archived_segments.len(), 2);

// Check that initializing archiver with initial state before last block results in the same
// archived segments once last block is added
// archived segments and mappings once last block is added
{
let mut archiver_with_initial_state = Archiver::with_initial_state(
kzg.clone(),
Expand All @@ -238,45 +255,33 @@ fn archiver() {
BlockObjectMapping::default(),
true
),
archived_segments,
block2_outcome,
);
}

assert_eq!(
archived_segments[0].object_mapping.len(),
RecordedHistorySegment::NUM_RAW_RECORDS
);
// 1 object fits into the second segment
assert_eq!(
archived_segments[0]
.object_mapping
.iter()
.filter(|object_mapping| !object_mapping.objects().is_empty())
.count(),
1
);
assert_eq!(
archived_segments[1].object_mapping.len(),
RecordedHistorySegment::NUM_RAW_RECORDS
);
// 0 object fits into the second segment
assert_eq!(
archived_segments[1]
.object_mapping
.iter()
.filter(|object_mapping| !object_mapping.objects().is_empty())
.count(),
0
);
// 1 object fits into the second segment, and there are no objects left for the third segment
let object_mapping = block2_outcome.object_mapping;
assert_eq!(object_mapping.len(), 1);
{
let block_objects =
iter::repeat(block_1.as_ref()).zip(block_1_object_mapping.objects().iter().skip(2));
let piece_objects = archived_segments[0]
.pieces
.source_pieces()
.zip(&archived_segments[0].object_mapping)
.flat_map(|(piece, object_mapping)| iter::repeat(piece).zip(object_mapping.objects()));

let block_objects = iter::repeat(block_2.as_ref())
.zip(block_1_object_mapping.objects())
.skip(2)
.take(1);
let piece_objects = object_mapping
.iter()
.map(|object_mapping| {
let piece = archived_segments[0]
.pieces
.source_pieces()
.nth(object_mapping.piece_index.position() as usize)
.unwrap();
let piece_object = PieceObject {
hash: object_mapping.hash,
offset: object_mapping.offset,
};
(piece, piece_object)
})
.take(1);
compare_block_objects_to_piece_objects(block_objects, piece_objects);
}

Expand Down Expand Up @@ -343,12 +348,15 @@ fn archiver() {
thread_rng().fill(block.as_mut_slice());
block
};
let archived_segments =
archiver.add_block(block_3.clone(), BlockObjectMapping::default(), true);
let block3_outcome = archiver.add_block(block_3.clone(), BlockObjectMapping::default(), true);
let archived_segments = block3_outcome.archived_segments;
assert_eq!(archived_segments.len(), 1);
// There are no objects left for the fourth segment
let object_mapping = block3_outcome.object_mapping;
assert_eq!(object_mapping.len(), 0);

// Check that initializing archiver with initial state before last block results in the same
// archived segments once last block is added
// archived segments and mappings once last block is added
{
let mut archiver_with_initial_state = Archiver::with_initial_state(
kzg.clone(),
Expand All @@ -361,11 +369,11 @@ fn archiver() {

assert_eq!(
archiver_with_initial_state.add_block(block_3, BlockObjectMapping::default(), true),
archived_segments,
block3_outcome,
);
}

// Archived segment should fit exactly into the last archived segment (rare case)
// Block should fit exactly into the last archived segment (rare case)
{
let archived_segment = archived_segments.first().unwrap();
let last_archived_block = archived_segment.segment_header.last_archived_block();
Expand Down Expand Up @@ -492,6 +500,7 @@ fn one_byte_smaller_segment() {
assert_eq!(
Archiver::new(kzg.clone(), erasure_coding.clone())
.add_block(vec![0u8; block_size], BlockObjectMapping::default(), true)
.archived_segments
.len(),
1
);
Expand All @@ -503,6 +512,7 @@ fn one_byte_smaller_segment() {
BlockObjectMapping::default(),
true
)
.archived_segments
.is_empty());
}

Expand Down Expand Up @@ -532,13 +542,14 @@ fn spill_over_edge_case() {
- 3;
assert!(archiver
.add_block(vec![0u8; block_size], BlockObjectMapping::default(), true)
.archived_segments
.is_empty());

// Here we add one more block with internal length that takes 4 bytes in compact length
// encoding + one more for enum variant, this should result in new segment being created, but
// the very first segment item will not include newly added block because it would result in
// subtracting with overflow when trying to slice internal bytes of the segment item
let archived_segments = archiver.add_block(
let block_outcome = archiver.add_block(
vec![0u8; RecordedHistorySegment::SIZE],
BlockObjectMapping::V0 {
objects: vec![BlockObject {
Expand All @@ -548,23 +559,14 @@ fn spill_over_edge_case() {
},
true,
);
let archived_segments = block_outcome.archived_segments;
let object_mapping = block_outcome.object_mapping;
assert_eq!(archived_segments.len(), 2);
// If spill over actually happened, we'll not find object mapping in the first segment
assert_eq!(object_mapping.len(), 1);
assert_eq!(
archived_segments[0]
.object_mapping
.iter()
.filter(|o| !o.objects().is_empty())
.count(),
0
);
assert_eq!(
archived_segments[1]
.object_mapping
.iter()
.filter(|o| !o.objects().is_empty())
.count(),
1
object_mapping.first().unwrap().piece_index.segment_index(),
SegmentIndex::ONE
);
}

Expand All @@ -578,8 +580,10 @@ fn object_on_the_edge_of_segment() {
.unwrap();
let mut archiver = Archiver::new(kzg, erasure_coding);
let first_block = vec![0u8; RecordedHistorySegment::SIZE];
let archived_segments =
let block1_outcome =
archiver.add_block(first_block.clone(), BlockObjectMapping::default(), true);
let archived_segments = block1_outcome.archived_segments;
let object_mapping = block1_outcome.object_mapping;
assert_eq!(archived_segments.len(), 1);
let archived_segment = archived_segments.into_iter().next().unwrap();
let left_unarchived_from_first_block = first_block.len() as u32
Expand All @@ -589,6 +593,7 @@ fn object_on_the_edge_of_segment() {
.archived_progress
.partial()
.unwrap();
assert_eq!(object_mapping.len(), 0);

let mut second_block = vec![0u8; RecordedHistorySegment::SIZE * 2];
let object_mapping = BlockObject {
Expand Down Expand Up @@ -632,7 +637,7 @@ fn object_on_the_edge_of_segment() {
// First ensure that any smaller offset will get translated into the first archived segment,
// this is a protection against code regressions
{
let archived_segments = archiver.clone().add_block(
let block2_outcome = archiver.clone().add_block(
second_block.clone(),
BlockObjectMapping::V0 {
objects: vec![BlockObject {
Expand All @@ -642,45 +647,34 @@ fn object_on_the_edge_of_segment() {
},
true,
);
let archived_segments = block2_outcome.archived_segments;
let object_mapping = block2_outcome.object_mapping;

assert_eq!(archived_segments.len(), 2);
assert_eq!(object_mapping.len(), 1);
assert_eq!(
archived_segments[0]
.object_mapping
.iter()
.filter(|o| !o.objects().is_empty())
.count(),
1
object_mapping.first().unwrap().piece_index.segment_index(),
SegmentIndex::ZERO
);
}

let archived_segments = archiver.add_block(
let block2_outcome = archiver.add_block(
second_block,
BlockObjectMapping::V0 {
objects: vec![object_mapping],
},
true,
);
let archived_segments = block2_outcome.archived_segments;
let object_mapping = block2_outcome.object_mapping;

assert_eq!(archived_segments.len(), 2);
assert_eq!(
archived_segments[0]
.object_mapping
.iter()
.filter(|o| !o.objects().is_empty())
.count(),
0
);
// Object should fall in the next archived segment
assert_eq!(object_mapping.len(), 1);
assert_eq!(
archived_segments[1]
.object_mapping
.iter()
.filter(|o| !o.objects().is_empty())
.count(),
1
object_mapping.first().unwrap().piece_index.segment_index(),
SegmentIndex::ONE
);
assert_eq!(archived_segments[1].object_mapping[0].objects().len(), 1);

// Ensure bytes are mapped correctly
assert_eq!(
Expand All @@ -689,7 +683,7 @@ fn object_on_the_edge_of_segment() {
.to_raw_record_chunks()
.flatten()
.copied()
.skip(archived_segments[1].object_mapping[0].objects()[0].offset as usize)
.skip(object_mapping[0].offset as usize)
.take(mapped_bytes.len())
.collect::<Vec<_>>(),
mapped_bytes
Expand Down

0 comments on commit b23a4a0

Please sign in to comment.