From 954d81aaa95eef083d6bea8657f748c47d4511ce Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 22 Nov 2021 16:23:05 +0100 Subject: [PATCH 01/58] WIP: translated bitmap (eventually) fixes #2892 --- rts/motoko-rts/src/gc/mark_compact.rs | 5 +++-- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 13 +++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index cf4de274035..1b10ef8c93b 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -5,7 +5,7 @@ pub mod bitmap; pub mod mark_stack; -use bitmap::{alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END}; +use bitmap::{alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, translate_bitmap, BITMAP_ITER_END}; use mark_stack::{alloc_mark_stack, free_mark_stack, pop_mark_stack, push_mark_stack}; use crate::constants::WORD_SIZE; @@ -99,6 +99,7 @@ unsafe fn mark_compact( let mem_size = Bytes(heap_end - heap_base); alloc_bitmap(mem, mem_size); + translate_bitmap(heap_base / WORD_SIZE); alloc_mark_stack(mem); mark_static_roots(mem, static_roots, heap_base); @@ -139,7 +140,7 @@ unsafe fn mark_object(mem: &mut M, obj: Value, heap_base: u32) { // Check object alignment to avoid undefined behavior. See also static_checks module. debug_assert_eq!(obj % WORD_SIZE, 0); - let obj_idx = (obj - heap_base) / WORD_SIZE; + let obj_idx = obj / WORD_SIZE; if get_bit(obj_idx) { // Already marked diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 90cd7dcff44..b5f0a5a6042 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -24,17 +24,22 @@ pub unsafe fn free_bitmap() { BITMAP_PTR = core::ptr::null_mut(); } -pub unsafe fn get_bit(idx: u32) -> bool { +/// Move the bitmap base such that accessing `idx` +/// will touch the beginning of the bitmap. +pub unsafe fn translate_bitmap(idx: u32) { let byte_idx = idx / 8; + BITMAP_PTR = BITMAP_PTR.sub(byte_idx as usize) +} + +pub unsafe fn get_bit(idx: u32) -> bool { + let (byte_idx, bit_idx) = (idx / 8, idx % 8); let byte = *BITMAP_PTR.add(byte_idx as usize); - let bit_idx = idx % 8; (byte >> bit_idx) & 0b1 == 0b1 } pub unsafe fn set_bit(idx: u32) { - let byte_idx = idx / 8; + let (byte_idx, bit_idx) = (idx / 8, idx % 8); let byte = *BITMAP_PTR.add(byte_idx as usize); - let bit_idx = idx % 8; let new_byte = byte | (0b1 << bit_idx); *BITMAP_PTR.add(byte_idx as usize) = new_byte; } From 9b76fed7f21b001280e00f7bc8ee44bc2899c471 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 22 Nov 2021 18:38:58 +0100 Subject: [PATCH 02/58] fmt --- rts/motoko-rts/src/gc/mark_compact.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 1b10ef8c93b..d4d705d6c66 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -5,7 +5,9 @@ pub mod bitmap; pub mod mark_stack; -use bitmap::{alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, translate_bitmap, BITMAP_ITER_END}; +use bitmap::{ + alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, translate_bitmap, BITMAP_ITER_END, +}; use mark_stack::{alloc_mark_stack, free_mark_stack, pop_mark_stack, push_mark_stack}; use crate::constants::WORD_SIZE; From e22bc43588c5bdd6d75e60cfcbef567a2f7cbb77 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 12:45:48 +0100 Subject: [PATCH 03/58] WIP: try allocating the heap such that the dynamic heap is 32-bit aligned --- rts/motoko-rts-tests/src/gc/heap.rs | 57 ++++++++++++++++---- rts/motoko-rts-tests/src/main.rs | 2 +- rts/motoko-rts/src/gc/copying.rs | 2 +- rts/motoko-rts/src/gc/mark_compact.rs | 9 +++- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 26 ++++++--- rts/motoko-rts/src/memory/ic.rs | 16 ++++-- 6 files changed, 88 insertions(+), 24 deletions(-) diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index d2911a35249..afbcb32f1d7 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -6,8 +6,10 @@ use motoko_rts::gc::mark_compact::mark_stack::INIT_STACK_SIZE; use motoko_rts::memory::Memory; use motoko_rts::types::*; +use std::alloc::*; use std::cell::{Ref, RefCell}; use std::convert::TryFrom; +use std::ptr::NonNull; use std::rc::Rc; use fxhash::{FxHashMap, FxHashSet}; @@ -90,7 +92,7 @@ impl MotokoHeap { } /// Get the heap as an array. Use `offset` values returned by the methods above to read. - pub fn heap(&self) -> Ref> { + pub(crate) fn heap(&self) -> Ref> { Ref::map(self.inner.borrow(), |heap| &heap.heap) } @@ -108,10 +110,38 @@ impl MotokoHeap { } } +pub(crate) struct Aligned32Bytes {} + +unsafe impl Allocator for Aligned32Bytes { + fn allocate(&self, layout_unaligned: Layout) -> Result, AllocError> { + //assert_eq!(layout_unaligned.size(), 232); + assert_eq!(layout_unaligned.align(), 1); + //let overhead = Layout::for_value(&[1; 0]); + let overhead = Layout::from_size_align(16, 8).unwrap(); + assert_eq!(overhead.size(), 16); + assert_eq!(overhead.align(), 8); + //let layout = Layout::from_size_align(layout_unaligned.size() + 32 - overhead.size(), 32).unwrap(); + let layout = layout_unaligned.align_to(32).unwrap(); + unsafe { + let ptr = std::alloc::alloc(layout); //.add(overhead.size()); + assert_eq!(ptr as usize % 32, 0); + let slice = std::slice::from_raw_parts_mut( + ptr, + layout_unaligned.size(), + ); + assert_eq!(slice.len(), layout_unaligned.size()); + Ok(NonNull::new(slice).unwrap()) + } + } + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + dealloc(ptr.as_ptr(), layout.align_to(32).unwrap()) + } +} + struct MotokoHeapInner { /// The heap. This is a boxed slice instead of a vector as growing this wouldn't make sense /// (all pointers would have to be updated). - heap: Box<[u8]>, + heap: Box<[u8], Aligned32Bytes>, /// Where the dynamic heap starts heap_base_offset: usize, @@ -196,7 +226,8 @@ impl MotokoHeapInner { .as_usize(); let total_heap_size_bytes = static_heap_size_bytes + dynamic_heap_size_bytes; - + let static_heap_size_aligned_bytes = (static_heap_size_bytes + 31) / 32 * 32; + let heap_size = heap_size_for_gc( gc, static_heap_size_bytes, @@ -204,25 +235,33 @@ impl MotokoHeapInner { map.len(), ); - let mut heap: Vec = vec![0; heap_size]; + let mut heap: Vec = Vec::with_capacity_in(heap_size + (static_heap_size_aligned_bytes - static_heap_size_bytes), Aligned32Bytes {}); + //let mut heap: Vec = vec![0; heap_size]; + heap.resize(heap_size + (static_heap_size_aligned_bytes - static_heap_size_bytes), 0); + //assert_eq!(heap.len(), heap_size); + assert_eq!(&mut heap[static_heap_size_aligned_bytes] as *mut u8 as usize % 32, 0); // Maps `ObjectIdx`s into their offsets in the heap let object_addrs: FxHashMap = - create_dynamic_heap(map, continuation_table, &mut heap[static_heap_size_bytes..]); + create_dynamic_heap(map, continuation_table, &mut heap[static_heap_size_aligned_bytes..]); // Closure table pointer is the last word in static heap - let continuation_table_ptr_offset = static_heap_size_bytes - WORD_SIZE; + let continuation_table_ptr_offset = static_heap_size_aligned_bytes - WORD_SIZE; create_static_heap( roots, &object_addrs, continuation_table_ptr_offset, static_heap_size_bytes + dynamic_heap_size_without_continuation_table_bytes, - &mut heap[..static_heap_size_bytes], + &mut heap[..static_heap_size_aligned_bytes], ); + let boxed_slice = heap.into_boxed_slice(); + let heap_start = boxed_slice.as_ptr() as usize; + //assert_eq!(heap_start, 0); + assert_eq!(heap_start % 32, 0); MotokoHeapInner { - heap: heap.into_boxed_slice(), - heap_base_offset: static_heap_size_bytes, + heap: boxed_slice, + heap_base_offset: static_heap_size_aligned_bytes, heap_ptr_offset: total_heap_size_bytes, static_root_array_offset: 0, continuation_table_ptr_offset: continuation_table_ptr_offset, diff --git a/rts/motoko-rts-tests/src/main.rs b/rts/motoko-rts-tests/src/main.rs index 9f77525e034..b7e16109b95 100644 --- a/rts/motoko-rts-tests/src/main.rs +++ b/rts/motoko-rts-tests/src/main.rs @@ -1,4 +1,4 @@ -#![feature(map_first_last)] +#![feature(map_first_last, allocator_api)] mod bigint; mod bitmap; diff --git a/rts/motoko-rts/src/gc/copying.rs b/rts/motoko-rts/src/gc/copying.rs index 1328e2f7c4e..620b42e47d8 100644 --- a/rts/motoko-rts/src/gc/copying.rs +++ b/rts/motoko-rts/src/gc/copying.rs @@ -23,7 +23,7 @@ unsafe fn copying_gc(mem: &mut M) { copying_gc_internal( mem, - ic::get_heap_base(), + ic::get_aligned_heap_base(), // get_hp || ic::HP as usize, // set_hp diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index d4d705d6c66..36ffe473ed2 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -38,9 +38,11 @@ unsafe fn schedule_compacting_gc(mem: &mut M) { unsafe fn compacting_gc(mem: &mut M) { use crate::memory::ic; + //ic::init(); + compacting_gc_internal( mem, - ic::get_heap_base(), + ic::get_aligned_heap_base(), // get_hp || ic::HP as usize, // set_hp @@ -72,6 +74,9 @@ pub unsafe fn compacting_gc_internal< note_live_size: NoteLiveSize, note_reclaimed: NoteReclaimed, ) { + assert_eq!(heap_base % 32, 0); + assert_eq!(heap_base / WORD_SIZE % 8, 0); + let old_hp = get_hp() as u32; mark_compact( @@ -135,7 +140,7 @@ unsafe fn mark_static_roots(mem: &mut M, static_roots: Value, heap_ba } } -unsafe fn mark_object(mem: &mut M, obj: Value, heap_base: u32) { +unsafe fn mark_object(mem: &mut M, obj: Value, _heap_base: u32) { let obj_tag = obj.tag(); let obj = obj.get_ptr() as u32; diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index b5f0a5a6042..12f7a909ab0 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -4,6 +4,7 @@ use crate::types::{size_of, Blob, Bytes, Obj}; /// Current bitmap static mut BITMAP_PTR: *mut u8 = core::ptr::null_mut(); +static mut BITMAP_BASE: usize = 0; pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes) { // We will have at most this many objects in the heap, each requiring a bit @@ -22,23 +23,28 @@ pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes) { pub unsafe fn free_bitmap() { BITMAP_PTR = core::ptr::null_mut(); + BITMAP_BASE = 0 } /// Move the bitmap base such that accessing `idx` /// will touch the beginning of the bitmap. pub unsafe fn translate_bitmap(idx: u32) { - let byte_idx = idx / 8; - BITMAP_PTR = BITMAP_PTR.sub(byte_idx as usize) + let (byte_idx, bit_idx) = (idx / 8, idx % 8); + assert_eq!(bit_idx, 0); + BITMAP_BASE = byte_idx as usize; + BITMAP_PTR = BITMAP_PTR.sub(BITMAP_BASE); } pub unsafe fn get_bit(idx: u32) -> bool { let (byte_idx, bit_idx) = (idx / 8, idx % 8); + debug_assert!(byte_idx as usize >= BITMAP_BASE); let byte = *BITMAP_PTR.add(byte_idx as usize); - (byte >> bit_idx) & 0b1 == 0b1 + (byte >> bit_idx) & 0b1 == 0b1 // TODO IDEA: != 0 (faster, also below!) } pub unsafe fn set_bit(idx: u32) { let (byte_idx, bit_idx) = (idx / 8, idx % 8); + debug_assert!(byte_idx as usize >= BITMAP_BASE); let byte = *BITMAP_PTR.add(byte_idx as usize); let new_byte = byte | (0b1 << bit_idx); *BITMAP_PTR.add(byte_idx as usize) = new_byte; @@ -59,7 +65,8 @@ pub struct BitmapIter { } pub unsafe fn iter_bits() -> BitmapIter { - let blob_len_bytes = (BITMAP_PTR.sub(size_of::().to_bytes().as_usize()) as *mut Obj) + let bitmap = BITMAP_PTR.add(BITMAP_BASE); + let blob_len_bytes = (bitmap.sub(size_of::().to_bytes().as_usize()) as *mut Obj) .as_blob() .len() .as_u32(); @@ -71,7 +78,7 @@ pub unsafe fn iter_bits() -> BitmapIter { let current_word = if blob_len_64bit_words == 0 { 0 } else { - *(BITMAP_PTR as *const u64) + *(bitmap as *const u64) }; BitmapIter { @@ -105,7 +112,9 @@ impl BitmapIter { // Inner loop iterates bits in the current word while self.current_word != 0 { - if self.current_word & 0b1 == 0b1 { + if self.current_word & 0b1 == 0b1 + /* TODO */ + { let bit_idx = (self.current_word_idx * 64) + (64 - self.bits_left); self.current_word >>= 1; self.bits_left -= 1; @@ -122,8 +131,9 @@ impl BitmapIter { if self.current_word_idx == self.size { return BITMAP_ITER_END; } - self.current_word = - unsafe { *(BITMAP_PTR as *const u64).add(self.current_word_idx as usize) }; + self.current_word = unsafe { + *(BITMAP_PTR.add(BITMAP_BASE) as *const u64).add(self.current_word_idx as usize) + }; self.bits_left = 64; } } diff --git a/rts/motoko-rts/src/memory/ic.rs b/rts/motoko-rts/src/memory/ic.rs index 39bb474e832..82aee0351ce 100644 --- a/rts/motoko-rts/src/memory/ic.rs +++ b/rts/motoko-rts/src/memory/ic.rs @@ -24,13 +24,23 @@ pub(crate) static mut LAST_HP: u32 = 0; // Provided by generated code extern "C" { - pub(crate) fn get_heap_base() -> u32; + fn get_heap_base() -> u32; pub(crate) fn get_static_roots() -> Value; } +pub(crate) unsafe fn get_aligned_heap_base() -> u32 { + assert!(false); + // align to 32 bytes + HP = ((get_heap_base() + 31) / 32) * 32; + assert_eq!(HP, 1); + HP +} + #[no_mangle] unsafe extern "C" fn init() { - HP = get_heap_base() as u32; + assert!(false); + HP = get_aligned_heap_base(); + assert_eq!(HP, 1); LAST_HP = HP; } @@ -51,7 +61,7 @@ unsafe extern "C" fn get_total_allocations() -> Bytes { #[no_mangle] unsafe extern "C" fn get_heap_size() -> Bytes { - Bytes(HP - get_heap_base()) + Bytes(HP - get_aligned_heap_base()) } /// Provides a `Memory` implementation, to be used in functions compiled for IC or WASI. The From b40f6b6350053c2139062a24b2a69908821393af Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 13:07:05 +0100 Subject: [PATCH 04/58] ruthlessly increment the static_heap_size_bytes THIS PASSES TESTS! --- rts/motoko-rts-tests/src/gc/heap.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index afbcb32f1d7..23cf2fb3983 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -212,7 +212,8 @@ impl MotokoHeapInner { // Each object will be 3 words per object + one word for each reference. Static heap will // have an array (header + length) with one element, one MutBox for each root. +1 for // continuation table pointer. - let static_heap_size_bytes = (2 + roots.len() + (roots.len() * 2) + 1) * WORD_SIZE; + let static_heap_size_bytes_unaligned = (2 + roots.len() + (roots.len() * 2) + 1) * WORD_SIZE; + let static_heap_size_bytes = (static_heap_size_bytes_unaligned + 31) / 32 * 32; let dynamic_heap_size_without_continuation_table_bytes = { let object_headers_words = map.len() * 3; @@ -226,7 +227,6 @@ impl MotokoHeapInner { .as_usize(); let total_heap_size_bytes = static_heap_size_bytes + dynamic_heap_size_bytes; - let static_heap_size_aligned_bytes = (static_heap_size_bytes + 31) / 32 * 32; let heap_size = heap_size_for_gc( gc, @@ -235,24 +235,23 @@ impl MotokoHeapInner { map.len(), ); - let mut heap: Vec = Vec::with_capacity_in(heap_size + (static_heap_size_aligned_bytes - static_heap_size_bytes), Aligned32Bytes {}); - //let mut heap: Vec = vec![0; heap_size]; - heap.resize(heap_size + (static_heap_size_aligned_bytes - static_heap_size_bytes), 0); - //assert_eq!(heap.len(), heap_size); - assert_eq!(&mut heap[static_heap_size_aligned_bytes] as *mut u8 as usize % 32, 0); + let mut heap: Vec = Vec::with_capacity_in(heap_size, Aligned32Bytes {}); + heap.resize(heap_size, 0); + assert_eq!(heap.len(), heap_size); + assert_eq!(&mut heap[static_heap_size_bytes] as *mut u8 as usize % 32, 0); // Maps `ObjectIdx`s into their offsets in the heap let object_addrs: FxHashMap = - create_dynamic_heap(map, continuation_table, &mut heap[static_heap_size_aligned_bytes..]); + create_dynamic_heap(map, continuation_table, &mut heap[static_heap_size_bytes..]); // Closure table pointer is the last word in static heap - let continuation_table_ptr_offset = static_heap_size_aligned_bytes - WORD_SIZE; + let continuation_table_ptr_offset = static_heap_size_bytes - WORD_SIZE; create_static_heap( roots, &object_addrs, continuation_table_ptr_offset, static_heap_size_bytes + dynamic_heap_size_without_continuation_table_bytes, - &mut heap[..static_heap_size_aligned_bytes], + &mut heap[..static_heap_size_bytes], ); let boxed_slice = heap.into_boxed_slice(); @@ -261,7 +260,7 @@ impl MotokoHeapInner { assert_eq!(heap_start % 32, 0); MotokoHeapInner { heap: boxed_slice, - heap_base_offset: static_heap_size_aligned_bytes, + heap_base_offset: static_heap_size_bytes, heap_ptr_offset: total_heap_size_bytes, static_root_array_offset: 0, continuation_table_ptr_offset: continuation_table_ptr_offset, From 373651771875af727a74c397332c4ff120a0e4c1 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 13:15:06 +0100 Subject: [PATCH 05/58] tweak assertions --- rts/motoko-rts/src/memory/ic.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/rts/motoko-rts/src/memory/ic.rs b/rts/motoko-rts/src/memory/ic.rs index 82aee0351ce..99a5b6735ac 100644 --- a/rts/motoko-rts/src/memory/ic.rs +++ b/rts/motoko-rts/src/memory/ic.rs @@ -29,18 +29,16 @@ extern "C" { } pub(crate) unsafe fn get_aligned_heap_base() -> u32 { - assert!(false); // align to 32 bytes - HP = ((get_heap_base() + 31) / 32) * 32; - assert_eq!(HP, 1); - HP + let hb = ((get_heap_base() + 31) / 32) * 32; + assert_eq!(hb % 32, 0); + hb } #[no_mangle] unsafe extern "C" fn init() { - assert!(false); HP = get_aligned_heap_base(); - assert_eq!(HP, 1); + assert_eq!(HP % 32, 0); LAST_HP = HP; } From 1873a2e3572e14e3cab567c314e2712f8aecc6fe Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 13:18:19 +0100 Subject: [PATCH 06/58] cleanup --- rts/motoko-rts/src/gc/mark_compact.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 36ffe473ed2..4ffce127455 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -38,8 +38,6 @@ unsafe fn schedule_compacting_gc(mem: &mut M) { unsafe fn compacting_gc(mem: &mut M) { use crate::memory::ic; - //ic::init(); - compacting_gc_internal( mem, ic::get_aligned_heap_base(), From 01a0cc8e6298bdf2f512dd1e868e01eb1600fe0c Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 13:39:15 +0100 Subject: [PATCH 07/58] fmt --- rts/motoko-rts-tests/src/gc/heap.rs | 41 +++++++++++++++-------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index 23cf2fb3983..3903a61a39e 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -116,20 +116,17 @@ unsafe impl Allocator for Aligned32Bytes { fn allocate(&self, layout_unaligned: Layout) -> Result, AllocError> { //assert_eq!(layout_unaligned.size(), 232); assert_eq!(layout_unaligned.align(), 1); - //let overhead = Layout::for_value(&[1; 0]); - let overhead = Layout::from_size_align(16, 8).unwrap(); + //let overhead = Layout::for_value(&[1; 0]); + let overhead = Layout::from_size_align(16, 8).unwrap(); assert_eq!(overhead.size(), 16); assert_eq!(overhead.align(), 8); //let layout = Layout::from_size_align(layout_unaligned.size() + 32 - overhead.size(), 32).unwrap(); - let layout = layout_unaligned.align_to(32).unwrap(); + let layout = layout_unaligned.align_to(32).unwrap(); unsafe { - let ptr = std::alloc::alloc(layout); //.add(overhead.size()); - assert_eq!(ptr as usize % 32, 0); - let slice = std::slice::from_raw_parts_mut( - ptr, - layout_unaligned.size(), - ); - assert_eq!(slice.len(), layout_unaligned.size()); + let ptr = std::alloc::alloc(layout); //.add(overhead.size()); + assert_eq!(ptr as usize % 32, 0); + let slice = std::slice::from_raw_parts_mut(ptr, layout_unaligned.size()); + assert_eq!(slice.len(), layout_unaligned.size()); Ok(NonNull::new(slice).unwrap()) } } @@ -212,8 +209,9 @@ impl MotokoHeapInner { // Each object will be 3 words per object + one word for each reference. Static heap will // have an array (header + length) with one element, one MutBox for each root. +1 for // continuation table pointer. - let static_heap_size_bytes_unaligned = (2 + roots.len() + (roots.len() * 2) + 1) * WORD_SIZE; - let static_heap_size_bytes = (static_heap_size_bytes_unaligned + 31) / 32 * 32; + let static_heap_size_bytes_unaligned = + (2 + roots.len() + (roots.len() * 2) + 1) * WORD_SIZE; + let static_heap_size_bytes = (static_heap_size_bytes_unaligned + 31) / 32 * 32; let dynamic_heap_size_without_continuation_table_bytes = { let object_headers_words = map.len() * 3; @@ -227,7 +225,7 @@ impl MotokoHeapInner { .as_usize(); let total_heap_size_bytes = static_heap_size_bytes + dynamic_heap_size_bytes; - + let heap_size = heap_size_for_gc( gc, static_heap_size_bytes, @@ -236,9 +234,12 @@ impl MotokoHeapInner { ); let mut heap: Vec = Vec::with_capacity_in(heap_size, Aligned32Bytes {}); - heap.resize(heap_size, 0); - assert_eq!(heap.len(), heap_size); - assert_eq!(&mut heap[static_heap_size_bytes] as *mut u8 as usize % 32, 0); + heap.resize(heap_size, 0); + assert_eq!(heap.len(), heap_size); + assert_eq!( + &mut heap[static_heap_size_bytes] as *mut u8 as usize % 32, + 0 + ); // Maps `ObjectIdx`s into their offsets in the heap let object_addrs: FxHashMap = @@ -254,10 +255,10 @@ impl MotokoHeapInner { &mut heap[..static_heap_size_bytes], ); - let boxed_slice = heap.into_boxed_slice(); - let heap_start = boxed_slice.as_ptr() as usize; - //assert_eq!(heap_start, 0); - assert_eq!(heap_start % 32, 0); + let boxed_slice = heap.into_boxed_slice(); + let heap_start = boxed_slice.as_ptr() as usize; + //assert_eq!(heap_start, 0); + assert_eq!(heap_start % 32, 0); MotokoHeapInner { heap: boxed_slice, heap_base_offset: static_heap_size_bytes, From a8b5310a9f4739068b2f07dc8146d2f17695b23e Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 16:27:22 +0100 Subject: [PATCH 08/58] WIP: minor refactoring --- rts/motoko-rts-tests/src/bitmap.rs | 10 ++++-- rts/motoko-rts/src/gc/mark_compact.rs | 5 ++- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 37 ++++++++++---------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/rts/motoko-rts-tests/src/bitmap.rs b/rts/motoko-rts-tests/src/bitmap.rs index aee40a8d18a..2fc60b4c687 100644 --- a/rts/motoko-rts-tests/src/bitmap.rs +++ b/rts/motoko-rts-tests/src/bitmap.rs @@ -2,7 +2,7 @@ use crate::memory::TestMemory; use motoko_rts::constants::WORD_SIZE; use motoko_rts::gc::mark_compact::bitmap::{ - alloc_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END, + alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END, }; use motoko_rts::memory::Memory; use motoko_rts::types::{Bytes, Words}; @@ -68,7 +68,7 @@ fn test_set_get(mem: &mut M, mut bits: Vec) -> Result<(), String unsafe { alloc_bitmap( mem, - Bytes((u32::from(*bits.iter().max().unwrap()) + 1) * WORD_SIZE), + Bytes((u32::from(*bits.iter().max().unwrap()) + 1) * WORD_SIZE), 0 // TODO: test with static heap ); for bit in &bits { @@ -98,6 +98,8 @@ fn test_set_get(mem: &mut M, mut bits: Vec) -> Result<(), String last_bit = Some(bit); } + + free_bitmap() } Ok(()) @@ -111,7 +113,7 @@ fn test_bit_iter(mem: &mut M, bits: HashSet) -> TestCaseResult { .to_bytes(); unsafe { - alloc_bitmap(mem, heap_size); + alloc_bitmap(mem, heap_size, 0); // TODO: test with static heap for bit in bits.iter() { set_bit(u32::from(*bit)); @@ -154,6 +156,8 @@ fn test_bit_iter(mem: &mut M, bits: HashSet) -> TestCaseResult { .into(), )); } + + free_bitmap() } Ok(()) diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 4ffce127455..e075020d7a4 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -6,7 +6,7 @@ pub mod bitmap; pub mod mark_stack; use bitmap::{ - alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, translate_bitmap, BITMAP_ITER_END, + alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END, }; use mark_stack::{alloc_mark_stack, free_mark_stack, pop_mark_stack, push_mark_stack}; @@ -103,8 +103,7 @@ unsafe fn mark_compact( ) { let mem_size = Bytes(heap_end - heap_base); - alloc_bitmap(mem, mem_size); - translate_bitmap(heap_base / WORD_SIZE); + alloc_bitmap(mem, mem_size, heap_base / WORD_SIZE); alloc_mark_stack(mem); mark_static_roots(mem, static_roots, heap_base); diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 12f7a909ab0..dfab785e053 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -4,47 +4,46 @@ use crate::types::{size_of, Blob, Bytes, Obj}; /// Current bitmap static mut BITMAP_PTR: *mut u8 = core::ptr::null_mut(); -static mut BITMAP_BASE: usize = 0; +static mut BITMAP_COMPENSATION: usize = 0; +static mut BITMAP_SIZE: u32 = 0; -pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes) { +pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_prefix_words: u32) { // We will have at most this many objects in the heap, each requiring a bit let n_bits = heap_size.to_words().as_u32(); + // Additional bits needed to account for + // unaligned heap start address + let prefix_bits = heap_prefix_words % 8; // Each byte will hold 8 bits. - let bitmap_bytes = (n_bits + 7) / 8; + BITMAP_SIZE = (prefix_bits + n_bits + 7) / 8; // Also round allocation up to 8-bytes to make iteration efficient. We want to be able to read // 64 bits in a single read and check as many bits as possible with a single `word != 0`. - let bitmap_bytes = Bytes(((bitmap_bytes + 7) / 8) * 8); + let bitmap_bytes = Bytes(((BITMAP_SIZE + 7) / 8) * 8); // Allocating an actual object here as otherwise dump_heap gets confused let blob = alloc_blob(mem, bitmap_bytes).get_ptr() as *mut Blob; memzero(blob.payload_addr() as usize, bitmap_bytes.to_words()); - BITMAP_PTR = blob.payload_addr() + BITMAP_COMPENSATION = (heap_prefix_words / 8) as usize; + BITMAP_PTR = blob.payload_addr().sub(BITMAP_COMPENSATION) } pub unsafe fn free_bitmap() { BITMAP_PTR = core::ptr::null_mut(); - BITMAP_BASE = 0 -} - -/// Move the bitmap base such that accessing `idx` -/// will touch the beginning of the bitmap. -pub unsafe fn translate_bitmap(idx: u32) { - let (byte_idx, bit_idx) = (idx / 8, idx % 8); - assert_eq!(bit_idx, 0); - BITMAP_BASE = byte_idx as usize; - BITMAP_PTR = BITMAP_PTR.sub(BITMAP_BASE); + BITMAP_SIZE = 0; + BITMAP_COMPENSATION = 0 } pub unsafe fn get_bit(idx: u32) -> bool { let (byte_idx, bit_idx) = (idx / 8, idx % 8); - debug_assert!(byte_idx as usize >= BITMAP_BASE); + debug_assert!(byte_idx as usize >= BITMAP_COMPENSATION); + debug_assert!(BITMAP_COMPENSATION + BITMAP_SIZE as usize > byte_idx as usize); let byte = *BITMAP_PTR.add(byte_idx as usize); (byte >> bit_idx) & 0b1 == 0b1 // TODO IDEA: != 0 (faster, also below!) } pub unsafe fn set_bit(idx: u32) { let (byte_idx, bit_idx) = (idx / 8, idx % 8); - debug_assert!(byte_idx as usize >= BITMAP_BASE); + debug_assert!(byte_idx as usize >= BITMAP_COMPENSATION); + debug_assert!(BITMAP_COMPENSATION + BITMAP_SIZE as usize > byte_idx as usize); let byte = *BITMAP_PTR.add(byte_idx as usize); let new_byte = byte | (0b1 << bit_idx); *BITMAP_PTR.add(byte_idx as usize) = new_byte; @@ -65,7 +64,7 @@ pub struct BitmapIter { } pub unsafe fn iter_bits() -> BitmapIter { - let bitmap = BITMAP_PTR.add(BITMAP_BASE); + let bitmap = BITMAP_PTR.add(BITMAP_COMPENSATION); let blob_len_bytes = (bitmap.sub(size_of::().to_bytes().as_usize()) as *mut Obj) .as_blob() .len() @@ -132,7 +131,7 @@ impl BitmapIter { return BITMAP_ITER_END; } self.current_word = unsafe { - *(BITMAP_PTR.add(BITMAP_BASE) as *const u64).add(self.current_word_idx as usize) + *(BITMAP_PTR.add(BITMAP_COMPENSATION) as *const u64).add(self.current_word_idx as usize) }; self.bits_left = 64; } From b5c6d82ea490f66c92b1850369227175183047d2 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 16:50:30 +0100 Subject: [PATCH 09/58] fmt --- rts/motoko-rts-tests/src/bitmap.rs | 7 ++++--- rts/motoko-rts/src/gc/mark_compact.rs | 4 +--- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rts/motoko-rts-tests/src/bitmap.rs b/rts/motoko-rts-tests/src/bitmap.rs index 2fc60b4c687..482532ff7d4 100644 --- a/rts/motoko-rts-tests/src/bitmap.rs +++ b/rts/motoko-rts-tests/src/bitmap.rs @@ -68,7 +68,8 @@ fn test_set_get(mem: &mut M, mut bits: Vec) -> Result<(), String unsafe { alloc_bitmap( mem, - Bytes((u32::from(*bits.iter().max().unwrap()) + 1) * WORD_SIZE), 0 // TODO: test with static heap + Bytes((u32::from(*bits.iter().max().unwrap()) + 1) * WORD_SIZE), + 0, // TODO: test with static heap ); for bit in &bits { @@ -99,7 +100,7 @@ fn test_set_get(mem: &mut M, mut bits: Vec) -> Result<(), String last_bit = Some(bit); } - free_bitmap() + free_bitmap() } Ok(()) @@ -157,7 +158,7 @@ fn test_bit_iter(mem: &mut M, bits: HashSet) -> TestCaseResult { )); } - free_bitmap() + free_bitmap() } Ok(()) diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index e075020d7a4..834ceab0ae1 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -5,9 +5,7 @@ pub mod bitmap; pub mod mark_stack; -use bitmap::{ - alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END, -}; +use bitmap::{alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END}; use mark_stack::{alloc_mark_stack, free_mark_stack, pop_mark_stack, push_mark_stack}; use crate::constants::WORD_SIZE; diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index dfab785e053..7ce54bdd864 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -131,7 +131,8 @@ impl BitmapIter { return BITMAP_ITER_END; } self.current_word = unsafe { - *(BITMAP_PTR.add(BITMAP_COMPENSATION) as *const u64).add(self.current_word_idx as usize) + *(BITMAP_PTR.add(BITMAP_COMPENSATION) as *const u64) + .add(self.current_word_idx as usize) }; self.bits_left = 64; } From 922d33091803cd8cebf8cbc59f08bffc916de234 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 21:41:37 +0100 Subject: [PATCH 10/58] WIP: remove the allocator based alignment and instead fill the heap array in such that it gets realigned --- rts/motoko-rts-tests/src/bitmap.rs | 4 +- rts/motoko-rts-tests/src/gc/heap.rs | 73 ++++++-------------- rts/motoko-rts-tests/src/main.rs | 2 +- rts/motoko-rts/src/gc/mark_compact.rs | 3 - rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 5 +- 5 files changed, 28 insertions(+), 59 deletions(-) diff --git a/rts/motoko-rts-tests/src/bitmap.rs b/rts/motoko-rts-tests/src/bitmap.rs index 482532ff7d4..50dd04a4937 100644 --- a/rts/motoko-rts-tests/src/bitmap.rs +++ b/rts/motoko-rts-tests/src/bitmap.rs @@ -69,7 +69,7 @@ fn test_set_get(mem: &mut M, mut bits: Vec) -> Result<(), String alloc_bitmap( mem, Bytes((u32::from(*bits.iter().max().unwrap()) + 1) * WORD_SIZE), - 0, // TODO: test with static heap + 1, // TODO: test with static heap ); for bit in &bits { @@ -114,7 +114,7 @@ fn test_bit_iter(mem: &mut M, bits: HashSet) -> TestCaseResult { .to_bytes(); unsafe { - alloc_bitmap(mem, heap_size, 0); // TODO: test with static heap + alloc_bitmap(mem, heap_size, 1); // TODO: test with static heap for bit in bits.iter() { set_bit(u32::from(*bit)); diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index 3903a61a39e..a7ed7b9ecb7 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -6,10 +6,8 @@ use motoko_rts::gc::mark_compact::mark_stack::INIT_STACK_SIZE; use motoko_rts::memory::Memory; use motoko_rts::types::*; -use std::alloc::*; use std::cell::{Ref, RefCell}; use std::convert::TryFrom; -use std::ptr::NonNull; use std::rc::Rc; use fxhash::{FxHashMap, FxHashSet}; @@ -92,7 +90,7 @@ impl MotokoHeap { } /// Get the heap as an array. Use `offset` values returned by the methods above to read. - pub(crate) fn heap(&self) -> Ref> { + pub fn heap(&self) -> Ref> { Ref::map(self.inner.borrow(), |heap| &heap.heap) } @@ -110,35 +108,10 @@ impl MotokoHeap { } } -pub(crate) struct Aligned32Bytes {} - -unsafe impl Allocator for Aligned32Bytes { - fn allocate(&self, layout_unaligned: Layout) -> Result, AllocError> { - //assert_eq!(layout_unaligned.size(), 232); - assert_eq!(layout_unaligned.align(), 1); - //let overhead = Layout::for_value(&[1; 0]); - let overhead = Layout::from_size_align(16, 8).unwrap(); - assert_eq!(overhead.size(), 16); - assert_eq!(overhead.align(), 8); - //let layout = Layout::from_size_align(layout_unaligned.size() + 32 - overhead.size(), 32).unwrap(); - let layout = layout_unaligned.align_to(32).unwrap(); - unsafe { - let ptr = std::alloc::alloc(layout); //.add(overhead.size()); - assert_eq!(ptr as usize % 32, 0); - let slice = std::slice::from_raw_parts_mut(ptr, layout_unaligned.size()); - assert_eq!(slice.len(), layout_unaligned.size()); - Ok(NonNull::new(slice).unwrap()) - } - } - unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - dealloc(ptr.as_ptr(), layout.align_to(32).unwrap()) - } -} - struct MotokoHeapInner { /// The heap. This is a boxed slice instead of a vector as growing this wouldn't make sense /// (all pointers would have to be updated). - heap: Box<[u8], Aligned32Bytes>, + heap: Box<[u8]>, /// Where the dynamic heap starts heap_base_offset: usize, @@ -209,9 +182,7 @@ impl MotokoHeapInner { // Each object will be 3 words per object + one word for each reference. Static heap will // have an array (header + length) with one element, one MutBox for each root. +1 for // continuation table pointer. - let static_heap_size_bytes_unaligned = - (2 + roots.len() + (roots.len() * 2) + 1) * WORD_SIZE; - let static_heap_size_bytes = (static_heap_size_bytes_unaligned + 31) / 32 * 32; + let static_heap_size_bytes = (2 + roots.len() + (roots.len() * 2) + 1) * WORD_SIZE; let dynamic_heap_size_without_continuation_table_bytes = { let object_headers_words = map.len() * 3; @@ -233,17 +204,21 @@ impl MotokoHeapInner { map.len(), ); - let mut heap: Vec = Vec::with_capacity_in(heap_size, Aligned32Bytes {}); - heap.resize(heap_size, 0); - assert_eq!(heap.len(), heap_size); - assert_eq!( - &mut heap[static_heap_size_bytes] as *mut u8 as usize % 32, - 0 - ); + let mut heap: Vec = vec![0; heap_size + 28]; + + // MarkCompact assumes that the dynamic heap starts at a 32-byte multiple + let misalign = match gc { + GC::Copying => 0, + GC::MarkCompact => (32 - &heap[static_heap_size_bytes] as *const u8 as usize % 32) % 32, + }; + assert_eq!(misalign % 4, 0); // Maps `ObjectIdx`s into their offsets in the heap - let object_addrs: FxHashMap = - create_dynamic_heap(map, continuation_table, &mut heap[static_heap_size_bytes..]); + let object_addrs: FxHashMap = create_dynamic_heap( + map, + continuation_table, + &mut heap[static_heap_size_bytes + misalign..heap_size + misalign], + ); // Closure table pointer is the last word in static heap let continuation_table_ptr_offset = static_heap_size_bytes - WORD_SIZE; @@ -252,19 +227,15 @@ impl MotokoHeapInner { &object_addrs, continuation_table_ptr_offset, static_heap_size_bytes + dynamic_heap_size_without_continuation_table_bytes, - &mut heap[..static_heap_size_bytes], + &mut heap[misalign..static_heap_size_bytes + misalign], ); - let boxed_slice = heap.into_boxed_slice(); - let heap_start = boxed_slice.as_ptr() as usize; - //assert_eq!(heap_start, 0); - assert_eq!(heap_start % 32, 0); MotokoHeapInner { - heap: boxed_slice, - heap_base_offset: static_heap_size_bytes, - heap_ptr_offset: total_heap_size_bytes, - static_root_array_offset: 0, - continuation_table_ptr_offset: continuation_table_ptr_offset, + heap: heap.into_boxed_slice(), + heap_base_offset: static_heap_size_bytes + misalign, + heap_ptr_offset: total_heap_size_bytes + misalign, + static_root_array_offset: misalign, + continuation_table_ptr_offset: continuation_table_ptr_offset + misalign, } } diff --git a/rts/motoko-rts-tests/src/main.rs b/rts/motoko-rts-tests/src/main.rs index b7e16109b95..9f77525e034 100644 --- a/rts/motoko-rts-tests/src/main.rs +++ b/rts/motoko-rts-tests/src/main.rs @@ -1,4 +1,4 @@ -#![feature(map_first_last, allocator_api)] +#![feature(map_first_last)] mod bigint; mod bitmap; diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 834ceab0ae1..41ccc97ce51 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -70,9 +70,6 @@ pub unsafe fn compacting_gc_internal< note_live_size: NoteLiveSize, note_reclaimed: NoteReclaimed, ) { - assert_eq!(heap_base % 32, 0); - assert_eq!(heap_base / WORD_SIZE % 8, 0); - let old_hp = get_hp() as u32; mark_compact( diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 7ce54bdd864..1955d00e5fe 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -12,8 +12,9 @@ pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_p let n_bits = heap_size.to_words().as_u32(); // Additional bits needed to account for // unaligned heap start address - let prefix_bits = heap_prefix_words % 8; - // Each byte will hold 8 bits. + let prefix_bits = 0; //heap_prefix_words % 8; + // Each byte will hold 8 bits. + //assert_eq!(prefix_bits, 1); BITMAP_SIZE = (prefix_bits + n_bits + 7) / 8; // Also round allocation up to 8-bytes to make iteration efficient. We want to be able to read // 64 bits in a single read and check as many bits as possible with a single `word != 0`. From be85715afd5053ef0bb1a352077259eb2523fca3 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 21:43:12 +0100 Subject: [PATCH 11/58] rename --- rts/motoko-rts-tests/src/gc/heap.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index a7ed7b9ecb7..1f9a5bbad20 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -207,17 +207,17 @@ impl MotokoHeapInner { let mut heap: Vec = vec![0; heap_size + 28]; // MarkCompact assumes that the dynamic heap starts at a 32-byte multiple - let misalign = match gc { + let realign = match gc { GC::Copying => 0, GC::MarkCompact => (32 - &heap[static_heap_size_bytes] as *const u8 as usize % 32) % 32, }; - assert_eq!(misalign % 4, 0); + assert_eq!(realign % 4, 0); // Maps `ObjectIdx`s into their offsets in the heap let object_addrs: FxHashMap = create_dynamic_heap( map, continuation_table, - &mut heap[static_heap_size_bytes + misalign..heap_size + misalign], + &mut heap[static_heap_size_bytes + realign..heap_size + realign], ); // Closure table pointer is the last word in static heap @@ -227,15 +227,15 @@ impl MotokoHeapInner { &object_addrs, continuation_table_ptr_offset, static_heap_size_bytes + dynamic_heap_size_without_continuation_table_bytes, - &mut heap[misalign..static_heap_size_bytes + misalign], + &mut heap[realign..static_heap_size_bytes + realign], ); MotokoHeapInner { heap: heap.into_boxed_slice(), - heap_base_offset: static_heap_size_bytes + misalign, - heap_ptr_offset: total_heap_size_bytes + misalign, - static_root_array_offset: misalign, - continuation_table_ptr_offset: continuation_table_ptr_offset + misalign, + heap_base_offset: static_heap_size_bytes + realign, + heap_ptr_offset: total_heap_size_bytes + realign, + static_root_array_offset: realign, + continuation_table_ptr_offset: continuation_table_ptr_offset + realign, } } From 3712928751a8292b6726908ebf774813f8c72b60 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 21:54:17 +0100 Subject: [PATCH 12/58] clean up --- rts/motoko-rts/src/memory/ic.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rts/motoko-rts/src/memory/ic.rs b/rts/motoko-rts/src/memory/ic.rs index 99a5b6735ac..abc06099ba8 100644 --- a/rts/motoko-rts/src/memory/ic.rs +++ b/rts/motoko-rts/src/memory/ic.rs @@ -30,15 +30,12 @@ extern "C" { pub(crate) unsafe fn get_aligned_heap_base() -> u32 { // align to 32 bytes - let hb = ((get_heap_base() + 31) / 32) * 32; - assert_eq!(hb % 32, 0); - hb + (get_heap_base() + 31) / 32 * 32 } #[no_mangle] unsafe extern "C" fn init() { HP = get_aligned_heap_base(); - assert_eq!(HP % 32, 0); LAST_HP = HP; } From 20defb8b3a3668eefdb4bd4a1cfdbdf2cc71b3d2 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 21:57:30 +0100 Subject: [PATCH 13/58] undo debugging --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 1955d00e5fe..7ce54bdd864 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -12,9 +12,8 @@ pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_p let n_bits = heap_size.to_words().as_u32(); // Additional bits needed to account for // unaligned heap start address - let prefix_bits = 0; //heap_prefix_words % 8; - // Each byte will hold 8 bits. - //assert_eq!(prefix_bits, 1); + let prefix_bits = heap_prefix_words % 8; + // Each byte will hold 8 bits. BITMAP_SIZE = (prefix_bits + n_bits + 7) / 8; // Also round allocation up to 8-bytes to make iteration efficient. We want to be able to read // 64 bits in a single read and check as many bits as possible with a single `word != 0`. From 64efc2e005f41a0025a4bcd592c412b49675a7cb Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 22:46:36 +0100 Subject: [PATCH 14/58] remove unneeded stuff --- rts/motoko-rts/src/gc/mark_compact.rs | 10 ++++++---- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 6 ++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 41ccc97ce51..9043b0bc445 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -72,6 +72,8 @@ pub unsafe fn compacting_gc_internal< ) { let old_hp = get_hp() as u32; + assert_eq!(heap_base % 32, 0); + mark_compact( mem, set_hp, @@ -105,7 +107,7 @@ unsafe fn mark_compact( if (*continuation_table_ptr_loc).is_ptr() { // TODO: No need to check if continuation table is already marked - mark_object(mem, *continuation_table_ptr_loc, heap_base); + mark_object(mem, *continuation_table_ptr_loc); // Similar to `mark_root_mutbox_fields`, `continuation_table_ptr_loc` is in static heap so // it will be readable when we unthread continuation table thread(continuation_table_ptr_loc); @@ -132,7 +134,7 @@ unsafe fn mark_static_roots(mem: &mut M, static_roots: Value, heap_ba } } -unsafe fn mark_object(mem: &mut M, obj: Value, _heap_base: u32) { +unsafe fn mark_object(mem: &mut M, obj: Value) { let obj_tag = obj.tag(); let obj = obj.get_ptr() as u32; @@ -159,7 +161,7 @@ unsafe fn mark_stack(mem: &mut M, heap_base: u32) { unsafe fn mark_fields(mem: &mut M, obj: *mut Obj, obj_tag: Tag, heap_base: u32) { visit_pointer_fields(obj, obj_tag, heap_base as usize, |field_addr| { let field_value = *field_addr; - mark_object(mem, field_value, heap_base); + mark_object(mem, field_value); // Thread if backwards or self pointer if field_value.get_ptr() <= obj as usize { @@ -175,7 +177,7 @@ unsafe fn mark_root_mutbox_fields(mem: &mut M, mutbox: *mut MutBox, h if pointer_to_dynamic_heap(field_addr, heap_base as usize) { // TODO: We should be able to omit the "already marked" check here as no two root MutBox // can point to the same object (I think) - mark_object(mem, *field_addr, heap_base); + mark_object(mem, *field_addr); // It's OK to thread forward pointers here as the static objects won't be moved, so we will // be able to unthread objects pointed by these fields later. thread(field_addr); diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 7ce54bdd864..8cd8cf15917 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -8,13 +8,11 @@ static mut BITMAP_COMPENSATION: usize = 0; static mut BITMAP_SIZE: u32 = 0; pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_prefix_words: u32) { + assert_eq!(heap_prefix_words % 8, 0); // We will have at most this many objects in the heap, each requiring a bit let n_bits = heap_size.to_words().as_u32(); - // Additional bits needed to account for - // unaligned heap start address - let prefix_bits = heap_prefix_words % 8; // Each byte will hold 8 bits. - BITMAP_SIZE = (prefix_bits + n_bits + 7) / 8; + BITMAP_SIZE = (n_bits + 7) / 8; // Also round allocation up to 8-bytes to make iteration efficient. We want to be able to read // 64 bits in a single read and check as many bits as possible with a single `word != 0`. let bitmap_bytes = Bytes(((BITMAP_SIZE + 7) / 8) * 8); From 22611f2c3ff0b92896f31f991e603478619ef28a Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 23 Nov 2021 23:03:07 +0100 Subject: [PATCH 15/58] these tests only make sense with no static heap portions --- rts/motoko-rts-tests/src/bitmap.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rts/motoko-rts-tests/src/bitmap.rs b/rts/motoko-rts-tests/src/bitmap.rs index 50dd04a4937..66e260e8cc2 100644 --- a/rts/motoko-rts-tests/src/bitmap.rs +++ b/rts/motoko-rts-tests/src/bitmap.rs @@ -69,7 +69,7 @@ fn test_set_get(mem: &mut M, mut bits: Vec) -> Result<(), String alloc_bitmap( mem, Bytes((u32::from(*bits.iter().max().unwrap()) + 1) * WORD_SIZE), - 1, // TODO: test with static heap + 0, ); for bit in &bits { @@ -114,7 +114,7 @@ fn test_bit_iter(mem: &mut M, bits: HashSet) -> TestCaseResult { .to_bytes(); unsafe { - alloc_bitmap(mem, heap_size, 1); // TODO: test with static heap + alloc_bitmap(mem, heap_size, 0); for bit in bits.iter() { set_bit(u32::from(*bit)); From 72d128da3313d54297e427caaa8a1da806fecd42 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 00:34:43 +0100 Subject: [PATCH 16/58] Update rts/motoko-rts/src/gc/mark_compact/bitmap.rs --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 8cd8cf15917..17547c51d32 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -8,7 +8,7 @@ static mut BITMAP_COMPENSATION: usize = 0; static mut BITMAP_SIZE: u32 = 0; pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_prefix_words: u32) { - assert_eq!(heap_prefix_words % 8, 0); + debug_assert_eq!(heap_prefix_words % 8, 0); // We will have at most this many objects in the heap, each requiring a bit let n_bits = heap_size.to_words().as_u32(); // Each byte will hold 8 bits. From bf1cd6ef8ac3e9820ef1b97041f82f4a892051f6 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 11:07:22 +0100 Subject: [PATCH 17/58] might be faster, but definitely shorter --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 17547c51d32..869e68dce57 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -35,7 +35,7 @@ pub unsafe fn get_bit(idx: u32) -> bool { debug_assert!(byte_idx as usize >= BITMAP_COMPENSATION); debug_assert!(BITMAP_COMPENSATION + BITMAP_SIZE as usize > byte_idx as usize); let byte = *BITMAP_PTR.add(byte_idx as usize); - (byte >> bit_idx) & 0b1 == 0b1 // TODO IDEA: != 0 (faster, also below!) + (byte >> bit_idx) & 0b1 != 0 } pub unsafe fn set_bit(idx: u32) { @@ -109,9 +109,7 @@ impl BitmapIter { // Inner loop iterates bits in the current word while self.current_word != 0 { - if self.current_word & 0b1 == 0b1 - /* TODO */ - { + if self.current_word & 0b1 != 0 { let bit_idx = (self.current_word_idx * 64) + (64 - self.bits_left); self.current_word >>= 1; self.bits_left -= 1; From 7cc89b1b885146954c3821f81b137ac45fb0917c Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 11:54:25 +0100 Subject: [PATCH 18/58] move invariant computation out of the loop --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 869e68dce57..23e7ca17e29 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -100,6 +100,7 @@ impl BitmapIter { /// Returns the next bit, or `BITMAP_ITER_END` if there are no more bits set. pub fn next(&mut self) -> u32 { debug_assert!(self.current_word_idx <= self.size); + let bitmap = BITMAP_PTR.add(BITMAP_COMPENSATION); // Outer loop iterates 64-bit words loop { @@ -126,10 +127,8 @@ impl BitmapIter { if self.current_word_idx == self.size { return BITMAP_ITER_END; } - self.current_word = unsafe { - *(BITMAP_PTR.add(BITMAP_COMPENSATION) as *const u64) - .add(self.current_word_idx as usize) - }; + self.current_word = + unsafe { *(bitmap as *const u64).add(self.current_word_idx as usize) }; self.bits_left = 64; } } From f670ba7260f89d20aea110da2cf45eb5e8f59926 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 12:05:29 +0100 Subject: [PATCH 19/58] it is usafe --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 23e7ca17e29..25cbe454453 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -100,7 +100,7 @@ impl BitmapIter { /// Returns the next bit, or `BITMAP_ITER_END` if there are no more bits set. pub fn next(&mut self) -> u32 { debug_assert!(self.current_word_idx <= self.size); - let bitmap = BITMAP_PTR.add(BITMAP_COMPENSATION); + let bitmap = unsafe { BITMAP_PTR.add(BITMAP_COMPENSATION) }; // Outer loop iterates 64-bit words loop { From 78bf0478162ece7c26dc432eef845e23f63be026 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 13:43:32 +0100 Subject: [PATCH 20/58] add an explanation how this scheme works --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 25cbe454453..f783954d8a0 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -2,12 +2,55 @@ use crate::mem_utils::memzero; use crate::memory::{alloc_blob, Memory}; use crate::types::{size_of, Blob, Bytes, Obj}; +/* How the Wasm-heap maps to the bitmap + + +---- Rts stack ----+---- Motoko statics ----+---- Dynamic heap --~~--+ Heap end + (prefix words) | BM | <- bitmap lives here + / \ + / \ + /...bits...\ each bit corresponds to a word + in the dynamic heap + +When marking, we pass an absolute pointer (i.e. address space relative), for speed. +Internally the bitmap is kept in a (non-moving) blob at the start of the dynamic heap (DH). +To efficiently mark the right bit in the bitmap, we maintain a pointer that points +_before the start of the bitmap_ such that using the `/%8`-operation on the DH +absolute word number will address the right bit: + + +---- BITMAP_PTR + v + | (forbidden) |...................... bitmap ................| + | heap_prefix_words / 8 bytes | heap_size / 32 bytes + | BITMAP_COMPENSATION | BITMAP_SIZE + +Debug assertions guard the forbidden bytes from access, as this area physically overlaps +with the Motoko static heap. + +## The alignment caveat + +For this scheme to work, it is essential that the start of the DH is an address that +is divisible by 32 (implying that `heap_prefix_words % 8 == 0`). Otherwise the +`/%8`-operation on the DH's starting address will not yield the least significant bit +in the BM, and thus the sweep operation will be off. + +## Example calculation: + +Assume the DH is at 0x80000. The BM thus could be at 0x80004. +Since the heap_prefix_words is 0x20000, BITMAP_PTR = 0x80004 - 0x20000 / 8 = 0x7c004. + +Now let's mark the address 0x80548 in the DH. Its absolute word number is 0x20152. +The `(0x20152 / 8, 0x20152 % 8)`-rule gives a bit position 2 with byte offset 0x402a, +thus we mark bit 2 in byte 0x7c004+0x402a = 0x8002e, which is physically in the BM. + + */ + /// Current bitmap static mut BITMAP_PTR: *mut u8 = core::ptr::null_mut(); static mut BITMAP_COMPENSATION: usize = 0; static mut BITMAP_SIZE: u32 = 0; pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_prefix_words: u32) { + // see Note "How the Wasm-heap maps to the bitmap" above debug_assert_eq!(heap_prefix_words % 8, 0); // We will have at most this many objects in the heap, each requiring a bit let n_bits = heap_size.to_words().as_u32(); From aa537381e8e2b605ebba685de0f4f57a3c56bbaa Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 16:23:10 +0100 Subject: [PATCH 21/58] Update rts/motoko-rts/src/gc/mark_compact/bitmap.rs --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index f783954d8a0..77bdd15a8d6 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -33,7 +33,7 @@ is divisible by 32 (implying that `heap_prefix_words % 8 == 0`). Otherwise the `/%8`-operation on the DH's starting address will not yield the least significant bit in the BM, and thus the sweep operation will be off. -## Example calculation: +## Example calculation Assume the DH is at 0x80000. The BM thus could be at 0x80004. Since the heap_prefix_words is 0x20000, BITMAP_PTR = 0x80004 - 0x20000 / 8 = 0x7c004. From 61ae218275447814b6a77f3a4c72e68cf3fd9808 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 17:38:26 +0100 Subject: [PATCH 22/58] tweak --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 77bdd15a8d6..72d3e17dcc8 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -50,7 +50,7 @@ static mut BITMAP_COMPENSATION: usize = 0; static mut BITMAP_SIZE: u32 = 0; pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_prefix_words: u32) { - // see Note "How the Wasm-heap maps to the bitmap" above + // See Note "How the Wasm-heap maps to the bitmap" above debug_assert_eq!(heap_prefix_words % 8, 0); // We will have at most this many objects in the heap, each requiring a bit let n_bits = heap_size.to_words().as_u32(); From 221c7c9e688138517f46fba63189fe6b83aeadb3 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 17:54:09 +0100 Subject: [PATCH 23/58] REVERT ME --- perf-delta.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-delta.nix b/perf-delta.nix index edf3b4f8153..b7c7450298e 100644 --- a/perf-delta.nix +++ b/perf-delta.nix @@ -36,7 +36,7 @@ let do # ignore all errors echo -n $file - if timeout 10s moc $file -no-check-ir -ref-system-api -o $file.wasm 2>/dev/null + if timeout 10s moc $file --compacting-gc -no-check-ir -ref-system-api -o $file.wasm 2>/dev/null then echo " failed (ignored)" else echo " ok" fi From 1d8e3bb082a260acf12c2e9a25d8c40761725f6c Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Wed, 24 Nov 2021 19:52:17 +0100 Subject: [PATCH 24/58] REVERT ME TOO --- test/profile-report.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/profile-report.sh b/test/profile-report.sh index 80543254dbe..51971c11934 100755 --- a/test/profile-report.sh +++ b/test/profile-report.sh @@ -51,7 +51,7 @@ __END__ for file in perf/*.mo; do base="$(basename "$file" .mo)" echo "Profiling $base..." - moc -g "$file" -o "_profile_build/$base.wasm" + moc --compacting-gc --force-gc -g "$file" -o "_profile_build/$base.wasm" wasm-profiler-instrument --ic-system-api -i "_profile_build/$base.wasm" -o "_profile_build/$base.instrumented.wasm" # qr.mo takes far too long with profiling instrumentation, so limit runtime From 394f4ce450398977955bc22ae7efd87ef589eb6f Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 20:31:31 +0100 Subject: [PATCH 25/58] define and use `get_bitmap_forbidden_size` --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 72d3e17dcc8..0a39d04da23 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -49,6 +49,11 @@ static mut BITMAP_PTR: *mut u8 = core::ptr::null_mut(); static mut BITMAP_COMPENSATION: usize = 0; static mut BITMAP_SIZE: u32 = 0; +#[cfg(debug_assertions)] +fn get_bitmap_forbidden_size() -> usize { + unsafe { BITMAP_COMPENSATION } +} + pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_prefix_words: u32) { // See Note "How the Wasm-heap maps to the bitmap" above debug_assert_eq!(heap_prefix_words % 8, 0); @@ -75,16 +80,20 @@ pub unsafe fn free_bitmap() { pub unsafe fn get_bit(idx: u32) -> bool { let (byte_idx, bit_idx) = (idx / 8, idx % 8); - debug_assert!(byte_idx as usize >= BITMAP_COMPENSATION); - debug_assert!(BITMAP_COMPENSATION + BITMAP_SIZE as usize > byte_idx as usize); + #[cfg(debug_assertions)] + debug_assert!(byte_idx as usize >= get_bitmap_forbidden_size()); + #[cfg(debug_assertions)] + debug_assert!(get_bitmap_forbidden_size() + BITMAP_SIZE as usize > byte_idx as usize); let byte = *BITMAP_PTR.add(byte_idx as usize); (byte >> bit_idx) & 0b1 != 0 } pub unsafe fn set_bit(idx: u32) { let (byte_idx, bit_idx) = (idx / 8, idx % 8); - debug_assert!(byte_idx as usize >= BITMAP_COMPENSATION); - debug_assert!(BITMAP_COMPENSATION + BITMAP_SIZE as usize > byte_idx as usize); + #[cfg(debug_assertions)] + debug_assert!(byte_idx as usize >= get_bitmap_forbidden_size()); + #[cfg(debug_assertions)] + debug_assert!(get_bitmap_forbidden_size() + BITMAP_SIZE as usize > byte_idx as usize); let byte = *BITMAP_PTR.add(byte_idx as usize); let new_byte = byte | (0b1 << bit_idx); *BITMAP_PTR.add(byte_idx as usize) = new_byte; From 1f85b6b3a5682a852a77815060b6fd65e8df802d Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 20:34:22 +0100 Subject: [PATCH 26/58] rename to `BITMAP_FORBIDDEN_PTR` --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 0a39d04da23..2b1f45576f9 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -17,7 +17,7 @@ To efficiently mark the right bit in the bitmap, we maintain a pointer that poin _before the start of the bitmap_ such that using the `/%8`-operation on the DH absolute word number will address the right bit: - +---- BITMAP_PTR + +---- BITMAP_FORBIDDEN_PTR v | (forbidden) |...................... bitmap ................| | heap_prefix_words / 8 bytes | heap_size / 32 bytes @@ -36,7 +36,7 @@ in the BM, and thus the sweep operation will be off. ## Example calculation Assume the DH is at 0x80000. The BM thus could be at 0x80004. -Since the heap_prefix_words is 0x20000, BITMAP_PTR = 0x80004 - 0x20000 / 8 = 0x7c004. +Since the heap_prefix_words is 0x20000, BITMAP_FORBIDDEN_PTR = 0x80004 - 0x20000 / 8 = 0x7c004. Now let's mark the address 0x80548 in the DH. Its absolute word number is 0x20152. The `(0x20152 / 8, 0x20152 % 8)`-rule gives a bit position 2 with byte offset 0x402a, @@ -45,7 +45,7 @@ thus we mark bit 2 in byte 0x7c004+0x402a = 0x8002e, which is physically in the */ /// Current bitmap -static mut BITMAP_PTR: *mut u8 = core::ptr::null_mut(); +static mut BITMAP_FORBIDDEN_PTR: *mut u8 = core::ptr::null_mut(); static mut BITMAP_COMPENSATION: usize = 0; static mut BITMAP_SIZE: u32 = 0; @@ -69,11 +69,11 @@ pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_p memzero(blob.payload_addr() as usize, bitmap_bytes.to_words()); BITMAP_COMPENSATION = (heap_prefix_words / 8) as usize; - BITMAP_PTR = blob.payload_addr().sub(BITMAP_COMPENSATION) + BITMAP_FORBIDDEN_PTR = blob.payload_addr().sub(BITMAP_COMPENSATION) } pub unsafe fn free_bitmap() { - BITMAP_PTR = core::ptr::null_mut(); + BITMAP_FORBIDDEN_PTR = core::ptr::null_mut(); BITMAP_SIZE = 0; BITMAP_COMPENSATION = 0 } @@ -84,7 +84,7 @@ pub unsafe fn get_bit(idx: u32) -> bool { debug_assert!(byte_idx as usize >= get_bitmap_forbidden_size()); #[cfg(debug_assertions)] debug_assert!(get_bitmap_forbidden_size() + BITMAP_SIZE as usize > byte_idx as usize); - let byte = *BITMAP_PTR.add(byte_idx as usize); + let byte = *BITMAP_FORBIDDEN_PTR.add(byte_idx as usize); (byte >> bit_idx) & 0b1 != 0 } @@ -94,9 +94,9 @@ pub unsafe fn set_bit(idx: u32) { debug_assert!(byte_idx as usize >= get_bitmap_forbidden_size()); #[cfg(debug_assertions)] debug_assert!(get_bitmap_forbidden_size() + BITMAP_SIZE as usize > byte_idx as usize); - let byte = *BITMAP_PTR.add(byte_idx as usize); + let byte = *BITMAP_FORBIDDEN_PTR.add(byte_idx as usize); let new_byte = byte | (0b1 << bit_idx); - *BITMAP_PTR.add(byte_idx as usize) = new_byte; + *BITMAP_FORBIDDEN_PTR.add(byte_idx as usize) = new_byte; } pub struct BitmapIter { @@ -114,7 +114,7 @@ pub struct BitmapIter { } pub unsafe fn iter_bits() -> BitmapIter { - let bitmap = BITMAP_PTR.add(BITMAP_COMPENSATION); + let bitmap = BITMAP_FORBIDDEN_PTR.add(BITMAP_COMPENSATION); let blob_len_bytes = (bitmap.sub(size_of::().to_bytes().as_usize()) as *mut Obj) .as_blob() .len() @@ -152,7 +152,7 @@ impl BitmapIter { /// Returns the next bit, or `BITMAP_ITER_END` if there are no more bits set. pub fn next(&mut self) -> u32 { debug_assert!(self.current_word_idx <= self.size); - let bitmap = unsafe { BITMAP_PTR.add(BITMAP_COMPENSATION) }; + let bitmap = unsafe { BITMAP_FORBIDDEN_PTR.add(BITMAP_COMPENSATION) }; // Outer loop iterates 64-bit words loop { From 0ab5d8b7256aef5cc49fa9909109be14b4f31bb5 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 20:54:44 +0100 Subject: [PATCH 27/58] get rid of `BITMAP_COMPENSATION` Now [`BITMAP_FORBIDDEN_PTR`, `BITMAP_PTR`] straddle the no-go area. `BITMAP_COMPENSATION` can be computed. --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 30 +++++++++----------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 2b1f45576f9..9426a4db8af 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -17,11 +17,11 @@ To efficiently mark the right bit in the bitmap, we maintain a pointer that poin _before the start of the bitmap_ such that using the `/%8`-operation on the DH absolute word number will address the right bit: - +---- BITMAP_FORBIDDEN_PTR - v - | (forbidden) |...................... bitmap ................| - | heap_prefix_words / 8 bytes | heap_size / 32 bytes - | BITMAP_COMPENSATION | BITMAP_SIZE + +---- BITMAP_FORBIDDEN_PTR +---- BITMAP_PTR + v v + , (forbidden) +...................... bitmap ................+ + | heap_prefix_words / 8 bytes | heap_size / 32 bytes | + | get_bitmap_forbidden_size() | BITMAP_SIZE | Debug assertions guard the forbidden bytes from access, as this area physically overlaps with the Motoko static heap. @@ -46,12 +46,12 @@ thus we mark bit 2 in byte 0x7c004+0x402a = 0x8002e, which is physically in the /// Current bitmap static mut BITMAP_FORBIDDEN_PTR: *mut u8 = core::ptr::null_mut(); -static mut BITMAP_COMPENSATION: usize = 0; +static mut BITMAP_PTR: *mut u8 = core::ptr::null_mut(); static mut BITMAP_SIZE: u32 = 0; #[cfg(debug_assertions)] -fn get_bitmap_forbidden_size() -> usize { - unsafe { BITMAP_COMPENSATION } +unsafe fn get_bitmap_forbidden_size() -> usize { + BITMAP_PTR as usize - BITMAP_FORBIDDEN_PTR as usize } pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_prefix_words: u32) { @@ -68,14 +68,14 @@ pub unsafe fn alloc_bitmap(mem: &mut M, heap_size: Bytes, heap_p let blob = alloc_blob(mem, bitmap_bytes).get_ptr() as *mut Blob; memzero(blob.payload_addr() as usize, bitmap_bytes.to_words()); - BITMAP_COMPENSATION = (heap_prefix_words / 8) as usize; - BITMAP_FORBIDDEN_PTR = blob.payload_addr().sub(BITMAP_COMPENSATION) + BITMAP_PTR = blob.payload_addr(); + BITMAP_FORBIDDEN_PTR = BITMAP_PTR.sub(heap_prefix_words as usize / 8) } pub unsafe fn free_bitmap() { + BITMAP_PTR = core::ptr::null_mut(); BITMAP_FORBIDDEN_PTR = core::ptr::null_mut(); BITMAP_SIZE = 0; - BITMAP_COMPENSATION = 0 } pub unsafe fn get_bit(idx: u32) -> bool { @@ -114,8 +114,7 @@ pub struct BitmapIter { } pub unsafe fn iter_bits() -> BitmapIter { - let bitmap = BITMAP_FORBIDDEN_PTR.add(BITMAP_COMPENSATION); - let blob_len_bytes = (bitmap.sub(size_of::().to_bytes().as_usize()) as *mut Obj) + let blob_len_bytes = (BITMAP_PTR.sub(size_of::().to_bytes().as_usize()) as *mut Obj) .as_blob() .len() .as_u32(); @@ -127,7 +126,7 @@ pub unsafe fn iter_bits() -> BitmapIter { let current_word = if blob_len_64bit_words == 0 { 0 } else { - *(bitmap as *const u64) + *(BITMAP_PTR as *const u64) }; BitmapIter { @@ -152,7 +151,6 @@ impl BitmapIter { /// Returns the next bit, or `BITMAP_ITER_END` if there are no more bits set. pub fn next(&mut self) -> u32 { debug_assert!(self.current_word_idx <= self.size); - let bitmap = unsafe { BITMAP_FORBIDDEN_PTR.add(BITMAP_COMPENSATION) }; // Outer loop iterates 64-bit words loop { @@ -180,7 +178,7 @@ impl BitmapIter { return BITMAP_ITER_END; } self.current_word = - unsafe { *(bitmap as *const u64).add(self.current_word_idx as usize) }; + unsafe { *(BITMAP_PTR as *const u64).add(self.current_word_idx as usize) }; self.bits_left = 64; } } From 989b6e24fddc306fb2adfe8964fed75ee42d9667 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 20:57:41 +0100 Subject: [PATCH 28/58] tweak --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 9426a4db8af..3d0a231674a 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -19,7 +19,7 @@ absolute word number will address the right bit: +---- BITMAP_FORBIDDEN_PTR +---- BITMAP_PTR v v - , (forbidden) +...................... bitmap ................+ + , (forbidden) ,...................... bitmap ................, | heap_prefix_words / 8 bytes | heap_size / 32 bytes | | get_bitmap_forbidden_size() | BITMAP_SIZE | From 191a144dd2a2e0f9fa9c9469e645f515eaa5d13b Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 22:19:22 +0100 Subject: [PATCH 29/58] refactor BitmapIter to keep `current_bit_idx` this makes the hot loop more efficient and opens further avenues --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 3d0a231674a..a19136a907f 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -100,10 +100,10 @@ pub unsafe fn set_bit(idx: u32) { } pub struct BitmapIter { - /// Size of the bitmap, in 64-bit words words. Does not change after initialization. + /// Size of the bitmap, in bits. Does not change after initialization. size: u32, - /// Current 64-bit word index - current_word_idx: u32, + /// Current bit index + current_bit_idx: u32, /// Current 64-bit word in the bitmap that we're iterating. We read in 64-bit chunks to be able /// to check as many bits as possible with a single `word != 0`. current_word: u64, @@ -130,8 +130,8 @@ pub unsafe fn iter_bits() -> BitmapIter { }; BitmapIter { - size: blob_len_64bit_words, - current_word_idx: 0, + size: blob_len_bytes * 8, + current_bit_idx: 0, current_word, bits_left: 64, } @@ -150,18 +150,18 @@ pub const BITMAP_ITER_END: u32 = 1024 * 1024 * 1024; impl BitmapIter { /// Returns the next bit, or `BITMAP_ITER_END` if there are no more bits set. pub fn next(&mut self) -> u32 { - debug_assert!(self.current_word_idx <= self.size); + debug_assert!(self.current_bit_idx <= self.size); // Outer loop iterates 64-bit words loop { - if self.current_word == 0 && self.current_word_idx == self.size { + if self.current_word == 0 && self.current_bit_idx == self.size { return BITMAP_ITER_END; } // Inner loop iterates bits in the current word while self.current_word != 0 { if self.current_word & 0b1 != 0 { - let bit_idx = (self.current_word_idx * 64) + (64 - self.bits_left); + let bit_idx = self.current_bit_idx + 64 - self.bits_left; self.current_word >>= 1; self.bits_left -= 1; return bit_idx; @@ -173,12 +173,12 @@ impl BitmapIter { } // Move on to next word - self.current_word_idx += 1; - if self.current_word_idx == self.size { + self.current_bit_idx += 64; + if self.current_bit_idx == self.size { return BITMAP_ITER_END; } self.current_word = - unsafe { *(BITMAP_PTR as *const u64).add(self.current_word_idx as usize) }; + unsafe { *(BITMAP_PTR as *const u64).add(self.current_bit_idx as usize / 64) }; self.bits_left = 64; } } From d03b4ef8072b0f5fc96971a15698c3803b355ce0 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 22:28:07 +0100 Subject: [PATCH 30/58] refactor to use `bits_visited` even faster hot loop --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index a19136a907f..f2c8a2e3683 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -107,10 +107,10 @@ pub struct BitmapIter { /// Current 64-bit word in the bitmap that we're iterating. We read in 64-bit chunks to be able /// to check as many bits as possible with a single `word != 0`. current_word: u64, - /// Bits left in the current 64-bit word. Used to compute index of a bit in the bitmap. We + /// Bits checked in the current 64-bit word. Used to compute index of a bit in the bitmap. We /// can't use a global index here as we don't know how much to bump it when `current_word` is /// 0 and we move to the next 64-bit word. - bits_left: u32, + bits_visited: u32, } pub unsafe fn iter_bits() -> BitmapIter { @@ -133,7 +133,7 @@ pub unsafe fn iter_bits() -> BitmapIter { size: blob_len_bytes * 8, current_bit_idx: 0, current_word, - bits_left: 64, + bits_visited: 0, } } @@ -161,14 +161,14 @@ impl BitmapIter { // Inner loop iterates bits in the current word while self.current_word != 0 { if self.current_word & 0b1 != 0 { - let bit_idx = self.current_bit_idx + 64 - self.bits_left; + let bit_idx = self.current_bit_idx + self.bits_visited; self.current_word >>= 1; - self.bits_left -= 1; + self.bits_visited += 1; return bit_idx; } else { let shift_amt = self.current_word.trailing_zeros(); self.current_word >>= shift_amt; - self.bits_left -= shift_amt; + self.bits_visited += shift_amt; } } @@ -179,7 +179,7 @@ impl BitmapIter { } self.current_word = unsafe { *(BITMAP_PTR as *const u64).add(self.current_bit_idx as usize / 64) }; - self.bits_left = 64; + self.bits_visited = 0; } } } From 7828b10e7f9a424dd379bd0e6495bc99ca58ec54 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 23:14:01 +0100 Subject: [PATCH 31/58] instead of counting `bits_visited`, add `current_word.leading_zeros()` this further speeds up the inner loop --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index f2c8a2e3683..e2d2543b423 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -107,10 +107,8 @@ pub struct BitmapIter { /// Current 64-bit word in the bitmap that we're iterating. We read in 64-bit chunks to be able /// to check as many bits as possible with a single `word != 0`. current_word: u64, - /// Bits checked in the current 64-bit word. Used to compute index of a bit in the bitmap. We - /// can't use a global index here as we don't know how much to bump it when `current_word` is - /// 0 and we move to the next 64-bit word. - bits_visited: u32, + /// How many leading bits are initially zeroed in the current_word + leading_zeros: u32, } pub unsafe fn iter_bits() -> BitmapIter { @@ -133,7 +131,7 @@ pub unsafe fn iter_bits() -> BitmapIter { size: blob_len_bytes * 8, current_bit_idx: 0, current_word, - bits_visited: 0, + leading_zeros: current_word.leading_zeros(), } } @@ -161,25 +159,25 @@ impl BitmapIter { // Inner loop iterates bits in the current word while self.current_word != 0 { if self.current_word & 0b1 != 0 { - let bit_idx = self.current_bit_idx + self.bits_visited; + let bit_idx = self.current_bit_idx; self.current_word >>= 1; - self.bits_visited += 1; + self.current_bit_idx += 1; return bit_idx; } else { let shift_amt = self.current_word.trailing_zeros(); self.current_word >>= shift_amt; - self.bits_visited += shift_amt; + self.current_bit_idx += shift_amt; } } // Move on to next word - self.current_bit_idx += 64; + self.current_bit_idx += self.leading_zeros; if self.current_bit_idx == self.size { return BITMAP_ITER_END; } self.current_word = unsafe { *(BITMAP_PTR as *const u64).add(self.current_bit_idx as usize / 64) }; - self.bits_visited = 0; + self.leading_zeros = self.current_word.leading_zeros(); } } } From e20da5a275cb099830462e48c2341bc6bd4a2e29 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 25 Nov 2021 23:55:04 +0100 Subject: [PATCH 32/58] WIP: make bits iterated correspond to absolute words --- rts/motoko-rts-tests/src/gc.rs | 4 ++-- rts/motoko-rts/src/gc/mark_compact.rs | 2 +- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 7 +++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/rts/motoko-rts-tests/src/gc.rs b/rts/motoko-rts-tests/src/gc.rs index 935ca85803a..5f0a8fda4d0 100644 --- a/rts/motoko-rts-tests/src/gc.rs +++ b/rts/motoko-rts-tests/src/gc.rs @@ -119,8 +119,8 @@ fn test_gc( heap.continuation_table_ptr_offset(), ); - for _ in 0..3 { - gc.run(heap.clone()); + for _ in 0..0 { + gc.run(heap.clone()); /* FIXME: not alignment-preserving! */ let heap_base_offset = heap.heap_base_offset(); let heap_ptr_offset = heap.heap_ptr_offset(); diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 9043b0bc445..920234b3afd 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -199,7 +199,7 @@ unsafe fn update_refs(set_hp: SetHp, heap_base: u32) { let mut bitmap_iter = iter_bits(); let mut bit = bitmap_iter.next(); while bit != BITMAP_ITER_END { - let p = (heap_base + (bit * WORD_SIZE)) as *mut Obj; + let p = (bit * WORD_SIZE) as *mut Obj; let p_new = free; // Update backwards references to the object's new location and restore object header diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index e2d2543b423..3ed77fe3ce3 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -127,9 +127,12 @@ pub unsafe fn iter_bits() -> BitmapIter { *(BITMAP_PTR as *const u64) }; + debug_assert!(BITMAP_PTR as usize >= BITMAP_FORBIDDEN_PTR as usize); + let forbidden_bits = (BITMAP_PTR as usize - BITMAP_FORBIDDEN_PTR as usize) as u32 * 8; + BitmapIter { - size: blob_len_bytes * 8, - current_bit_idx: 0, + size: blob_len_bytes * 8 + forbidden_bits, + current_bit_idx: forbidden_bits, current_word, leading_zeros: current_word.leading_zeros(), } From 49c0b641cd54f2a3910d71193a6e6ab797e4fc1d Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Fri, 26 Nov 2021 00:14:10 +0100 Subject: [PATCH 33/58] Update perf-delta.nix --- perf-delta.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-delta.nix b/perf-delta.nix index b7c7450298e..3cd15319403 100644 --- a/perf-delta.nix +++ b/perf-delta.nix @@ -36,7 +36,7 @@ let do # ignore all errors echo -n $file - if timeout 10s moc $file --compacting-gc -no-check-ir -ref-system-api -o $file.wasm 2>/dev/null + if timeout 10s moc $file --force-gc --compacting-gc -no-check-ir -ref-system-api -o $file.wasm 2>/dev/null then echo " failed (ignored)" else echo " ok" fi From 3d7c9995c43fe40ea16ded022ab9d40170ede8b3 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Fri, 26 Nov 2021 17:14:54 +0100 Subject: [PATCH 34/58] fix the glaring bug --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 3ed77fe3ce3..ca8cdecd363 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -179,7 +179,7 @@ impl BitmapIter { return BITMAP_ITER_END; } self.current_word = - unsafe { *(BITMAP_PTR as *const u64).add(self.current_bit_idx as usize / 64) }; + unsafe { *(BITMAP_FORBIDDEN_PTR.add(self.current_bit_idx as usize / 8) as *const u64) }; self.leading_zeros = self.current_word.leading_zeros(); } } From 2716eefa20394a75e8e77939844dde1fae30b6df Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Fri, 26 Nov 2021 17:18:38 +0100 Subject: [PATCH 35/58] fmt --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index ca8cdecd363..40fb93cd441 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -178,8 +178,9 @@ impl BitmapIter { if self.current_bit_idx == self.size { return BITMAP_ITER_END; } - self.current_word = - unsafe { *(BITMAP_FORBIDDEN_PTR.add(self.current_bit_idx as usize / 8) as *const u64) }; + self.current_word = unsafe { + *(BITMAP_FORBIDDEN_PTR.add(self.current_bit_idx as usize / 8) as *const u64) + }; self.leading_zeros = self.current_word.leading_zeros(); } } From c49fc1863a6a76f7c392e9c1f371ea5ef1afd569 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Fri, 26 Nov 2021 19:33:08 +0100 Subject: [PATCH 36/58] minor nomenclature fixes --- rts/motoko-rts/src/debug.rs | 10 +++++----- rts/motoko-rts/src/gc/mark_compact.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rts/motoko-rts/src/debug.rs b/rts/motoko-rts/src/debug.rs index 92dbd523987..f92bf86bbaf 100644 --- a/rts/motoko-rts/src/debug.rs +++ b/rts/motoko-rts/src/debug.rs @@ -31,21 +31,21 @@ pub unsafe fn dump_heap( print_heap(heap_base, hp); } -pub(crate) unsafe fn print_continuation_table(closure_tbl_loc: *mut Value) { +pub(crate) unsafe fn print_continuation_table(continuation_tbl_loc: *mut Value) { if !crate::continuation_table::table_initialized() { println!(100, "Continuation table not initialized"); return; } - let arr = (*closure_tbl_loc).as_array() as *mut Array; + let arr = (*continuation_tbl_loc).as_array(); let len = (*arr).len; if len == 0 { - println!(50, "Closure table empty"); + println!(50, "Continuation table empty"); return; } - println!(50, "Closure table: {}", len); + println!(50, "Continuation table: {}", len); let mut buf = [0u8; 1000]; let mut write_buf = WriteBuf::new(&mut buf); @@ -59,7 +59,7 @@ pub(crate) unsafe fn print_continuation_table(closure_tbl_loc: *mut Value) { write_buf.reset(); } } - println!(50, "End of closure table"); + println!(50, "End of continuation table"); } pub(crate) unsafe fn print_static_roots(static_roots: Value) { diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 920234b3afd..b80277a2c12 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -109,7 +109,7 @@ unsafe fn mark_compact( // TODO: No need to check if continuation table is already marked mark_object(mem, *continuation_table_ptr_loc); // Similar to `mark_root_mutbox_fields`, `continuation_table_ptr_loc` is in static heap so - // it will be readable when we unthread continuation table + // it will be readable when we unthread the continuation table thread(continuation_table_ptr_loc); } From a460d3232f1e4e1c4ac73de4c0ef08b924685676 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Fri, 26 Nov 2021 20:20:31 +0100 Subject: [PATCH 37/58] tweak graphics --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 40fb93cd441..f8db4c6000a 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -4,12 +4,14 @@ use crate::types::{size_of, Blob, Bytes, Obj}; /* How the Wasm-heap maps to the bitmap - +---- Rts stack ----+---- Motoko statics ----+---- Dynamic heap --~~--+ Heap end - (prefix words) | BM | <- bitmap lives here - / \ - / \ - /...bits...\ each bit corresponds to a word - in the dynamic heap + +---- Rts stack ----+---- Motoko statics ----+---- Dynamic heap --~~--+ Heap limit + (prefix words) bitmap lives here -> | BM | + / \ + / \ + each bit corresponds to a word: /...bits...\ + in the dynamic heap + +## Marking with absolute addresses When marking, we pass an absolute pointer (i.e. address space relative), for speed. Internally the bitmap is kept in a (non-moving) blob at the start of the dynamic heap (DH). From f45e67641309dbaca53c8c3ac1a7e662b2f7cd02 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Sat, 27 Nov 2021 00:44:43 +0100 Subject: [PATCH 38/58] move comparison before the loop this shaves off a few more instructions from loop and function --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index f8db4c6000a..5f7adb251f7 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -148,19 +148,19 @@ pub unsafe fn iter_bits() -> BitmapIter { // // (We actually need less bits than that as when the heap is full we can't allocate bitmap and mark // stack and can't do GC) -pub const BITMAP_ITER_END: u32 = 1024 * 1024 * 1024; +pub const BITMAP_ITER_END: u32 = u32::MAX; impl BitmapIter { /// Returns the next bit, or `BITMAP_ITER_END` if there are no more bits set. pub fn next(&mut self) -> u32 { debug_assert!(self.current_bit_idx <= self.size); + if self.current_bit_idx == self.size { + return BITMAP_ITER_END; + } + // Outer loop iterates 64-bit words loop { - if self.current_word == 0 && self.current_bit_idx == self.size { - return BITMAP_ITER_END; - } - // Inner loop iterates bits in the current word while self.current_word != 0 { if self.current_word & 0b1 != 0 { From c3b3b185e407402059b60156424a8c11cc21e401 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Sat, 27 Nov 2021 00:56:38 +0100 Subject: [PATCH 39/58] for copying collector no alignment necessary --- rts/motoko-rts/src/gc/copying.rs | 2 +- rts/motoko-rts/src/memory/ic.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rts/motoko-rts/src/gc/copying.rs b/rts/motoko-rts/src/gc/copying.rs index 620b42e47d8..1328e2f7c4e 100644 --- a/rts/motoko-rts/src/gc/copying.rs +++ b/rts/motoko-rts/src/gc/copying.rs @@ -23,7 +23,7 @@ unsafe fn copying_gc(mem: &mut M) { copying_gc_internal( mem, - ic::get_aligned_heap_base(), + ic::get_heap_base(), // get_hp || ic::HP as usize, // set_hp diff --git a/rts/motoko-rts/src/memory/ic.rs b/rts/motoko-rts/src/memory/ic.rs index abc06099ba8..779a014c0df 100644 --- a/rts/motoko-rts/src/memory/ic.rs +++ b/rts/motoko-rts/src/memory/ic.rs @@ -24,7 +24,7 @@ pub(crate) static mut LAST_HP: u32 = 0; // Provided by generated code extern "C" { - fn get_heap_base() -> u32; + pub(crate) fn get_heap_base() -> u32; pub(crate) fn get_static_roots() -> Value; } From af2385de6c559ee0cdc1a820b2b232939255072d Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Sat, 27 Nov 2021 01:49:41 +0100 Subject: [PATCH 40/58] fix wording --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 5f7adb251f7..7c11e136d60 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -14,7 +14,7 @@ use crate::types::{size_of, Blob, Bytes, Obj}; ## Marking with absolute addresses When marking, we pass an absolute pointer (i.e. address space relative), for speed. -Internally the bitmap is kept in a (non-moving) blob at the start of the dynamic heap (DH). +Internally the bitmap is kept in a (non-moving) blob behind the dynamic heap (DH). To efficiently mark the right bit in the bitmap, we maintain a pointer that points _before the start of the bitmap_ such that using the `/%8`-operation on the DH absolute word number will address the right bit: From c87ae9b1b54d924a1cce6e7a0d945ffa1f98098c Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Sat, 27 Nov 2021 02:21:17 +0100 Subject: [PATCH 41/58] soup up the graphic --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 7c11e136d60..b39764153cc 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -19,11 +19,21 @@ To efficiently mark the right bit in the bitmap, we maintain a pointer that poin _before the start of the bitmap_ such that using the `/%8`-operation on the DH absolute word number will address the right bit: + +---- BITMAP_FORBIDDEN_PTR +---- BITMAP_PTR v v - , (forbidden) ,...................... bitmap ................, - | heap_prefix_words / 8 bytes | heap_size / 32 bytes | - | get_bitmap_forbidden_size() | BITMAP_SIZE | + , (forbidden) ,...................... bitmap ..........~~...., + | heap_prefix_words / 8 bytes | heap_size / 32 bytes | + | get_bitmap_forbidden_size() | BITMAP_SIZE | + ^ ^ ^ + / ^^^^^^^^^^^ \ | + / corresponds \ ^^^^^^^^^^^ / + / \ corresponds / + / \ / + +---- Rts stack ----+---- Motoko statics ----+--------- Dynamic heap ----------~~--+ + ! + ! 32-byte aligned + Debug assertions guard the forbidden bytes from access, as this area physically overlaps with the Motoko static heap. From 3223b362053de139570502707a62ba542789be41 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 29 Nov 2021 15:45:00 +0100 Subject: [PATCH 42/58] Update rts/motoko-rts/src/gc/mark_compact/bitmap.rs --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index b39764153cc..48649651905 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -4,7 +4,7 @@ use crate::types::{size_of, Blob, Bytes, Obj}; /* How the Wasm-heap maps to the bitmap - +---- Rts stack ----+---- Motoko statics ----+---- Dynamic heap --~~--+ Heap limit + +---- RTS stack ----+---- Motoko statics ----+---- Dynamic heap --~~--+ Heap limit (prefix words) bitmap lives here -> | BM | / \ / \ From bf975837012e549d25f4750bacb121ae797801bc Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 29 Nov 2021 16:53:53 +0100 Subject: [PATCH 43/58] update example --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 48649651905..bd4a6dd4619 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -36,7 +36,7 @@ absolute word number will address the right bit: Debug assertions guard the forbidden bytes from access, as this area physically overlaps -with the Motoko static heap. +with the Motoko dynamic heap. ## The alignment caveat @@ -47,12 +47,12 @@ in the BM, and thus the sweep operation will be off. ## Example calculation -Assume the DH is at 0x80000. The BM thus could be at 0x80004. -Since the heap_prefix_words is 0x20000, BITMAP_FORBIDDEN_PTR = 0x80004 - 0x20000 / 8 = 0x7c004. +Assume the DH is at 0x80000. The BM thus could be at 0xB0004. +Since the heap_prefix_words is 0x20000, BITMAP_FORBIDDEN_PTR = 0xB0004 - 0x20000 / 8 = 0xAC004. Now let's mark the address 0x80548 in the DH. Its absolute word number is 0x20152. -The `(0x20152 / 8, 0x20152 % 8)`-rule gives a bit position 2 with byte offset 0x402a, -thus we mark bit 2 in byte 0x7c004+0x402a = 0x8002e, which is physically in the BM. +The `(0x20152 / 8, 0x20152 % 8)`-rule gives a bit position 2 with byte offset 0x402A, +thus we mark bit 2 in byte 0xAC004 + 0x402A = 0xB002E, which is physically in the BM. */ From 165555b38088b81e09f6f34ff0d6893077e2a401 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 29 Nov 2021 17:00:11 +0100 Subject: [PATCH 44/58] wordsmithing --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index bd4a6dd4619..07c2631fc43 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -8,7 +8,7 @@ use crate::types::{size_of, Blob, Bytes, Obj}; (prefix words) bitmap lives here -> | BM | / \ / \ - each bit corresponds to a word: /...bits...\ + each bit represents a word -> /...bits...\ in the dynamic heap ## Marking with absolute addresses @@ -35,8 +35,8 @@ absolute word number will address the right bit: ! 32-byte aligned -Debug assertions guard the forbidden bytes from access, as this area physically overlaps -with the Motoko dynamic heap. +Debug assertions guard the forbidden bytes from access, as this area potentially +physically overlaps with the Motoko dynamic heap. ## The alignment caveat @@ -47,8 +47,9 @@ in the BM, and thus the sweep operation will be off. ## Example calculation -Assume the DH is at 0x80000. The BM thus could be at 0xB0004. -Since the heap_prefix_words is 0x20000, BITMAP_FORBIDDEN_PTR = 0xB0004 - 0x20000 / 8 = 0xAC004. +Assume the DH is at 0x80000. Assume heap limit being at 0xB0000. Then the BM thus +could be placed at 0xB0004. Since the heap_prefix_words is 0x20000, +BITMAP_FORBIDDEN_PTR = 0xB0004 - 0x20000 / 8 = 0xAC004. Now let's mark the address 0x80548 in the DH. Its absolute word number is 0x20152. The `(0x20152 / 8, 0x20152 % 8)`-rule gives a bit position 2 with byte offset 0x402A, From 7b24bfad451051411fcb259a4468beccfcf29b79 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 29 Nov 2021 17:13:53 +0100 Subject: [PATCH 45/58] make "next word" precise --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 07c2631fc43..426be8a0a09 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -186,7 +186,7 @@ impl BitmapIter { } } - // Move on to next word + // Move on to next word (always 64-bit boundary) self.current_bit_idx += self.leading_zeros; if self.current_bit_idx == self.size { return BITMAP_ITER_END; From 140e93d3e7a345153a428b29811586e3d165eac0 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 29 Nov 2021 17:48:09 +0100 Subject: [PATCH 46/58] undo this but be watchful `make -C rts test` passes now on my Mac, maybe this was a red herring? --- rts/motoko-rts-tests/src/gc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rts/motoko-rts-tests/src/gc.rs b/rts/motoko-rts-tests/src/gc.rs index 5f0a8fda4d0..935ca85803a 100644 --- a/rts/motoko-rts-tests/src/gc.rs +++ b/rts/motoko-rts-tests/src/gc.rs @@ -119,8 +119,8 @@ fn test_gc( heap.continuation_table_ptr_offset(), ); - for _ in 0..0 { - gc.run(heap.clone()); /* FIXME: not alignment-preserving! */ + for _ in 0..3 { + gc.run(heap.clone()); let heap_base_offset = heap.heap_base_offset(); let heap_ptr_offset = heap.heap_ptr_offset(); From 9872e4dc30f578a7d18e46b7acdacdb30ec5a77a Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 29 Nov 2021 19:34:19 +0100 Subject: [PATCH 47/58] pass bool to `init` meaning mandatory alignment --- rts/motoko-rts/src/memory/ic.rs | 8 ++++++-- src/codegen/compile.ml | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/rts/motoko-rts/src/memory/ic.rs b/rts/motoko-rts/src/memory/ic.rs index 779a014c0df..6eb86d9c4f8 100644 --- a/rts/motoko-rts/src/memory/ic.rs +++ b/rts/motoko-rts/src/memory/ic.rs @@ -34,8 +34,12 @@ pub(crate) unsafe fn get_aligned_heap_base() -> u32 { } #[no_mangle] -unsafe extern "C" fn init() { - HP = get_aligned_heap_base(); +unsafe extern "C" fn init(align: bool) { + HP = if align { + get_aligned_heap_base() + } else { + get_heap_base() + }; LAST_HP = HP; } diff --git a/src/codegen/compile.ml b/src/codegen/compile.ml index 6853e83c1b3..dbce25ec585 100644 --- a/src/codegen/compile.ml +++ b/src/codegen/compile.ml @@ -864,7 +864,7 @@ module RTS = struct E.add_func_import env "rts" "alloc_words" [I32Type] [I32Type]; E.add_func_import env "rts" "get_total_allocations" [] [I64Type]; E.add_func_import env "rts" "get_heap_size" [] [I32Type]; - E.add_func_import env "rts" "init" [] []; + E.add_func_import env "rts" "init" [I32Type] []; E.add_func_import env "rts" "alloc_blob" [I32Type] [I32Type]; E.add_func_import env "rts" "alloc_array" [I32Type] [I32Type]; () @@ -8846,6 +8846,7 @@ and conclude_module env start_fi_o = (* Wrap the start function with the RTS initialization *) let rts_start_fi = E.add_fun env "rts_start" (Func.of_body env [] [] (fun env1 -> + Bool.lit (!Flags.gc_strategy = Mo_config.Flags.MarkCompact) ^^ E.call_import env "rts" "init" ^^ match start_fi_o with | Some fi -> From a9bca2b7553fed00fbfbaaee3f90e2414e8ab27a Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 14:18:19 +0100 Subject: [PATCH 48/58] Update rts/motoko-rts/src/gc/mark_compact/bitmap.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ömer Sinan Ağacan --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 426be8a0a09..63d841db55f 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -4,7 +4,7 @@ use crate::types::{size_of, Blob, Bytes, Obj}; /* How the Wasm-heap maps to the bitmap - +---- RTS stack ----+---- Motoko statics ----+---- Dynamic heap --~~--+ Heap limit + +---- RTS stack ----+---- Motoko statics ----+---- Dynamic heap ------+ Heap limit (prefix words) bitmap lives here -> | BM | / \ / \ From 2d9f509ebac8c2fce3168dd77e971ed4205c8bda Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 14:46:01 +0100 Subject: [PATCH 49/58] Update rts/motoko-rts/src/memory/ic.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ömer Sinan Ağacan --- rts/motoko-rts/src/memory/ic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/memory/ic.rs b/rts/motoko-rts/src/memory/ic.rs index 6eb86d9c4f8..e23899fd31c 100644 --- a/rts/motoko-rts/src/memory/ic.rs +++ b/rts/motoko-rts/src/memory/ic.rs @@ -30,7 +30,7 @@ extern "C" { pub(crate) unsafe fn get_aligned_heap_base() -> u32 { // align to 32 bytes - (get_heap_base() + 31) / 32 * 32 + ((get_heap_base() + 31) / 32) * 32 } #[no_mangle] From d3a2610e69b5e2c58dddd8384137f8160b38c27f Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 14:54:08 +0100 Subject: [PATCH 50/58] Update rts/motoko-rts/src/gc/mark_compact/bitmap.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ömer Sinan Ağacan --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 63d841db55f..fd957ef3370 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -13,11 +13,11 @@ use crate::types::{size_of, Blob, Bytes, Obj}; ## Marking with absolute addresses -When marking, we pass an absolute pointer (i.e. address space relative), for speed. -Internally the bitmap is kept in a (non-moving) blob behind the dynamic heap (DH). -To efficiently mark the right bit in the bitmap, we maintain a pointer that points -_before the start of the bitmap_ such that using the `/%8`-operation on the DH -absolute word number will address the right bit: +When marking, we need to map an address to a bit in the bitmap. Internally the +bitmap is kept in a (non-moving) blob after the dynamic heap (DH). To +efficiently mark the right bit in the bitmap, we maintain a pointer that points +_before the start of the bitmap_ such that `address / 8` and `address % 8` +will address the right bit: +---- BITMAP_FORBIDDEN_PTR +---- BITMAP_PTR From a3eb0e3d8c87542a490891517f42826b00fb7891 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 14:54:39 +0100 Subject: [PATCH 51/58] Update rts/motoko-rts/src/gc/mark_compact/bitmap.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ömer Sinan Ağacan --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index fd957ef3370..18cb114cd65 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -36,7 +36,7 @@ will address the right bit: Debug assertions guard the forbidden bytes from access, as this area potentially -physically overlaps with the Motoko dynamic heap. +overlaps with the Motoko dynamic heap. ## The alignment caveat From adb8c0aa994f7e73ae6edfa7edc3cbeed9aa68c9 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 14:57:07 +0100 Subject: [PATCH 52/58] Update rts/motoko-rts/src/gc/mark_compact/bitmap.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ömer Sinan Ağacan --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index 18cb114cd65..e4a4f1ae566 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -41,9 +41,9 @@ overlaps with the Motoko dynamic heap. ## The alignment caveat For this scheme to work, it is essential that the start of the DH is an address that -is divisible by 32 (implying that `heap_prefix_words % 8 == 0`). Otherwise the -`/%8`-operation on the DH's starting address will not yield the least significant bit -in the BM, and thus the sweep operation will be off. +is divisible by 32 (`heap_prefix_words % 8 == 0`). Otherwise the `address / 8` +and `address % 8` operations on the DH's starting address will not yield the +least significant bit in the BM. ## Example calculation From e7f6c342e919d70bb4ca1cbb5f3cf92d3abac5f5 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 15:06:12 +0100 Subject: [PATCH 53/58] fix BM start --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index e4a4f1ae566..c0b1e95f7f1 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -48,8 +48,8 @@ least significant bit in the BM. ## Example calculation Assume the DH is at 0x80000. Assume heap limit being at 0xB0000. Then the BM thus -could be placed at 0xB0004. Since the heap_prefix_words is 0x20000, -BITMAP_FORBIDDEN_PTR = 0xB0004 - 0x20000 / 8 = 0xAC004. +could be placed at 0xB0008. Since the heap_prefix_words is 0x20000, +BITMAP_FORBIDDEN_PTR = 0xB0008 - 0x20000 / 8 = 0xAC008. Now let's mark the address 0x80548 in the DH. Its absolute word number is 0x20152. The `(0x20152 / 8, 0x20152 % 8)`-rule gives a bit position 2 with byte offset 0x402A, From c3f11474a610b4b32e1fc788fdc57a518e0457cb Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 15:07:22 +0100 Subject: [PATCH 54/58] tweak --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index c0b1e95f7f1..daffc4742dd 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -53,7 +53,7 @@ BITMAP_FORBIDDEN_PTR = 0xB0008 - 0x20000 / 8 = 0xAC008. Now let's mark the address 0x80548 in the DH. Its absolute word number is 0x20152. The `(0x20152 / 8, 0x20152 % 8)`-rule gives a bit position 2 with byte offset 0x402A, -thus we mark bit 2 in byte 0xAC004 + 0x402A = 0xB002E, which is physically in the BM. +thus we mark bit 2 in byte 0xAC004 + 0x402A = 0xB002E, which is in the BM. */ From cf98977a97670e007f23eb3868526f3df916da79 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 15:21:06 +0100 Subject: [PATCH 55/58] doc and code tweaks review feedback --- rts/motoko-rts-tests/src/gc/heap.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index 1f9a5bbad20..5f8573fb944 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -204,12 +204,14 @@ impl MotokoHeapInner { map.len(), ); + // The Worst-case unalignment w.r.t. 32-byte alignment is 28 (assuming + // that we have general word alignment). So we over-allocate 28 bytes. let mut heap: Vec = vec![0; heap_size + 28]; // MarkCompact assumes that the dynamic heap starts at a 32-byte multiple let realign = match gc { GC::Copying => 0, - GC::MarkCompact => (32 - &heap[static_heap_size_bytes] as *const u8 as usize % 32) % 32, + GC::MarkCompact => (32 - heap.as_ptr() as usize % 32) % 32, }; assert_eq!(realign % 4, 0); From b94cf22ba7d8bf1dccbb6710215a2cd5e50fb591 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 17:25:24 +0100 Subject: [PATCH 56/58] typo --- rts/motoko-rts/src/gc/mark_compact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index b80277a2c12..78c6ff57b58 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -127,7 +127,7 @@ unsafe fn mark_static_roots(mem: &mut M, static_roots: Value, heap_ba // Static objects are not in the dynamic heap so don't need marking. for i in 0..root_array.len() { let obj = root_array.get(i).as_obj(); - // Root array should only has pointers to other static MutBoxes + // Root array should only have pointers to other static MutBoxes debug_assert_eq!(obj.tag(), TAG_MUTBOX); // check tag debug_assert!((obj as u32) < heap_base); // check that MutBox is static mark_root_mutbox_fields(mem, obj as *mut MutBox, heap_base); From 86fcfd488cf95989162fc1b8a783e3ef4edb3d5e Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 19:53:14 +0100 Subject: [PATCH 57/58] oooops --- rts/motoko-rts-tests/src/gc/heap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/motoko-rts-tests/src/gc/heap.rs b/rts/motoko-rts-tests/src/gc/heap.rs index 5f8573fb944..8f553457edb 100644 --- a/rts/motoko-rts-tests/src/gc/heap.rs +++ b/rts/motoko-rts-tests/src/gc/heap.rs @@ -211,7 +211,7 @@ impl MotokoHeapInner { // MarkCompact assumes that the dynamic heap starts at a 32-byte multiple let realign = match gc { GC::Copying => 0, - GC::MarkCompact => (32 - heap.as_ptr() as usize % 32) % 32, + GC::MarkCompact => (32 - (heap.as_ptr() as usize + static_heap_size_bytes) % 32) % 32, }; assert_eq!(realign % 4, 0); From f37b9af30098496ae6edd8c541a9f8c76d59c6b4 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 30 Nov 2021 20:48:59 +0100 Subject: [PATCH 58/58] simplify --- rts/motoko-rts/src/gc/mark_compact/bitmap.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs index daffc4742dd..2607a4baa87 100644 --- a/rts/motoko-rts/src/gc/mark_compact/bitmap.rs +++ b/rts/motoko-rts/src/gc/mark_compact/bitmap.rs @@ -62,7 +62,6 @@ static mut BITMAP_FORBIDDEN_PTR: *mut u8 = core::ptr::null_mut(); static mut BITMAP_PTR: *mut u8 = core::ptr::null_mut(); static mut BITMAP_SIZE: u32 = 0; -#[cfg(debug_assertions)] unsafe fn get_bitmap_forbidden_size() -> usize { BITMAP_PTR as usize - BITMAP_FORBIDDEN_PTR as usize } @@ -93,9 +92,7 @@ pub unsafe fn free_bitmap() { pub unsafe fn get_bit(idx: u32) -> bool { let (byte_idx, bit_idx) = (idx / 8, idx % 8); - #[cfg(debug_assertions)] debug_assert!(byte_idx as usize >= get_bitmap_forbidden_size()); - #[cfg(debug_assertions)] debug_assert!(get_bitmap_forbidden_size() + BITMAP_SIZE as usize > byte_idx as usize); let byte = *BITMAP_FORBIDDEN_PTR.add(byte_idx as usize); (byte >> bit_idx) & 0b1 != 0 @@ -103,9 +100,7 @@ pub unsafe fn get_bit(idx: u32) -> bool { pub unsafe fn set_bit(idx: u32) { let (byte_idx, bit_idx) = (idx / 8, idx % 8); - #[cfg(debug_assertions)] debug_assert!(byte_idx as usize >= get_bitmap_forbidden_size()); - #[cfg(debug_assertions)] debug_assert!(get_bitmap_forbidden_size() + BITMAP_SIZE as usize > byte_idx as usize); let byte = *BITMAP_FORBIDDEN_PTR.add(byte_idx as usize); let new_byte = byte | (0b1 << bit_idx); @@ -141,7 +136,7 @@ pub unsafe fn iter_bits() -> BitmapIter { }; debug_assert!(BITMAP_PTR as usize >= BITMAP_FORBIDDEN_PTR as usize); - let forbidden_bits = (BITMAP_PTR as usize - BITMAP_FORBIDDEN_PTR as usize) as u32 * 8; + let forbidden_bits = get_bitmap_forbidden_size() as u32 * 8; BitmapIter { size: blob_len_bytes * 8 + forbidden_bits,