diff --git a/crates/bevy_ecs/src/query/mod.rs b/crates/bevy_ecs/src/query/mod.rs index 857fe033ab1d47..d037c4a5419745 100644 --- a/crates/bevy_ecs/src/query/mod.rs +++ b/crates/bevy_ecs/src/query/mod.rs @@ -11,7 +11,7 @@ pub use iter::*; pub use state::*; #[allow(unreachable_code)] -unsafe fn debug_checked_unreachable() -> ! { +pub(crate) unsafe fn debug_checked_unreachable() -> ! { #[cfg(debug_assertions)] unreachable!(); std::hint::unreachable_unchecked(); diff --git a/crates/bevy_ecs/src/storage/blob_vec.rs b/crates/bevy_ecs/src/storage/blob_vec.rs index 8ce77d73a64a7d..14d584b0483646 100644 --- a/crates/bevy_ecs/src/storage/blob_vec.rs +++ b/crates/bevy_ecs/src/storage/blob_vec.rs @@ -209,6 +209,27 @@ impl BlobVec { OwningPtr::new(self.swap_scratch) } + /// Removes the value at `index` and copies the value stored into `ptr`. + /// Does not do any bounds checking on `index`. + /// + /// # Safety + /// It is the caller's responsibility to ensure that `index` is < `self.len()` + /// and that `self[index]` has been properly initialized. + #[inline] + pub unsafe fn swap_remove_unchecked(&mut self, index: usize, ptr: PtrMut<'_>) { + debug_assert!(index < self.len()); + let last = self.get_unchecked_mut(self.len - 1).as_ptr(); + let target = self.get_unchecked_mut(index).as_ptr(); + // Copy the item at the index into the provided ptr + std::ptr::copy_nonoverlapping::(target, ptr.as_ptr(), self.item_layout.size()); + // Recompress the storage by moving the previous last element into the + // now-free row overwriting the previous data. The removed row may be the last + // one so a non-overlapping copy must not be used here. + std::ptr::copy::(last, target, self.item_layout.size()); + // Invalidate the data stored in the last row, as it has been moved + self.len -= 1; + } + /// # Safety /// It is the caller's responsibility to ensure that `index` is < self.len() #[inline] diff --git a/crates/bevy_ecs/src/storage/table.rs b/crates/bevy_ecs/src/storage/table.rs index 37ee9498656c0d..4b716c6814b043 100644 --- a/crates/bevy_ecs/src/storage/table.rs +++ b/crates/bevy_ecs/src/storage/table.rs @@ -1,6 +1,7 @@ use crate::{ component::{ComponentId, ComponentInfo, ComponentTicks, Components}, entity::Entity, + query::debug_checked_unreachable, storage::{blob_vec::BlobVec, SparseSet}, }; use bevy_ptr::{OwningPtr, Ptr, PtrMut}; @@ -122,6 +123,30 @@ impl Column { (data, ticks) } + /// Removes the element from `other` at `src_row` and inserts it + /// into the current column to initialize the values at `dst_row`. + /// Does not do any bounds checking. + /// + /// # Safety + /// + /// - `other` must have the same data layout as `self` + /// - `src_row` must be in bounds for `other` + /// - `dst_row` must be in bounds for `self` + /// - `other[src_row]` must be initialized to a valid value. + /// - `self[dst_row]` must not be initalized yet. + #[inline] + pub(crate) unsafe fn initialize_from_unchecked( + &mut self, + other: &mut Column, + src_row: usize, + dst_row: usize, + ) { + debug_assert!(self.data.layout() == other.data.layout()); + let ptr = self.data.get_unchecked_mut(dst_row); + other.data.swap_remove_unchecked(src_row, ptr); + *self.ticks.get_unchecked_mut(dst_row) = other.ticks.swap_remove(src_row); + } + // # Safety // - ptr must point to valid data of this column's component type pub(crate) unsafe fn push(&mut self, ptr: OwningPtr<'_>, ticks: ComponentTicks) { @@ -249,9 +274,11 @@ impl Table { let is_last = row == self.entities.len() - 1; let new_row = new_table.allocate(self.entities.swap_remove(row)); for (component_id, column) in self.columns.iter_mut() { - let (data, ticks) = column.swap_remove_and_forget_unchecked(row); if let Some(new_column) = new_table.get_column_mut(*component_id) { - new_column.initialize(new_row, data, ticks); + new_column.initialize_from_unchecked(column, row, new_row); + } else { + // It's the caller's responsibility to drop these cases. + let (_, _) = column.swap_remove_and_forget_unchecked(row); } } TableMoveResult { @@ -280,8 +307,7 @@ impl Table { let new_row = new_table.allocate(self.entities.swap_remove(row)); for (component_id, column) in self.columns.iter_mut() { if let Some(new_column) = new_table.get_column_mut(*component_id) { - let (data, ticks) = column.swap_remove_and_forget_unchecked(row); - new_column.initialize(new_row, data, ticks); + new_column.initialize_from_unchecked(column, row, new_row); } else { column.swap_remove_unchecked(row); } @@ -311,9 +337,10 @@ impl Table { let is_last = row == self.entities.len() - 1; let new_row = new_table.allocate(self.entities.swap_remove(row)); for (component_id, column) in self.columns.iter_mut() { - let new_column = new_table.get_column_mut(*component_id).unwrap(); - let (data, ticks) = column.swap_remove_and_forget_unchecked(row); - new_column.initialize(new_row, data, ticks); + new_table + .get_column_mut(*component_id) + .unwrap_or_else(|| debug_checked_unreachable()) + .initialize_from_unchecked(column, row, new_row); } TableMoveResult { new_row,