Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use wrapping pointer arithmetic in mem/impls.rs #713

Merged
merged 1 commit into from
Oct 17, 2024
Merged

Use wrapping pointer arithmetic in mem/impls.rs #713

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 61 additions & 44 deletions src/mem/impls.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
// In C and Rust it is UB to read or write to usize::MAX because if an allocation extends to the
// last byte of address space (there must be an allocation to do the read or write), in C computing
// its one-past-the-end pointer would be equal to NULL and in Rust computing the address of a
// trailing ZST member with a safe place projection would wrap (place projection address computation
// is non-wrapping).
//
// However, some embedded systems have special memory at usize::MAX, and need to access that
// memory. If they do that with the intrinsics provided by compiler-builtins (such as memcpy!), the
// ptr::add in these loops will wrap. And if compiler-builtins is compiled with cfg(ub_checks),
// this will fail a UB check at runtime.
//
// Since this scenario is UB, we are within our rights hit this check and halt execution...
// But we are also within our rights to try to make it work.
// We use wrapping_add/wrapping_sub for pointer arithmetic in this module in an attempt to support
// this use. Of course this is not a guarantee that such use will work, it just means that this
// crate doing wrapping pointer arithmetic with a method that must not wrap won't be the problem if
// something does go wrong at runtime.
use core::intrinsics::likely;

const WORD_SIZE: usize = core::mem::size_of::<usize>();
Expand All @@ -9,7 +26,7 @@ const WORD_MASK: usize = WORD_SIZE - 1;
// word-wise copy.
// * The word-wise copy logic needs to perform some checks so it has some small overhead.
// ensures that even on 32-bit platforms we have copied at least 8 bytes through
// word-wise copy so the saving of word-wise copy outweights the fixed overhead.
// word-wise copy so the saving of word-wise copy outweighs the fixed overhead.
const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 {
2 * WORD_SIZE
} else {
Expand All @@ -28,32 +45,32 @@ unsafe fn read_usize_unaligned(x: *const usize) -> usize {
pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) {
#[inline(always)]
unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
let dest_end = dest.add(n);
let dest_end = dest.wrapping_add(n);
while dest < dest_end {
*dest = *src;
dest = dest.add(1);
src = src.add(1);
dest = dest.wrapping_add(1);
src = src.wrapping_add(1);
}
}

#[inline(always)]
unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_end = dest.add(n) as *mut usize;
let dest_end = dest.wrapping_add(n) as *mut usize;

while dest_usize < dest_end {
*dest_usize = *src_usize;
dest_usize = dest_usize.add(1);
src_usize = src_usize.add(1);
dest_usize = dest_usize.wrapping_add(1);
src_usize = src_usize.wrapping_add(1);
}
}

#[cfg(not(feature = "mem-unaligned"))]
#[inline(always)]
unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let dest_end = dest.add(n) as *mut usize;
let dest_end = dest.wrapping_add(n) as *mut usize;

// Calculate the misalignment offset and shift needed to reassemble value.
let offset = src as usize & WORD_MASK;
Expand All @@ -70,7 +87,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
let mut prev_word = core::ptr::read_volatile(src_aligned);

while dest_usize < dest_end {
src_aligned = src_aligned.add(1);
src_aligned = src_aligned.wrapping_add(1);
let cur_word = *src_aligned;
#[cfg(target_endian = "little")]
let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift);
Expand All @@ -79,7 +96,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
prev_word = cur_word;

*dest_usize = resembled;
dest_usize = dest_usize.add(1);
dest_usize = dest_usize.wrapping_add(1);
}
}

Expand All @@ -88,12 +105,12 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_end = dest.add(n) as *mut usize;
let dest_end = dest.wrapping_add(n) as *mut usize;

while dest_usize < dest_end {
*dest_usize = read_usize_unaligned(src_usize);
dest_usize = dest_usize.add(1);
src_usize = src_usize.add(1);
dest_usize = dest_usize.wrapping_add(1);
src_usize = src_usize.wrapping_add(1);
}
}

Expand All @@ -102,8 +119,8 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
let dest_misalignment = (dest as usize).wrapping_neg() & WORD_MASK;
copy_forward_bytes(dest, src, dest_misalignment);
dest = dest.add(dest_misalignment);
src = src.add(dest_misalignment);
dest = dest.wrapping_add(dest_misalignment);
src = src.wrapping_add(dest_misalignment);
n -= dest_misalignment;

let n_words = n & !WORD_MASK;
Expand All @@ -113,8 +130,8 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
} else {
copy_forward_misaligned_words(dest, src, n_words);
}
dest = dest.add(n_words);
src = src.add(n_words);
dest = dest.wrapping_add(n_words);
src = src.wrapping_add(n_words);
n -= n_words;
}
copy_forward_bytes(dest, src, n);
Expand All @@ -126,10 +143,10 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
// as their inputs instead of pointers to the start!
#[inline(always)]
unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
let dest_start = dest.sub(n);
let dest_start = dest.wrapping_sub(n);
while dest_start < dest {
dest = dest.sub(1);
src = src.sub(1);
dest = dest.wrapping_sub(1);
src = src.wrapping_sub(1);
*dest = *src;
}
}
Expand All @@ -138,11 +155,11 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_start = dest.sub(n) as *mut usize;
let dest_start = dest.wrapping_sub(n) as *mut usize;

while dest_start < dest_usize {
dest_usize = dest_usize.sub(1);
src_usize = src_usize.sub(1);
dest_usize = dest_usize.wrapping_sub(1);
src_usize = src_usize.wrapping_sub(1);
*dest_usize = *src_usize;
}
}
Expand All @@ -151,7 +168,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
#[inline(always)]
unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let dest_start = dest.sub(n) as *mut usize;
let dest_start = dest.wrapping_sub(n) as *mut usize;

// Calculate the misalignment offset and shift needed to reassemble value.
let offset = src as usize & WORD_MASK;
Expand All @@ -168,15 +185,15 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
let mut prev_word = core::ptr::read_volatile(src_aligned);

while dest_start < dest_usize {
src_aligned = src_aligned.sub(1);
src_aligned = src_aligned.wrapping_sub(1);
let cur_word = *src_aligned;
#[cfg(target_endian = "little")]
let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift;
#[cfg(target_endian = "big")]
let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift;
prev_word = cur_word;

dest_usize = dest_usize.sub(1);
dest_usize = dest_usize.wrapping_sub(1);
*dest_usize = resembled;
}
}
Expand All @@ -186,25 +203,25 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
let mut dest_usize = dest as *mut usize;
let mut src_usize = src as *mut usize;
let dest_start = dest.sub(n) as *mut usize;
let dest_start = dest.wrapping_sub(n) as *mut usize;

while dest_start < dest_usize {
dest_usize = dest_usize.sub(1);
src_usize = src_usize.sub(1);
dest_usize = dest_usize.wrapping_sub(1);
src_usize = src_usize.wrapping_sub(1);
*dest_usize = read_usize_unaligned(src_usize);
}
}

let mut dest = dest.add(n);
let mut src = src.add(n);
let mut dest = dest.wrapping_add(n);
let mut src = src.wrapping_add(n);

if n >= WORD_COPY_THRESHOLD {
// Align dest
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
let dest_misalignment = dest as usize & WORD_MASK;
copy_backward_bytes(dest, src, dest_misalignment);
dest = dest.sub(dest_misalignment);
src = src.sub(dest_misalignment);
dest = dest.wrapping_sub(dest_misalignment);
src = src.wrapping_sub(dest_misalignment);
n -= dest_misalignment;

let n_words = n & !WORD_MASK;
Expand All @@ -214,8 +231,8 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
} else {
copy_backward_misaligned_words(dest, src, n_words);
}
dest = dest.sub(n_words);
src = src.sub(n_words);
dest = dest.wrapping_sub(n_words);
src = src.wrapping_sub(n_words);
n -= n_words;
}
copy_backward_bytes(dest, src, n);
Expand All @@ -225,10 +242,10 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
#[inline(always)]
pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) {
let end = s.add(n);
let end = s.wrapping_add(n);
while s < end {
*s = c;
s = s.add(1);
s = s.wrapping_add(1);
}
}

Expand All @@ -242,11 +259,11 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
}

let mut s_usize = s as *mut usize;
let end = s.add(n) as *mut usize;
let end = s.wrapping_add(n) as *mut usize;

while s_usize < end {
*s_usize = broadcast;
s_usize = s_usize.add(1);
s_usize = s_usize.wrapping_add(1);
}
}

Expand All @@ -255,12 +272,12 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
let misalignment = (s as usize).wrapping_neg() & WORD_MASK;
set_bytes_bytes(s, c, misalignment);
s = s.add(misalignment);
s = s.wrapping_add(misalignment);
n -= misalignment;

let n_words = n & !WORD_MASK;
set_bytes_words(s, c, n_words);
s = s.add(n_words);
s = s.wrapping_add(n_words);
n -= n_words;
}
set_bytes_bytes(s, c, n);
Expand All @@ -270,8 +287,8 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 {
let mut i = 0;
while i < n {
let a = *s1.add(i);
let b = *s2.add(i);
let a = *s1.wrapping_add(i);
let b = *s2.wrapping_add(i);
if a != b {
return a as i32 - b as i32;
}
Expand All @@ -285,7 +302,7 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize {
let mut n = 0;
while *s != 0 {
n += 1;
s = s.add(1);
s = s.wrapping_add(1);
}
n
}
Loading