From 202aea5acd61ec9d264defffca09c1377950274f Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Tue, 26 Jun 2018 22:59:10 -0400 Subject: [PATCH 1/9] Speed up compilation of large constant arrays This is a different approach to #51672 as suggested by @oli-obk. Rather than write each repeated value one-by-one, we write the first one and then copy its value directly into the remaining memory. --- src/librustc_mir/interpret/eval_context.rs | 12 ++++++---- src/librustc_mir/interpret/memory.rs | 26 ++++++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/librustc_mir/interpret/eval_context.rs b/src/librustc_mir/interpret/eval_context.rs index ea667273ecead..1faa23637a5ed 100644 --- a/src/librustc_mir/interpret/eval_context.rs +++ b/src/librustc_mir/interpret/eval_context.rs @@ -591,10 +591,14 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M let (dest, dest_align) = self.force_allocation(dest)?.to_ptr_align(); - // FIXME: speed up repeat filling - for i in 0..length { - let elem_dest = dest.ptr_offset(elem_size * i as u64, &self)?; - self.write_value_to_ptr(value, elem_dest, dest_align, elem_ty)?; + if length > 0 { + //write the first value + self.write_value_to_ptr(value, dest, dest_align, elem_ty)?; + + if length > 1 { + let rest = dest.ptr_offset(elem_size * 1 as u64, &self)?; + self.memory.copy_repeatedly(dest, dest_align, rest, dest_align, elem_size, length - 1, false)?; + } } } diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs index ad571fbe90d5d..7a547c406a92b 100644 --- a/src/librustc_mir/interpret/memory.rs +++ b/src/librustc_mir/interpret/memory.rs @@ -594,6 +594,19 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { dest_align: Align, size: Size, nonoverlapping: bool, + ) -> EvalResult<'tcx> { + self.copy_repeatedly(src, src_align, dest, dest_align, size, 1, nonoverlapping) + } + + pub fn copy_repeatedly( + &mut self, + src: Scalar, + src_align: Align, + dest: Scalar, + dest_align: Align, + size: Size, + length: u64, + nonoverlapping: bool, ) -> EvalResult<'tcx> { // Empty accesses don't need to be valid pointers, but they should still be aligned self.check_align(src, src_align)?; @@ -617,7 +630,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { .collect(); let src_bytes = self.get_bytes_unchecked(src, size, src_align)?.as_ptr(); - let dest_bytes = self.get_bytes_mut(dest, size, dest_align)?.as_mut_ptr(); + let dest_bytes = self.get_bytes_mut(dest, size * length, dest_align)?.as_mut_ptr(); // SAFE: The above indexing would have panicked if there weren't at least `size` bytes // behind `src` and `dest`. Also, we use the overlapping-safe `ptr::copy` if `src` and @@ -634,13 +647,18 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { )); } } - ptr::copy(src_bytes, dest_bytes, size.bytes() as usize); + + for i in 0..length { + ptr::copy(src_bytes, dest_bytes.offset((size.bytes() * i) as isize), size.bytes() as usize); + } } else { - ptr::copy_nonoverlapping(src_bytes, dest_bytes, size.bytes() as usize); + for i in 0..length { + ptr::copy_nonoverlapping(src_bytes, dest_bytes.offset((size.bytes() * i) as isize), size.bytes() as usize); + } } } - self.copy_undef_mask(src, dest, size)?; + self.copy_undef_mask(src, dest, size * length)?; // copy back the relocations self.get_mut(dest.alloc_id)?.relocations.insert_presorted(relocations); From 63ab0cba7d36c4768bc64a27382fc56ca3a390ce Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Wed, 27 Jun 2018 00:33:38 -0400 Subject: [PATCH 2/9] Inline `abi::Size::bytes()` This save an additional 6 seconds on the test program. --- src/librustc_target/abi/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/librustc_target/abi/mod.rs b/src/librustc_target/abi/mod.rs index 9003e30357cbd..4deac0444b925 100644 --- a/src/librustc_target/abi/mod.rs +++ b/src/librustc_target/abi/mod.rs @@ -240,6 +240,7 @@ impl Size { } } + #[inline] pub fn bytes(self) -> u64 { self.raw } From 429bc8d0cc72eb30541886610d5fb9e8969244fa Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Wed, 27 Jun 2018 19:34:42 -0400 Subject: [PATCH 3/9] Inline all methods on `abi::Size` This save 3 seconds on the test program. --- src/librustc_target/abi/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/librustc_target/abi/mod.rs b/src/librustc_target/abi/mod.rs index 4deac0444b925..5762269242663 100644 --- a/src/librustc_target/abi/mod.rs +++ b/src/librustc_target/abi/mod.rs @@ -229,11 +229,13 @@ pub struct Size { impl Size { pub const ZERO: Size = Self::from_bytes(0); + #[inline] pub fn from_bits(bits: u64) -> Size { // Avoid potential overflow from `bits + 7`. Size::from_bytes(bits / 8 + ((bits % 8) + 7) / 8) } + #[inline] pub const fn from_bytes(bytes: u64) -> Size { Size { raw: bytes @@ -245,22 +247,26 @@ impl Size { self.raw } + #[inline] pub fn bits(self) -> u64 { self.bytes().checked_mul(8).unwrap_or_else(|| { panic!("Size::bits: {} bytes in bits doesn't fit in u64", self.bytes()) }) } + #[inline] pub fn abi_align(self, align: Align) -> Size { let mask = align.abi() - 1; Size::from_bytes((self.bytes() + mask) & !mask) } + #[inline] pub fn is_abi_aligned(self, align: Align) -> bool { let mask = align.abi() - 1; self.bytes() & mask == 0 } + #[inline] pub fn checked_add(self, offset: Size, cx: C) -> Option { let dl = cx.data_layout(); @@ -273,6 +279,7 @@ impl Size { } } + #[inline] pub fn checked_mul(self, count: u64, cx: C) -> Option { let dl = cx.data_layout(); @@ -290,6 +297,7 @@ impl Size { impl Add for Size { type Output = Size; + #[inline] fn add(self, other: Size) -> Size { Size::from_bytes(self.bytes().checked_add(other.bytes()).unwrap_or_else(|| { panic!("Size::add: {} + {} doesn't fit in u64", self.bytes(), other.bytes()) @@ -299,6 +307,7 @@ impl Add for Size { impl Sub for Size { type Output = Size; + #[inline] fn sub(self, other: Size) -> Size { Size::from_bytes(self.bytes().checked_sub(other.bytes()).unwrap_or_else(|| { panic!("Size::sub: {} - {} would result in negative size", self.bytes(), other.bytes()) @@ -308,6 +317,7 @@ impl Sub for Size { impl Mul for u64 { type Output = Size; + #[inline] fn mul(self, size: Size) -> Size { size * self } @@ -315,6 +325,7 @@ impl Mul for u64 { impl Mul for Size { type Output = Size; + #[inline] fn mul(self, count: u64) -> Size { match self.bytes().checked_mul(count) { Some(bytes) => Size::from_bytes(bytes), @@ -326,6 +337,7 @@ impl Mul for Size { } impl AddAssign for Size { + #[inline] fn add_assign(&mut self, other: Size) { *self = *self + other; } From 1ffa99d218ecff7b996d71a86e7790f5c3ba2614 Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Fri, 29 Jun 2018 20:22:35 -0400 Subject: [PATCH 4/9] Optimize `copy_undef_mask()` by lifting some loop invariant operations This saves 4.5 seconds and takes the compile time down to 5.5 seconds. --- src/librustc_mir/interpret/memory.rs | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs index 7a547c406a92b..49ad4b49b88b7 100644 --- a/src/librustc_mir/interpret/memory.rs +++ b/src/librustc_mir/interpret/memory.rs @@ -883,17 +883,26 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { // The bits have to be saved locally before writing to dest in case src and dest overlap. assert_eq!(size.bytes() as usize as u64, size.bytes()); let mut v = Vec::with_capacity(size.bytes() as usize); - for i in 0..size.bytes() { - let defined = self.get(src.alloc_id)?.undef_mask.get(src.offset + Size::from_bytes(i)); - v.push(defined); + + { + let src_allocation = self.get(src.alloc_id)?; + for i in 0..size.bytes() { + let defined = src_allocation.undef_mask.get(src.offset + Size::from_bytes(i)); + v.push(defined); + } } - for (i, defined) in v.into_iter().enumerate() { - self.get_mut(dest.alloc_id)?.undef_mask.set( - dest.offset + - Size::from_bytes(i as u64), - defined, - ); + + { + let dest_allocation = self.get_mut(dest.alloc_id)?; + for (i, defined) in v.into_iter().enumerate() { + dest_allocation.undef_mask.set( + dest.offset + + Size::from_bytes(i as u64), + defined, + ); + } } + Ok(()) } From 8f969ed3766ee159739d99dcb850f90ce9af39d0 Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Fri, 29 Jun 2018 22:26:15 -0400 Subject: [PATCH 5/9] Optimize `copy_undef_mask()` to use one pass This saves 0.5 seconds on the test compilation. --- src/librustc_mir/interpret/memory.rs | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs index 49ad4b49b88b7..2f961ef9d9782 100644 --- a/src/librustc_mir/interpret/memory.rs +++ b/src/librustc_mir/interpret/memory.rs @@ -882,25 +882,16 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { ) -> EvalResult<'tcx> { // The bits have to be saved locally before writing to dest in case src and dest overlap. assert_eq!(size.bytes() as usize as u64, size.bytes()); - let mut v = Vec::with_capacity(size.bytes() as usize); - { - let src_allocation = self.get(src.alloc_id)?; - for i in 0..size.bytes() { - let defined = src_allocation.undef_mask.get(src.offset + Size::from_bytes(i)); - v.push(defined); - } - } + let undef_mask = self.get(src.alloc_id)?.undef_mask.clone(); + let dest_allocation = self.get_mut(dest.alloc_id)?; - { - let dest_allocation = self.get_mut(dest.alloc_id)?; - for (i, defined) in v.into_iter().enumerate() { - dest_allocation.undef_mask.set( - dest.offset + - Size::from_bytes(i as u64), - defined, - ); - } + for i in 0..size.bytes() { + let defined = undef_mask.get(src.offset + Size::from_bytes(i)); + dest_allocation.undef_mask.set( + dest.offset + Size::from_bytes(i), + defined + ); } Ok(()) From c431f3fab90efa2100c5b7189d01a47bb69e4ed6 Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Fri, 29 Jun 2018 23:33:17 -0400 Subject: [PATCH 6/9] Inline a few `UndefMask` methods. This saves 2.5 seconds on the test program. --- src/librustc/mir/interpret/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/librustc/mir/interpret/mod.rs b/src/librustc/mir/interpret/mod.rs index 6bd5814799ae0..37580aa4efdb3 100644 --- a/src/librustc/mir/interpret/mod.rs +++ b/src/librustc/mir/interpret/mod.rs @@ -635,11 +635,13 @@ impl UndefMask { } } + #[inline] pub fn get(&self, i: Size) -> bool { let (block, bit) = bit_index(i); (self.blocks[block] & 1 << bit) != 0 } + #[inline] pub fn set(&mut self, i: Size, new_state: bool) { let (block, bit) = bit_index(i); if new_state { @@ -664,6 +666,7 @@ impl UndefMask { } } +#[inline] fn bit_index(bits: Size) -> (usize, usize) { let bits = bits.bytes(); let a = bits / BLOCK_SIZE; From 84fe0c40a1829ebac169a1f7804da4b557b31359 Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Sat, 30 Jun 2018 00:44:58 -0400 Subject: [PATCH 7/9] Fix relocations to include repeated values --- src/librustc_mir/interpret/memory.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs index 2f961ef9d9782..8aff58e09bae1 100644 --- a/src/librustc_mir/interpret/memory.rs +++ b/src/librustc_mir/interpret/memory.rs @@ -621,13 +621,21 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { // first copy the relocations to a temporary buffer, because // `get_bytes_mut` will clear the relocations, which is correct, // since we don't want to keep any relocations at the target. - let relocations: Vec<_> = self.relocations(src, size)? - .iter() - .map(|&(offset, alloc_id)| { - // Update relocation offsets for the new positions in the destination allocation. - (offset + dest.offset - src.offset, alloc_id) - }) - .collect(); + let relocations = { + let relocations = self.relocations(src, size)?; + let mut new_relocations = Vec::with_capacity(relocations.len() * (length as usize)); + for i in 0..length { + new_relocations.extend( + relocations + .iter() + .map(|&(offset, alloc_id)| { + (offset + dest.offset - src.offset + (i * size * relocations.len() as u64), alloc_id) + }) + ); + } + + new_relocations + }; let src_bytes = self.get_bytes_unchecked(src, size, src_align)?.as_ptr(); let dest_bytes = self.get_bytes_mut(dest, size * length, dest_align)?.as_mut_ptr(); From faef6a30e6a470fd46444642c62f4a1ff378ca7f Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Sat, 30 Jun 2018 14:23:41 -0400 Subject: [PATCH 8/9] Copy undef_masks correctly for repeated bytes --- src/librustc_mir/interpret/memory.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs index 8aff58e09bae1..bf720540bdcd4 100644 --- a/src/librustc_mir/interpret/memory.rs +++ b/src/librustc_mir/interpret/memory.rs @@ -666,7 +666,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { } } - self.copy_undef_mask(src, dest, size * length)?; + self.copy_undef_mask(src, dest, size, length)?; // copy back the relocations self.get_mut(dest.alloc_id)?.relocations.insert_presorted(relocations); @@ -887,6 +887,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { src: Pointer, dest: Pointer, size: Size, + repeat: u64, ) -> EvalResult<'tcx> { // The bits have to be saved locally before writing to dest in case src and dest overlap. assert_eq!(size.bytes() as usize as u64, size.bytes()); @@ -896,10 +897,13 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> { for i in 0..size.bytes() { let defined = undef_mask.get(src.offset + Size::from_bytes(i)); - dest_allocation.undef_mask.set( - dest.offset + Size::from_bytes(i), - defined - ); + + for j in 0..repeat { + dest_allocation.undef_mask.set( + dest.offset + Size::from_bytes(i + (size.bytes() * j)), + defined + ); + } } Ok(()) From 46512e09c9f97a68a36e2d72e3be766b1e76a1a0 Mon Sep 17 00:00:00 2001 From: Wesley Wiser Date: Sat, 30 Jun 2018 15:25:07 -0400 Subject: [PATCH 9/9] Add two regression tests for const eval --- src/test/compile-fail/const-err4.rs | 24 ++++++++++++++++++++++ src/test/run-pass/const-repeated-values.rs | 19 +++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 src/test/compile-fail/const-err4.rs create mode 100644 src/test/run-pass/const-repeated-values.rs diff --git a/src/test/compile-fail/const-err4.rs b/src/test/compile-fail/const-err4.rs new file mode 100644 index 0000000000000..09ebf1681c5e0 --- /dev/null +++ b/src/test/compile-fail/const-err4.rs @@ -0,0 +1,24 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[derive(Copy, Clone)] +union Foo { + a: isize, + b: (), +} + +enum Bar { + Boo = [unsafe { Foo { b: () }.a }; 4][3], + //~^ ERROR constant evaluation of enum discriminant resulted in non-integer +} + +fn main() { + assert_ne!(Bar::Boo as isize, 0); +} diff --git a/src/test/run-pass/const-repeated-values.rs b/src/test/run-pass/const-repeated-values.rs new file mode 100644 index 0000000000000..1d749a2626e9f --- /dev/null +++ b/src/test/run-pass/const-repeated-values.rs @@ -0,0 +1,19 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +const FOO: isize = 42; + +enum Bar { + Boo = *[&FOO; 4][3], +} + +fn main() { + assert_eq!(Bar::Boo as isize, 42); +}