Improve __copy_to_user and __copy_from_user performance

Provide a __copy_from_user that uses memcpy. On BCM2708, use
optimised memcpy/memmove/memcmp/memset implementations.

arch/arm: Add mmiocpy/set aliases for memcpy/set

See: #1082

copy_from_user: CPU_SW_DOMAIN_PAN compatibility

The downstream copy_from_user acceleration must also play nice with
CONFIG_CPU_SW_DOMAIN_PAN.

See: #1381

Signed-off-by: Phil Elwell <phil@raspberrypi.org>

arch/arm/include/asm/string.h

@@ -24,6 +24,11 @@ extern void * memchr(const void *, int, __kernel_size_t);
 #define __HAVE_ARCH_MEMSET
 extern void * memset(void *, int, __kernel_size_t);
 
+#ifdef CONFIG_MACH_BCM2708
+#define __HAVE_ARCH_MEMCMP
+extern int memcmp(const void *, const void *, size_t);
+#endif
+
 extern void __memzero(void *ptr, __kernel_size_t n);
 
 #define memset(p,v,n)							\

arch/arm/include/asm/uaccess.h

@@ -477,6 +477,9 @@ do {									\
 extern unsigned long __must_check
 arm_copy_from_user(void *to, const void __user *from, unsigned long n);
 
+extern unsigned long __must_check
+__copy_from_user_std(void *to, const void __user *from, unsigned long n);
+
 static inline unsigned long __must_check
 __copy_from_user(void *to, const void __user *from, unsigned long n)
 {

arch/arm/lib/Makefile

@@ -6,9 +6,8 @@
 
 lib-y		:= backtrace.o changebit.o csumipv6.o csumpartial.o   \
 		   csumpartialcopy.o csumpartialcopyuser.o clearbit.o \
-		   delay.o delay-loop.o findbit.o memchr.o memcpy.o   \
-		   memmove.o memset.o memzero.o setbit.o              \
-		   strchr.o strrchr.o                                 \
+		   delay.o delay-loop.o findbit.o memchr.o memzero.o  \
+		   setbit.o strchr.o strrchr.o                        \
 		   testchangebit.o testclearbit.o testsetbit.o        \
 		   ashldi3.o ashrdi3.o lshrdi3.o muldi3.o             \
 		   ucmpdi2.o lib1funcs.o div64.o                      \
@@ -18,6 +17,16 @@ lib-y		:= backtrace.o changebit.o csumipv6.o csumpartial.o   \
 mmu-y		:= clear_user.o copy_page.o getuser.o putuser.o       \
 		   copy_from_user.o copy_to_user.o
 
+# Choose optimised implementations for Raspberry Pi
+ifeq ($(CONFIG_MACH_BCM2708),y)
+  CFLAGS_uaccess_with_memcpy.o += -DCOPY_FROM_USER_THRESHOLD=1600
+  CFLAGS_uaccess_with_memcpy.o += -DCOPY_TO_USER_THRESHOLD=672
+  obj-$(CONFIG_MODULES) += exports_rpi.o
+  lib-y        += memcpy_rpi.o memmove_rpi.o memset_rpi.o memcmp_rpi.o
+else
+  lib-y        += memcpy.o memmove.o memset.o
+endif
+
 # using lib_ here won't override already available weak symbols
 obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o
 

arch/arm/lib/arm-mem.h

@@ -0,0 +1,159 @@
+/*
+Copyright (c) 2013, Raspberry Pi Foundation
+Copyright (c) 2013, RISC OS Open Ltd
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the copyright holder nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+.macro myfunc fname
+ .func fname
+ .global fname
+fname:
+.endm
+
+.macro preload_leading_step1  backwards, ptr, base
+/* If the destination is already 16-byte aligned, then we need to preload
+ * between 0 and prefetch_distance (inclusive) cache lines ahead so there
+ * are no gaps when the inner loop starts.
+ */
+ .if backwards
+        sub     ptr, base, #1
+        bic     ptr, ptr, #31
+ .else
+        bic     ptr, base, #31
+ .endif
+ .set OFFSET, 0
+ .rept prefetch_distance+1
+        pld     [ptr, #OFFSET]
+  .if backwards
+   .set OFFSET, OFFSET-32
+  .else
+   .set OFFSET, OFFSET+32
+  .endif
+ .endr
+.endm
+
+.macro preload_leading_step2  backwards, ptr, base, leading_bytes, tmp
+/* However, if the destination is not 16-byte aligned, we may need to
+ * preload one more cache line than that. The question we need to ask is:
+ * are the leading bytes more than the amount by which the source
+ * pointer will be rounded down for preloading, and if so, by how many
+ * cache lines?
+ */
+ .if backwards
+/* Here we compare against how many bytes we are into the
+ * cache line, counting down from the highest such address.
+ * Effectively, we want to calculate
+ *     leading_bytes = dst&15
+ *     cacheline_offset = 31-((src-leading_bytes-1)&31)
+ *     extra_needed = leading_bytes - cacheline_offset
+ * and test if extra_needed is <= 0, or rearranging:
+ *     leading_bytes + (src-leading_bytes-1)&31 <= 31
+ */
+        mov     tmp, base, lsl #32-5
+        sbc     tmp, tmp, leading_bytes, lsl #32-5
+        adds    tmp, tmp, leading_bytes, lsl #32-5
+        bcc     61f
+        pld     [ptr, #-32*(prefetch_distance+1)]
+ .else
+/* Effectively, we want to calculate
+ *     leading_bytes = (-dst)&15
+ *     cacheline_offset = (src+leading_bytes)&31
+ *     extra_needed = leading_bytes - cacheline_offset
+ * and test if extra_needed is <= 0.
+ */
+        mov     tmp, base, lsl #32-5
+        add     tmp, tmp, leading_bytes, lsl #32-5
+        rsbs    tmp, tmp, leading_bytes, lsl #32-5
+        bls     61f
+        pld     [ptr, #32*(prefetch_distance+1)]
+ .endif
+61:
+.endm
+
+.macro preload_trailing  backwards, base, remain, tmp
+        /* We need either 0, 1 or 2 extra preloads */
+ .if backwards
+        rsb     tmp, base, #0
+        mov     tmp, tmp, lsl #32-5
+ .else
+        mov     tmp, base, lsl #32-5
+ .endif
+        adds    tmp, tmp, remain, lsl #32-5
+        adceqs  tmp, tmp, #0
+        /* The instruction above has two effects: ensures Z is only
+         * set if C was clear (so Z indicates that both shifted quantities
+         * were 0), and clears C if Z was set (so C indicates that the sum
+         * of the shifted quantities was greater and not equal to 32) */
+        beq     82f
+ .if backwards
+        sub     tmp, base, #1
+        bic     tmp, tmp, #31
+ .else
+        bic     tmp, base, #31
+ .endif
+        bcc     81f
+ .if backwards
+        pld     [tmp, #-32*(prefetch_distance+1)]
+81:
+        pld     [tmp, #-32*prefetch_distance]
+ .else
+        pld     [tmp, #32*(prefetch_distance+2)]
+81:
+        pld     [tmp, #32*(prefetch_distance+1)]
+ .endif
+82:
+.endm
+
+.macro preload_all    backwards, narrow_case, shift, base, remain, tmp0, tmp1
+ .if backwards
+        sub     tmp0, base, #1
+        bic     tmp0, tmp0, #31
+        pld     [tmp0]
+        sub     tmp1, base, remain, lsl #shift
+ .else
+        bic     tmp0, base, #31
+        pld     [tmp0]
+        add     tmp1, base, remain, lsl #shift
+        sub     tmp1, tmp1, #1
+ .endif
+        bic     tmp1, tmp1, #31
+        cmp     tmp1, tmp0
+        beq     92f
+ .if narrow_case
+        /* In this case, all the data fits in either 1 or 2 cache lines */
+        pld     [tmp1]
+ .else
+91:
+  .if backwards
+        sub     tmp0, tmp0, #32
+  .else
+        add     tmp0, tmp0, #32
+  .endif
+        cmp     tmp0, tmp1
+        pld     [tmp0]
+        bne     91b
+ .endif
+92:
+.endm

arch/arm/lib/copy_from_user.S

@@ -89,11 +89,13 @@
 
 	.text
 
-ENTRY(arm_copy_from_user)
+ENTRY(__copy_from_user_std)
+WEAK(arm_copy_from_user)
 
 #include "copy_template.S"
 
 ENDPROC(arm_copy_from_user)
+ENDPROC(__copy_from_user_std)
 
 	.pushsection .fixup,"ax"
 	.align 0

arch/arm/lib/exports_rpi.c

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) 2014, Raspberry Pi (Trading) Ltd.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+EXPORT_SYMBOL(memcmp);