From 69ef03d52500c0ebda9b4b7a0c7434e816d80775 Mon Sep 17 00:00:00 2001 From: Sebastian Apel <13675545+SebastianApel@users.noreply.github.com> Date: Sun, 2 Apr 2023 22:13:08 +0200 Subject: [PATCH 1/3] Performance improvement of AVX2 code --- ggml.c | 100 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/ggml.c b/ggml.c index 63aa5eb6eb0f8..f310382b4f68f 100644 --- a/ggml.c +++ b/ggml.c @@ -1959,45 +1959,75 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest // Horizontal sum of all lanes of the accumulator sumf = _mm512_reduce_add_ps( acc0 ) + _mm512_reduce_add_ps( acc1 ); #elif defined(__AVX2__) + // Initialize accumulator with zeros __m256 acc = _mm256_setzero_ps(); - // Main loop - // TODO: figure a way to do this in a portable way - #ifdef __GNUC__ - #pragma GCC unroll 16 - #endif - for (int i = 0; i < nb; ++i) { - // Compute combined scale for the block - const __m256 d = _mm256_mul_ps( _mm256_broadcast_ss( &x[i].d ), _mm256_broadcast_ss( &y[i].d ) ); - - // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes - __m256i bx = bytesFromNibbles( x[i].qs ); - __m256i by = bytesFromNibbles( y[i].qs ); - - // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval. - const __m256i off = _mm256_set1_epi8( 8 ); - bx = _mm256_sub_epi8( bx, off ); - by = _mm256_sub_epi8( by, off ); - - // Get absolute values of x vectors - const __m256i ax = _mm256_sign_epi8(bx, bx); - - // Sign the values of the y vectors - const __m256i sy = _mm256_sign_epi8(by, bx); - - // Perform multiplication and create 16-bit values - const __m256i dot = _mm256_maddubs_epi16(ax, sy); - - const __m256i ones = _mm256_set1_epi16(1); - const __m256i i32 = _mm256_madd_epi16(ones, dot); + /* Prepare the constants we will need during execution */ + const __m256i lowMask = _mm256_set1_epi8( 0xF ); + const __m256i offset_8 = _mm256_set1_epi16( 8 ); - // Convert int32_t to float - const __m256 p = _mm256_cvtepi32_ps( i32 ); +#define UNROLL_COUNT 8 + // make sure we only unroll multiples of the block count + assert(nb % UNROLL_COUNT == 0); - // Apply the scale, and accumulate - acc = _mm256_fmadd_ps( d, p, acc ); - } + // Main loop + for (int i = 0; i < nb; i+=UNROLL_COUNT) { + + // This loop will be unrolled by the compiler + for (int u=0;u we now have a vector of 8 int_32t */ + __m256i xy_q = _mm256_add_epi32( xy_high_q, xy_low_q ); + + /* Convert to vectore of 8 int32_t to 8 floats */ + __m256 q = _mm256_cvtepi32_ps( xy_q ); + + /* Multiply q with scale and accumulate */ + acc = _mm256_fmadd_ps( scale, q, acc );; + } + + } // Return horizontal sum of the acc vector __m128 res = _mm256_extractf128_ps( acc, 1 ); @@ -2026,7 +2056,7 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest bx = _mm_sub_epi8( bx, off ); by = _mm_sub_epi8( by, off ); - // Get absolute values of x vectors + // Get absolute values of x vectors const __m128i ax = _mm_sign_epi8(bx, bx); // Sign the values of the y vectors From b589e34f92806bfd5e1af5a45834ae5bb66e00d1 Mon Sep 17 00:00:00 2001 From: Sebastian Apel <13675545+SebastianApel@users.noreply.github.com> Date: Mon, 3 Apr 2023 08:33:03 +0200 Subject: [PATCH 2/3] Fixed problem with MSVC compiler --- ggml.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml.c b/ggml.c index f310382b4f68f..068bf32f8c5f6 100644 --- a/ggml.c +++ b/ggml.c @@ -1986,7 +1986,7 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest Output: 2 vectors with 16 values of type int16_t (x_high_q, x_low_q) */ /* Load 16 bytes from memory */ - const __m128i tmp_x = _mm_loadu_si128( (const __m128i_u *) x[i+u].qs); + const __m128i tmp_x = _mm_loadu_si128( ( const __m128i* ) x[i+u].qs); /* Expand bytes into uint16_t values */ const __m256i bytes_x = _mm256_cvtepu8_epi16(tmp_x); /* Unpack values into individual bytes */ @@ -2002,7 +2002,7 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest Output: 2 vectors with 16 values of type int16_t (y_high_q, y_low_q) */ /* Load 16 bytes from memory */ - const __m128i tmp_y = _mm_loadu_si128( (const __m128i_u *) y[i+u].qs); + const __m128i tmp_y = _mm_loadu_si128( (const __m128i* ) y[i+u].qs); /* Expand bytes into uint16_t values */ const __m256i bytes_y = _mm256_cvtepu8_epi16(tmp_y); /* Unpack values into individual bytes */ From 1ed8878a4cbd3ffca7a2bb0c2fe58213459cbd7a Mon Sep 17 00:00:00 2001 From: Sebastian Apel <13675545+SebastianApel@users.noreply.github.com> Date: Mon, 3 Apr 2023 09:31:15 +0200 Subject: [PATCH 3/3] Reviewer comments: removed double semicolon, deleted empty line 1962 --- ggml.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ggml.c b/ggml.c index 068bf32f8c5f6..59e84ab45d120 100644 --- a/ggml.c +++ b/ggml.c @@ -1959,7 +1959,6 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest // Horizontal sum of all lanes of the accumulator sumf = _mm512_reduce_add_ps( acc0 ) + _mm512_reduce_add_ps( acc1 ); #elif defined(__AVX2__) - // Initialize accumulator with zeros __m256 acc = _mm256_setzero_ps(); @@ -2024,7 +2023,7 @@ static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void * rest __m256 q = _mm256_cvtepi32_ps( xy_q ); /* Multiply q with scale and accumulate */ - acc = _mm256_fmadd_ps( scale, q, acc );; + acc = _mm256_fmadd_ps( scale, q, acc ); } }