From 58e8c74f1ead1cadf3a9061f1b488d41b3036c80 Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Wed, 2 Oct 2019 19:30:22 +0000
Subject: [PATCH 1/9] Disable AVX-512 if using a version of the GNU Assembler
 with a known AVX-512 bug. Needs testing on other platforms. Fixes #193.

---
 common.mk | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/common.mk b/common.mk
index 003bb2e4..9de5b014 100644
--- a/common.mk
+++ b/common.mk
@@ -255,8 +255,22 @@ ifeq ($(DO_CHECKS), 1)
   endif
   ## done with check for conflicting options
 
+  # The GNU Assembler (GAS) has an AVX-512 bug in versions 2.30 and 2.31
+  # So we turn off AVX-512 if one of the bugged GAS versions is present.
+  # Note that both gcc and icc might use GAS!  Clang has its own assembler.
+  # If we are using the clang assembler,  we will detect this below and re-enable AVX-512
+  # See: https://github.com/manodeep/Corrfunc/issues/193
+  GAS_VERSION := $(shell as --version | head -n1 |\grep -oP '\d+\.\d+$$')
+  GAS_BUG_DISABLE_AVX512 := 0
+  ifeq ($(GAS_VERSION),$(filter $(GAS_VERSION),2.30 2.31))
+    GAS_BUG_DISABLE_AVX512 := 1
+  endif
+
   ifeq (icc,$(findstring icc,$(CC)))
-    CFLAGS += -xhost -xCORE-AVX512
+    # Normally, one would use -xHost with icc instead of -march=native
+    # But -xHost does not allow you to later turn off just AVX-512,
+    # which we may decide is necessary if the GAS bug is present
+    CFLAGS := -march=native
     ifeq (USE_OMP,$(findstring USE_OMP,$(OPT)))
       CFLAGS += -qopenmp
       CLINK  += -qopenmp
@@ -287,6 +301,9 @@ ifeq ($(DO_CHECKS), 1)
             $(warning $(ccmagenta) Either install clang ($(ccgreen)for Macports use, "sudo port install clang-3.8"$(ccmagenta)) or add option $(ccgreen)"-mno-avx"$(ccreset) to "CFLAGS" in $(ccmagenta)"common.mk"$(ccreset))
             export CLANG_ASM_WARNING_PRINTED := 1
           endif # warning printed
+
+          # Using clang as, not GAS
+          GAS_BUG_DISABLE_AVX512 := 0
         endif
 
         ifeq (USE_OMP,$(findstring USE_OMP,$(OPT)))
@@ -296,6 +313,10 @@ ifeq ($(DO_CHECKS), 1)
       endif #gcc findstring
     else ##CC is clang
       ### compiler specific flags for clang
+
+      # Using clang as, not gas
+      GAS_BUG_DISABLE_AVX512 := 0
+
       CLANG_OMP_AVAIL := false
       export APPLE_CLANG := 0
       ifeq (USE_OMP,$(findstring USE_OMP,$(OPT)))
@@ -385,6 +406,19 @@ ifeq ($(DO_CHECKS), 1)
     CLINK += -lm
   endif #not icc
 
+  # Do we need to turn off AVX-512?
+  ifeq ($(GAS_BUG_DISABLE_AVX512),1)
+    # Did the compiler support AVX-512 in the first place? Otherwise -mno-avx512f is not a valid option!
+    CC_SUPPORTS_AVX512 := $(shell $(CC) $(CFLAGS) -dM -E - < /dev/null | \grep -Pcm1 __AVX512F__)
+    ifeq ($(CC_SUPPORTS_AVX512),1)
+      CFLAGS += -mno-avx512f
+
+      ifneq ($(GAS_BUG_WARNING_PRINTED),1)
+        $(warning $(ccmagenta)DISABLING AVX-512 SUPPORT DUE TO GNU ASSEMBLER BUG$(ccreset))
+      endif
+      export GAS_BUG_WARNING_PRINTED := 1
+    endif
+  endif
 
   # All of the python/numpy checks follow
   export PYTHON_CHECKED ?= 0

From a789cd494c975459663ef42ed33d7189b88da721 Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Wed, 2 Oct 2019 20:23:04 +0000
Subject: [PATCH 2/9] Change escape sequences to use tput instead of echo for
 better cross-shell compatibility

---
 common.mk | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/common.mk b/common.mk
index 9de5b014..610b0297 100644
--- a/common.mk
+++ b/common.mk
@@ -66,33 +66,25 @@ endif
 ## Only set everything if the command is not "make clean" (or related to "make clean")
 ifeq ($(DO_CHECKS), 1)
   UNAME := $(shell uname)
-  ## Colored text output
-  ## Taken from: http://stackoverflow.com/questions/24144440/color-highlighting-of-makefile-warnings-and-errors
-  ## Except, you have to use "echo -e" on linux and "echo" on Mac
-  ECHO_COMMAND := echo -e
-  ifeq ($(UNAME), Darwin)
-    ECHO_COMMAND := echo
-  endif
-  ifeq ($(TRAVIS_OS_NAME), linux)
-    ECHO_COMMAND := echo
-  endif
 
   ifneq ($(UNAME), Darwin)
     CLINK += -lrt # need real time library for the nano-second timers. Not required on OSX
   endif
 
+  ## Colored text output
+  ## https://stackoverflow.com/questions/4332478/read-the-current-text-color-in-a-xterm/4332530#4332530
   ## Broadly speaking, here's the color convention (not strictly adhered to yet):
   ## green - shell commands
   ## red - error messages
   ## magenta - general highlight
   ## blue - related to code/compile option
   ## bold - only used with printing out compilation options
-  ccred:=$(shell $(ECHO_COMMAND) "\033[0;31m")
-  ccmagenta:=$(shell $(ECHO_COMMAND) "\033[0;35m")
-  ccgreen:=$(shell $(ECHO_COMMAND) "\033[0;32m")
-  ccblue:=$(shell $(ECHO_COMMAND) "\033[0;34m")
-  ccreset:=$(shell $(ECHO_COMMAND) "\033[0;0m")
-  boldfont:=$(shell $(ECHO_COMMAND) "\033[1m")
+  ccred:=$(shell tput setaf 1)
+  ccmagenta:=$(shell tput setaf 5)
+  ccgreen:=$(shell tput setaf 2)
+  ccblue:=$(shell tput setaf 4)
+  ccreset:=$(shell tput sgr0)
+  boldfont:=$(shell tput bold)
   ## end of colored text output
 
   ## First check make version. Versions of make older than 3.80 will crash
@@ -614,7 +606,7 @@ ifeq ($(DO_CHECKS), 1)
     ifeq (USE_MKL,$(findstring USE_MKL,$(OPT)))
       MAKEFILE_VARS += BLAS_INCLUDE BLAS_LINK
     endif
-    tabvar:= $(shell $(ECHO_COMMAND) "\t")
+    tabvar:= $(shell printf "\t")
     $(info )
     $(info $(ccmagenta)$(boldfont)-------COMPILE SETTINGS------------$(ccreset))
     $(foreach var, $(MAKEFILE_VARS), $(info $(tabvar) $(boldfont)$(var)$(ccreset)$(tabvar)$(tabvar) = ["$(ccblue)$(boldfont)${${var}}$(ccreset)"]))

From 87c70e782f136cd04aad3f86d77690f9e82ad94a Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Wed, 2 Oct 2019 21:27:04 +0000
Subject: [PATCH 3/9] Fix GNU assembler version detection

---
 common.mk | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/common.mk b/common.mk
index 610b0297..4564dd31 100644
--- a/common.mk
+++ b/common.mk
@@ -247,14 +247,14 @@ ifeq ($(DO_CHECKS), 1)
   endif
   ## done with check for conflicting options
 
-  # The GNU Assembler (GAS) has an AVX-512 bug in versions 2.30 and 2.31
+  # The GNU Assembler (GAS) has an AVX-512 bug in versions 2.30 to 2.31.1
   # So we turn off AVX-512 if one of the bugged GAS versions is present.
   # Note that both gcc and icc might use GAS!  Clang has its own assembler.
   # If we are using the clang assembler,  we will detect this below and re-enable AVX-512
   # See: https://github.com/manodeep/Corrfunc/issues/193
-  GAS_VERSION := $(shell as --version | head -n1 |\grep -oP '\d+\.\d+$$')
+  GAS_VERSION := $(shell as --version | head -n1 | \grep -P 'GNU assembler' |\grep -oP '\d+\.\d+$$')
   GAS_BUG_DISABLE_AVX512 := 0
-  ifeq ($(GAS_VERSION),$(filter $(GAS_VERSION),2.30 2.31))
+  ifneq (,$(filter $(GAS_VERSION),2.30 2.31 2.31.1))
     GAS_BUG_DISABLE_AVX512 := 1
   endif
 
@@ -262,7 +262,7 @@ ifeq ($(DO_CHECKS), 1)
     # Normally, one would use -xHost with icc instead of -march=native
     # But -xHost does not allow you to later turn off just AVX-512,
     # which we may decide is necessary if the GAS bug is present
-    CFLAGS := -march=native
+    CFLAGS += -march=native
     ifeq (USE_OMP,$(findstring USE_OMP,$(OPT)))
       CFLAGS += -qopenmp
       CLINK  += -qopenmp
@@ -406,7 +406,7 @@ ifeq ($(DO_CHECKS), 1)
       CFLAGS += -mno-avx512f
 
       ifneq ($(GAS_BUG_WARNING_PRINTED),1)
-        $(warning $(ccmagenta)DISABLING AVX-512 SUPPORT DUE TO GNU ASSEMBLER BUG$(ccreset))
+        $(warning $(ccmagenta)DISABLING AVX-512 SUPPORT DUE TO GNU ASSEMBLER BUG.  UPGRADE TO BINUTILS >=2.32 TO FIX THIS.$(ccreset))
       endif
       export GAS_BUG_WARNING_PRINTED := 1
     endif

From 4817b83d2529736a0dda367db68ac6834cd50058 Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Thu, 3 Oct 2019 14:52:03 +0000
Subject: [PATCH 4/9] Update GAS version detection to query runtime compiler

---
 common.mk | 26 +++++++-------------------
 1 file changed, 7 insertions(+), 19 deletions(-)

diff --git a/common.mk b/common.mk
index 4564dd31..ee3d5d3b 100644
--- a/common.mk
+++ b/common.mk
@@ -247,17 +247,6 @@ ifeq ($(DO_CHECKS), 1)
   endif
   ## done with check for conflicting options
 
-  # The GNU Assembler (GAS) has an AVX-512 bug in versions 2.30 to 2.31.1
-  # So we turn off AVX-512 if one of the bugged GAS versions is present.
-  # Note that both gcc and icc might use GAS!  Clang has its own assembler.
-  # If we are using the clang assembler,  we will detect this below and re-enable AVX-512
-  # See: https://github.com/manodeep/Corrfunc/issues/193
-  GAS_VERSION := $(shell as --version | head -n1 | \grep -P 'GNU assembler' |\grep -oP '\d+\.\d+$$')
-  GAS_BUG_DISABLE_AVX512 := 0
-  ifneq (,$(filter $(GAS_VERSION),2.30 2.31 2.31.1))
-    GAS_BUG_DISABLE_AVX512 := 1
-  endif
-
   ifeq (icc,$(findstring icc,$(CC)))
     # Normally, one would use -xHost with icc instead of -march=native
     # But -xHost does not allow you to later turn off just AVX-512,
@@ -293,9 +282,6 @@ ifeq ($(DO_CHECKS), 1)
             $(warning $(ccmagenta) Either install clang ($(ccgreen)for Macports use, "sudo port install clang-3.8"$(ccmagenta)) or add option $(ccgreen)"-mno-avx"$(ccreset) to "CFLAGS" in $(ccmagenta)"common.mk"$(ccreset))
             export CLANG_ASM_WARNING_PRINTED := 1
           endif # warning printed
-
-          # Using clang as, not GAS
-          GAS_BUG_DISABLE_AVX512 := 0
         endif
 
         ifeq (USE_OMP,$(findstring USE_OMP,$(OPT)))
@@ -305,10 +291,6 @@ ifeq ($(DO_CHECKS), 1)
       endif #gcc findstring
     else ##CC is clang
       ### compiler specific flags for clang
-
-      # Using clang as, not gas
-      GAS_BUG_DISABLE_AVX512 := 0
-
       CLANG_OMP_AVAIL := false
       export APPLE_CLANG := 0
       ifeq (USE_OMP,$(findstring USE_OMP,$(OPT)))
@@ -398,7 +380,13 @@ ifeq ($(DO_CHECKS), 1)
     CLINK += -lm
   endif #not icc
 
-  # Do we need to turn off AVX-512?
+  # The GNU Assembler (GAS) has an AVX-512 bug in versions 2.30 to 2.31.1
+  # So we turn off AVX-512 if the compiler reports it is using one of these versions.
+  # This works for gcc and icc.
+  # clang typically uses its own assembler, but if it is using the system assembler, this will also detect that.
+  # See: https://github.com/manodeep/Corrfunc/issues/193
+  GAS_BUG_DISABLE_AVX512 := $(shell $(CC) $(CFLAGS) -xc -Wa,-v -c /dev/null -o /dev/null 2>&1 | \grep -Pcm1 'GNU assembler version (2\.30|2\.31|2\.31\.1)(\s|$$)')
+
   ifeq ($(GAS_BUG_DISABLE_AVX512),1)
     # Did the compiler support AVX-512 in the first place? Otherwise -mno-avx512f is not a valid option!
     CC_SUPPORTS_AVX512 := $(shell $(CC) $(CFLAGS) -dM -E - < /dev/null | \grep -Pcm1 __AVX512F__)

From 7791b615edb2d79bf244c93c46bd641420af4951 Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Thu, 3 Oct 2019 15:01:53 +0000
Subject: [PATCH 5/9] Update changelog [ci skip]

---
 CHANGES.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index 1314fe9b..1dfe4ad0 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -25,6 +25,7 @@ Enhancements
 Bug fixes
 ----------
 - Incorrect calculations for non-native endian data [#191]
+- Workaround for GNU Assembler bug causing incorrect calculations [#196]
 
 
 

From adfb79ffc0f41a1e29823b83d73e283308570d5d Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Thu, 3 Oct 2019 15:12:14 +0000
Subject: [PATCH 6/9] Fix grep for osx

---
 common.mk | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.mk b/common.mk
index ee3d5d3b..cd3beb2c 100644
--- a/common.mk
+++ b/common.mk
@@ -385,11 +385,11 @@ ifeq ($(DO_CHECKS), 1)
   # This works for gcc and icc.
   # clang typically uses its own assembler, but if it is using the system assembler, this will also detect that.
   # See: https://github.com/manodeep/Corrfunc/issues/193
-  GAS_BUG_DISABLE_AVX512 := $(shell $(CC) $(CFLAGS) -xc -Wa,-v -c /dev/null -o /dev/null 2>&1 | \grep -Pcm1 'GNU assembler version (2\.30|2\.31|2\.31\.1)(\s|$$)')
+  GAS_BUG_DISABLE_AVX512 := $(shell $(CC) $(CFLAGS) -xc -Wa,-v -c /dev/null -o /dev/null 2>&1 | \grep -Ecm1 'GNU assembler version (2\.30|2\.31|2\.31\.1)(\s|$$)')
 
   ifeq ($(GAS_BUG_DISABLE_AVX512),1)
     # Did the compiler support AVX-512 in the first place? Otherwise -mno-avx512f is not a valid option!
-    CC_SUPPORTS_AVX512 := $(shell $(CC) $(CFLAGS) -dM -E - < /dev/null | \grep -Pcm1 __AVX512F__)
+    CC_SUPPORTS_AVX512 := $(shell $(CC) $(CFLAGS) -dM -E - < /dev/null | \grep -Ecm1 __AVX512F__)
     ifeq ($(CC_SUPPORTS_AVX512),1)
       CFLAGS += -mno-avx512f
 

From 3f2ed46090e7231042947e95e0ed287e9564eea9 Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Sat, 5 Oct 2019 23:03:24 -0400
Subject: [PATCH 7/9] Update message color [ci skip]

---
 common.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common.mk b/common.mk
index cd3beb2c..d262f97b 100644
--- a/common.mk
+++ b/common.mk
@@ -394,7 +394,7 @@ ifeq ($(DO_CHECKS), 1)
       CFLAGS += -mno-avx512f
 
       ifneq ($(GAS_BUG_WARNING_PRINTED),1)
-        $(warning $(ccmagenta)DISABLING AVX-512 SUPPORT DUE TO GNU ASSEMBLER BUG.  UPGRADE TO BINUTILS >=2.32 TO FIX THIS.$(ccreset))
+        $(warning $(ccred)DISABLING AVX-512 SUPPORT DUE TO GNU ASSEMBLER BUG.  UPGRADE TO BINUTILS >=2.32 TO FIX THIS.$(ccreset))
       endif
       export GAS_BUG_WARNING_PRINTED := 1
     endif

From 8bfcdebd4a8b136efd299b842abb2619aa460a24 Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Sun, 6 Oct 2019 14:37:30 +0000
Subject: [PATCH 8/9] Use -xCORE-AVX2 to disable AVX-512 for old Intel
 compilers

---
 common.mk | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/common.mk b/common.mk
index d262f97b..ba57dce9 100644
--- a/common.mk
+++ b/common.mk
@@ -256,6 +256,8 @@ ifeq ($(DO_CHECKS), 1)
       CFLAGS += -qopenmp
       CLINK  += -qopenmp
     endif ##openmp with icc
+
+    ICC_MAJOR_VER = $(shell icc -V 2>&1 | \grep -oP '(?<=Version )\d+')
   else ## not icc -> gcc or clang follow
 
     ## Warning that w(theta) with OUTPUT_THETAAVG is very slow without icc
@@ -385,13 +387,17 @@ ifeq ($(DO_CHECKS), 1)
   # This works for gcc and icc.
   # clang typically uses its own assembler, but if it is using the system assembler, this will also detect that.
   # See: https://github.com/manodeep/Corrfunc/issues/193
-  GAS_BUG_DISABLE_AVX512 := $(shell $(CC) $(CFLAGS) -xc -Wa,-v -c /dev/null -o /dev/null 2>&1 | \grep -Ecm1 'GNU assembler version (2\.30|2\.31|2\.31\.1)(\s|$$)')
+  GAS_BUG_DISABLE_AVX512 := $(shell $(CC) $(CFLAGS) -xc -Wa,-v -c /dev/null -o /dev/null 2>&1 | \grep -Ecm1 'GNU assembler version (2\.30|2\.31|2\.31\.1)')
 
   ifeq ($(GAS_BUG_DISABLE_AVX512),1)
-    # Did the compiler support AVX-512 in the first place? Otherwise -mno-avx512f is not a valid option!
+    # Did the compiler support AVX-512 in the first place? Otherwise no need to disable it!
     CC_SUPPORTS_AVX512 := $(shell $(CC) $(CFLAGS) -dM -E - < /dev/null | \grep -Ecm1 __AVX512F__)
     ifeq ($(CC_SUPPORTS_AVX512),1)
-      CFLAGS += -mno-avx512f
+      ifeq ($(shell test 0$(ICC_MAJOR_VER) -ge 019; echo $$?),0)
+        CFLAGS += -xCORE-AVX2
+      else
+        CFLAGS += -mno-avx512f
+      endif
 
       ifneq ($(GAS_BUG_WARNING_PRINTED),1)
         $(warning $(ccred)DISABLING AVX-512 SUPPORT DUE TO GNU ASSEMBLER BUG.  UPGRADE TO BINUTILS >=2.32 TO FIX THIS.$(ccreset))

From 7b3ef2dc3bf43e9338bead8b3e80f2b3039f8ef5 Mon Sep 17 00:00:00 2001
From: Lehman Garrison <lgarrison@flatironinstitute.org>
Date: Sun, 6 Oct 2019 16:47:25 +0000
Subject: [PATCH 9/9] Fix sense of icc version test

---
 common.mk | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/common.mk b/common.mk
index ba57dce9..744e1288 100644
--- a/common.mk
+++ b/common.mk
@@ -394,9 +394,10 @@ ifeq ($(DO_CHECKS), 1)
     CC_SUPPORTS_AVX512 := $(shell $(CC) $(CFLAGS) -dM -E - < /dev/null | \grep -Ecm1 __AVX512F__)
     ifeq ($(CC_SUPPORTS_AVX512),1)
       ifeq ($(shell test 0$(ICC_MAJOR_VER) -ge 019; echo $$?),0)
-        CFLAGS += -xCORE-AVX2
-      else
+      	# If gcc, clang, or new icc, we can use this
         CFLAGS += -mno-avx512f
+      else
+        CFLAGS += -xCORE-AVX2
       endif
 
       ifneq ($(GAS_BUG_WARNING_PRINTED),1)