QMCPACK · ye-luo · Aug 1, 2018 · May 25, 2018 · May 25, 2018 · May 25, 2018
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,59 @@
 
 Notable changes to QMCPACK will be documented in this file.
 
+## [3.5.0] - 2018-08-02
+
+### Notes
+
+This release includes support for the latest Quantum Espresso version 6.3,
+an initial implementation of periodic Gaussian support via PySCF, and a new
+version of the hybrid or "APW" representation of orbitals. Many minor
+bugs have been fixed, configuration and documentation improved. 
+
+Note that the PDF manuals are no longer included with the source. Versions
+are available online via https://qmcpack.org . The PDFs can be built
+using manual/build_manual.sh and
+nexus/documentation/user_guide_source/build_nexus_user_guide.sh
+
+Attention developers: This version contains substantially fewer source
+lines than previous versions due to clean out of old code and unused
+execution paths. Refactoring to improve the internal structure of
+QMCPACK is ongoing. Track the develop branch and follow discussion on
+GitHub closely to avoid difficult merges.
+
+* Support for Quantum Espresso 6.3 and 6.2.1. Check documentation to
+  ensure compiled with required HDF5 support.
+* Support for periodic gaussians and PySCF generated
+  wavefunctions. Initial version is limited to Gamma-point.
+* Improved hybrid representation of single particle orbitals
+  (APW-like) for significantly reduced memory usage and possible
+  accuracy increase compared to conventional spline representation.
+* Norms of orbitals are checked inside QMCPACK to catch conversion errors.
+* Added verbosity setting to QMCPACK output.
+* CUDA can now be enabled with SoA builds.
+* Many improvements to QMCPACK manual, including all new features, CIPSI, 3-body
+  jastrow factor description, spack package, and enabling HTML generation.
+* CMake configuration improvements, particularly around MKL handling.
+* Extensive cleanup of unused source files and unused code paths
+  removed, reducing the number of source lines by over 30 percent.
+
+### Known bugs
+
+* Weight of first block of DMC density is incorrect in CU
+  code. DMC densities in CUDA GPU code are incorrect for all
+  blocks. [\#934](https://github.com/QMCPACK/qmcpack/issues/934) and [\#925](https://github.com/QMCPACK/qmcpack/issues/925)
+* Runs with only a single electron may crash. [\#945](https://github.com/QMCPACK/qmcpack/issues/945)
+
+### NEXUS
+
+* Support for GAMESS HDF5 workflows.
+* Nexus accepts command line inputs.
+* Added GAMESS-NEXUS examples for RHF, CISD, and CASSCF wavefunction.
+* Added support for -nojastrow workflows.
+* Added support for Stampede supercomputer.
+* Added script to build NEXUS user guide.
+* Various bugfixes including to GAMESS input parsing.
+
 ## [3.4.0] - 2018-01-29
 
 ### Notes

diff --git a/CMake/ClangCompilers.cmake b/CMake/ClangCompilers.cmake
@@ -8,23 +8,22 @@ ENDIF()
 SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -std=c99")
 
 # Enable OpenMP
-SET(ENABLE_OPENMP 1)
-IF ( ENABLE_OPENMP )
-    SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -fopenmp")
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
-ENDIF()
+IF(QMC_OMP)
+  SET(ENABLE_OPENMP 1)
+  SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -fopenmp")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
+ENDIF(QMC_OMP)
 
 # Set clang specfic flags (which we always want)
 ADD_DEFINITIONS( -Drestrict=__restrict__ )
 
 SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -fomit-frame-pointer -fstrict-aliasing")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fomit-frame-pointer -fstrict-aliasing -D__forceinline=inline")
-SET( HAVE_POSIX_MEMALIGN 0 )    # Clang doesn't support -malign-double
 
 # Suppress compile warnings
 SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -Wno-deprecated -Wno-unused-value")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated -Wno-unused-value")
-IF ( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 3.8 )
+IF ( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "3.8.0" )
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-undefined-var-template")
 ENDIF()
 
@@ -34,10 +33,10 @@ SET( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -ffast-math" )
 SET( CMAKE_C_FLAGS_RELWITHDEBINFO     "${CMAKE_C_FLAGS_RELWITHDEBINFO} -ffast-math" )
 SET( CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -ffast-math" )
 
-#------------------------
-# Not on Cray's machine
-#------------------------
-IF(NOT $ENV{CRAYPE_VERSION} MATCHES ".")
+#--------------------------------------
+# Neither on Cray's machine nor PowerPC
+#--------------------------------------
+IF((NOT $ENV{CRAYPE_VERSION} MATCHES ".") AND (NOT CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64"))
 
 #check if the user has already specified -march=XXXX option for cross-compiling.
 if(CMAKE_CXX_FLAGS MATCHES "-march=" OR CMAKE_C_FLAGS MATCHES "-march=")
@@ -52,7 +51,7 @@ else() #(CMAKE_CXX_FLAGS MATCHES "-march=" OR CMAKE_C_FLAGS MATCHES "-march=")
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
 endif() #(CMAKE_CXX_FLAGS MATCHES "-march=" OR CMAKE_C_FLAGS MATCHES "-march=")
 
-ENDIF(NOT $ENV{CRAYPE_VERSION} MATCHES ".")
+ENDIF((NOT $ENV{CRAYPE_VERSION} MATCHES ".") AND (NOT CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64"))
 
 # Add static flags if necessary
 IF(QMC_BUILD_STATIC)
@@ -68,3 +67,16 @@ IF (ENABLE_GCOV)
   SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage")
 ENDIF(ENABLE_GCOV)
 
+SET(XRAY_PROFILE FALSE CACHE BOOL "Use llvm xray profiling")
+SET(XRAY_INSTRUCTION_THRESHOLD 200 CACHE INT "Instruction threshold for xray instrumentation")
+SET(XRAY_GPU_MOST FALSE CACHE BOOL "Manually instrument almost all of the CUDA implementations calls on the CPU")
+
+IF(XRAY_PROFILE)
+  set(XRAY_FLAGS "-fxray-instrument -fxray-instruction-threshold=${XRAY_INSTRUCTION_THRESHOLD}")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${XRAY_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${XRAY_FLAGS}")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${XRAY_FLAGS}")
+  if(XRAY_GPU_MOST)
+    set_property(DIRECTORY ${CMAKE_SOURCE_DIR} APPEND PROPERTY COMPILE_DEFINITIONS GPU_XRAY_TRACE_ON)
+  endif(XRAY_GPU_MOST)
+ENDIF(XRAY_PROFILE)
diff --git a/CMake/FindMKL.cmake b/CMake/FindMKL.cmake
@@ -2,42 +2,112 @@
 # This needs a lot of work to make it robust
 INCLUDE( CheckCXXSourceCompiles )
 
+# Extremely Basic Support of common mkl module environment variables
+# or -DMKLROOT/-DMKL_HOME instead of prefered -DMKL_ROOT
+if (NOT MKL_ROOT)
+  find_path(MKL_ROOT "mkl.h"
+    HINTS ${MKLROOT} ${MKL_HOME} $ENV{MKLROOT} $ENV{MKL_ROOT} $ENV{MKL_HOME}
+    PATH_SUFFIXES include)
+  string(REPLACE "/include" "" MKL_ROOT ${MKL_ROOT})
+endif (NOT MKL_ROOT)
+
+if (NOT MKL_ROOT)
+  if (CMAKE_CXX_COMPILER_ID MATCHES "Intel")
+    message(FATAL_ERROR "Intel's standard compilervar.sh sets the env variable MKLROOT.\n"
+            "If you are invoking icc without the customary environment\n"
+            "you must set the the environment variable or pass cmake MKL_ROOT.")
+  else(CMAKE_CXX_COMPILER_ID MATCHES "Intel")
+    message (FATAL_ERROR "ENABLE_MKL is TRUE and mkl directory not found. Set MKL_ROOT." )
+  endif(CMAKE_CXX_COMPILER_ID MATCHES "Intel")
+endif (NOT MKL_ROOT)
+
+# Finding and setting the MKL_INCLUDE_DIRECTORIES
+set(SUFFIXES include)
+find_path(MKL_INCLUDE_DIRECTORIES name "mkl.h" HINTS ${MKL_ROOT}
+  PATH_SUFFIXES ${SUFFIXES})
+if (MKL_INCLUDE_DIRECTORIES-NOTFOUND)
+  message(FATAL_ERROR "MKL_INCLUDE_DIRECTORIES not set. \"mkl.h\" not found in MKL_ROOT/(${SUFFIXES})")
+endif (MKL_INCLUDE_DIRECTORIES-NOTFOUND)
+message("MKL_INCLUDE_DIRECTORIES: ${MKL_INCLUDE_DIRECTORIES}")
+
+if ( NOT CMAKE_CXX_COMPILER_ID MATCHES "Intel" )
+  # Finding and setting the MKL_LINK_DIRECTORIES
+  # the directory organization varies with platform and targets
+  # these suffixes are not exhaustive
+  set(MKL_FIND_LIB "libmkl_intel_lp64${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  set(SUFFIXES lib lib/intel64)
+  find_path(MKL_LINK_DIRECTORIES name "${MKL_FIND_LIB}" HINTS ${MKL_ROOT}
+    PATH_SUFFIXES ${SUFFIXES})
+  if (MKL_LINK_DIRECTORIES-NOTFOUND)
+    message(FATAL_ERROR "MKL_LINK_DIRECTORIES not set. ${MKL_FIND_LIB} "
+      "not found in MKL_ROOT/(${SUFFIXES})")
+  endif (MKL_LINK_DIRECTORIES-NOTFOUND)
+  message("MKL_LINK_DIRECTORIES: ${MKL_LINK_DIRECTORIES}")
+  set(MKL_LINKER_FLAGS "-L${MKL_LINK_DIRECTORIES} -Wl,-rpath,${MKL_LINK_DIRECTORIES}")
+  set(MKL_LIBRARIES "-lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl")
+else ( NOT CMAKE_CXX_COMPILER_ID MATCHES "Intel" )
+  # this takes away link control for intel but is convenient
+  # perhaps we should consider dropping it since it will more or less
+  # unify the MKL setup.
+  # Note -mkl implicitly includes that icc's mkl/include
+  set(MKL_COMPILE_DEFINITIONS "-mkl")
+endif (NOT CMAKE_CXX_COMPILER_ID MATCHES "Intel" )
+
 # Check for mkl.h
 FILE( WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl.cxx"
-    "#include <iostream>\n #include <mkl.h>\n int main() { return 0; }\n" )
+  "#include <iostream>\n #include <mkl.h>\n int main() { return 0; }\n" )
 try_compile(HAVE_MKL ${CMAKE_BINARY_DIR}
-      ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl.cxx
-      COMPILE_DEFINITIONS "-mkl" )
+  ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl.cxx
+  CMAKE_FLAGS
+  "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE_DIRECTORIES} "
+  "-DLINK_DIRECTORIES=${MKL_LINK_DIRECTORIES}"
+  LINK_LIBRARIES "${MKL_LIBRARIES}"
+  COMPILE_DEFINITIONS "${MKL_COMPILE_DEFINITIONS}"
+  OUTPUT_VARIABLE MKL_OUT)
+if ( NOT HAVE_MKL )
+  MESSAGE( "${MKL_OUT}" )
+endif ( NOT HAVE_MKL )
 
 # Check for mkl_vml_functions.h
 FILE( WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_vml.cxx"
-    "#include <iostream>\n #include <mkl_vml_functions.h>\n int main() { return 0; }\n" )
+  "#include <iostream>\n #include <mkl_vml_functions.h>\n int main() { return 0; }\n" )
 try_compile(HAVE_MKL_VML ${CMAKE_BINARY_DIR}
-      ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_vml.cxx
-      COMPILE_DEFINITIONS "-mkl" )
+  ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_vml.cxx
+  CMAKE_FLAGS
+  "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE_DIRECTORIES} "
+  "-DLINK_DIRECTORIES=${MKL_LINK_DIRECTORIES}"
+  COMPILE_DEFINITIONS "${MKL_COMPILE_DEFINITIONS}"
+  OUTPUT_VARIABLE MKL_OUT)
 
 # Check for fftw3
-FILE( WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_fftw3.cxx"
-    "#include <iostream>\n #include <fftw/fftw3.h>\n int main() { return 0; }\n" )
-try_compile(HAVE_MKL_FFTW3 ${CMAKE_BINARY_DIR}
-      ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_fftw3.cxx
-      COMPILE_DEFINITIONS "-mkl" )
+find_path(MKL_FFTW3 "fftw3.h"
+  HINTS ${MKL_INCLUDE_DIRECTORIES}
+  PATH_SUFFIXES fftw)
+if(MKL_FFTW3)
+  list(APPEND MKL_INCLUDE_DIRECTORIES ${MKL_FFTW3})
+  FILE( WRITE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_fftw3.cxx"
+    "#include <iostream>\n #include <fftw3.h>\n int main() { return 0; }\n" )
+  try_compile(HAVE_MKL_FFTW3 ${CMAKE_BINARY_DIR}
+    ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/src_mkl_fftw3.cxx
+    CMAKE_FLAGS
+    "-DINCLUDE_DIRECTORIES=${MKL_INCLUDE_DIRECTORIES} "
+    "-DLINK_DIRECTORIES=${MKL_LINK_DIRECTORIES}"
+    LINK_LIBRARIES "${MKL_LIBRARIES}"
+    COMPILE_DEFINITIONS "${MKL_COMPILE_DEFINITIONS}"
+    OUTPUT_VARIABLE MKL_OUT)
+else(MKL_FFTW3)
+  UNSET(HAVE_MKL_FFTW3)
+endif(MKL_FFTW3)
 
 IF ( HAVE_MKL )
-    SET( MKL_FOUND 1 )
-    SET( MKL_FLAGS -mkl )
-    SET( MKL_LIBRARIES )
-    SET( MKL_LINKER_FLAGS -mkl )
-    IF ( HAVE_MKL_FFTW3 )
-        FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/include/fftw3.h" "#include <fftw/fftw3.h>\n" )
-        INCLUDE_DIRECTORIES( "${CMAKE_CURRENT_BINARY_DIR}/include" )
-    ENDIF()
-    MESSAGE(STATUS "MKL found: HAVE_MKL=${HAVE_MKL}, HAVE_MKL_VML=${HAVE_MKL_VML}, HAVE_MKL_FFTW3=${HAVE_MKL_FFTW3}")
-ELSE()
-    SET( MKL_FOUND 0 )
-    SET( MKL_FLAGS )
-    SET( MKL_LIBRARIES )
-    SET( MKL_LINKER_FLAGS )
-    MESSAGE(STATUS "MKL not found")
-ENDIF()
-
+  SET( MKL_FOUND 1 )
+  SET( MKL_FLAGS ${MKL_COMPILE_DEFINITIONS} )
+  include_directories( ${MKL_INCLUDE_DIRECTORIES} )
+  MESSAGE(STATUS "MKL found: HAVE_MKL=${HAVE_MKL}, HAVE_MKL_VML=${HAVE_MKL_VML}, HAVE_MKL_FFTW3=${HAVE_MKL_FFTW3}")
+ELSE( HAVE_MKL )
+  SET( MKL_FOUND 0 )
+  SET( MKL_FLAGS )
+  SET( MKL_LIBRARIES )
+  SET( MKL_LINKER_FLAGS )
+  MESSAGE("MKL not found")
+ENDIF( HAVE_MKL )
diff --git a/CMake/GNUCompilers.cmake b/CMake/GNUCompilers.cmake
@@ -7,9 +7,11 @@ ENDIF()
 SET(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -std=c99")
 
 # Enable OpenMP
-SET(ENABLE_OPENMP 1)
-SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -fopenmp")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
+IF(QMC_OMP)
+  SET(ENABLE_OPENMP 1)
+  SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -fopenmp")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
+ENDIF(QMC_OMP)
 
 # Set gnu specfic flags (which we always want)
 ADD_DEFINITIONS( -Drestrict=__restrict__ )
@@ -27,10 +29,10 @@ SET( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -ffast-math" )
 SET( CMAKE_C_FLAGS_RELWITHDEBINFO     "${CMAKE_C_FLAGS_RELWITHDEBINFO} -ffast-math" )
 SET( CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -ffast-math" )
 
-#------------------------
-# Not on Cray's machine
-#------------------------
-IF(NOT $ENV{CRAYPE_VERSION} MATCHES ".")
+#--------------------------------------
+# Neither on Cray's machine nor PowerPC
+#--------------------------------------
+IF((NOT $ENV{CRAYPE_VERSION} MATCHES ".") AND (NOT CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64"))
 
 #check if the user has already specified -march=XXXX option for cross-compiling.
 if(CMAKE_CXX_FLAGS MATCHES "-march=" OR CMAKE_C_FLAGS MATCHES "-march=")
@@ -45,7 +47,7 @@ else() #(CMAKE_CXX_FLAGS MATCHES "-march=" OR CMAKE_C_FLAGS MATCHES "-march=")
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
 endif() #(CMAKE_CXX_FLAGS MATCHES "-march=" OR CMAKE_C_FLAGS MATCHES "-march=")
 
-ENDIF(NOT $ENV{CRAYPE_VERSION} MATCHES ".")
+ENDIF((NOT $ENV{CRAYPE_VERSION} MATCHES ".") AND (NOT CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64"))
 
 # Add static flags if necessary
 IF(QMC_BUILD_STATIC)

diff --git a/CMake/IntelCompilers.cmake b/CMake/IntelCompilers.cmake
@@ -8,14 +8,16 @@ ENDIF()
 SET(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -std=c99")
 
 # Enable OpenMP
-SET(ENABLE_OPENMP 1)
-IF ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16 )
-SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -openmp")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -openmp")
-ELSE()
-SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -qopenmp")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopenmp")
-ENDIF()
+IF(QMC_OMP)
+  SET(ENABLE_OPENMP 1)
+  IF ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16 )
+  SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -openmp")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -openmp")
+  ELSE()
+  SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -qopenmp")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopenmp")
+  ENDIF()
+ENDIF(QMC_OMP)
 
 # Suppress compile warnings
 SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -Wno-deprecated")

diff --git a/CMake/ctest_script.cmake b/CMake/ctest_script.cmake
@@ -83,7 +83,8 @@ ELSEIF( ${HOSTNAME} MATCHES "ryzen-box" )
     SET( QMC_OPTIONS "${QMC_OPTIONS};-DCMAKE_PREFIX_PATH=/opt/OpenBLAS" )
     SET( N_PROCS 16)
 ELSE()
-    MESSAGE( FATAL_ERROR "Unknown host: ${HOSTNAME}" )
+    MESSAGE( MESSAGE "Unknown host: ${HOSTNAME}. Using generic setting." )
+    SET( CTEST_CMAKE_GENERATOR "Unix Makefiles")
 ENDIF()
 
 # Get the source directory based on the current directory
@@ -236,6 +237,10 @@ IF ( ENABLE_SOA )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_SOA='${ENABLE_SOA}'" )
 ENDIF()
 
+IF ( CUDA_ARCH )
+  SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCUDA_ARCH='${CUDA_ARCH}'" )
+ENDIF()
+
 IF ( BUILD_AFQMC )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DBUILD_AFQMC='${BUILD_AFQMC}'" )
 ENDIF()