Merge branch 'develop' into support-continue-opt

CMakeLists.txt

@@ -18,6 +18,10 @@ project(
   VERSION 3.14.9
   LANGUAGES C CXX)
 
+# add the automatically determined parts of the RPATH
+# which point to directories outside the build tree to the install RPATH
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
 #--------------------------------------------------------------------
 # Directory where customize cmake files reside
 #--------------------------------------------------------------------

config/build_alcf_polaris_Clang.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+# This recipe is intended for ALCF Polaris https://www.alcf.anl.gov/polaris
+# It builds all the varaints of QMCPACK in the current directory
+# last revision: Aug 17th 2022
+#
+# How to invoke this script?
+# build_alcf_polaris_Clang.sh # build all the variants assuming the current directory is the source directory.
+# build_alcf_polaris_Clang.sh <source_dir> # build all the variants with a given source directory <source_dir>
+# build_alcf_polaris_Clang.sh <source_dir> <install_dir> # build all the variants with a given source directory <source_dir> and install to <install_dir>
+
+module load mpiwrappers/cray-mpich-llvm llvm/main-20220317
+module load cudatoolkit-standalone/11.2.2
+module load cray-fftw/3.3.8.13
+module load cray-hdf5-parallel/1.12.1.3
+module load cmake/3.23.2
+
+export BOOST_ROOT=/soft/applications/qmcpack/boost_1_79_0
+export CMAKE_PREFIX_PATH=/soft/libraries/openblas/0.3.20-omp:$CMAKE_PREFIX_PATH
+
+
+echo "**********************************"
+echo '$ clang -v'
+clang -v
+echo "**********************************"
+
+TYPE=Release
+Machine=polaris
+Compiler=Clang
+
+if [[ $# -eq 0 ]]; then
+  source_folder=`pwd`
+elif [[ $# -eq 1 ]]; then
+  source_folder=$1
+else
+  source_folder=$1
+  install_folder=$2
+fi
+
+if [[ -f $source_folder/CMakeLists.txt ]]; then
+  echo Using QMCPACK source directory $source_folder
+else
+  echo "Source directory $source_folder doesn't contain CMakeLists.txt. Pass QMCPACK source directory as the first argument."
+  exit
+fi
+
+for name in offload_cuda_real_MP offload_cuda_real offload_cuda_cplx_MP offload_cuda_cplx \
+            cpu_real_MP cpu_real cpu_cplx_MP cpu_cplx
+do
+
+CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=$TYPE"
+
+if [[ $name == *"cplx"* ]]; then
+  CMAKE_FLAGS="$CMAKE_FLAGS -DQMC_COMPLEX=ON"
+fi
+
+if [[ $name == *"_MP"* ]]; then
+  CMAKE_FLAGS="$CMAKE_FLAGS -DQMC_MIXED_PRECISION=ON"
+fi
+
+if [[ $name == *"offload"* ]]; then
+  CMAKE_FLAGS="$CMAKE_FLAGS -DENABLE_OFFLOAD=ON -DUSE_OBJECT_TARGET=ON -DOFFLOAD_ARCH=sm_80"
+fi
+
+if [[ $name == *"cuda"* ]]; then
+  CMAKE_FLAGS="$CMAKE_FLAGS -DENABLE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=80"
+fi
+
+folder=build_${Machine}_${Compiler}_${name}
+
+if [[ -v install_folder ]]; then
+  CMAKE_FLAGS="$CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX=$install_folder/$folder"
+fi
+
+echo "**********************************"
+echo "$folder"
+echo "$CMAKE_FLAGS"
+echo "**********************************"
+
+mkdir $folder
+cd $folder
+
+if [ ! -f CMakeCache.txt ] ; then
+cmake $CMAKE_FLAGS -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx $source_folder
+fi
+
+if [[ -v install_folder ]]; then
+  make -j16 install && chmod -R -w $install_folder/$folder
+else
+  make -j16
+fi
+
+cd ..
+
+echo
+done

docs/installation.rst

@@ -882,6 +882,12 @@ Each node features a second-generation Intel Xeon Phi 7230 processor and 192 GB
   make -j 24
   ls -l bin/qmcpack
 
+Installing on ALCF Polaris
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Polaris is a HPE Apollo Gen10+ based 44 petaflops system.
+Each node features a AMD EPYC 7543P CPU and 4 NVIDIA A100 GPUs.
+A build recipe for Polaris can be found at ``<qmcpack_source>/config/build_alcf_polaris_Clang.sh``
+
 Installing on ORNL OLCF Summit
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/Containers/OhmmsPETE/ompReductionTinyVector.hpp

@@ -0,0 +1,32 @@
+//////////////////////////////////////////////////////////////////////////////////////
+// This file is distributed under the University of Illinois/NCSA Open Source License.
+// See LICENSE file in top directory for details.
+//
+// Copyright (c) 2022 QMCPACK developers.
+//
+// File developed by: Ye Luo, yeluo@anl.gov, Argonne National Laboratory
+//
+// File created by: Ye Luo, yeluo@anl.gov, Argonne National Laboratory
+//////////////////////////////////////////////////////////////////////////////////////
+
+
+#ifndef QMCPLUSPLUS_OMPREDUCTION_TINYVECTOR_H
+#define QMCPLUSPLUS_OMPREDUCTION_TINYVECTOR_H
+
+#include <complex>
+#include "config.h"
+#include "TinyVector.h"
+
+namespace qmcplusplus
+{
+#if !defined(OPENMP_NO_UDR)
+PRAGMA_OFFLOAD("omp declare reduction(+: TinyVector<float, OHMMS_DIM>: omp_out += omp_in)")
+PRAGMA_OFFLOAD("omp declare reduction(+: TinyVector<double, OHMMS_DIM>: omp_out += omp_in)")
+#endif
+
+#if !defined(OPENMP_NO_COMPLEX) && !defined(OPENMP_NO_UDR)
+PRAGMA_OFFLOAD("omp declare reduction(+: TinyVector<std::complex<float>, OHMMS_DIM>: omp_out += omp_in)")
+PRAGMA_OFFLOAD("omp declare reduction(+: TinyVector<std::complex<double>, OHMMS_DIM>: omp_out += omp_in)")
+#endif
+}
+#endif // QMCPLUSPLUS_OMPREDUCTION_TINYVECTOR_H

src/Particle/ParticleIO/XMLParticleIO.cpp

@@ -274,7 +274,9 @@ bool XMLParticleParser::readXML(xmlNodePtr cur)
           {
             int old_idx = map_storage_to_input[new_idx];
             if (new_idx != old_idx)
+            {
               app_debug() << "    " << d_in[old_idx] << " : " << old_idx << " -> " << new_idx << std::endl;
+            }
           }
         }
       }

src/Platforms/OMPTarget/ompBLAS.cpp

@@ -14,7 +14,7 @@
 #include <stdexcept>
 #include "config.h"
 #if !defined(OPENMP_NO_COMPLEX)
-#include "ompReduction.hpp"
+#include "ompReductionComplex.hpp"
 #endif
 
 namespace qmcplusplus

src/QMCApp/QMCAppBase.cpp

@@ -71,7 +71,7 @@ void QMCAppBase::saveXml()
 {
   if (!XmlDocStack.empty())
   {
-    std::string newxml(myProject.CurrentMainRoot());
+    std::string newxml(myProject.currentMainRoot());
     newxml.append(".cont.xml");
     app_log() << "\n========================================================="
               << "\n  A new xml input file : " << newxml << std::endl;

src/QMCApp/QMCMain.cpp

@@ -307,7 +307,7 @@ bool QMCMain::execute()
     HDFVersion cur_version;
     v_str << cur_version[0] << " " << cur_version[1];
     xmlNodePtr newmcptr = xmlNewNode(NULL, (const xmlChar*)"mcwalkerset");
-    xmlNewProp(newmcptr, (const xmlChar*)"fileroot", (const xmlChar*)myProject.CurrentMainRoot().c_str());
+    xmlNewProp(newmcptr, (const xmlChar*)"fileroot", (const xmlChar*)myProject.currentMainRoot().c_str());
     xmlNewProp(newmcptr, (const xmlChar*)"node", (const xmlChar*)"-1");
     xmlNewProp(newmcptr, (const xmlChar*)"nprocs", (const xmlChar*)np_str.str().c_str());
     xmlNewProp(newmcptr, (const xmlChar*)"version", (const xmlChar*)v_str.str().c_str());
@@ -638,7 +638,7 @@ bool QMCMain::runQMC(xmlNodePtr cur, bool reuse)
     if (!FirstQMC && !append_run)
       myProject.advance();
 
-    qmc_driver->setStatus(myProject.CurrentMainRoot(), "", append_run);
+    qmc_driver->setStatus(myProject.currentMainRoot(), "", append_run);
     // PD:
     // Q: How does m_walkerset_in end up being non empty?
     // A: Anytime that we aren't doing a restart.

src/QMCDrivers/QMCDriverFactory.cpp

@@ -102,7 +102,7 @@ QMCDriverFactory::DriverAssemblyState QMCDriverFactory::readSection(xmlNodePtr c
   const int nchars = qmc_mode.size();
 
   using DV = ProjectData::DriverVersion;
-  switch (project_data_.get_driver_version())
+  switch (project_data_.getDriverVersion())
   {
   case DV::BATCH:
 #if defined(QMC_CUDA)

src/QMCDrivers/QMCDriverNew.h

@@ -308,7 +308,7 @@ class QMCDriverNew : public QMCDriverInterface, public MPIObjectBase
   /// check logpsi and grad and lap against values computed from scratch
   static void checkLogAndGL(Crowd& crowd, const std::string_view location);
 
-  const std::string& get_root_name() const override { return project_data_.CurrentMainRoot(); }
+  const std::string& get_root_name() const override { return project_data_.currentMainRoot(); }
 
   /** The timers for the driver.
    *

src/QMCDrivers/WFOpt/HamiltonianRef.cpp

@@ -23,8 +23,8 @@ void HamiltonianRef::addOperator(OperatorBase& op) { Hrefs_.emplace_back(op); }
 
 FullPrecRealType HamiltonianRef::evaluateValueAndDerivatives(ParticleSet& P,
                                                              const opt_variables_type& optvars,
-                                                             std::vector<ValueType>& dlogpsi,
-                                                             std::vector<ValueType>& dhpsioverpsi,
+                                                             Vector<ValueType>& dlogpsi,
+                                                             Vector<ValueType>& dhpsioverpsi,
                                                              bool compute_deriv)
 {
   FullPrecRealType LocalEnergy = Hrefs_[0].get().evaluate(P);

src/QMCDrivers/WFOpt/HamiltonianRef.h

@@ -36,8 +36,8 @@ class HamiltonianRef
   /// the same evaluateValueAndDerivatives as QMCHamiltonian
   FullPrecRealType evaluateValueAndDerivatives(ParticleSet& P,
                                                const opt_variables_type& optvars,
-                                               std::vector<ValueType>& dlogpsi,
-                                               std::vector<ValueType>& dhpsioverpsi,
+                                               Vector<ValueType>& dlogpsi,
+                                               Vector<ValueType>& dhpsioverpsi,
                                                bool compute_deriv);
 
   /// the same evaluate as QMCHamiltonian

src/QMCDrivers/WFOpt/QMCCostFunction.cpp

@@ -279,11 +279,11 @@ void QMCCostFunction::checkConfigurations(EngineHandle& handle)
       if (needGrads)
       {
         //allocate vector
-        std::vector<Return_rt> rDsaved(NumOptimizables, 0.0);
-        std::vector<Return_rt> rHDsaved(NumOptimizables, 0.0);
+        Vector<Return_rt> rDsaved(NumOptimizables, 0.0);
+        Vector<Return_rt> rHDsaved(NumOptimizables, 0.0);
 
-        std::vector<Return_t> Dsaved(NumOptimizables, 0.0);
-        std::vector<Return_t> HDsaved(NumOptimizables, 0.0);
+        Vector<Return_t> Dsaved(NumOptimizables, 0.0);
+        Vector<Return_t> HDsaved(NumOptimizables, 0.0);
 
         psiClones[ip]->evaluateDerivatives(wRef, OptVariablesForPsi, Dsaved, HDsaved);
         etmp = hClones[ip]->evaluateValueAndDerivatives(wRef, OptVariablesForPsi, Dsaved, HDsaved, compute_nlpp);
@@ -295,8 +295,8 @@ void QMCCostFunction::checkConfigurations(EngineHandle& handle)
           rDsaved[i]  = std::real(Dsaved[i]);
           rHDsaved[i] = std::real(HDsaved[i]);
         }
-        copy(rDsaved.begin(), rDsaved.end(), (*DerivRecords[ip])[iw]);
-        copy(rHDsaved.begin(), rHDsaved.end(), (*HDerivRecords[ip])[iw]);
+        std::copy(rDsaved.begin(), rDsaved.end(), (*DerivRecords[ip])[iw]);
+        std::copy(rHDsaved.begin(), rHDsaved.end(), (*HDerivRecords[ip])[iw]);
       }
       else
         etmp = hClones[ip]->evaluate(wRef);
@@ -404,8 +404,8 @@ void QMCCostFunction::engine_checkConfigurations(cqmc::engine::LMYEngine<Return_
       if (needGrads)
       {
         //allocate vector
-        std::vector<Return_t> Dsaved(NumOptimizables, 0.0);
-        std::vector<Return_t> HDsaved(NumOptimizables, 0.0);
+        Vector<Return_t> Dsaved(NumOptimizables, 0.0);
+        Vector<Return_t> HDsaved(NumOptimizables, 0.0);
 
         psiClones[ip]->evaluateDerivatives(wRef, OptVariablesForPsi, Dsaved, HDsaved);
         etmp = hClones[ip]->evaluateValueAndDerivatives(wRef, OptVariablesForPsi, Dsaved, HDsaved, compute_nlpp);
@@ -417,12 +417,12 @@ void QMCCostFunction::engine_checkConfigurations(cqmc::engine::LMYEngine<Return_
         // dervative vectors
         der_rat_samp.at(0) = 1.0;
         for (int i = 0; i < Dsaved.size(); i++)
-          der_rat_samp.at(i + 1) = Dsaved.at(i);
+          der_rat_samp[i + 1] = Dsaved[i];
 
         // energy dervivatives
         le_der_samp.at(0) = etmp;
         for (int i = 0; i < HDsaved.size(); i++)
-          le_der_samp.at(i + 1) = HDsaved.at(i) + etmp * Dsaved.at(i);
+          le_der_samp[i + 1] = HDsaved[i] + etmp * Dsaved[i];
 
 #ifdef HAVE_LMY_ENGINE
         if (MinMethod == "adaptive")
@@ -548,11 +548,11 @@ QMCCostFunction::EffectiveWeight QMCCostFunction::correlatedSampling(bool needGr
       Return_rt weight = saved[REWEIGHT] = vmc_or_dmc * (logpsi - saved[LOGPSI_FREE]);
       if (needGrad)
       {
-        std::vector<Return_t> Dsaved(NumOptimizables, 0);
-        std::vector<Return_t> HDsaved(NumOptimizables, 0);
+        Vector<Return_t> Dsaved(NumOptimizables, 0);
+        Vector<Return_t> HDsaved(NumOptimizables, 0);
 
-        std::vector<Return_rt> rDsaved(NumOptimizables, 0);
-        std::vector<Return_rt> rHDsaved(NumOptimizables, 0);
+        Vector<Return_rt> rDsaved(NumOptimizables, 0);
+        Vector<Return_rt> rHDsaved(NumOptimizables, 0);
         psiClones[ip]->evaluateDerivatives(wRef, OptVariablesForPsi, Dsaved, HDsaved);
 
         saved[ENERGY_NEW] =

src/QMCDrivers/WaveFunctionTester.cpp

@@ -1726,8 +1726,8 @@ void WaveFunctionTester::runDerivTest()
   wfvar_prime = wfVars;
   wfVars.print(fout);
   int Nvars = wfVars.size();
-  std::vector<ValueType> Dsaved(Nvars);
-  std::vector<ValueType> HDsaved(Nvars);
+  Vector<ValueType> Dsaved(Nvars);
+  Vector<ValueType> HDsaved(Nvars);
   std::vector<RealType> PGradient(Nvars);
   std::vector<RealType> HGradient(Nvars);
   Psi.resetParameters(wfVars);
@@ -1833,8 +1833,8 @@ void WaveFunctionTester::runDerivNLPPTest()
   wfvar_prime = wfVars;
   wfVars.print(nlout);
   int Nvars = wfVars.size();
-  std::vector<ValueType> Dsaved(Nvars);
-  std::vector<ValueType> HDsaved(Nvars);
+  Vector<ValueType> Dsaved(Nvars);
+  Vector<ValueType> HDsaved(Nvars);
   std::vector<RealType> PGradient(Nvars);
   std::vector<RealType> HGradient(Nvars);
   Psi.resetParameters(wfVars);
@@ -1935,8 +1935,8 @@ void WaveFunctionTester::runDerivCloneTest()
   wfvar_prime.print(fout);
   psi_clone->resetParameters(wfvar_prime);
   Psi.resetParameters(wfVars);
-  std::vector<ValueType> Dsaved(Nvars, 0), og_Dsaved(Nvars, 0);
-  std::vector<ValueType> HDsaved(Nvars, 0), og_HDsaved(Nvars, 0);
+  Vector<ValueType> Dsaved(Nvars, 0), og_Dsaved(Nvars, 0);
+  Vector<ValueType> HDsaved(Nvars, 0), og_HDsaved(Nvars, 0);
   std::vector<RealType> PGradient(Nvars, 0), og_PGradient(Nvars, 0);
   std::vector<RealType> HGradient(Nvars, 0), og_HGradient(Nvars, 0);
   ValueType logpsi2 = psi_clone->evaluateLog(*w_clone);

src/QMCHamiltonians/ACForce.h

@@ -34,6 +34,9 @@ class ACForce : public OperatorBase
   /** Destructor, "final" triggers a clang warning **/
   ~ACForce() override = default;
 
+  bool dependsOnWaveFunction() const override { return true; }
+  std::string getClassName() const override { return "ACForce"; }
+
   /** I/O Routines */
   bool put(xmlNodePtr cur) final;
 

src/QMCHamiltonians/BareKineticEnergy.h

@@ -77,6 +77,8 @@ class BareKineticEnergy : public OperatorBase
   ///destructor
   ~BareKineticEnergy() override;
 
+  //FIXME bool dependsOnWaveFunction() const override { return true; }
+  std::string getClassName() const override { return "BareKineticEnergy"; }
   void resetTargetParticleSet(ParticleSet& P) override {}
 
 #if !defined(REMOVE_TRACEMANAGER)

src/QMCHamiltonians/ChiesaCorrection.h

@@ -28,6 +28,7 @@ class ChiesaCorrection : public OperatorBase
 public:
   ChiesaCorrection(ParticleSet& ptcl, const TrialWaveFunction& psi) : psi_ref(psi), ptcl_ref(ptcl) {}
 
+  std::string getClassName() const override { return "ChiesaCorrection"; }
   void resetTargetParticleSet(ParticleSet& P) override;
 
   Return_t evaluate(ParticleSet& P) override;

src/QMCHamiltonians/ConservedEnergy.h

@@ -76,6 +76,8 @@ struct ConservedEnergy : public OperatorBase
 
   void resetTargetParticleSet(ParticleSet& P) override {}
 
+  std::string getClassName() const override { return "ConservedEnergy"; }
+
   Return_t evaluate(ParticleSet& P) override
   {
     RealType gradsq = Dot(P.G, P.G);

src/QMCHamiltonians/CoulombPBCAA.h

@@ -92,6 +92,8 @@ struct CoulombPBCAA : public OperatorBase, public ForceBase
 
   ~CoulombPBCAA() override;
 
+  std::string getClassName() const override { return "CoulombPBCAA"; }
+
   void resetTargetParticleSet(ParticleSet& P) override;
 
   Return_t evaluate(ParticleSet& P) override;

src/QMCHamiltonians/CoulombPBCAB.h

@@ -134,6 +134,7 @@ class CoulombPBCAB : public OperatorBase, public ForceBase
 
   void resetTargetParticleSet(ParticleSet& P) override;
 
+  std::string getClassName() const override { return "CoulombPBCAB"; }
 
 #if !defined(REMOVE_TRACEMANAGER)
   void contributeParticleQuantities() override;