From 3191d0911fe44694e9fac5e78deab1b3f0612f3e Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Tue, 5 Oct 2021 20:38:22 -0500 Subject: [PATCH 1/2] Revert integration with DiracMatrixComputeOMPTarget --- src/QMCWaveFunctions/Fermion/DiracMatrix.h | 22 ++++++++++++++----- .../Fermion/MatrixUpdateOMPTarget.h | 18 ++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/QMCWaveFunctions/Fermion/DiracMatrix.h b/src/QMCWaveFunctions/Fermion/DiracMatrix.h index da8fd9e8f6..9c67b77cf7 100644 --- a/src/QMCWaveFunctions/Fermion/DiracMatrix.h +++ b/src/QMCWaveFunctions/Fermion/DiracMatrix.h @@ -180,9 +180,14 @@ class DiracMatrix * @tparam TMAT matrix value type * @tparam TREAL real type */ - template - inline std::enable_if_t::value> invert_transpose(const Matrix& amat, - Matrix& invMat, + template::is_host_accessible>, + typename = std::enable_if_t::is_host_accessible>> + inline std::enable_if_t::value> invert_transpose(const Matrix& amat, + Matrix& invMat, std::complex& LogDet) { const int n = invMat.rows(); @@ -196,9 +201,14 @@ class DiracMatrix * @tparam TMAT matrix value type * @tparam TREAL real type */ - template - inline std::enable_if_t::value> invert_transpose(const Matrix& amat, - Matrix& invMat, + template::is_host_accessible>, + typename = std::enable_if_t::is_host_accessible>> + inline std::enable_if_t::value> invert_transpose(const Matrix& amat, + Matrix& invMat, std::complex& LogDet) { const int n = invMat.rows(); diff --git a/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h b/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h index f70650d91f..7516358fbc 100644 --- a/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h +++ b/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h @@ -178,7 +178,7 @@ class MatrixUpdateOMPTarget } static void mw_invertTranspose(const RefVectorWithLeader& engines, - RefVector>& logdetT_list, + RefVector>& psiM_list, RefVector>& a_inv_refs, OffloadVector& log_values, const std::vector& compute_mask) @@ -186,16 +186,18 @@ class MatrixUpdateOMPTarget auto& engine_leader = engines.getLeader(); auto& det_inverter = engine_leader.get_det_inverter(); - a_inv_refs.reserve(engines.size()); - for (int iw = 0; iw < engines.size(); iw++) { - a_inv_refs.emplace_back(engines[iw].get_ref_psiMinv()); - const Value* a_inv_ptr = a_inv_refs.back().get().data(); - PRAGMA_OFFLOAD("omp target update to(a_inv_ptr[:a_inv_refs.back().get().size()])") + auto& Ainv = a_inv_refs[iw].get(); + engine_leader.detEng.invert_transpose(psiM_list[iw].get(), Ainv, log_values[iw]); + Value* Ainv_ptr = Ainv.data(); + PRAGMA_OFFLOAD("omp target update to(Ainv_ptr[:Ainv.size()])") } - typename DetInverter::HandleResource dummy; - det_inverter.mw_invertTranspose(dummy, logdetT_list, a_inv_refs, log_values, compute_mask); + PRAGMA_OFFLOAD("omp taskwait") + + //FIXME DiracMatrixComputeOMPTarget is either broken or connected incorrectly + //typename DetInverter::HandleResource dummy; + //det_inverter.mw_invertTranspose(dummy, psiM_list, a_inv_refs, log_values, compute_mask); } template From 4b2ef728fc892d2701051a6b278a13ee9108549f Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Tue, 5 Oct 2021 20:43:26 -0500 Subject: [PATCH 2/2] Mark all the RefVector const. --- .../Fermion/DiracDeterminantBatched.cpp | 4 ++-- .../Fermion/DiracDeterminantBatched.h | 4 ++-- .../Fermion/DiracMatrixComputeCUDA.hpp | 12 ++++++------ .../Fermion/DiracMatrixComputeOMPTarget.hpp | 4 ++-- .../Fermion/MatrixDelayedUpdateCUDA.h | 4 ++-- src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp index e4724567f7..8cb3ff7dc6 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp @@ -54,8 +54,8 @@ void DiracDeterminantBatched::invertPsiM(const DualMatrix& ps template void DiracDeterminantBatched::mw_invertPsiM(const RefVectorWithLeader& wfc_list, - RefVector>& logdetT_list, - RefVector>& a_inv_list, + const RefVector>& logdetT_list, + const RefVector>& a_inv_list, const std::vector& compute_mask) const { auto& wfc_leader = wfc_list.getCastedLeader>(); diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.h b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.h index f27fd03bc6..9af46d4585 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.h +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.h @@ -279,8 +279,8 @@ class DiracDeterminantBatched : public DiracDeterminantBase * the compute mask. See future PR for those changes, or drop of compute_mask argument. */ void mw_invertPsiM(const RefVectorWithLeader& wfc_list, - RefVector>& logdetT_list, - RefVector>& a_inv_list, + const RefVector>& logdetT_list, + const RefVector>& a_inv_list, const std::vector& compute_mask) const; /** Does a Phi->mw_evaluate_notranspose then mw_invertPsiM over a set of diff --git a/src/QMCWaveFunctions/Fermion/DiracMatrixComputeCUDA.hpp b/src/QMCWaveFunctions/Fermion/DiracMatrixComputeCUDA.hpp index bfcd70794c..c29f921410 100644 --- a/src/QMCWaveFunctions/Fermion/DiracMatrixComputeCUDA.hpp +++ b/src/QMCWaveFunctions/Fermion/DiracMatrixComputeCUDA.hpp @@ -87,8 +87,8 @@ class DiracMatrixComputeCUDA : public Resource */ template inline void mw_computeInvertAndLog(CUDALinearAlgebraHandles& cuda_handles, - RefVector>& a_mats, - RefVector>& inv_a_mats, + const RefVector>& a_mats, + const RefVector>& inv_a_mats, const int n, const int lda, DualVector& log_values) @@ -276,8 +276,8 @@ class DiracMatrixComputeCUDA : public Resource template inline std::enable_if_t::value> mw_invertTranspose( CUDALinearAlgebraHandles& cuda_handles, - RefVector>& a_mats, - RefVector>& inv_a_mats, + const RefVector>& a_mats, + const RefVector>& inv_a_mats, DualVector& log_values, const std::vector& compute_mask) { @@ -319,8 +319,8 @@ class DiracMatrixComputeCUDA : public Resource template inline std::enable_if_t::value> mw_invertTranspose( CUDALinearAlgebraHandles& cuda_handles, - RefVector>& a_mats, - RefVector>& inv_a_mats, + const RefVector>& a_mats, + const RefVector>& inv_a_mats, DualVector& log_values, const std::vector& compute_mask) { diff --git a/src/QMCWaveFunctions/Fermion/DiracMatrixComputeOMPTarget.hpp b/src/QMCWaveFunctions/Fermion/DiracMatrixComputeOMPTarget.hpp index 7db43c7e8c..1b5a527584 100644 --- a/src/QMCWaveFunctions/Fermion/DiracMatrixComputeOMPTarget.hpp +++ b/src/QMCWaveFunctions/Fermion/DiracMatrixComputeOMPTarget.hpp @@ -224,8 +224,8 @@ class DiracMatrixComputeOMPTarget : public Resource */ template inline void mw_invertTranspose(HandleResource& resource, - RefVector>& a_mats, - RefVector>& inv_a_mats, + const RefVector>& a_mats, + const RefVector>& inv_a_mats, OffloadPinnedVector& log_values, const std::vector& recompute) { diff --git a/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h b/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h index 06762ff9ac..4b0bb03bb5 100644 --- a/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h +++ b/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h @@ -412,8 +412,8 @@ class MatrixDelayedUpdateCUDA } static void mw_invertTranspose(const RefVectorWithLeader& engines, - RefVector>& psiM_list, - RefVector>& psiMinv_list, + const RefVector>& psiM_list, + const RefVector>& psiMinv_list, DualVector& log_values, const std::vector& compute_mask) { diff --git a/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h b/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h index 7516358fbc..7c5be2e5d9 100644 --- a/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h +++ b/src/QMCWaveFunctions/Fermion/MatrixUpdateOMPTarget.h @@ -178,8 +178,8 @@ class MatrixUpdateOMPTarget } static void mw_invertTranspose(const RefVectorWithLeader& engines, - RefVector>& psiM_list, - RefVector>& a_inv_refs, + const RefVector>& psiM_list, + const RefVector>& a_inv_refs, OffloadVector& log_values, const std::vector& compute_mask) {