From 7635cfc85898534667b05c41601bc0a9a3ada502 Mon Sep 17 00:00:00 2001 From: Giovanni Pizzi Date: Thu, 25 Jul 2019 13:06:18 +0200 Subject: [PATCH] Gathering matrices before dumping chk file Fixes #278 In the parallel code, the `u_matrix` is distributed in `u_matrix_loc`. However, the `write_chkpt` routine writes on file the content of the `u_matrix` that was not updated during the wannierisation but only at the end. Therefore, the code was always restarting from the results of the disentanglement (unless it was stop appropriately with a fixed number of steps). Now we gather the `u_matrix_loc` and `m_matrix_loc` just before dumping, in the `wannierise.F90` file. Note that gathering is not needed in the main program file because the dumping of the .chk file there happens right after the disentanglement, where the u_matrix is still not scattered. Also, I'm not 100% sure that also the `m_matrix_loc` needs to be gathered, but anyway this is what is done at the end of the cycle in `wann_main` so it shouldn't harm (at worst it can be a bit less efficient). Final note: this is very hard to test automatically, so I couldn't do a proper testing of this PR. --- src/parameters.F90 | 3 +++ src/wannierise.F90 | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/parameters.F90 b/src/parameters.F90 index f36ad8018..6a7b901c7 100644 --- a/src/parameters.F90 +++ b/src/parameters.F90 @@ -3760,6 +3760,9 @@ subroutine param_write_chkpt(chkpt) !! Write checkpoint file !! IMPORTANT! If you change the chkpt format, adapt !! accordingly also the w90chk2chk.x utility! + !! Also, note that this routine writes the u_matrix and the m_matrix - in parallel + !! mode these are however stored in distributed form in, e.g., u_matrix_loc only, so + !! if you are changing the u_matrix, remember to gather it from u_matrix_loc first! !=================================================! use w90_io, only: io_file_unit, io_date, seedname diff --git a/src/wannierise.F90 b/src/wannierise.F90 index de1675c79..f1b3b5ce2 100644 --- a/src/wannierise.F90 +++ b/src/wannierise.F90 @@ -602,7 +602,17 @@ subroutine wann_main !omega_tilde = wann_spread%om_d + wann_spread%om_nu end if - if (ldump .and. on_root) call param_write_chkpt('postdis') + if (ldump) then + ! Before calling param_write_chkpt, I need to gather on the root node + ! the u_matrix from the u_matrix_loc. No need to broadcast it since + ! it's printed by the root node only + call comms_gatherv(u_matrix_loc, num_wann*num_wann*counts(my_node_id), & + u_matrix, num_wann*num_wann*counts, num_wann*num_wann*displs) + ! I also transfer the M matrix + call comms_gatherv(m_matrix_loc, num_wann*num_wann*nntot*counts(my_node_id), & + m_matrix, num_wann*num_wann*nntot*counts, num_wann*num_wann*nntot*displs) + if (on_root) call param_write_chkpt('postdis') + endif if (conv_window .gt. 1) call internal_test_convergence()