From 6ad71d9e0aa05e24d9356dfcc40c8686d0a8b05c Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 9 Jan 2023 16:21:31 -0700 Subject: [PATCH 1/3] redo multiinstance support --- cesm/driver/ensemble_driver.F90 | 37 ++++++++++++++-------- cesm/driver/esm.F90 | 2 +- cesm/nuopc_cap_share/nuopc_shr_methods.F90 | 6 ++-- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/cesm/driver/ensemble_driver.F90 b/cesm/driver/ensemble_driver.F90 index 7e64c1cc6..02a0a517e 100644 --- a/cesm/driver/ensemble_driver.F90 +++ b/cesm/driver/ensemble_driver.F90 @@ -79,7 +79,7 @@ subroutine SetModelServices(ensemble_driver, rc) use ESMF , only : ESMF_CalendarSetDefault use ESMF , only : ESMF_CALKIND_NOLEAP, ESMF_CALKIND_GREGORIAN use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd - use NUOPC_Driver , only : NUOPC_DriverAddComp + use NUOPC_Driver , only : NUOPC_DriverAddComp, NUOPC_DriverGetComp use esm , only : ESMSetServices => SetServices, ReadAttributes use esm_time_mod , only : esm_time_clockInit @@ -89,7 +89,7 @@ subroutine SetModelServices(ensemble_driver, rc) ! local variables type(ESMF_VM) :: vm - type(ESMF_GridComp) :: driver, gridcomptmp + type(ESMF_GridComp) :: driver type(ESMF_Config) :: config integer :: n integer, pointer :: petList(:) @@ -169,6 +169,10 @@ subroutine SetModelServices(ensemble_driver, rc) call NUOPC_CompAttributeSet(ensemble_driver, name='read_restart', value=trim(read_restart_string), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return + + call NUOPC_CompAttributeSet(ensemble_driver, name='Profiling', value='max', rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + !------------------------------------------- ! Extract the config object from the ensemble_driver !------------------------------------------- @@ -200,23 +204,30 @@ subroutine SetModelServices(ensemble_driver, rc) !------------------------------------------- allocate(petList(ntasks_per_member)) - ! which driver instance is this? - inst = localPet/ntasks_per_member + 1 + ! We need to loop over instances + do inst = 1, number_of_members + + ! Determine pet list for driver instance + petList(1) = (inst-1) * ntasks_per_member + do n=2,ntasks_per_member + petList(n) = petList(n-1) + 1 + enddo + + ! Add driver instance to ensemble driver + write(drvrinst,'(a,i4.4)') "ESM",inst + call NUOPC_DriverAddComp(ensemble_driver, drvrinst, ESMSetServices, petList=petList, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + enddo - ! Determine pet list for driver instance + inst = localPet/ntasks_per_member + 1 petList(1) = (inst-1) * ntasks_per_member do n=2,ntasks_per_member petList(n) = petList(n-1) + 1 enddo - - ! Add driver instance to ensemble driver - write(drvrinst,'(a,i4.4)') "ESM",inst - call NUOPC_DriverAddComp(ensemble_driver, drvrinst, ESMSetServices, petList=petList, comp=gridcomptmp, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - if (localpet >= petlist(1) .and. localpet <= petlist(ntasks_per_member)) then - - driver = gridcomptmp + write(drvrinst,'(a,i4.4)') "ESM",inst + call NUOPC_DriverGetComp(ensemble_driver, drvrinst, comp=driver, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return if(number_of_members > 1) then call NUOPC_CompAttributeAdd(driver, attrList=(/'inst_suffix'/), rc=rc) diff --git a/cesm/driver/esm.F90 b/cesm/driver/esm.F90 index 1c73ea17d..cc2e6f4f1 100644 --- a/cesm/driver/esm.F90 +++ b/cesm/driver/esm.F90 @@ -636,7 +636,7 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, n if (chkerr(rc,__LINE__,u_FILE_u)) return !------ - ! Add driver restart flag a to gcomp attributes + ! Add driver restart flag to gcomp attributes !------ attribute = 'read_restart' call NUOPC_CompAttributeGet(driver, name=trim(attribute), value=cvalue, rc=rc) diff --git a/cesm/nuopc_cap_share/nuopc_shr_methods.F90 b/cesm/nuopc_cap_share/nuopc_shr_methods.F90 index 0ed53f22b..b6b0245ac 100644 --- a/cesm/nuopc_cap_share/nuopc_shr_methods.F90 +++ b/cesm/nuopc_cap_share/nuopc_shr_methods.F90 @@ -145,6 +145,7 @@ subroutine set_component_logging(gcomp, mastertask, logunit, shrlogunit, rc) character(len=CL) :: logfile character(len=CL) :: inst_suffix integer :: inst_index ! not used here + integer :: n !----------------------------------------------------------------------- rc = ESMF_SUCCESS @@ -157,8 +158,9 @@ subroutine set_component_logging(gcomp, mastertask, logunit, shrlogunit, rc) call get_component_instance(gcomp, inst_suffix, inst_index, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return ! Multiinstance logfile name needs a correction - if(logfile(4:4) == '_') then - logfile = logfile(1:3)//trim(inst_suffix)//logfile(9:) + if(len_trim(inst_suffix) > 0) then + n = index(logfile, '.') + logfile = logfile(1:n-1)//trim(inst_suffix)//logfile(n:) endif open(newunit=logunit,file=trim(diro)//"/"//trim(logfile)) From d17cd8956bb36736a4f1a44febb5ca58896a0db3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 11 Jan 2023 12:52:31 -0700 Subject: [PATCH 2/3] move timer init function --- cesm/driver/ensemble_driver.F90 | 23 +++++++++++++++++------ cesm/driver/esm.F90 | 8 -------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cesm/driver/ensemble_driver.F90 b/cesm/driver/ensemble_driver.F90 index 02a0a517e..42d34c438 100644 --- a/cesm/driver/ensemble_driver.F90 +++ b/cesm/driver/ensemble_driver.F90 @@ -82,7 +82,7 @@ subroutine SetModelServices(ensemble_driver, rc) use NUOPC_Driver , only : NUOPC_DriverAddComp, NUOPC_DriverGetComp use esm , only : ESMSetServices => SetServices, ReadAttributes use esm_time_mod , only : esm_time_clockInit - + use perf_mod , only : t_startf, t_stopf, t_initf ! input/output variables type(ESMF_GridComp) :: ensemble_driver integer, intent(out) :: rc @@ -102,6 +102,7 @@ subroutine SetModelServices(ensemble_driver, rc) integer :: inst integer :: number_of_members integer :: ntasks_per_member + integer :: Global_Comm character(CL) :: start_type ! Type of startup character(len=7) :: drvrinst character(len=5) :: inst_suffix @@ -116,10 +117,21 @@ subroutine SetModelServices(ensemble_driver, rc) rc = ESMF_SUCCESS call ESMF_LogWrite(trim(subname)//": called", ESMF_LOGMSG_INFO) - call ESMF_GridCompGet(ensemble_driver, config=config, vm=vm, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_VMGet(vm, localPet=localPet, mpiCommunicator=global_comm, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + + if (localPet == 0) then + mastertask=.true. + else + mastertask = .false. + end if + + call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=mastertask) + call t_startf(subname) + !------------------------------------------- ! Initialize clocks !------------------------------------------- @@ -169,10 +181,6 @@ subroutine SetModelServices(ensemble_driver, rc) call NUOPC_CompAttributeSet(ensemble_driver, name='read_restart', value=trim(read_restart_string), rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - - call NUOPC_CompAttributeSet(ensemble_driver, name='Profiling', value='max', rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return - !------------------------------------------- ! Extract the config object from the ensemble_driver !------------------------------------------- @@ -205,6 +213,7 @@ subroutine SetModelServices(ensemble_driver, rc) allocate(petList(ntasks_per_member)) ! We need to loop over instances + call t_startf('compute_drivers') do inst = 1, number_of_members ! Determine pet list for driver instance @@ -218,6 +227,7 @@ subroutine SetModelServices(ensemble_driver, rc) call NUOPC_DriverAddComp(ensemble_driver, drvrinst, ESMSetServices, petList=petList, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return enddo + call t_stopf('compute_drivers') inst = localPet/ntasks_per_member + 1 petList(1) = (inst-1) * ntasks_per_member @@ -275,6 +285,7 @@ subroutine SetModelServices(ensemble_driver, rc) endif deallocate(petList) + call t_stopf(subname) call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO) diff --git a/cesm/driver/esm.F90 b/cesm/driver/esm.F90 index cc2e6f4f1..ce768b6d2 100644 --- a/cesm/driver/esm.F90 +++ b/cesm/driver/esm.F90 @@ -10,7 +10,6 @@ module ESM use shr_mem_mod , only : shr_mem_init use shr_log_mod , only : shr_log_setLogunit use esm_utils_mod, only : logunit, mastertask, dbug_flag, chkerr - use perf_mod , only : t_initf, t_setLogUnit implicit none private @@ -151,8 +150,6 @@ subroutine SetModelServices(driver, rc) call ESMF_VMGet(vm, localPet=localPet, mpiCommunicator=global_comm, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return - call ESMF_VMGet(vm, localPet=localPet, rc=rc) - if (chkerr(rc,__LINE__,u_FILE_u)) return if (localPet == 0) then mastertask=.true. else @@ -211,11 +208,6 @@ subroutine SetModelServices(driver, rc) write(logunit,*) trim(meminitstr) end if - !------------------------------------------- - ! Timer initialization (has to be after pelayouts are determined) - !------------------------------------------- - call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=mastertask, MaxThreads=maxthreads) - call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO) end subroutine SetModelServices From c7fec3b21710b76366ac8c0120b5c8be6910743d Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 11 Jan 2023 13:10:42 -0700 Subject: [PATCH 3/3] fix merge issues --- cesm/driver/ensemble_driver.F90 | 6 +++--- cesm/driver/esm.F90 | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cesm/driver/ensemble_driver.F90 b/cesm/driver/ensemble_driver.F90 index 8bb3b1154..58b9d58a1 100644 --- a/cesm/driver/ensemble_driver.F90 +++ b/cesm/driver/ensemble_driver.F90 @@ -124,12 +124,12 @@ subroutine SetModelServices(ensemble_driver, rc) if (chkerr(rc,__LINE__,u_FILE_u)) return if (localPet == 0) then - mastertask=.true. + maintask=.true. else - mastertask = .false. + maintask = .false. end if - call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=mastertask) + call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=maintask) call t_startf(subname) !------------------------------------------- diff --git a/cesm/driver/esm.F90 b/cesm/driver/esm.F90 index 2e951bfa5..da2f6f6d3 100644 --- a/cesm/driver/esm.F90 +++ b/cesm/driver/esm.F90 @@ -9,7 +9,7 @@ module ESM use shr_mpi_mod , only : shr_mpi_bcast use shr_mem_mod , only : shr_mem_init use shr_log_mod , only : shr_log_setLogunit - use esm_utils_mod, only : logunit, mastertask, dbug_flag, chkerr + use esm_utils_mod, only : logunit, maintask, dbug_flag, chkerr implicit none private