field_RANKSUFF_gang_module.fypp

#! (C) Copyright 2022- ECMWF.
#! (C) Copyright 2022- Meteo-France.
#!
#! This software is licensed under the terms of the Apache Licence Version 2.0
#! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#! In applying this licence, ECMWF does not waive the privileges and immunities
#! granted to it by virtue of its status as an intergovernmental organisation
#! nor does it submit to any jurisdiction.

MODULE FIELD_${RANK}$${SUFF}$_GANG_MODULE

USE FIELD_BASIC_MODULE
USE FIELD_CONSTANTS_MODULE
USE DEV_ALLOC_MODULE

#:set fieldTypeList = fieldType.getFieldTypeList (ranks=[RANK], kinds=['JP' + str (SUFF)], ganged=True)

#:for ft in fieldTypeList

#:set fieldTypeList1 = fieldType.getFieldTypeList (ranks=[RANK-1], kinds=['JP' + str (SUFF)])
#:set ft1 = fieldTypeList1[0]

USE ${ft.name}$_MODULE
USE ${ft1.name}$_MODULE

${fieldType.useParkind1 ()}$

IMPLICIT NONE

PRIVATE

#:set ftn = ft.name
#:set ftn1 = ft1.name

TYPE, EXTENDS (${ftn1}$_WRAPPER) :: ${ftn1}$_WRAPPER_HELPER
  CLASS (FIELD_BASIC), POINTER :: PARENT => NULL ()
CONTAINS
  PROCEDURE :: DELETE_DEVICE_DATA => ${ftn1}$_DELETE_DEVICE_DATA_WRAPPER_HELPER
  PROCEDURE :: CREATE_DEVICE_DATA => ${ftn1}$_CREATE_DEVICE_DATA_WRAPPER_HELPER
  PROCEDURE :: SET_DEVICE_DIRTY => ${ftn1}$_SET_DEVICE_DIRTY_DATA_WRAPPER_HELPER
  PROCEDURE :: GET_DEVICE_DATA => ${ftn1}$_GET_DEVICE_DATA_WRAPPER_HELPER
  PROCEDURE :: GET_HOST_DATA => ${ftn1}$_GET_HOST_DATA_WRAPPER_HELPER
END TYPE

TYPE ${ftn1}$_WRAPPER_HELPER_PTR
  CLASS (${ftn1}$_WRAPPER_HELPER), POINTER :: PTR => NULL ()
END TYPE

#:for type in ['WRAPPER', 'OWNER']

TYPE, EXTENDS (${ftn}$_${type}$) :: ${ftn}$_GANG_${type}$
  TYPE (${ftn1}$_WRAPPER_HELPER_PTR), POINTER :: CHILDREN (:) => NULL ()
CONTAINS
  PROCEDURE :: INIT => ${ftn}$_GANG_${type}$_INIT
  PROCEDURE :: FINAL => ${ftn}$_GANG_${type}$_FINAL
  PROCEDURE :: CREATE_DEVICE_DATA => ${ftn}$_GANG_${type}$_CREATE_DEVICE_DATA
  PROCEDURE :: DELETE_DEVICE_DATA => ${ftn}$_GANG_${type}$_DELETE_DEVICE_DATA
  PROCEDURE :: SET_STATUS => ${ftn}$_GANG_${type}$_SET_STATUS
END TYPE ${ftn}$_GANG_${type}$

PUBLIC :: ${ftn}$_GANG_${type}$

#:endfor

CONTAINS

  SUBROUTINE ${ftn1}$_CREATE_DEVICE_DATA_WRAPPER_HELPER (SELF)
    CLASS(${ftn1}$_WRAPPER_HELPER) :: SELF

    IF (ASSOCIATED (SELF%PARENT)) THEN
      CALL SELF%PARENT%CREATE_DEVICE_DATA ()
    ENDIF

  END SUBROUTINE

  SUBROUTINE ${ftn1}$_DELETE_DEVICE_DATA_WRAPPER_HELPER (SELF)
    CLASS(${ftn1}$_WRAPPER_HELPER) :: SELF

    IF (ASSOCIATED (SELF%PARENT)) THEN
      CALL SELF%PARENT%DELETE_DEVICE_DATA ()
    ENDIF

  END SUBROUTINE

  SUBROUTINE ${ftn1}$_SET_DEVICE_DIRTY_DATA_WRAPPER_HELPER (SELF)

    CLASS (${ftn1}$_WRAPPER_HELPER) :: SELF

    IF (ASSOCIATED (SELF%PARENT)) THEN
      CALL SELF%PARENT%SET_DEVICE_DIRTY ()
    ENDIF

  END SUBROUTINE 

#:for what in ['HOST', 'DEVICE']
  SUBROUTINE ${ftn1}$_GET_${what}$_DATA_WRAPPER_HELPER (SELF, MODE, PTR, QUEUE)
    CLASS(${ftn1}$_WRAPPER_HELPER) :: SELF
    INTEGER (KIND=JPIM),           INTENT(IN)    :: MODE
    ${ft1.type}$, POINTER,         INTENT(INOUT) :: PTR(${ft1.shape}$)
    INTEGER (KIND=JPIM), OPTIONAL, INTENT(IN)    :: QUEUE
    INTEGER(KIND=JPIM)                           :: LBOUNDS(${ft1.rank}$)

    IF (ASSOCIATED (SELF%PARENT)) THEN
      IF (IAND (MODE, NWR) /= 0) THEN
        CALL SELF%PARENT%SYNC_${what}$_RDWR (QUEUE)
      ELSEIF (IAND (MODE, NRD) /= 0) THEN
        CALL SELF%PARENT%SYNC_${what}$_RDONLY (QUEUE)
      ENDIF
    ENDIF

    CALL SELF%${ftn1}$_WRAPPER%GET_${what}$_DATA (MODE, PTR, QUEUE)

  END SUBROUTINE 

#:endfor

  SUBROUTINE ${ftn}$_GANG_WRAPPER_INIT(SELF, DATA, PERSISTENT, LBOUNDS, MAP_DEVPTR, SYNC_ON_FINAL, INITIALIZED)

    USE FIELD_STATISTICS_MODULE

    CLASS(${ftn}$_GANG_WRAPPER) :: SELF
    ${ft.type}$, TARGET, INTENT(IN) :: DATA(${ft.shape}$)
    LOGICAL, INTENT(IN), OPTIONAL :: PERSISTENT
    INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: LBOUNDS(${ft.rank}$)
    LOGICAL, INTENT(IN), OPTIONAL :: MAP_DEVPTR
    LOGICAL, INTENT(IN), OPTIONAL :: SYNC_ON_FINAL
    LOGICAL, INTENT(IN), OPTIONAL :: INITIALIZED

    LOGICAL :: LLPERSISTENT
    INTEGER (KIND=JPIM) :: IFLR, JFLD, NFLD
    INTEGER (KIND=JPIM) :: LLBOUNDS (${ft.rank}$)
    CLASS (${ftn1}$_WRAPPER_HELPER), POINTER :: YLFW

  
    CALL SELF%${ftn}$_WRAPPER%INIT (DATA=DATA, PERSISTENT=PERSISTENT, LBOUNDS=LBOUNDS, MAP_DEVPTR=MAP_DEVPTR, & 
                                  & SYNC_ON_FINAL=SYNC_ON_FINAL, INITIALIZED=INITIALIZED)

    LLBOUNDS = 1
    IF (PRESENT (LBOUNDS)) LLBOUNDS = LBOUNDS

    IFLR = ${ft.rank}$-1
    NFLD = SIZE (DATA, IFLR)

    LLBOUNDS(IFLR) = LLBOUNDS (IFLR+1)
    ALLOCATE (SELF%CHILDREN (NFLD))
    
#:set ar = ', '.join ([':'] * (ft.rank-2))

    DO JFLD = 1, NFLD
      ALLOCATE (YLFW)
       
      IF (FIELD_STATISTICS_ENABLE) CALL FIELD_STATISTICS_NEW ()

      CALL YLFW%INIT (DATA=DATA(${ar}$, JFLD, :), PERSISTENT=PERSISTENT, LBOUNDS=LLBOUNDS(1:IFLR), MAP_DEVPTR=MAP_DEVPTR, &
                    & SYNC_ON_FINAL=SYNC_ON_FINAL, INITIALIZED=INITIALIZED)
      SELF%CHILDREN (JFLD)%PTR => YLFW
      CALL SELF%CHILDREN(JFLD)%PTR%SET_STATUS (SELF%GET_STATUS ())
    ENDDO

  END SUBROUTINE ${ftn}$_GANG_WRAPPER_INIT

  SUBROUTINE ${ftn}$_GANG_OWNER_INIT(SELF, LBOUNDS, UBOUNDS, PERSISTENT, DELAYED, INIT_VALUE, PINNED, MAP_DEVPTR, POOLED)

    USE FIELD_STATISTICS_MODULE

    CLASS(${ftn}$_GANG_OWNER) :: SELF
    INTEGER(KIND=JPIM), INTENT(IN), OPTIONAL :: LBOUNDS(${ft.rank}$)
    INTEGER(KIND=JPIM), INTENT(IN) :: UBOUNDS(${ft.rank}$)
    LOGICAL, INTENT(IN), OPTIONAL :: PERSISTENT
    LOGICAL, OPTIONAL,  INTENT(IN) :: DELAYED
    ${ft.type}$, OPTIONAL, INTENT(IN) :: INIT_VALUE
    LOGICAL, OPTIONAL,  INTENT(IN) :: PINNED
    LOGICAL, OPTIONAL,  INTENT(IN) :: MAP_DEVPTR
    LOGICAL, OPTIONAL,  INTENT(IN) :: POOLED

    LOGICAL :: LLPERSISTENT
    INTEGER (KIND=JPIM) :: IFLR, JFLD, NFLD
    INTEGER (KIND=JPIM) :: LLBOUNDS (${ft.rank}$)
    CLASS (${ftn1}$_WRAPPER_HELPER), POINTER :: YLFW

    CALL SELF%${ftn}$_OWNER%INIT (PERSISTENT=PERSISTENT, LBOUNDS=LBOUNDS, UBOUNDS=UBOUNDS, &
        & DELAYED=.FALSE., INIT_VALUE=INIT_VALUE, PINNED=PINNED, MAP_DEVPTR=MAP_DEVPTR, POOLED=POOLED)

    LLBOUNDS = 1
    IF (PRESENT (LBOUNDS)) LLBOUNDS = LBOUNDS

    IFLR = ${ft.rank}$-1
    NFLD = UBOUNDS (IFLR)-LLBOUNDS (IFLR)+1

    LLBOUNDS(IFLR) = LLBOUNDS (IFLR+1)

    ALLOCATE (SELF%CHILDREN (NFLD))
    
#:set ar = ', '.join ([':'] * (ft.rank-2))

    DO JFLD = 1, NFLD
      ALLOCATE (YLFW)

      IF (FIELD_STATISTICS_ENABLE) CALL FIELD_STATISTICS_NEW ()

      CALL YLFW%INIT (DATA=SELF%PTR (${ar}$, JFLD, :), PERSISTENT=PERSISTENT, LBOUNDS=LLBOUNDS(1:IFLR), SYNC_ON_FINAL=.FALSE.)
      SELF%CHILDREN (JFLD)%PTR => YLFW
      CALL SELF%CHILDREN(JFLD)%PTR%SET_STATUS (SELF%GET_STATUS ())
    ENDDO
    
  END SUBROUTINE ${ftn}$_GANG_OWNER_INIT

#:for type in ['WRAPPER', 'OWNER']
  SUBROUTINE ${ftn}$_GANG_${type}$_FINAL(SELF)

    USE FIELD_STATISTICS_MODULE

    CLASS(${ftn}$_GANG_${type}$) :: SELF
    ${ft.type}$, POINTER :: PTR(${ft.shape}$)
    INTEGER (KIND=JPIM) :: JFLD
    CLASS (${ftn1}$_WRAPPER_HELPER), POINTER :: YLF

    DO JFLD = 1, SIZE (SELF%CHILDREN)
 
      IF (FIELD_STATISTICS_ENABLE) CALL FIELD_STATISTICS_DELETE ()

      YLF => SELF%CHILDREN(JFLD)%PTR
      NULLIFY (YLF%DEVPTR)
      NULLIFY (YLF%PARENT)
! Mark wrapper as fresh on host, synchronization has to be handled by then parent
      CALL YLF%SET_STATUS (IOR (SELF%GET_STATUS (), NHSTFRESH))
      CALL YLF%FINAL ()
      DEALLOCATE (YLF)
    ENDDO
  
    CALL SELF%${ftn}$_${type}$%FINAL ()

    DEALLOCATE (SELF%CHILDREN)  
    NULLIFY (SELF%CHILDREN)  

  END SUBROUTINE ${ftn}$_GANG_${type}$_FINAL

  SUBROUTINE ${ftn}$_GANG_${type}$_DELETE_DEVICE_DATA(SELF)
    ! Delete the copy of this field on GPU device
    CLASS(${ftn}$_GANG_${type}$) :: SELF

    INTEGER (KIND=JPIM) :: JFLD

    IF (ASSOCIATED (SELF%DEVPTR)) THEN
      DO JFLD = 1, SIZE (SELF%CHILDREN)
        NULLIFY (SELF%CHILDREN (JFLD)%PTR%DEVPTR)
      ENDDO
      CALL DEV_DEALLOCATE (SELF%DEVPTR, SELF%MAP_DEVPTR)
    ENDIF

  END SUBROUTINE ${ftn}$_GANG_${type}$_DELETE_DEVICE_DATA

  SUBROUTINE ${ftn}$_GANG_${type}$_CREATE_DEVICE_DATA (SELF)
    CLASS(${ftn}$_GANG_${type}$) :: SELF
    INTEGER (KIND=JPIM) :: JFLD

    CALL SELF%${ftn}$_${type}$%CREATE_DEVICE_DATA ()

#:set ar = ', '.join ([':'] * (ft.rank-2))
    DO JFLD = 1, SIZE (SELF%CHILDREN)
      SELF%CHILDREN (JFLD)%PTR%DEVPTR (${','.join (list (map (lambda i: "LBOUND(SELF%DEVPTR," + str(i) + "):", range (1, ft1.rank))))}$,1:) => SELF%DEVPTR (${ar}$, JFLD, :)
    ENDDO

  END SUBROUTINE

  SUBROUTINE ${ftn}$_GANG_${type}$_SET_STATUS (SELF, KSTATUS)
    CLASS(${ftn}$_GANG_${type}$) :: SELF
    INTEGER (KIND=JPIM), INTENT (IN) :: KSTATUS

    INTEGER (KIND=JPIM) :: JFLD

    CALL SELF%${ftn}$_${type}$%SET_STATUS (KSTATUS)

    DO JFLD = 1, SIZE (SELF%CHILDREN)
      CALL SELF%CHILDREN(JFLD)%PTR%SET_STATUS (KSTATUS)
    ENDDO

  END SUBROUTINE

#:endfor

#:endfor

END MODULE FIELD_${RANK}$${SUFF}$_GANG_MODULE