From cb0a3e56d67fd307547c3fcda48e109f7f038d67 Mon Sep 17 00:00:00 2001
From: Ming Hu <Ming.Hu@noaa.gov>
Date: Thu, 17 Feb 2022 08:11:55 -0700
Subject: [PATCH] Update FVCOM code to handle sub-domain restart files using
 multiple cores. (#624)

The large domain FV3LAM generates multiple sub-domain restart files
instead of a single file. Update FVCOM to work with sub-domain restart
files and with multiple cores.

Update the unit test for sub-domain files and to check new variables
processed by FVCOM (roughness length and sea ice depth).

Fixes #623.
---
 sorc/fvcom_tools.fd/module_ncio.f90        | 286 ++++++++++----------
 sorc/fvcom_tools.fd/module_nwp.f90         | 115 ++++----
 sorc/fvcom_tools.fd/module_nwp_base.f90    |  10 +-
 sorc/fvcom_tools.fd/process_FVCOM.f90      | 295 ++++++++++++++-------
 tests/CMakeLists.txt                       |   1 +
 tests/fvcom_tools/ftst_readfvcomnetcdf.F90 |  47 +++-
 6 files changed, 444 insertions(+), 310 deletions(-)

diff --git a/sorc/fvcom_tools.fd/module_ncio.f90 b/sorc/fvcom_tools.fd/module_ncio.f90
index 548878979..00fc79be8 100644
--- a/sorc/fvcom_tools.fd/module_ncio.f90
+++ b/sorc/fvcom_tools.fd/module_ncio.f90
@@ -121,14 +121,14 @@ subroutine open_nc(this,filename,action,debug_level)
     elseif(action=="w" .or. action=="W") then
        status = nf90_open(path = trim(filename), mode = nf90_write, ncid = ncid)
     else
-       write(*,*) 'unknow action :', action
+       write(6,*) 'unknow action :', action
        stop 123
     endif
     if (status /= nf90_noerr) call this%handle_err(status)
     this%ncid=ncid
 
     if(this%debug_level>0) then
-       write(*,*) '>>> open file: ',trim(this%filename)
+       write(6,*) '>>> open file: ',trim(this%filename)
     endif
 
   end subroutine open_nc
@@ -280,8 +280,8 @@ subroutine replace_var_nc_char_1d(this,varname,nd1,field)
     ilength=nd1
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) (field(i),i=1,min(nd1,10))
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) (field(i),i=1,min(nd1,10))
     endif
 
     call this%replace_var_nc_char(varname,ilength,field)
@@ -325,8 +325,8 @@ subroutine replace_var_nc_char_2d(this,varname,nd1,nd2,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) field(1,1)
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) field(1,1)
     endif
     !
     call this%replace_var_nc_char(varname,ilength,temp)
@@ -377,8 +377,8 @@ subroutine replace_var_nc_char_3d(this,varname,nd1,nd2,nd3,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) field(1,1,1)
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) field(1,1,1)
     endif
 
     call this%replace_var_nc_char(varname,ilength,temp)
@@ -439,7 +439,7 @@ subroutine replace_var_nc_char(this,varname,ilength,field)
     if(xtype==NF90_CHAR) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
        stop 123
     endif
 
@@ -458,7 +458,7 @@ subroutine replace_var_nc_char(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -466,7 +466,7 @@ subroutine replace_var_nc_char(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -476,18 +476,18 @@ subroutine replace_var_nc_char(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>replace variable: ',trim(varname)
+       write(6,'(a,a)') '>>>replace variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -520,8 +520,8 @@ subroutine replace_var_nc_real_1d(this,varname,nd1,field)
     ilength=nd1
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) (field(i),i=1,min(nd1,10))
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) (field(i),i=1,min(nd1,10))
     endif
     !
     call this%replace_var_nc_real(varname,ilength,field)
@@ -565,8 +565,8 @@ subroutine replace_var_nc_real_2d(this,varname,nd1,nd2,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
     endif
 
     call this%replace_var_nc_real(varname,ilength,temp)
@@ -620,9 +620,9 @@ subroutine replace_var_nc_real_3d(this,varname,nd1,nd2,nd3,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
+       write(6,*) trim(thissubname),' show samples:'
        do k=1,nd3
-          write(*,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
+          write(6,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
        enddo
     endif
 
@@ -684,7 +684,7 @@ subroutine replace_var_nc_real(this,varname,ilength,field)
     if(xtype==NF90_FLOAT) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
        stop 123
     endif
 
@@ -703,7 +703,7 @@ subroutine replace_var_nc_real(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -711,7 +711,7 @@ subroutine replace_var_nc_real(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -721,18 +721,18 @@ subroutine replace_var_nc_real(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>replace variable: ',trim(varname)
+       write(6,'(a,a)') '>>>replace variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -767,8 +767,8 @@ subroutine replace_var_nc_double_1d(this,varname,nd1,field)
     ilength=nd1
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) (field(i),i=1,min(nd1,10))
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) (field(i),i=1,min(nd1,10))
     endif
     !
     call this%replace_var_nc_double(varname,ilength,field)
@@ -815,8 +815,8 @@ subroutine replace_var_nc_double_2d(this,varname,nd1,nd2,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
     endif
 
     call this%replace_var_nc_double(varname,ilength,temp)
@@ -870,9 +870,9 @@ subroutine replace_var_nc_double_3d(this,varname,nd1,nd2,nd3,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
+       write(6,*) trim(thissubname),' show samples:'
        do k=1,nd3
-          write(*,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
+          write(6,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
        enddo
     endif
 
@@ -935,7 +935,7 @@ subroutine replace_var_nc_double(this,varname,ilength,field)
     if(xtype==NF90_DOUBLE) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
        stop 123
     endif
 
@@ -954,7 +954,7 @@ subroutine replace_var_nc_double(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -962,7 +962,7 @@ subroutine replace_var_nc_double(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -972,18 +972,18 @@ subroutine replace_var_nc_double(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>replace variable: ',trim(varname)
+       write(6,'(a,a)') '>>>replace variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -1015,8 +1015,8 @@ subroutine replace_var_nc_int_1d(this,varname,nd1,field)
     ilength=nd1
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) (field(i),i=1,min(nd1,10))
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) (field(i),i=1,min(nd1,10))
     endif
 
     call this%replace_var_nc_int(varname,ilength,field)
@@ -1063,8 +1063,8 @@ subroutine replace_var_nc_int_2d(this,varname,nd1,nd2,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
-       write(*,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
+       write(6,*) trim(thissubname),' show samples:'
+       write(6,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
     endif
 
     call this%replace_var_nc_int(varname,ilength,temp)
@@ -1115,9 +1115,9 @@ subroutine replace_var_nc_int_3d(this,varname,nd1,nd2,nd3,field)
     enddo
     !
     if(this%debug_level>100) then
-       write(*,*) trim(thissubname),' show samples:'
+       write(6,*) trim(thissubname),' show samples:'
        do k=1,nd3
-          write(*,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
+          write(6,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
        enddo
     endif
 
@@ -1179,7 +1179,7 @@ subroutine replace_var_nc_int(this,varname,ilength,field)
     if(xtype==NF90_INT) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
        stop 123
     endif
 
@@ -1198,7 +1198,7 @@ subroutine replace_var_nc_int(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -1206,7 +1206,7 @@ subroutine replace_var_nc_int(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -1216,18 +1216,18 @@ subroutine replace_var_nc_int(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
-       stop 1234
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
+       stop 1634
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>replace variable: ',trim(varname)
+       write(6,'(a,a)') '>>>replace variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -1261,11 +1261,11 @@ subroutine get_var_nc_double_1d(this,varname,nd1,field)
     !
     if(nd1==this%ends(1)) then
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) (field(i),i=1,min(nd1,10))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) (field(i),i=1,min(nd1,10))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
     endif
     !
   end subroutine get_var_nc_double_1d
@@ -1314,8 +1314,8 @@ subroutine get_var_nc_double_2d(this,varname,nd1,nd2,field)
 !          write(*,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
 !       endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2)
     endif
     deallocate(temp)
     !
@@ -1372,8 +1372,8 @@ subroutine get_var_nc_double_3d(this,varname,nd1,nd2,nd3,field)
 !          enddo
 !       endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
     endif
     deallocate(temp)
     !
@@ -1431,7 +1431,7 @@ subroutine get_var_nc_double(this,varname,ilength,field)
     if(xtype==NF90_DOUBLE) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_DOUBLE,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_DOUBLE,' but read in ',xtype
        stop 123
     endif
 
@@ -1444,14 +1444,14 @@ subroutine get_var_nc_double(this,varname,ilength,field)
     if(status /= nf90_NoErr) call this%handle_err(status)
     do i=1,nDims
        dimname="       "
-       write(*,*) 'dimids(i) = ', dimids(i)
+       write(6,*) 'dimids(i) = ', dimids(i)
        status = nf90_inquire_dimension(ncid, dimids(i), dimname, len = ndim)
        if (status /= nf90_noerr) call this%handle_err(status)
        ends(i)=ndim
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -1459,7 +1459,7 @@ subroutine get_var_nc_double(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -1469,18 +1469,18 @@ subroutine get_var_nc_double(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>read in variable: ',trim(varname)
+       write(6,'(a,a)') '>>>read in variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(a,I10)') '        data type : ',this%xtype
-       write(*,'(a,I10)')'        dimension size: ',this%nDims
+       write(6,'(a,I10)') '        data type : ',this%xtype
+       write(6,'(a,I10)')'        dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(a,I5,I10,2x,a)') '        rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(a,I5,I10,2x,a)') '        rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -1514,11 +1514,11 @@ subroutine get_var_nc_real_1d(this,varname,nd1,field)
     !
     if(nd1==this%ends(1)) then
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) (field(i),i=1,min(nd1,10))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) (field(i),i=1,min(nd1,10))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
     endif
     !
   end subroutine get_var_nc_real_1d
@@ -1566,12 +1566,12 @@ subroutine get_var_nc_real_2d(this,varname,nd1,nd2,field)
        enddo
        !
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2)
     endif
     deallocate(temp)
     !
@@ -1622,14 +1622,14 @@ subroutine get_var_nc_real_3d(this,varname,nd1,nd2,nd3,field)
        enddo
        !
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
+          write(6,*) trim(thissubname),' show samples:'
           do k=1,nd3
-             write(*,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
+             write(6,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
           enddo
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
     endif
     deallocate(temp)
     !
@@ -1690,7 +1690,7 @@ subroutine get_var_nc_real(this,varname,ilength,field)
     if(xtype==NF90_FLOAT) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_FLOAT,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_FLOAT,' but read in ',xtype
        stop 123
     endif
 
@@ -1709,7 +1709,7 @@ subroutine get_var_nc_real(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -1717,7 +1717,7 @@ subroutine get_var_nc_real(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -1727,18 +1727,18 @@ subroutine get_var_nc_real(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>read in variable: ',trim(varname)
+       write(6,'(a,a)') '>>>read in variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -1772,11 +1772,11 @@ subroutine get_var_nc_int_1d(this,varname,nd1,field)
     !
     if(nd1==this%ends(1)) then
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) (field(i),i=1,min(nd1,10))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) (field(i),i=1,min(nd1,10))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
     endif
     !
   end subroutine get_var_nc_int_1d
@@ -1824,12 +1824,12 @@ subroutine get_var_nc_int_2d(this,varname,nd1,nd2,field)
        enddo
        !
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2)
     endif
     deallocate(temp)
     !
@@ -1880,14 +1880,14 @@ subroutine get_var_nc_int_3d(this,varname,nd1,nd2,nd3,field)
        enddo
        !
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
+          write(6,*) trim(thissubname),' show samples:'
           do k=1,nd3
-             write(*,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
+             write(6,*) 'k,max,min:',k,maxval(field(:,:,k)),minval(field(:,:,k))
           enddo
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
     endif
     deallocate(temp)
     !
@@ -1948,7 +1948,7 @@ subroutine get_var_nc_int(this,varname,ilength,field)
     if(xtype==NF90_INT) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_INT,' but read in ',xtype
        stop 123
     endif
 
@@ -1967,7 +1967,7 @@ subroutine get_var_nc_int(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -1975,7 +1975,7 @@ subroutine get_var_nc_int(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -1985,18 +1985,18 @@ subroutine get_var_nc_int(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>read in variable: ',trim(varname)
+       write(6,'(a,a)') '>>>read in variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -2030,11 +2030,11 @@ subroutine get_var_nc_short_1d(this,varname,nd1,field)
     !
     if(nd1==this%ends(1)) then
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) (field(i),i=1,min(nd1,10))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) (field(i),i=1,min(nd1,10))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
     endif
     !
   end subroutine get_var_nc_short_1d
@@ -2082,12 +2082,12 @@ subroutine get_var_nc_short_2d(this,varname,nd1,nd2,field)
        enddo
        !
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) 'max,min:',maxval(field(:,:)),minval(field(:,:))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2)
     endif
     deallocate(temp)
     !
@@ -2145,7 +2145,7 @@ subroutine get_var_nc_short(this,varname,ilength,field)
     if(xtype==NF90_SHORT) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_SHORT,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_SHORT,' but read in ',xtype
        stop 123
     endif
 
@@ -2164,7 +2164,7 @@ subroutine get_var_nc_short(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -2172,7 +2172,7 @@ subroutine get_var_nc_short(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -2182,18 +2182,18 @@ subroutine get_var_nc_short(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>read in variable: ',trim(varname)
+       write(6,'(a,a)') '>>>read in variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
@@ -2227,11 +2227,11 @@ subroutine get_var_nc_char_1d(this,varname,nd1,field)
     !
     if(nd1==this%ends(1)) then
        if(this%debug_level>100) then
-          write(*,*) trim(thissubname),' show samples:'
-          write(*,*) (field(i),i=1,min(nd1,10))
+          write(6,*) trim(thissubname),' show samples:'
+          write(6,*) (field(i),i=1,min(nd1,10))
        endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
     endif
     !
   end subroutine get_var_nc_char_1d
@@ -2280,8 +2280,8 @@ subroutine get_var_nc_char_2d(this,varname,nd1,nd2,field)
 !          write(*,*) field(1,1)
 !       endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2)
     endif
     deallocate(temp)
     !
@@ -2336,8 +2336,8 @@ subroutine get_var_nc_char_3d(this,varname,nd1,nd2,nd3,field)
 !          write(*,*) field(1,1,1)
 !       endif
     else
-       write(*,*) trim(thissubname),' ERROR: dimension does not match.'
-       write(*,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
+       write(6,*) trim(thissubname),' ERROR: dimension does not match.'
+       write(6,*) nd1,this%ends(1),nd2,this%ends(2),nd3,this%ends(3)
     endif
     deallocate(temp)
     !
@@ -2398,7 +2398,7 @@ subroutine get_var_nc_char(this,varname,ilength,field)
     if(xtype==NF90_CHAR) then
        this%xtype=xtype
     else
-       write(*,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_CHAR,' but read in ',xtype
+       write(6,*) trim(thissubname),' ERROR: wrong data type, expect ',NF90_CHAR,' but read in ',xtype
        stop 123
     endif
 
@@ -2417,7 +2417,7 @@ subroutine get_var_nc_char(this,varname,ilength,field)
        this%ends(i)=ends(i)
        this%dimname(i)=trim(dimname)
        if(this%ends(i) < 1) then
-          write(*,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
+          write(6,*) trim(thissubname),' Error, ends dimension should larger than 0 :', ends(i)
           stop 1234
        endif
     enddo
@@ -2425,7 +2425,7 @@ subroutine get_var_nc_char(this,varname,ilength,field)
     length3d=length2d*ends(3)
     length4d=length3d*ends(4)
     if(ilength .ne. length4d) then
-       write(*,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
+       write(6,*) trim(thissubname),'ERROR: ',ilength,' should equal to ',length4d
        stop 123
     endif
     !
@@ -2435,18 +2435,18 @@ subroutine get_var_nc_char(this,varname,ilength,field)
             count = ends(1:4))
        if(status /= nf90_NoErr) call this%handle_err(status)
     else
-       write(*,*) trim(thissubname),'Error: too many dimensions:',nDims
+       write(6,*) trim(thissubname),'Error: too many dimensions:',nDims
        stop 1234
     endif
     !
     if(this%debug_level>0) then
-       write(*,'(a,a)') '>>>read in variable: ',trim(varname)
+       write(6,'(a,a)') '>>>read in variable: ',trim(varname)
     endif
     if(this%debug_level>10) then
-       write(*,'(8x,a,I10)') 'data type : ',this%xtype
-       write(*,'(8x,a,I10)') 'dimension size: ',this%nDims
+       write(6,'(8x,a,I10)') 'data type : ',this%xtype
+       write(6,'(8x,a,I10)') 'dimension size: ',this%nDims
        do i=1,this%nDims
-          write(*,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
+          write(6,'(8x,a,I5,I10,2x,a)') 'rank, ends, name=',i,this%ends(i),trim(this%dimname(i))
        enddo
     endif
     !
diff --git a/sorc/fvcom_tools.fd/module_nwp.f90 b/sorc/fvcom_tools.fd/module_nwp.f90
index d0b7b2376..19d8cd52f 100644
--- a/sorc/fvcom_tools.fd/module_nwp.f90
+++ b/sorc/fvcom_tools.fd/module_nwp.f90
@@ -208,15 +208,15 @@ subroutine initial_nwp(this,itype,wcstart)
 !        If the data type does not match one of the known types, exit.
 
          else
-            write(*,*) 'Unknown data type:', itype
+            write(6,*) 'Unknown data type:', itype
             stop 1234
          end if
 
          this%head => NULL()
          this%tail => NULL()
 
-         write(*,*) 'Finished initial_nwp'
-         write(*,*) ' '
+         write(6,*) 'Finished initial_nwp'
+         write(6,*) ' '
 
       end subroutine initial_nwp
 
@@ -231,18 +231,18 @@ subroutine list_initial_nwp(this)
 
          integer :: k
 
-         write(*,*) 'List initial setup for ', this%datatype
-         write(*,*) 'number of variables ', this%numvar
-         write(*,*) 'variable index: mask, sst, ice, sfcT, sfcTl'
-         write(*,'(15x,10I3)') this%i_mask, this%i_sst, this%i_ice, &
+         write(6,*) 'List initial setup for ', this%datatype
+         write(6,*) 'number of variables ', this%numvar
+         write(6,*) 'variable index: mask, sst, ice, sfcT, sfcTl'
+         write(6,'(15x,10I3)') this%i_mask, this%i_sst, this%i_ice, &
       &      this%i_sfcT, this%i_sfcTl
-         write(*,*) 'variable name:'
+         write(6,*) 'variable name:'
          do k=1,this%numvar
-            write(*,*) k,trim(this%varnames(k))
+            write(6,*) k,trim(this%varnames(k))
          enddo
 
-         write(*,*) 'Finished list_initial_nwp'
-         write(*,*) ' '
+         write(6,*) 'Finished list_initial_nwp'
+         write(6,*) ' '
 
       end subroutine list_initial_nwp
 
@@ -265,9 +265,11 @@ end subroutine list_initial_nwp
       !! @param[inout] sfcTl Skin Temperature in restart file
       !! @param[inout] zorl Surface roughness length
       !! @param[inout] hice Ice thickness
+      !! @param[in]    ybegin Start grid point in Y direction for the domain
+      !! @param[in]    yend   End grid point in Y direction for the domain
       !!
       !! @author David Wright, University of Michigan and GLERL
-      subroutine read_nwp(this,filename,itype,wcstart,numlon,numlat,numtimes,time_to_get,mask,sst,ice,sfcT,iceT,sfcTl,zorl,hice)
+      subroutine read_nwp(this,filename,itype,wcstart,numlon,numlat,numtimes,time_to_get,mask,sst,ice,sfcT,iceT,sfcTl,zorl,hice,ybegin,yend)
 
          class(fcst_nwp) :: this
 
@@ -276,11 +278,13 @@ subroutine read_nwp(this,filename,itype,wcstart,numlon,numlat,numtimes,time_to_g
          character(len=4), intent(in) :: wcstart
 
          integer, intent(in) :: time_to_get
+         integer, intent(in) :: ybegin,yend
          integer, intent(inout) :: numlon, numlat, numtimes
 !         real(r_single), intent(inout) :: mask(:,:), sst(:,:), ice(:,:), sfcT(:,:)
          real(r_kind), intent(inout) :: mask(:,:),sst(:,:),ice(:,:),sfcT(:,:) &
                                         ,iceT(:,:),sfcTl(:,:),zorl(:,:),hice(:,:)
 
+!
 !        Open the file using module_ncio.f90 code, and find the number of
 !        lat/lon points
 
@@ -289,12 +293,14 @@ subroutine read_nwp(this,filename,itype,wcstart,numlon,numlat,numtimes,time_to_g
          call ncdata%get_dim(this%dimnameNS,this%xlat)
          call ncdata%get_dim(this%dimnameTIME,this%xtime)
 
-         write(*,*) 'number of longitudes for file ', filename, this%xlon
+         write(6,*) 'number of longitudes for file ', filename, this%xlon
          numlon = this%xlon
-         write(*,*) 'number of latitudes for file ', filename, this%xlat
-         numlat = this%xlat
-         write(*,*) 'number of times for file ', filename, this%xtime
+         write(6,*) 'number of latitudes for file ', filename, this%xlat
+         !numlat = this%xlat
+         numlat = yend-ybegin+1
+         write(6,*) 'number of times for file ', filename, this%xtime
          numtimes = this%xtime
+         write(6,*) 'the range of Y for this domain is=',ybegin,yend
 
 !        Allocate all the arrays to receive data
          if (wcstart == 'cold' .OR. itype == ' FVCOM') then
@@ -309,47 +315,47 @@ subroutine read_nwp(this,filename,itype,wcstart,numlon,numlat,numtimes,time_to_g
 !        Get variables from the data file, but only if the variable is
 !        defined for that data type.
 
-            write(*,*) 'itype = ', itype
-            write(*,*) 'wcstart = ', wcstart
-            write(*,*) 'xlat = ', this%xlat
-            write(*,*) 'xlon = ', this%xlon
-            write(*,*) 'xtime = ', this%xtime
+            write(6,*) 'itype = ', itype
+            write(6,*) 'wcstart = ', wcstart
+            write(6,*) 'xlat = ', this%xlat
+            write(6,*) 'xlon = ', this%xlon
+            write(6,*) 'xtime = ', this%xtime
 
             if (this%i_mask .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_mask),this%xlon,  &
                                    this%xlat,this%nwp_mask_c)
-               mask = this%nwp_mask_c(:,:)
+               mask = this%nwp_mask_c(:,ybegin:yend)
             end if
             if (this%i_sst .gt. 0) then
-               write(*,*) 'get sst for cold or FVCOM'
+               write(6,*) 'get sst for cold or FVCOM'
                call ncdata%get_var(this%varnames(this%i_sst),this%xlon,  &
                                    this%xlat,this%xtime,this%nwp_sst_c)
-               sst = this%nwp_sst_c(:,:,time_to_get)
+               sst = this%nwp_sst_c(:,ybegin:yend,time_to_get)
             end if
             if (this%i_ice .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_ice),this%xlon,  &
                                    this%xlat,this%xtime,this%nwp_ice_c)
-               ice = this%nwp_ice_c(:,:,time_to_get)
+               ice = this%nwp_ice_c(:,ybegin:yend,time_to_get)
             end if
             if (this%i_sfcT .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_sfcT),this%xlon,  &
                                    this%xlat,this%xtime,this%nwp_sfcT_c)
-               sfcT = this%nwp_sfcT_c(:,:,time_to_get)
+               sfcT = this%nwp_sfcT_c(:,ybegin:yend,time_to_get)
             end if
             if (this%i_iceT .gt. 0) then
                 call ncdata%get_var(this%varnames(this%i_iceT),this%xlon,  &
                                     this%xlat,this%xtime,this%nwp_iceT_c)
-                iceT = this%nwp_iceT_c(:,:,time_to_get)
+                iceT = this%nwp_iceT_c(:,ybegin:yend,time_to_get)
             end if
             if (this%i_zorl .gt. 0) then
                 call ncdata%get_var(this%varnames(this%i_zorl),this%xlon,  &
                                     this%xlat,this%xtime,this%nwp_zorl_c)
-                zorl = this%nwp_zorl_c(:,:,time_to_get)
+                zorl = this%nwp_zorl_c(:,ybegin:yend,time_to_get)
             end if 
             if (this%i_hice .gt. 0) then
                 call ncdata%get_var(this%varnames(this%i_hice),this%xlon,  &
                                     this%xlat,this%xtime,this%nwp_hice_c)
-                hice = this%nwp_hice_c(:,:,time_to_get)
+                hice = this%nwp_hice_c(:,ybegin:yend,time_to_get)
             end if 
  
          else if (wcstart == 'warm') then
@@ -364,63 +370,63 @@ subroutine read_nwp(this,filename,itype,wcstart,numlon,numlat,numtimes,time_to_g
 !        Get variables from the data file, but only if the variable is
 !        defined for that data type.
       
-            write(*,*) 'itype = ', itype
-            write(*,*) 'wcstart =', wcstart
-            write(*,*) 'xlat = ', this%xlat
-            write(*,*) 'xlon = ', this%xlon
-            write(*,*) 'xtime = ', this%xtime
+            write(6,*) 'itype = ', itype
+            write(6,*) 'wcstart =', wcstart
+            write(6,*) 'xlat = ', this%xlat
+            write(6,*) 'xlon = ', this%xlon
+            write(6,*) 'xtime = ', this%xtime
 
             if (this%i_mask .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_mask),this%xlon,  &
                                    this%xlat,this%nwp_mask_w)
-               mask = this%nwp_mask_w(:,:)
+               mask = this%nwp_mask_w(:,ybegin:yend)
             end if
             if (this%i_sst .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_sst),this%xlon,  &
                                    this%xlat,this%nwp_sst_w)
-               sst = this%nwp_sst_w(:,:)
+               sst = this%nwp_sst_w(:,ybegin:yend)
             end if
             if (this%i_ice .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_ice),this%xlon,  &
                                    this%xlat,this%nwp_ice_w)
-               ice = this%nwp_ice_w(:,:)
+               ice = this%nwp_ice_w(:,ybegin:yend)
             end if
             if (this%i_sfcT .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_sfcT),this%xlon,  &
                                    this%xlat,this%nwp_sfcT_w)
-               sfcT = this%nwp_sfcT_w(:,:)
+               sfcT = this%nwp_sfcT_w(:,ybegin:yend)
             end if
             if (this%i_iceT .gt. 0) then
                 call ncdata%get_var(this%varnames(this%i_iceT),this%xlon,  &
                                     this%xlat,this%nwp_iceT_w)
-                iceT = this%nwp_iceT_w(:,:)
+                iceT = this%nwp_iceT_w(:,ybegin:yend)
             end if 
             if (this%i_sfcTl .gt. 0) then
                call ncdata%get_var(this%varnames(this%i_sfcTl),this%xlon,  &
                                    this%xlat,this%nwp_sfcTl_w)
-               sfcTl = this%nwp_sfcTl_w(:,:)
+               sfcTl = this%nwp_sfcTl_w(:,ybegin:yend)
             end if
             if (this%i_zorl .gt. 0) then
                 call ncdata%get_var(this%varnames(this%i_zorl),this%xlon,  &
                                     this%xlat,this%nwp_zorl_w)
-                zorl = this%nwp_zorl_w(:,:)
+                zorl = this%nwp_zorl_w(:,ybegin:yend)
             end if 
             if (this%i_hice .gt. 0) then
                 call ncdata%get_var(this%varnames(this%i_hice),this%xlon,  &
                                     this%xlat,this%nwp_hice_w)
-                hice = this%nwp_hice_w(:,:)
+                hice = this%nwp_hice_w(:,ybegin:yend)
             end if 
 
          else
-            write(*,*) 'Choose either "warm" or "cold" for file'
+            write(6,*) 'Choose either "warm" or "cold" for file'
             stop 'Error in wcstart. Check spelling or if variable was assigned'
          end if
 !        Close the netCDF file.
 
          call ncdata%close
 
-         write(*,*) 'Finished read_nwp'
-         write(*,*) ' '
+         write(6,*) 'Finished read_nwp'
+         write(6,*) ' '
 
       end subroutine read_nwp
 
@@ -452,33 +458,34 @@ subroutine finish_nwp(this,itype,wcstart)
             deallocate(this%nwp_iceT_c)
             deallocate(this%nwp_zorl_c)
             deallocate(this%nwp_hice_c)
+            if (itype==' FVCOM') deallocate(this%dimnameDATE)
          else if (wcstart == 'warm') then
             deallocate(this%nwp_mask_w)
             deallocate(this%nwp_sst_w)
             deallocate(this%nwp_ice_w)
             deallocate(this%nwp_sfcT_w)
             deallocate(this%nwp_iceT_w)
+            deallocate(this%nwp_sfcTl_w)
             deallocate(this%nwp_zorl_w)
             deallocate(this%nwp_hice_w)
          else
-            write(*,*) 'no deallocation'
+            write(6,*) 'no deallocation'
          end if
 
          thisobs => this%head
          if(.NOT.associated(thisobs)) then
-            write(*,*) 'No memory to release'
+            write(6,*) 'No memory to release'
             return
          endif
          do while(associated(thisobs))
-!            write(*,*) 'destroy ==',thisobs%name
 
             thisobsnext => thisobs%next
             call thisobs%destroy()
             thisobs => thisobsnext
          enddo
 
-         write(*,*) 'Finished finish_nwp'
-         write(*,*) ' '
+         write(6,*) 'Finished finish_nwp'
+         write(6,*) ' '
 
       end subroutine finish_nwp
 
@@ -508,8 +515,8 @@ subroutine get_time_ind_nwp(this,filename,instr,outindex)
          call ncdata%open(trim(filename),'r',200)
          call ncdata%get_dim(this%dimnameTIME,this%xtime)
          call ncdata%get_dim(this%dimnameDATE,this%datelen)
-         write(*,*) 'xtime = ', this%xtime
-         write(*,*) 'datelen = ', this%datelen
+         write(6,*) 'xtime = ', this%xtime
+         write(6,*) 'datelen = ', this%datelen
          allocate(this%times(this%datelen,this%xtime))
          call ncdata%get_var('Times',this%datelen,this%xtime,this%times)
 
@@ -528,8 +535,8 @@ subroutine get_time_ind_nwp(this,filename,instr,outindex)
             outindex = -999
             deallocate(this%times)
             call ncdata%close
-            write(*,*) 'WARNING: Supplied time not found in file: ', trim(instr)
-            write(*,*) 'Stoppping fvcom_to_FV3 and proceeding without using FVCOM data'
+            write(6,*) 'WARNING: Supplied time not found in file: ', trim(instr)
+            write(6,*) 'Stoppping fvcom_to_FV3 and proceeding without using FVCOM data'
             stop
          end if
 
diff --git a/sorc/fvcom_tools.fd/module_nwp_base.f90 b/sorc/fvcom_tools.fd/module_nwp_base.f90
index 382826458..42214f262 100644
--- a/sorc/fvcom_tools.fd/module_nwp_base.f90
+++ b/sorc/fvcom_tools.fd/module_nwp_base.f90
@@ -60,7 +60,7 @@ subroutine list_obsbase(this)
 
 !        Write out the lon, lat, and time of the ob
 
-         write(*,'(a,3f10.3)') 'LIGHTNING OB: longitude, latitude, time =', &
+         write(6,'(a,3f10.3)') 'LIGHTNING OB: longitude, latitude, time =', &
             this%lon, this%lat, this%time
 
 !        Loop through all variables and print out obs and quality
@@ -68,11 +68,11 @@ subroutine list_obsbase(this)
          numvar = this%numvar
          if (numvar >= 1) then
 ! MULTI-DIMENSIONAL EXAMPLE IN  module_obs_base.f90
-            write(*,'(a4,10F12.2)') 'obs=', (this%obs(i),i=1,numvar)
+            write(6,'(a4,10F12.2)') 'obs=', (this%obs(i),i=1,numvar)
             if(this%ifquality) &
-            write(*,'(a4,10I12)') 'qul=', (this%quality(i),i=1,numvar)
+            write(6,'(a4,10I12)') 'qul=', (this%quality(i),i=1,numvar)
          else
-            write(*,*) 'No obs for this location'
+            write(6,*) 'No obs for this location'
          endif
 
       end subroutine list_obsbase
@@ -105,7 +105,7 @@ subroutine alloc_obsbase(this,numvar,ifquality)
             if(this%ifquality) allocate(this%quality(numvar))
 
          else
-            write(*,*) 'alloc_obsbase Error: dimension must be larger than 0:', numvar
+            write(6,*) 'alloc_obsbase Error: dimension must be larger than 0:', numvar
             stop 1234
          endif
 
diff --git a/sorc/fvcom_tools.fd/process_FVCOM.f90 b/sorc/fvcom_tools.fd/process_FVCOM.f90
index a9a017717..c82121927 100755
--- a/sorc/fvcom_tools.fd/process_FVCOM.f90
+++ b/sorc/fvcom_tools.fd/process_FVCOM.f90
@@ -39,7 +39,8 @@ program process_FVCOM
    implicit none
 
 ! MPI variables
-  integer :: npe, mype, mypeLocal,ierror
+   integer :: npe, mype, mypeLocal,ierror
+   integer,allocatable :: mpi_layout_begin(:),mpi_layout_end(:)
 !
 
 !  New object-oriented declarations
@@ -49,7 +50,7 @@ program process_FVCOM
 
 !  Grid variables
 
-   character*180 :: geofile
+   character*180 :: thisfv3file
    character*2 :: workPath
    character*1 :: char1
 
@@ -73,6 +74,8 @@ program process_FVCOM
    character(len=180) :: wcstart
    character(len=180) :: inputFVCOMselStr
    character(len=180), dimension(:), allocatable :: args
+   integer            :: fv3_io_layout_y
+   integer,allocatable :: fv3_layout_begin(:),fv3_layout_end(:)
 
    real(r_kind), allocatable :: fv3ice(:,:), fv3sst(:,:)
    real(r_kind), allocatable :: fv3sfcT(:,:), fv3mask(:,:)
@@ -87,18 +90,21 @@ program process_FVCOM
 !  SETUP (general control namelist) :
 
    integer :: update_type
+   character(len=4) :: clayout_y
+   integer :: iy,iblock
 
 !   namelist/setup/update_type, t2
 
 ! MPI setup
-  call MPI_INIT(ierror)
-  call MPI_COMM_SIZE(mpi_comm_world,npe,ierror)
-  call MPI_COMM_RANK(mpi_comm_world,mype,ierror)
+   call MPI_INIT(ierror)
+   call MPI_COMM_SIZE(mpi_comm_world,npe,ierror)
+   call MPI_COMM_RANK(mpi_comm_world,mype,ierror)
 !
 ! NCEP LSF has to use all cores allocated to run this application 
 ! but this if check can make sure only one core run through the real code.
 !
-if(mype==0) then
+
+   write(*,*) 'number of cores=',npe
 
    zero = 0.0
 !  Get file names from command line arguements
@@ -113,99 +119,180 @@ program process_FVCOM
 ! wcstart: warm (restart) or cold start
 ! inputFVCOMtimes: string of time to use
    fv3file=trim(args(1))
-   write(*,*) trim(fv3file)
+   write(*,*) 'surface file=',trim(fv3file)
    fvcomfile=trim(args(2))    
-   write(*,*) trim(fvcomfile)
+   write(*,*) 'fvcom file=',trim(fvcomfile)
    wcstart=trim(args(3))
-   write(*,*) 'warm or cold start = ', wcstart
+   write(*,*) 'warm or cold start = ', trim(wcstart)
    inputFVCOMselStr=trim(args(4))
 !   read(inputFVCOMselStr,*) inputFVCOMsel
-   write(*,*) 'select time = ', inputFVCOMselStr
-
-!  Obtain grid parameters
+   write(*,*) 'select time = ', trim(inputFVCOMselStr)
+   clayout_y=trim(args(5))
+   read(clayout_y,'(I2)') fv3_io_layout_y
+   if (wcstart == 'cold') then
+     fv3_io_layout_y=1
+   endif
+   write(*,*) 'subdomain number=',fv3_io_layout_y
+   if(mype < fv3_io_layout_y) then
+      write(thisfv3file,'(a,I3.3)') 'stdout_fvcom.',mype
+      open(6, file=trim(thisfv3file),form='formatted',status='unknown')
+      write(6,*) '===> start process for core = ', mype
+   endif
+! 
+! figure out subdomain grid begin and end grid index in Y for each core  
+! The match the subdomains to each core
+!
+   allocate(fv3_layout_begin(fv3_io_layout_y))
+   allocate(fv3_layout_end(fv3_io_layout_y))
+   allocate(mpi_layout_begin(npe))
+   allocate(mpi_layout_end(npe))
+   fv3_layout_begin=0
+   fv3_layout_end=0
 
    workPath='./'
-   write(geofile,'(a,a)') trim(workPath), trim(fv3file)
-   write(*,*) 'sfc data file', trim(geofile)
-   call geo%open(trim(geofile),'r',200)
-   call geo%get_dim("xaxis_1",NLON)
-   call geo%get_dim("yaxis_1",NLAT)
-   write(*,*) 'NLON,NLAT:', NLON, NLAT
-   call geo%close
-
-   write(*,*) 'Finished reading sfc_data grid information.'
-   write(*,*) ' '
+   iy=0
+   do ix=1,fv3_io_layout_y
+      if(fv3_io_layout_y > 1) then
+          write(thisfv3file,'(a,a,a1,I4.4)') trim(workPath), trim(fv3file),'.',ix-1
+      else
+          write(thisfv3file,'(a,a)') trim(workPath), trim(fv3file)
+      endif
+      call geo%open(trim(thisfv3file),'r',200)
+      call geo%get_dim("yaxis_1",NLAT)
+      call geo%close
+      fv3_layout_begin(ix)=iy+1
+      fv3_layout_end(ix)=iy+nlat
+      iy=fv3_layout_end(ix)
+   enddo  ! find dimension
+   write(6,'(a,20I5)') 'begin index for each subdomain',fv3_layout_begin
+   write(6,'(a,20I5)') '  end index for each subdomain',fv3_layout_end
+
+   mypeLocal=mype+1
+   if(npe==1) then
+      mpi_layout_begin(mypeLocal)=1
+      mpi_layout_end(mypeLocal)=fv3_io_layout_y
+   else
+      mpi_layout_begin=0
+      mpi_layout_end=0
+      do ix=1,fv3_io_layout_y
+         iy=mod(ix-1,npe)+1
+         mpi_layout_end(iy)=mpi_layout_end(iy)+1
+      enddo
+      iy=0
+      do ix=1,npe
+         if(mpi_layout_end(ix) > 0) then
+           mpi_layout_begin(ix)=iy+1  
+           mpi_layout_end(ix)=iy+mpi_layout_end(ix)
+           iy=mpi_layout_end(ix)
+         else
+           mpi_layout_begin(ix)=0
+           mpi_layout_end(ix)=0
+         endif
+      enddo 
+   endif
+   write(6,'(a)') 'begin and end domain index for each core:'
+   write(6,'(20I5)') mpi_layout_begin
+   write(6,'(20I5)') mpi_layout_end
+
+if(mypeLocal <= fv3_io_layout_y) then
+
+   do ix=mpi_layout_begin(mypeLocal),mpi_layout_end(mypeLocal)
+
+      write(6,*) 'process subdomain ',ix,' with core ',mype
+      if(fv3_io_layout_y > 1) then
+          write(thisfv3file,'(a,a,a1,I4.4)') trim(workPath), trim(fv3file),'.',ix-1
+      else
+          write(thisfv3file,'(a,a)') trim(workPath), trim(fv3file)
+      endif
+      write(6,*) 'sfc data file', trim(thisfv3file)
+
+      call geo%open(trim(thisfv3file),'r',200)
+      call geo%get_dim("xaxis_1",NLON)
+      call geo%get_dim("yaxis_1",NLAT)
+      write(6,*) 'NLON,NLAT:', NLON, NLAT
+      call geo%close
+
+      write(6,*) 'Finished reading sfc_data grid information.'
+      write(6,*) ' '
+!
+!  figure out subdomain grid dimension in Y
+!
+      write(6,*) 'the Y dimensions=',fv3_layout_begin(ix),fv3_layout_end(ix)
 
 !  Allocate variables for I/O
 
-   allocate(fv3ice(nlon,nlat))
-   allocate(fv3sfcT(nlon,nlat))
-   allocate(fv3sst(nlon,nlat))
-   allocate(fv3mask(nlon,nlat))
-   allocate(fv3iceT(nlon,nlat))
-   allocate(fv3sfcTl(nlon,nlat))
-   allocate(fv3zorl(nlon,nlat))
-   allocate(fv3hice(nlon,nlat))
-
-   allocate(lbcice(nlon,nlat))
-   allocate(lbcsfcT(nlon,nlat))
-   allocate(lbcsst(nlon,nlat))
-   allocate(lbcmask(nlon,nlat))
-   allocate(lbciceT(nlon,nlat))
-   allocate(lbczorl(nlon,nlat))
-   allocate(lbchice(nlon,nlat))
+      allocate(fv3ice(nlon,nlat))
+      allocate(fv3sfcT(nlon,nlat))
+      allocate(fv3sst(nlon,nlat))
+      allocate(fv3mask(nlon,nlat))
+      allocate(fv3iceT(nlon,nlat))
+      allocate(fv3sfcTl(nlon,nlat))
+      allocate(fv3zorl(nlon,nlat))
+      allocate(fv3hice(nlon,nlat))
+
+      allocate(lbcice(nlon,nlat))
+      allocate(lbcsfcT(nlon,nlat))
+      allocate(lbcsst(nlon,nlat))
+      allocate(lbcmask(nlon,nlat))
+      allocate(lbciceT(nlon,nlat))
+      allocate(lbczorl(nlon,nlat))
+      allocate(lbchice(nlon,nlat))
 !  Read fv3 sfc_data.nc before update
 
 !   fv3file='sfc_data.nc'
 !   fv3times: length of time dimension of UFS atmospheric grid (should be 1)
 !   t1: index of time dimension to pull (should be 1)
-   fv3times=1  
-   t1=1
+      fv3times=1  
+      t1=1
 
-   call fcst%initial('FV3LAM',wcstart)
-   call fcst%list_initial
-   call fcst%read_n(trim(fv3file),'FV3LAM',wcstart,fv3lon,fv3lat,fv3times,t1,fv3mask,fv3sst,fv3ice,fv3sfcT,fv3iceT,fv3sfcTl,fv3zorl,fv3hice)
-   call fcst%finish('FV3LAM',wcstart)
+      call fcst%initial('FV3LAM',wcstart)
+      call fcst%list_initial
+      call fcst%read_n(trim(thisfv3file),'FV3LAM',wcstart,fv3lon,fv3lat,fv3times,t1,&
+                   fv3mask,fv3sst,fv3ice,fv3sfcT,fv3iceT,fv3sfcTl,fv3zorl,fv3hice,  &
+                   1,nlat)
+      call fcst%finish('FV3LAM',wcstart)
 
 
-   write(*,*) 'fv3times: ', fv3times
-   write(*,*) 'time to use: ', t1
+      write(6,*) 'fv3times: ', fv3times
+      write(6,*) 'time to use: ', t1
 
 !  Read FVCOM input datasets
 
 !   fvcomfile='fvcom.nc'
 !   lbctimes: length of time dimension of FVCOM input data (command line input)
 ! Space infront of ' FVCOM' below is important!!
-   call fcst%initial(' FVCOM',wcstart)
-   call fcst%list_initial
-   call fcst%get_time_ind(trim(fvcomfile),inputFVCOMselStr,indexFVCOMsel)
+      call fcst%initial(' FVCOM',wcstart)
+      call fcst%list_initial
+      call fcst%get_time_ind(trim(fvcomfile),inputFVCOMselStr,indexFVCOMsel)
 !   t2: index of time dimension to pull from FVCOM
-   t2=indexFVCOMsel
-   write(*,*) 'time asked for =', trim(inputFVCOMselStr)
-   write(*,*) 'time index selected = ', t2
-   call fcst%read_n(trim(fvcomfile),' FVCOM',wcstart,lbclon,lbclat,lbctimes,t2,lbcmask,lbcsst,lbcice,lbcsfcT,lbciceT,fv3sfcTl,lbczorl,lbchice)
-   call fcst%finish(' FVCOM',wcstart)
+      t2=indexFVCOMsel
+      write(6,*) 'time asked for =', trim(inputFVCOMselStr)
+      write(6,*) 'time index selected = ', t2
+      call fcst%read_n(trim(fvcomfile),' FVCOM',wcstart,lbclon,lbclat,lbctimes,t2, &
+                   lbcmask,lbcsst,lbcice,lbcsfcT,lbciceT,fv3sfcTl,lbczorl,lbchice, &
+                   fv3_layout_begin(ix),fv3_layout_end(ix))
+      call fcst%finish(' FVCOM',wcstart)
 
 !  Check that the dimensions match
 
-   if (lbclon .ne. nlon .or. lbclat .ne. nlat) then
-     write(*,*) 'ERROR: FVCOM/FV3 dimensions do not match:'
-     write(*,*) 'lbclon: ', lbclon
-     write(*,*) 'nlon: ', nlon
-     write(*,*) 'lbclat: ', lbclat
-     write(*,*) 'nlat: ', nlat
-     stop 'error'
-   endif
-
-   write(*,*) 'lbctimes: ', lbctimes
-   write(*,*) 'time to use: ', t2
-
-   if (t2 .gt. lbctimes) then
-     write(*,*) 'ERROR: Requested time dimension out of range'
-     write(*,*) 'Length of time dimension: ', lbctimes
-     write(*,*) 'Time index to use: ', t2
-     stop 'error'
-   endif
+      if (lbclon .ne. nlon .or. lbclat .ne. nlat) then
+        write(6,*) 'ERROR: FVCOM/FV3 dimensions do not match:'
+        write(6,*) 'lbclon: ', lbclon
+        write(6,*) 'nlon: ', nlon
+        write(6,*) 'lbclat: ', lbclat
+        write(6,*) 'nlat: ', nlat
+        stop 'error'
+      endif
+
+      write(6,*) 'lbctimes: ', lbctimes
+      write(6,*) 'time to use: ', t2
+
+      if (t2 .gt. lbctimes) then
+        write(6,*) 'ERROR: Requested time dimension out of range'
+        write(6,*) 'Length of time dimension: ', lbctimes
+        write(6,*) 'Time index to use: ', t2
+        stop 'error'
+      endif
 
 !  Update with FVCOM fields and process
 !   ice cover data. Ice fraction is set
@@ -276,33 +363,53 @@ program process_FVCOM
         enddo
      enddo
    else
-     write(*,*) 'Variable wcstart is not set to either warm or cold'
+     write(6,*) 'Variable wcstart is not set to either warm or cold'
    end if
 
 ! Write out sfc file again
 
-   call geo%open(trim(fv3file),'w',300)
-   call geo%replace_var("tsea",NLON,NLAT,fv3sst)
-   call geo%replace_var("fice",NLON,NLAT,fv3ice)
-   call geo%replace_var("slmsk",NLON,NLAT,fv3mask)
-   call geo%replace_var("tisfc",NLON,NLAT,fv3iceT)
-   call geo%replace_var("hice",NLON,NLAT,fv3hice)
-   if (wcstart == 'cold') then
+      write(6,*) "udpate file=",trim(thisfv3file)
+      call geo%open(trim(thisfv3file),'w',300)
+      call geo%replace_var("tsea",NLON,NLAT,fv3sst)
+      call geo%replace_var("fice",NLON,NLAT,fv3ice)
+      call geo%replace_var("slmsk",NLON,NLAT,fv3mask)
+      call geo%replace_var("tisfc",NLON,NLAT,fv3iceT)
+      call geo%replace_var("hice",NLON,NLAT,fv3hice)
+      if (wcstart == 'cold') then
 ! Add_New_Var takes names of (Variable,Dim1,Dim2,Dim3,Long_Name,Units)
-      call geo%replace_var("zorl",NLON,NLAT,fv3zorl)
-      call geo%add_new_var('glmsk','xaxis_1','yaxis_1','Time','glmsk','none')
-      call geo%replace_var('glmsk',NLON,NLAT,lbcmask)
-   end if
-   if (wcstart == 'warm') then
-      call geo%replace_var("zorli",NLON,NLAT,fv3zorl)
-      call geo%replace_var("tsfc",NLON,NLAT,fv3sfcT)
-      call geo%replace_var("tsfcl",NLON,NLAT,fv3sfcTl)
-      call geo%add_new_var('glmsk','xaxis_1','yaxis_1','glmsk','none')
-      call geo%replace_var('glmsk',NLON,NLAT,lbcmask)
-   end if
-   call geo%close
-
-   write(6,*) "=== LOWBC UPDATE SUCCESS ==="
+         call geo%replace_var("zorl",NLON,NLAT,fv3zorl)
+         call geo%add_new_var('glmsk','xaxis_1','yaxis_1','Time','glmsk','none')
+         call geo%replace_var('glmsk',NLON,NLAT,lbcmask)
+      end if
+      if (wcstart == 'warm') then
+         call geo%replace_var("zorli",NLON,NLAT,fv3zorl)
+         call geo%replace_var("tsfc",NLON,NLAT,fv3sfcT)
+         call geo%replace_var("tsfcl",NLON,NLAT,fv3sfcTl)
+         call geo%add_new_var('glmsk','xaxis_1','yaxis_1','Time','glmsk','none')
+         call geo%replace_var('glmsk',NLON,NLAT,lbcmask)
+      end if
+      call geo%close
+
+      deallocate(fv3ice)
+      deallocate(fv3sfcT)
+      deallocate(fv3sst)
+      deallocate(fv3mask)
+      deallocate(fv3iceT)
+      deallocate(fv3sfcTl)
+      deallocate(fv3zorl)
+      deallocate(fv3hice)
+
+      deallocate(lbcice)
+      deallocate(lbcsfcT)
+      deallocate(lbcsst)
+      deallocate(lbcmask)
+      deallocate(lbciceT)
+      deallocate(lbczorl)
+      deallocate(lbchice)
+
+      write(6,*) "=== LOWBC UPDATE SUCCESS ===",ix
+
+   enddo ! loop through subdomain
 
 endif ! mype==0
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 30fefaa32..894b01b5f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -4,6 +4,7 @@
 # Ed Hartnett 2/11/21
 
 # Add the test subdirecotries.
+#add_subdirectory(fvcom_tools)
 add_subdirectory(filter_topo)
 add_subdirectory(chgres_cube)
 add_subdirectory(fre-nctools)
diff --git a/tests/fvcom_tools/ftst_readfvcomnetcdf.F90 b/tests/fvcom_tools/ftst_readfvcomnetcdf.F90
index 0f8f68e2c..d43fb1f7f 100644
--- a/tests/fvcom_tools/ftst_readfvcomnetcdf.F90
+++ b/tests/fvcom_tools/ftst_readfvcomnetcdf.F90
@@ -31,29 +31,36 @@ program readfvcomnetcdf
  integer :: fv3sst_expected(NUM_VALUES) !expected fv3 sst values
  real :: fv3ice_expected(NUM_VALUES) !expected fv3 ice values
  real :: fv3iceT_expected(NUM_VALUES) !expected fv3 ice temp values
+ real :: fv3zorl_expected(NUM_VALUES) !expected fv3 surface roughness values
+ real :: fv3hice_expected(NUM_VALUES) !exepcted fv3 ice thickness values
  integer :: lbcmask_expected(NUM_VALUES) !expected fvcom mask values
  real :: lbcsst_expected(NUM_VALUES) !expected fvcom sst values
  real :: lbcice_expected(NUM_VALUES) !expected fvcom ice values
  real :: lbciceT_expected(NUM_VALUES) !expected fvcom ice temp values
+ real :: lbcvice_expected(NUM_VALUES) !expected fvcom ice thickness values
 
 ! Create allocabable arrays to read from .nc files
  real, allocatable :: fv3ice(:,:), fv3sst(:,:)
  real, allocatable :: fv3sfcT(:,:), fv3mask(:,:)
  real, allocatable :: fv3iceT(:,:), fv3sfcTl(:,:)
+ real, allocatable :: fv3zorl(:,:), fv3hice(:,:)
  real, allocatable :: lbcice(:,:), lbcsst(:,:)
  real, allocatable :: lbcsfcT(:,:), lbcmask(:,:)
- real, allocatable :: lbciceT(:,:)
-
+ real, allocatable :: lbciceT(:,:), lbchice(:,:)
+ real, allocatable :: lbczorl(:,:)
 ! Expected values from the dummy files
  data lat_lon_expected_values /5, 5/
  data fv3mask_expected /1, 0/
  data fv3sst_expected /1, 0/
  data fv3ice_expected /.1, 0/
  data fv3iceT_expected /.1, 0/
+ data fv3zorl_expected /1.1, 0/
+ data fv3hice_expected /.1, 0/
  data lbcmask_expected /1, 0/
  data lbcsst_expected /1, -99.99999/
  data lbcice_expected /1, -99.99999/
  data lbciceT_expected /1, -99.99999/
+ data lbcvice_expected /1, -99.99999/
  data t2_expected /2 / !expect second time index from fvcom file
 
  type(ncio) :: geo !grid data object
@@ -93,16 +100,20 @@ program readfvcomnetcdf
  allocate(fv3mask(nlon,nlat))
  allocate(fv3iceT(nlon,nlat))
  allocate(fv3sfcTl(nlon,nlat))
+ allocate(fv3zorl(nlon,nlat))
+ allocate(fv3hice(nlon,nlat))
 
  allocate(lbcice(nlon,nlat))
  allocate(lbcsfcT(nlon,nlat))
  allocate(lbcsst(nlon,nlat))
  allocate(lbcmask(nlon,nlat))
  allocate(lbciceT(nlon,nlat))
+ allocate(lbczorl(nlon,nlat))
+ allocate(lbchice(nlon,nlat))
 
 !Initialize and read in fv3 sfc data
  call fcst%initial('FV3LAM',wcstart)
- call fcst%read_n(trim(fv3file),'FV3LAM',wcstart,fv3lon,fv3lat,fv3times,t1,fv3mask,fv3sst,fv3ice,fv3sfcT,fv3iceT,fv3sfcTl)
+ call fcst%read_n(trim(fv3file),'FV3LAM',wcstart,fv3lon,fv3lat,fv3times,t1,fv3mask,fv3sst,fv3ice,fv3sfcT,fv3iceT,fv3sfcTl,fv3zorl,fv3hice,1,nlat)
  call fcst%finish('FV3LAM',wcstart)
 !If variables in fv3 sfc file do not match expected, stop
  if (abs(fv3mask(1,1) - fv3mask_expected(1)) > EPSILON) stop 5
@@ -117,27 +128,35 @@ program readfvcomnetcdf
  if (abs(fv3iceT(1,1) - fv3iceT_expected(1)) > EPSILON) stop 9
  if (abs(fv3iceT(5,5) - fv3iceT_expected(2)) > EPSILON) stop 10
 
+ if (abs(fv3zorl(1,1) - fv3zorl_expected(1)) > EPSILON) stop 11
+ if (abs(fv3zorl(5,5) - fv3zorl_expected(2)) > EPSILON) stop 12
+
+ if (abs(fv3hice(1,1) - fv3hice_expected(1)) > EPSILON) stop 13
+ if (abs(fv3hice(5,5) - fv3hice_expected(2)) > EPSILON) stop 14
+
 !Initialize and read in fvcom data
  call fcst%initial(' FVCOM',wcstart)
  call fcst%get_time_ind(trim(fvcomfile),inputFVCOMselStr,t2)
 !If second time index is not returned, stop
- if  (abs(t2 - t2_expected) > EPSILON) stop 11
- call fcst%read_n(trim(fvcomfile),' FVCOM',wcstart,lbclon,lbclat,lbctimes,t2,lbcmask,lbcsst,lbcice,lbcsfcT,lbciceT,fv3sfcTl)
+ if  (abs(t2 - t2_expected) > EPSILON) stop 15
+ call fcst%read_n(trim(fvcomfile),' FVCOM',wcstart,lbclon,lbclat,lbctimes,t2,lbcmask,lbcsst,lbcice,lbcsfcT,lbciceT,fv3sfcTl,lbczorl,lbchice,1,nlat)
  call fcst%finish(' FVCOM',wcstart)
 !If variables in fvcom file do not match expected, stop
- if (abs(lbcmask(1,1) - lbcmask_expected(1)) > EPSILON) stop 12
- if (abs(lbcmask(5,5) - lbcmask_expected(2)) > EPSILON) stop 13
+ if (abs(lbcmask(1,1) - lbcmask_expected(1)) > EPSILON) stop 16
+ if (abs(lbcmask(5,5) - lbcmask_expected(2)) > EPSILON) stop 17
 
- if (abs(lbcsst(1,1) - lbcsst_expected(1)) > EPSILON) stop 14
- if (abs(lbcsst(5,5) - lbcsst_expected(2)) > EPSILON) stop 15
+ if (abs(lbcsst(1,1) - lbcsst_expected(1)) > EPSILON) stop 18
+ if (abs(lbcsst(5,5) - lbcsst_expected(2)) > EPSILON) stop 19
  
- if (abs(lbcice(1,1) - lbcice_expected(1)) > EPSILON) stop 16
- if (abs(lbcice(5,5) - lbcice_expected(2)) > EPSILON) stop 17
+ if (abs(lbcice(1,1) - lbcice_expected(1)) > EPSILON) stop 20
+ if (abs(lbcice(5,5) - lbcice_expected(2)) > EPSILON) stop 21
 
- if (abs(lbciceT(1,1) - lbciceT_expected(1)) > EPSILON) stop 18
- if (abs(lbciceT(5,5) - lbciceT_expected(2)) > EPSILON) stop 19
+ if (abs(lbciceT(1,1) - lbciceT_expected(1)) > EPSILON) stop 22
+ if (abs(lbciceT(5,5) - lbciceT_expected(2)) > EPSILON) stop 23
+
+ if (abs(lbchice(1,1) - lbcvice_expected(1)) > EPSILON) stop 24
+ if (abs(lbchice(5,5) - lbcvice_expected(2)) > EPSILON) stop 25
 
- 
  print*,"OK"
 
  print*,"SUCCESS!"