diff --git a/.gitignore b/.gitignore index f32b5b0ea..2a2a5fdb6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ doc/build # Ignore macOS cache files .DS_Store + # Ignore testsuite file/directories testsuite* caselist* diff --git a/cice.setup b/cice.setup index d70aece61..f7ff12bd6 100755 --- a/cice.setup +++ b/cice.setup @@ -504,11 +504,18 @@ EOF0 cat >! ${tsdir}/report_codecov.csh << EOF0 #!/bin/csh -f -#setenv CODECOV_TOKEN "1d09241f-ed9e-47d8-847c-038bab024b53" # consortium cice -#setenv CODECOV_TOKEN "f3236008-0b92-4707-9ad5-ad906f5d2ba7" # apcraig cice -setenv CODECOV_TOKEN "0dcc6066-fdce-47b6-b84a-c55e2a0af4c0" # apcraig test_cice_icepack -set report_name = "${shhash}:${branch}:${machine} ${testsuite}" +# define CODECOV_TOKEN env variable +if !(\$?CODECOV_TOKEN) then + if (-e ~/.codecov_cice_token) then + source ~/.codecov_cice_token + endif + if !(\$?CODECOV_TOKEN) then + echo "\${0}: ERROR, CODECOV_TOKEN env variable not defined" + exit 2 + endif +endif +set report_name = "${shhash}:${branch}:${machine} ${testsuite}" set use_curl = 1 EOF0 diff --git a/cicecore/cicedynB/general/ice_forcing.F90 b/cicecore/cicedynB/general/ice_forcing.F90 index 64f4b4834..52d4124b4 100755 --- a/cicecore/cicedynB/general/ice_forcing.F90 +++ b/cicecore/cicedynB/general/ice_forcing.F90 @@ -116,7 +116,7 @@ module ice_forcing ocn_data_format, & ! 'bin'=binary or 'nc'=netcdf atm_data_type, & ! 'default', 'monthly', 'ncar', ! 'LYq' or 'hadgem' or 'oned' or - ! 'JRA55_gx1' or 'JRA55_gx3' + ! 'JRA55_gx1' or 'JRA55_gx3' or 'JRA55_tx1' bgc_data_type, & ! 'default', 'clim' ocn_data_type, & ! 'default', 'clim', 'ncar', 'oned', ! 'hadgem_sst' or 'hadgem_sst_uvocn' @@ -241,6 +241,7 @@ subroutine init_forcing_atmo if (use_leap_years .and. (trim(atm_data_type) /= 'JRA55_gx1' .and. & trim(atm_data_type) /= 'JRA55_gx3' .and. & + trim(atm_data_type) /= 'JRA55_tx1' .and. & trim(atm_data_type) /= 'hycom' .and. & trim(atm_data_type) /= 'box2001')) then write(nu_diag,*) 'use_leap_years option is currently only supported for' @@ -261,6 +262,8 @@ subroutine init_forcing_atmo call JRA55_gx1_files(fyear) elseif (trim(atm_data_type) == 'JRA55_gx3') then call JRA55_gx3_files(fyear) + elseif (trim(atm_data_type) == 'JRA55_tx1') then + call JRA55_tx1_files(fyear) elseif (trim(atm_data_type) == 'hadgem') then call hadgem_files(fyear) elseif (trim(atm_data_type) == 'monthly') then @@ -560,6 +563,8 @@ subroutine get_forcing_atmo call JRA55_data(fyear) elseif (trim(atm_data_type) == 'JRA55_gx3') then call JRA55_data(fyear) + elseif (trim(atm_data_type) == 'JRA55_tx1') then + call JRA55_data(fyear) elseif (trim(atm_data_type) == 'hadgem') then call hadgem_data elseif (trim(atm_data_type) == 'monthly') then @@ -1405,6 +1410,10 @@ subroutine file_year (data_file, yr) i = index(data_file,'.nc') - 5 tmpname = data_file write(data_file,'(a,i4.4,a)') tmpname(1:i), yr, '.nc' + elseif (trim(atm_data_type) == 'JRA55_tx1') then ! netcdf + i = index(data_file,'.nc') - 5 + tmpname = data_file + write(data_file,'(a,i4.4,a)') tmpname(1:i), yr, '.nc' else ! LANL/NCAR naming convention i = index(data_file,'.dat') - 5 tmpname = data_file @@ -2047,6 +2056,22 @@ subroutine JRA55_gx1_files(yr) write (nu_diag,*) trim(uwind_file) endif end subroutine JRA55_gx1_files + subroutine JRA55_tx1_files(yr) +! + integer (kind=int_kind), intent(in) :: & + yr ! current forcing year + + character(len=*), parameter :: subname = '(JRA55_tx1_files)' + + uwind_file = & + trim(atm_data_dir)//'/8XDAILY/JRA55_03hr_forcing_tx1_2005.nc' + call file_year(uwind_file,yr) + if (my_task == master_task) then + write (nu_diag,*) ' ' + write (nu_diag,*) 'Atmospheric data files:' + write (nu_diag,*) trim(uwind_file) + endif + end subroutine JRA55_tx1_files subroutine JRA55_gx3_files(yr) ! integer (kind=int_kind), intent(in) :: & diff --git a/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 b/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 index 43e5dab74..23baeb40a 100644 --- a/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 @@ -53,7 +53,7 @@ subroutine ice_write_hist (ns) dxu, dxt, dyu, dyt, HTN, HTE, ANGLE, ANGLET, tmask, & lont_bounds, latt_bounds, lonu_bounds, latu_bounds use ice_history_shared - use ice_arrays_column, only: hin_max + use ice_arrays_column, only: hin_max, floe_rad_c use ice_restart_shared, only: runid, lcdf64 use netcdf #endif @@ -67,13 +67,13 @@ subroutine ice_write_hist (ns) #ifdef ncdf integer (kind=int_kind) :: i,j,k,ic,n,nn, & ncid,status,imtid,jmtid,kmtidi,kmtids,kmtidb, cmtid,timid, & - length,nvertexid,ivertex,kmtida + length,nvertexid,ivertex,kmtida,fmtid integer (kind=int_kind), dimension(2) :: dimid2 integer (kind=int_kind), dimension(3) :: dimid3 integer (kind=int_kind), dimension(4) :: dimidz integer (kind=int_kind), dimension(5) :: dimidcz integer (kind=int_kind), dimension(3) :: dimid_nverts - integer (kind=int_kind), dimension(5) :: dimidex + integer (kind=int_kind), dimension(6) :: dimidex real (kind=real_kind) :: ltime real (kind= dbl_kind) :: ltime2 character (char_len) :: title @@ -89,7 +89,8 @@ subroutine ice_write_hist (ns) type(file_desc_t) :: File type(io_desc_t) :: iodesc2d, & iodesc3dc, iodesc3dv, iodesc3di, iodesc3db, iodesc3da, & - iodesc4di, iodesc4ds + iodesc3df, & + iodesc4di, iodesc4ds, iodesc4df type(var_desc_t) :: varid ! 4 coordinate variables: TLON, TLAT, ULON, ULAT @@ -168,9 +169,11 @@ subroutine ice_write_hist (ns) call ice_pio_initdecomp(ndim3=nzilyr, iodesc=iodesc3di) call ice_pio_initdecomp(ndim3=nzblyr, iodesc=iodesc3db) call ice_pio_initdecomp(ndim3=nzalyr, iodesc=iodesc3da) + call ice_pio_initdecomp(ndim3=nfsd_hist, iodesc=iodesc3df) call ice_pio_initdecomp(ndim3=nverts, inner_dim=.true., iodesc=iodesc3dv) call ice_pio_initdecomp(ndim3=nzilyr, ndim4=ncat_hist, iodesc=iodesc4di) call ice_pio_initdecomp(ndim3=nzslyr, ndim4=ncat_hist, iodesc=iodesc4ds) + call ice_pio_initdecomp(ndim3=nzslyr, ndim4=nfsd_hist, iodesc=iodesc4df) ltime2 = time/int(secday) ltime = real(time/int(secday),kind=real_kind) @@ -192,6 +195,7 @@ subroutine ice_write_hist (ns) status = pio_def_dim(File,'nkaer',nzalyr,kmtida) status = pio_def_dim(File,'time',PIO_UNLIMITED,timid) status = pio_def_dim(File,'nvertices',nverts,nvertexid) + status = pio_def_dim(File,'nf',nfsd_hist,fmtid) !----------------------------------------------------------------- ! define coordinate variables: time, time_bounds @@ -261,6 +265,7 @@ subroutine ice_write_hist (ns) var_nz(3) = coord_attributes('VGRDs', 'vertical snow levels', '1') var_nz(4) = coord_attributes('VGRDb', 'vertical ice-bio levels', '1') var_nz(5) = coord_attributes('VGRDa', 'vertical snow-ice-bio levels', '1') + var_nz(6) = coord_attributes('NFSD', 'category floe size (center)', 'm') !----------------------------------------------------------------- ! define information for optional time-invariant variables @@ -342,12 +347,13 @@ subroutine ice_write_hist (ns) endif enddo - ! Extra dimensions (NCAT, NZILYR, NZSLYR, NZBLYR, NZALYR) + ! Extra dimensions (NCAT, NZILYR, NZSLYR, NZBLYR, NZALYR, NFSD) dimidex(1)=cmtid dimidex(2)=kmtidi dimidex(3)=kmtids dimidex(4)=kmtidb dimidex(5)=kmtida + dimidex(6)=fmtid do i = 1, nvarz if (igrdz(i)) then @@ -605,6 +611,43 @@ subroutine ice_write_hist (ns) endif enddo ! num_avail_hist_fields_3Da + !----------------------------------------------------------------- + ! 3D (fsd) + !----------------------------------------------------------------- + + dimidz(1) = imtid + dimidz(2) = jmtid + dimidz(3) = fmtid + dimidz(4) = timid + + do n = n3Dacum + 1, n3Dfcum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_def_var(File, trim(avail_hist_fields(n)%vname), & + pio_real, dimidz, varid) + status = pio_put_att(File,varid,'units', & + trim(avail_hist_fields(n)%vunit)) + status = pio_put_att(File,varid, 'long_name', & + trim(avail_hist_fields(n)%vdesc)) + status = pio_put_att(File,varid,'coordinates', & + trim(avail_hist_fields(n)%vcoord)) + status = pio_put_att(File,varid,'cell_measures', & + trim(avail_hist_fields(n)%vcellmeas)) + status = pio_put_att(File,varid,'missing_value',spval) + status = pio_put_att(File,varid,'_FillValue',spval) + + ! Add cell_methods attribute to variables if averaged + if (hist_avg .and. histfreq(ns) /= '1') then + status = pio_put_att(File,varid,'cell_methods','time: mean') + endif + + if (histfreq(ns) == '1' .or. .not. hist_avg) then + status = pio_put_att(File,varid,'time_rep','instantaneous') + else + status = pio_put_att(File,varid,'time_rep','averaged') + endif + endif + enddo ! num_avail_hist_fields_3Df + !----------------------------------------------------------------- ! define attributes for 4D variables ! time coordinate is dropped @@ -620,7 +663,7 @@ subroutine ice_write_hist (ns) dimidcz(4) = cmtid dimidcz(5) = timid - do n = n3Dacum + 1, n4Dicum + do n = n3Dfcum + 1, n4Dicum if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then status = pio_def_var(File, trim(avail_hist_fields(n)%vname), & pio_real, dimidcz, varid) @@ -686,6 +729,45 @@ subroutine ice_write_hist (ns) endif enddo ! num_avail_hist_fields_4Ds + + !----------------------------------------------------------------- + ! 4D (fsd layers) + !----------------------------------------------------------------- + + dimidcz(1) = imtid + dimidcz(2) = jmtid + dimidcz(3) = fmtid + dimidcz(4) = cmtid + dimidcz(5) = timid + + do n = n4Dscum + 1, n4Dfcum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_def_var(File, trim(avail_hist_fields(n)%vname), & + pio_real, dimidcz, varid) + status = pio_put_att(File,varid,'units', & + trim(avail_hist_fields(n)%vunit)) + status = pio_put_att(File,varid, 'long_name', & + trim(avail_hist_fields(n)%vdesc)) + status = pio_put_att(File,varid,'coordinates', & + trim(avail_hist_fields(n)%vcoord)) + status = pio_put_att(File,varid,'cell_measures', & + trim(avail_hist_fields(n)%vcellmeas)) + status = pio_put_att(File,varid,'missing_value',spval) + status = pio_put_att(File,varid,'_FillValue',spval) + + ! Add cell_methods attribute to variables if averaged + if (hist_avg .and. histfreq(ns) /= '1') then + status = pio_put_att(File,varid,'cell_methods','time: mean') + endif + + if (histfreq(ns) == '1' .or. .not. hist_avg) then + status = pio_put_att(File,varid,'time_rep','instantaneous') + else + status = pio_put_att(File,varid,'time_rep','averaged') + endif + endif + enddo ! num_avail_hist_fields_4Df + !----------------------------------------------------------------- ! global attributes !----------------------------------------------------------------- @@ -780,7 +862,7 @@ subroutine ice_write_hist (ns) workr2, status, fillval=spval_dbl) enddo - ! Extra dimensions (NCAT, VGRD*) + ! Extra dimensions (NCAT, NFSD, VGRD*) do i = 1, nvarz if (igrdz(i)) then @@ -788,6 +870,8 @@ subroutine ice_write_hist (ns) SELECT CASE (var_nz(i)%short_name) CASE ('NCAT') status = pio_put_var(File, varid, hin_max(1:ncat_hist)) + CASE ('NFSD') + status = pio_put_var(File, varid, floe_rad_c(1:nfsd_hist)) CASE ('VGRDi') status = pio_put_var(File, varid, (/(k, k=1,nzilyr)/)) CASE ('VGRDs') @@ -984,10 +1068,31 @@ subroutine ice_write_hist (ns) enddo ! num_avail_hist_fields_3Db deallocate(workr3) - allocate(workr4(nx_block,ny_block,nblocks,ncat_hist,nzilyr)) - ! 4D (categories, vertical ice) - do n = n3Dacum+1, n4Dicum + ! 3D (fsd) + allocate(workr3(nx_block,ny_block,nblocks,nfsd_hist)) + do n = n3Dacum+1, n3Dfcum nn = n - n3Dacum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_inq_varid(File,avail_hist_fields(n)%vname,varid) + if (status /= pio_noerr) call abort_ice(subname// & + 'ERROR: getting varid for '//avail_hist_fields(n)%vname) + do j = 1, nblocks + do i = 1, nfsd_hist + workr3(:,:,j,i) = a3Df(:,:,i,nn,j) + enddo + enddo + call pio_setframe(varid, int(1,kind=PIO_OFFSET)) +! call pio_setframe(File, varid, int(1,kind=PIO_OFFSET_KIND)) + call pio_write_darray(File, varid, iodesc3df,& + workr3, status, fillval=spval_dbl) + endif + enddo ! num_avail_hist_fields_3Df + deallocate(workr3) + + allocate(workr4(nx_block,ny_block,nblocks,ncat_hist,nzilyr)) + ! 4D (categories, fsd) + do n = n3Dfcum+1, n4Dicum + nn = n - n3Dfcum if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then status = pio_inq_varid(File,avail_hist_fields(n)%vname,varid) if (status /= pio_noerr) call abort_ice(subname// & @@ -1029,6 +1134,29 @@ subroutine ice_write_hist (ns) enddo ! num_avail_hist_fields_4Di deallocate(workr4) + allocate(workr4(nx_block,ny_block,nblocks,ncat_hist,nfsd_hist)) + ! 4D (categories, vertical ice) + do n = n4Dscum+1, n4Dfcum + nn = n - n4Dscum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_inq_varid(File,avail_hist_fields(n)%vname,varid) + if (status /= pio_noerr) call abort_ice(subname// & + 'ERROR: getting varid for '//avail_hist_fields(n)%vname) + do j = 1, nblocks + do i = 1, ncat_hist + do k = 1, nfsd_hist + workr4(:,:,j,i,k) = a4Ds(:,:,k,i,nn,j) + enddo ! k + enddo ! i + enddo ! j + call pio_setframe(varid, int(1,kind=PIO_OFFSET)) +! call pio_setframe(File, varid, int(1,kind=PIO_OFFSET_KIND)) + call pio_write_darray(File, varid, iodesc4df,& + workr4, status, fillval=spval_dbl) + endif + enddo ! num_avail_hist_fields_4Di + deallocate(workr4) + ! similarly for num_avail_hist_fields_4Db (define workr4b, iodesc4db) @@ -1042,8 +1170,10 @@ subroutine ice_write_hist (ns) call pio_freedecomp(File,iodesc3di) call pio_freedecomp(File,iodesc3db) call pio_freedecomp(File,iodesc3da) + call pio_freedecomp(File,iodesc3df) call pio_freedecomp(File,iodesc4di) call pio_freedecomp(File,iodesc4ds) + call pio_freedecomp(File,iodesc4df) !----------------------------------------------------------------- ! close output dataset diff --git a/cicecore/cicedynB/infrastructure/io/io_pio/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_pio/ice_restart.F90 index d673c7f7a..c207d5b25 100644 --- a/cicecore/cicedynB/infrastructure/io/io_pio/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_pio/ice_restart.F90 @@ -126,7 +126,7 @@ subroutine init_restart_write(filename_spec) use ice_communicate, only: my_task, master_task use ice_domain_size, only: nx_global, ny_global, ncat, nilyr, nslyr, & n_iso, n_aero, nblyr, n_zaero, n_algae, n_doc, & - n_dic, n_don, n_fed, n_fep + n_dic, n_don, n_fed, n_fep, nfsd use ice_dyn_shared, only: kdyn use ice_arrays_column, only: oceanmixed_ice @@ -138,10 +138,10 @@ subroutine init_restart_write(filename_spec) tr_pond_topo, tr_pond_lvl, tr_brine, & tr_bgc_N, tr_bgc_C, tr_bgc_Nit, & tr_bgc_Sil, tr_bgc_DMS, & - tr_bgc_chl, tr_bgc_Am, & + tr_bgc_chl, tr_bgc_Am, & tr_bgc_PON, tr_bgc_DON, & - tr_zaero, tr_bgc_Fe, & - tr_bgc_hum + tr_zaero, tr_bgc_Fe, & + tr_bgc_hum, tr_fsd integer (kind=int_kind) :: & nbtrcr @@ -178,7 +178,7 @@ subroutine init_restart_write(filename_spec) tr_bgc_chl_out=tr_bgc_chl, tr_bgc_Am_out=tr_bgc_Am, & tr_bgc_PON_out=tr_bgc_PON, tr_bgc_DON_out=tr_bgc_DON, & tr_zaero_out=tr_zaero, tr_bgc_Fe_out=tr_bgc_Fe, & - tr_bgc_hum_out=tr_bgc_hum) + tr_bgc_hum_out=tr_bgc_hum, tr_fsd_out=tr_fsd) call icepack_query_parameters(solve_zsal_out=solve_zsal, skl_bgc_out=skl_bgc, & z_tracers_out=z_tracers) call icepack_warnings_flush(nu_diag) @@ -473,6 +473,13 @@ subroutine init_restart_write(filename_spec) call define_rest_field(File,'qsno'//trim(nchar),dims) enddo + if (tr_fsd) then + do k=1,nfsd + write(nchar,'(i3.3)') k + call define_rest_field(File,'fsd'//trim(nchar),dims) + enddo + endif + if (tr_iso) then do k=1,n_iso write(nchar,'(i3.3)') k diff --git a/cicecore/cicedynB/infrastructure/io/io_pio2/ice_history_write.F90 b/cicecore/cicedynB/infrastructure/io/io_pio2/ice_history_write.F90 index 771d0e313..229aa9e51 100644 --- a/cicecore/cicedynB/infrastructure/io/io_pio2/ice_history_write.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_pio2/ice_history_write.F90 @@ -53,7 +53,7 @@ subroutine ice_write_hist (ns) dxu, dxt, dyu, dyt, HTN, HTE, ANGLE, ANGLET, tmask, & lont_bounds, latt_bounds, lonu_bounds, latu_bounds use ice_history_shared - use ice_arrays_column, only: hin_max + use ice_arrays_column, only: hin_max, floe_rad_c use ice_restart_shared, only: runid, lcdf64 use netcdf #endif @@ -67,13 +67,13 @@ subroutine ice_write_hist (ns) #ifdef ncdf integer (kind=int_kind) :: i,j,k,ic,n,nn, & ncid,status,imtid,jmtid,kmtidi,kmtids,kmtidb, cmtid,timid, & - length,nvertexid,ivertex,kmtida + length,nvertexid,ivertex,kmtida,fmtid integer (kind=int_kind), dimension(2) :: dimid2 integer (kind=int_kind), dimension(3) :: dimid3 integer (kind=int_kind), dimension(4) :: dimidz integer (kind=int_kind), dimension(5) :: dimidcz integer (kind=int_kind), dimension(3) :: dimid_nverts - integer (kind=int_kind), dimension(5) :: dimidex + integer (kind=int_kind), dimension(6) :: dimidex real (kind=real_kind) :: ltime real (kind= dbl_kind) :: ltime2 character (char_len) :: title @@ -89,7 +89,8 @@ subroutine ice_write_hist (ns) type(file_desc_t) :: File type(io_desc_t) :: iodesc2d, & iodesc3dc, iodesc3dv, iodesc3di, iodesc3db, iodesc3da, & - iodesc4di, iodesc4ds + iodesc3df, & + iodesc4di, iodesc4ds, iodesc4df type(var_desc_t) :: varid ! 4 coordinate variables: TLON, TLAT, ULON, ULAT @@ -168,9 +169,11 @@ subroutine ice_write_hist (ns) call ice_pio_initdecomp(ndim3=nzilyr, iodesc=iodesc3di) call ice_pio_initdecomp(ndim3=nzblyr, iodesc=iodesc3db) call ice_pio_initdecomp(ndim3=nzalyr, iodesc=iodesc3da) + call ice_pio_initdecomp(ndim3=nfsd_hist, iodesc=iodesc3df) call ice_pio_initdecomp(ndim3=nverts, inner_dim=.true., iodesc=iodesc3dv) call ice_pio_initdecomp(ndim3=nzilyr, ndim4=ncat_hist, iodesc=iodesc4di) call ice_pio_initdecomp(ndim3=nzslyr, ndim4=ncat_hist, iodesc=iodesc4ds) + call ice_pio_initdecomp(ndim3=nzslyr, ndim4=nfsd_hist, iodesc=iodesc4df) ltime2 = time/int(secday) ltime = real(time/int(secday),kind=real_kind) @@ -192,6 +195,7 @@ subroutine ice_write_hist (ns) status = pio_def_dim(File,'nkaer',nzalyr,kmtida) status = pio_def_dim(File,'time',PIO_UNLIMITED,timid) status = pio_def_dim(File,'nvertices',nverts,nvertexid) + status = pio_def_dim(File,'nf',nfsd_hist,fmtid) !----------------------------------------------------------------- ! define coordinate variables: time, time_bounds @@ -261,6 +265,7 @@ subroutine ice_write_hist (ns) var_nz(3) = coord_attributes('VGRDs', 'vertical snow levels', '1') var_nz(4) = coord_attributes('VGRDb', 'vertical ice-bio levels', '1') var_nz(5) = coord_attributes('VGRDa', 'vertical snow-ice-bio levels', '1') + var_nz(6) = coord_attributes('NFSD', 'category floe size (center)', 'm') !----------------------------------------------------------------- ! define information for optional time-invariant variables @@ -342,12 +347,13 @@ subroutine ice_write_hist (ns) endif enddo - ! Extra dimensions (NCAT, NZILYR, NZSLYR, NZBLYR, NZALYR) + ! Extra dimensions (NCAT, NZILYR, NZSLYR, NZBLYR, NZALYR, NFSD) dimidex(1)=cmtid dimidex(2)=kmtidi dimidex(3)=kmtids dimidex(4)=kmtidb dimidex(5)=kmtida + dimidex(6)=fmtid do i = 1, nvarz if (igrdz(i)) then @@ -605,6 +611,43 @@ subroutine ice_write_hist (ns) endif enddo ! num_avail_hist_fields_3Da + !----------------------------------------------------------------- + ! 3D (fsd) + !----------------------------------------------------------------- + + dimidz(1) = imtid + dimidz(2) = jmtid + dimidz(3) = fmtid + dimidz(4) = timid + + do n = n3Dacum + 1, n3Dfcum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_def_var(File, trim(avail_hist_fields(n)%vname), & + pio_real, dimidz, varid) + status = pio_put_att(File,varid,'units', & + trim(avail_hist_fields(n)%vunit)) + status = pio_put_att(File,varid, 'long_name', & + trim(avail_hist_fields(n)%vdesc)) + status = pio_put_att(File,varid,'coordinates', & + trim(avail_hist_fields(n)%vcoord)) + status = pio_put_att(File,varid,'cell_measures', & + trim(avail_hist_fields(n)%vcellmeas)) + status = pio_put_att(File,varid,'missing_value',spval) + status = pio_put_att(File,varid,'_FillValue',spval) + + ! Add cell_methods attribute to variables if averaged + if (hist_avg .and. histfreq(ns) /= '1') then + status = pio_put_att(File,varid,'cell_methods','time: mean') + endif + + if (histfreq(ns) == '1' .or. .not. hist_avg) then + status = pio_put_att(File,varid,'time_rep','instantaneous') + else + status = pio_put_att(File,varid,'time_rep','averaged') + endif + endif + enddo ! num_avail_hist_fields_3Df + !----------------------------------------------------------------- ! define attributes for 4D variables ! time coordinate is dropped @@ -620,7 +663,7 @@ subroutine ice_write_hist (ns) dimidcz(4) = cmtid dimidcz(5) = timid - do n = n3Dacum + 1, n4Dicum + do n = n3Dfcum + 1, n4Dicum if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then status = pio_def_var(File, trim(avail_hist_fields(n)%vname), & pio_real, dimidcz, varid) @@ -686,6 +729,45 @@ subroutine ice_write_hist (ns) endif enddo ! num_avail_hist_fields_4Ds + + !----------------------------------------------------------------- + ! 4D (fsd layers) + !----------------------------------------------------------------- + + dimidcz(1) = imtid + dimidcz(2) = jmtid + dimidcz(3) = fmtid + dimidcz(4) = cmtid + dimidcz(5) = timid + + do n = n4Dscum + 1, n4Dfcum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_def_var(File, trim(avail_hist_fields(n)%vname), & + pio_real, dimidcz, varid) + status = pio_put_att(File,varid,'units', & + trim(avail_hist_fields(n)%vunit)) + status = pio_put_att(File,varid, 'long_name', & + trim(avail_hist_fields(n)%vdesc)) + status = pio_put_att(File,varid,'coordinates', & + trim(avail_hist_fields(n)%vcoord)) + status = pio_put_att(File,varid,'cell_measures', & + trim(avail_hist_fields(n)%vcellmeas)) + status = pio_put_att(File,varid,'missing_value',spval) + status = pio_put_att(File,varid,'_FillValue',spval) + + ! Add cell_methods attribute to variables if averaged + if (hist_avg .and. histfreq(ns) /= '1') then + status = pio_put_att(File,varid,'cell_methods','time: mean') + endif + + if (histfreq(ns) == '1' .or. .not. hist_avg) then + status = pio_put_att(File,varid,'time_rep','instantaneous') + else + status = pio_put_att(File,varid,'time_rep','averaged') + endif + endif + enddo ! num_avail_hist_fields_4Df + !----------------------------------------------------------------- ! global attributes !----------------------------------------------------------------- @@ -780,7 +862,7 @@ subroutine ice_write_hist (ns) workr2, status, fillval=spval_dbl) enddo - ! Extra dimensions (NCAT, VGRD*) + ! Extra dimensions (NCAT, NFSD, VGRD*) do i = 1, nvarz if (igrdz(i)) then @@ -788,6 +870,8 @@ subroutine ice_write_hist (ns) SELECT CASE (var_nz(i)%short_name) CASE ('NCAT') status = pio_put_var(File, varid, hin_max(1:ncat_hist)) + CASE ('NFSD') + status = pio_put_var(File, varid, floe_rad_c(1:nfsd_hist)) CASE ('VGRDi') status = pio_put_var(File, varid, (/(k, k=1,nzilyr)/)) CASE ('VGRDs') @@ -983,10 +1067,30 @@ subroutine ice_write_hist (ns) enddo ! num_avail_hist_fields_3Db deallocate(workr3) - allocate(workr4(nx_block,ny_block,nblocks,ncat_hist,nzilyr)) - ! 4D (categories, vertical ice) - do n = n3Dacum+1, n4Dicum + ! 3D (fsd) + allocate(workr3(nx_block,ny_block,nblocks,nfsd_hist)) + do n = n3Dacum+1, n3Dfcum nn = n - n3Dacum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_inq_varid(File,avail_hist_fields(n)%vname,varid) + if (status /= pio_noerr) call abort_ice(subname// & + 'ERROR: getting varid for '//avail_hist_fields(n)%vname) + do j = 1, nblocks + do i = 1, nfsd_hist + workr3(:,:,j,i) = a3Df(:,:,i,nn,j) + enddo + enddo + call pio_setframe(File, varid, int(1,kind=PIO_OFFSET_KIND)) + call pio_write_darray(File, varid, iodesc3df,& + workr3, status, fillval=spval_dbl) + endif + enddo ! num_avail_hist_fields_3Df + deallocate(workr3) + + allocate(workr4(nx_block,ny_block,nblocks,ncat_hist,nzilyr)) + ! 4D (categories, fsd) + do n = n3Dfcum+1, n4Dicum + nn = n - n3Dfcum if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then status = pio_inq_varid(File,avail_hist_fields(n)%vname,varid) if (status /= pio_noerr) call abort_ice(subname// & @@ -1027,6 +1131,28 @@ subroutine ice_write_hist (ns) enddo ! num_avail_hist_fields_4Di deallocate(workr4) + allocate(workr4(nx_block,ny_block,nblocks,ncat_hist,nfsd_hist)) + ! 4D (categories, vertical ice) + do n = n4Dscum+1, n4Dfcum + nn = n - n4Dscum + if (avail_hist_fields(n)%vhistfreq == histfreq(ns) .or. write_ic) then + status = pio_inq_varid(File,avail_hist_fields(n)%vname,varid) + if (status /= pio_noerr) call abort_ice(subname// & + 'ERROR: getting varid for '//avail_hist_fields(n)%vname) + do j = 1, nblocks + do i = 1, ncat_hist + do k = 1, nfsd_hist + workr4(:,:,j,i,k) = a4Ds(:,:,k,i,nn,j) + enddo ! k + enddo ! i + enddo ! j + call pio_setframe(File, varid, int(1,kind=PIO_OFFSET_KIND)) + call pio_write_darray(File, varid, iodesc4df,& + workr4, status, fillval=spval_dbl) + endif + enddo ! num_avail_hist_fields_4Di + deallocate(workr4) + ! similarly for num_avail_hist_fields_4Db (define workr4b, iodesc4db) @@ -1040,8 +1166,10 @@ subroutine ice_write_hist (ns) call pio_freedecomp(File,iodesc3di) call pio_freedecomp(File,iodesc3db) call pio_freedecomp(File,iodesc3da) + call pio_freedecomp(File,iodesc3df) call pio_freedecomp(File,iodesc4di) call pio_freedecomp(File,iodesc4ds) + call pio_freedecomp(File,iodesc4df) !----------------------------------------------------------------- ! close output dataset diff --git a/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 index d673c7f7a..c207d5b25 100644 --- a/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90 @@ -126,7 +126,7 @@ subroutine init_restart_write(filename_spec) use ice_communicate, only: my_task, master_task use ice_domain_size, only: nx_global, ny_global, ncat, nilyr, nslyr, & n_iso, n_aero, nblyr, n_zaero, n_algae, n_doc, & - n_dic, n_don, n_fed, n_fep + n_dic, n_don, n_fed, n_fep, nfsd use ice_dyn_shared, only: kdyn use ice_arrays_column, only: oceanmixed_ice @@ -138,10 +138,10 @@ subroutine init_restart_write(filename_spec) tr_pond_topo, tr_pond_lvl, tr_brine, & tr_bgc_N, tr_bgc_C, tr_bgc_Nit, & tr_bgc_Sil, tr_bgc_DMS, & - tr_bgc_chl, tr_bgc_Am, & + tr_bgc_chl, tr_bgc_Am, & tr_bgc_PON, tr_bgc_DON, & - tr_zaero, tr_bgc_Fe, & - tr_bgc_hum + tr_zaero, tr_bgc_Fe, & + tr_bgc_hum, tr_fsd integer (kind=int_kind) :: & nbtrcr @@ -178,7 +178,7 @@ subroutine init_restart_write(filename_spec) tr_bgc_chl_out=tr_bgc_chl, tr_bgc_Am_out=tr_bgc_Am, & tr_bgc_PON_out=tr_bgc_PON, tr_bgc_DON_out=tr_bgc_DON, & tr_zaero_out=tr_zaero, tr_bgc_Fe_out=tr_bgc_Fe, & - tr_bgc_hum_out=tr_bgc_hum) + tr_bgc_hum_out=tr_bgc_hum, tr_fsd_out=tr_fsd) call icepack_query_parameters(solve_zsal_out=solve_zsal, skl_bgc_out=skl_bgc, & z_tracers_out=z_tracers) call icepack_warnings_flush(nu_diag) @@ -473,6 +473,13 @@ subroutine init_restart_write(filename_spec) call define_rest_field(File,'qsno'//trim(nchar),dims) enddo + if (tr_fsd) then + do k=1,nfsd + write(nchar,'(i3.3)') k + call define_rest_field(File,'fsd'//trim(nchar),dims) + enddo + endif + if (tr_iso) then do k=1,n_iso write(nchar,'(i3.3)') k diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index e8ec14cab..73e50bf47 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -196,7 +196,7 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB -else if (${ICE_MACHINE} =~ brooks*) then +else if (${ICE_MACHINE} =~ daley* || ${ICE_MACHINE} =~ banting*) then cat >> ${jobfile} << EOFB #PBS -N ${ICE_CASENAME} #PBS -j oe @@ -213,7 +213,20 @@ cat >> ${jobfile} << EOFB #SBATCH -N ${nnodes} #SBATCH -e slurm%j.err #SBATCH -o slurm%j.out -#SBATCH --mail-type END,FAIL +#SBATCH --mail-type FAIL +#SBATCH --mail-user=robert.grumbine@noaa.gov +EOFB + +else if (${ICE_MACHINE} =~ hera*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t `echo ${batchtime} | cut -f1-2 -d:` +#SBATCH -q batch +#SBATCH -A marine-cpu +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +#SBATCH --mail-type FAIL #SBATCH --mail-user=robert.grumbine@noaa.gov EOFB @@ -223,8 +236,25 @@ cat >> ${jobfile} << EOFB EOFB else if (${ICE_MACHINE} =~ phase3*) then +if ( ${nnodes} > 15) then + setenv p3tile 16 + setenv mem `expr 100 \* 1024 / $nnodes` +else + setenv p3tile ${nnodes} + setenv mem 8192 +endif +echo mem = ${mem} nnodes and p3tiles ${nnodes} ${p3tile} p3tile must be le nnodes cat >> ${jobfile} << EOFB -# nothing to do +#BSUB -J ${ICE_CASENAME} +#BSUB -q "dev_shared" +#BSUB -P RTO-T2O +#BSUB -W `echo ${batchtime} | cut -f1-2 -d:` +#BSUB -n ${nnodes} +#BSUB -R "affinity[core]" +#BSUB -R "span[ptile=${p3tile}]" +#BSUB -R "rusage[mem=${mem}]" +#BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J +#BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J EOFB else if (${ICE_MACHINE} =~ high_Sierra*) then diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 17a260e06..48d6c0062 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -153,7 +153,7 @@ EOFR endif #======= -else if (${ICE_MACHINE} =~ brooks*) then +else if (${ICE_MACHINE} =~ daley* || ${ICE_MACHINE} =~ banting*) then if (${ICE_COMMDIR} =~ serial*) then cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE @@ -197,6 +197,20 @@ cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE EOFR +#======= +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +======= + #======= else if (${ICE_MACHINE} =~ travisCI*) then if (${ICE_COMMDIR} =~ serial*) then diff --git a/configuration/scripts/machines/Macros.banting_gnu b/configuration/scripts/machines/Macros.banting_gnu new file mode 100644 index 000000000..769470f6d --- /dev/null +++ b/configuration/scripts/machines/Macros.banting_gnu @@ -0,0 +1,39 @@ +#============================================================================== +# Makefile macros for ECCC banting +#============================================================================== +# For use with GNU compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 +#-xHost + +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.banting_intel b/configuration/scripts/machines/Macros.banting_intel new file mode 100644 index 000000000..d16ed432a --- /dev/null +++ b/configuration/scripts/machines/Macros.banting_intel @@ -0,0 +1,40 @@ +#============================================================================== +# Makefile macros for ECCC banting +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif diff --git a/configuration/scripts/machines/Macros.brooks_intel b/configuration/scripts/machines/Macros.brooks_intel deleted file mode 100644 index a6fecdbbb..000000000 --- a/configuration/scripts/machines/Macros.brooks_intel +++ /dev/null @@ -1,73 +0,0 @@ -#============================================================================== -# Makefile macros for ECCC brooks -#============================================================================== -# For use with intel compiler -#============================================================================== - -CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -fp-model precise -#-xHost - -FIXEDFLAGS := -132 -FREEFLAGS := -FR -FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -#-xHost -FFLAGS_NOOPT:= -O0 - -ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -# -heap-arrays 1024 -else - FFLAGS += -O2 -endif - -SCC := cc -SFC := ftn -MPICC := cc -MPIFC := ftn - -ifeq ($(ICE_COMMDIR), mpi) - FC := $(MPIFC) - CC := $(MPICC) -else - FC := $(SFC) - CC := $(SCC) -endif -LD:= $(FC) - -#NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ - -PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs - -#PNETCDF_PATH := $(PNETCDF) -#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib - -INCLDIR := $(INCLDIR) - -LIB_NETCDF := $(NETCDF_PATH)/lib -LIB_PNETCDF := $(PNETCDF_PATH)/lib -LIB_MPI := $(IMPILIBDIR) - -#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl -#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf - -ifeq ($(ICE_IOTYPE), netcdf) - INCLDIR += $(shell nf-config --fflags) - SLIBS := $(shell nf-config --flibs) -endif - - -ifeq ($(ICE_THREADED), true) - LDFLAGS += -qopenmp - CFLAGS += -qopenmp - FFLAGS += -qopenmp -endif - -### if using parallel I/O, load all 3 libraries. PIO must be first! -ifeq ($(ICE_IOTYPE), pio) - PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib - INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include - SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof -endif - diff --git a/configuration/scripts/machines/Macros.daley_gnu b/configuration/scripts/machines/Macros.daley_gnu new file mode 100644 index 000000000..be01e67fe --- /dev/null +++ b/configuration/scripts/machines/Macros.daley_gnu @@ -0,0 +1,39 @@ +#============================================================================== +# Makefile macros for ECCC daley +#============================================================================== +# For use with GNU compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 +#-xHost + +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.daley_intel b/configuration/scripts/machines/Macros.daley_intel new file mode 100644 index 000000000..5aeb4c438 --- /dev/null +++ b/configuration/scripts/machines/Macros.daley_intel @@ -0,0 +1,40 @@ +#============================================================================== +# Makefile macros for ECCC daley +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif diff --git a/configuration/scripts/machines/Macros.hera_intel b/configuration/scripts/machines/Macros.hera_intel new file mode 100644 index 000000000..38cc62ed8 --- /dev/null +++ b/configuration/scripts/machines/Macros.hera_intel @@ -0,0 +1,60 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := + + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.banting_gnu b/configuration/scripts/machines/env.banting_gnu new file mode 100755 index 000000000..7b1a55eac --- /dev/null +++ b/configuration/scripts/machines/env.banting_gnu @@ -0,0 +1,36 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /opt/modules/default/init/csh # Initialize modules for csh +# Clear environment +module unload craype-x86-skylake +module unload PrgEnv-intel + +module load PrgEnv-gnu # GNU compiler +module load craype-x86-skylake # Reload +module load cray-mpich # MPI (Cray MPICH) +module load cray-netcdf # NetCDF +module load cray-hdf5 # HDF5 +setenv HDF5_USE_FILE_LOCKING FALSE # necessary since data is on an NFS filesystem + +endif + +setenv ICE_MACHINE_ENVNAME banting +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/banting/cice/runs +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/banting/cice/baselines +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 40 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 18 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.banting_intel b/configuration/scripts/machines/env.banting_intel new file mode 100755 index 000000000..48dce9d1e --- /dev/null +++ b/configuration/scripts/machines/env.banting_intel @@ -0,0 +1,31 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /opt/modules/default/init/csh # Initialize modules for csh +module load PrgEnv-intel # Intel compiler +module load cray-mpich # MPI (Cray MPICH) +module load cray-netcdf # NetCDF +module load cray-hdf5 # HDF5 +setenv HDF5_USE_FILE_LOCKING FALSE # necessary since data is on an NFS filesystem + +endif + +setenv ICE_MACHINE_ENVNAME banting +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/banting/cice/runs +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/banting/cice/baselines +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 40 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 18 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.brooks_intel b/configuration/scripts/machines/env.brooks_intel deleted file mode 100755 index 4a51b5bb1..000000000 --- a/configuration/scripts/machines/env.brooks_intel +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/csh -f - -source /opt/modules/default/init/csh -module load PrgEnv-intel # Intel compiler -module load cray-mpich # MPI (Cray MPICH) -module load cray-netcdf # NetCDF -module load cray-hdf5 # HDF5 - -setenv ICE_MACHINE_ENVNAME brooks -setenv ICE_MACHINE_COMPILER intel -setenv ICE_MACHINE_MAKE make -setenv ICE_MACHINE_WKDIR ~/data/brooks/cice/runs -setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/phb001/ -setenv ICE_MACHINE_BASELINE ~/data/brooks/cice/baselines -setenv ICE_MACHINE_SUBMIT "qsub" -setenv ICE_MACHINE_TPNODE 36 -setenv ICE_MACHINE_MAXRUNLENGTH 3 -setenv ICE_MACHINE_ACCT P0000000 -setenv ICE_MACHINE_QUEUE "development" -setenv ICE_MACHINE_BLDTHRDS 4 -setenv ICE_MACHINE_QSTAT "qstat " - -if (-e ~/.cice_proj) then - set account_name = `head -1 ~/.cice_proj` - setenv CICE_ACCT ${account_name} -endif diff --git a/configuration/scripts/machines/env.daley_gnu b/configuration/scripts/machines/env.daley_gnu new file mode 100755 index 000000000..f77bf2d88 --- /dev/null +++ b/configuration/scripts/machines/env.daley_gnu @@ -0,0 +1,36 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /opt/modules/default/init/csh # Initialize modules for csh +# Clear environment +module unload craype-x86-skylake +module unload PrgEnv-intel + +module load PrgEnv-gnu # GNU compiler +module load craype-x86-skylake # Reload +module load cray-mpich # MPI (Cray MPICH) +module load cray-netcdf # NetCDF +module load cray-hdf5 # HDF5 +setenv HDF5_USE_FILE_LOCKING FALSE # necessary since data is on an NFS filesystem + +endif + +setenv ICE_MACHINE_ENVNAME daley +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/daley/cice/runs +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/daley/cice/baselines +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 40 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 18 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.daley_intel b/configuration/scripts/machines/env.daley_intel new file mode 100755 index 000000000..0d25f983f --- /dev/null +++ b/configuration/scripts/machines/env.daley_intel @@ -0,0 +1,31 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /opt/modules/default/init/csh # Initialize modules for csh +module load PrgEnv-intel # Intel compiler +module load cray-mpich # MPI (Cray MPICH) +module load cray-netcdf # NetCDF +module load cray-hdf5 # HDF5 +setenv HDF5_USE_FILE_LOCKING FALSE # necessary since data is on an NFS filesystem + +endif + +setenv ICE_MACHINE_ENVNAME daley +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/daley/cice/runs +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/daley/cice/baselines +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 40 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 18 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.hera_intel b/configuration/scripts/machines/env.hera_intel new file mode 100755 index 000000000..ef3afdb7a --- /dev/null +++ b/configuration/scripts/machines/env.hera_intel @@ -0,0 +1,26 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +#module list +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load esmf/7.1.0r +module load hdf5/1.10.4 netcdf/4.6.1 +module load wgrib wgrib2 +#echo renewed modules: +#module list + +setenv ICE_MACHINE_ENVNAME hera +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/rgdev/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/rgdev/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/rgdev/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "batch" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu index cabd66944..db8bc01a3 100755 --- a/configuration/scripts/machines/env.high_Sierra_gnu +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -4,7 +4,7 @@ setenv ICE_MACHINE_ENVNAME high_Sierra setenv ICE_MACHINE_COMPILER gnu setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data +setenv ICE_MACHINE_INPUTDATA /Volumes/Data/ setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " setenv ICE_MACHINE_TPNODE 4 diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 03aa7083f..4c8f65eee 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -1,8 +1,12 @@ -#!/bin/csh -f --login +#!/bin/csh -f -source /etc/profile.d/lmod.csh +#. $MODULESHOME/init/csh +source $MODULESHOME/init/csh +echo done with init.csh +module use -a $MODULESHOME/modulefiles module purge +module load lsf/10.1 module load ips/18.0.1.163 module load impi/18.0.1 module load NetCDF/4.5.0 @@ -15,10 +19,10 @@ setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_SUBMIT "bsub < " setenv ICE_MACHINE_TPNODE 4 setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QSTAT "bjobs " diff --git a/configuration/scripts/options/set_nml.tx1 b/configuration/scripts/options/set_nml.tx1 index 120f7249f..dfdf6f19b 100644 --- a/configuration/scripts/options/set_nml.tx1 +++ b/configuration/scripts/options/set_nml.tx1 @@ -5,8 +5,8 @@ grid_format = 'bin' grid_type = 'tripole' grid_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/tx1/grid_tx1.bin' kmt_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/tx1/kmt_tx1.bin' -fyear_init = 1997 -atm_data_format = 'bin' -atm_data_type = 'clim' -atm_data_dir = 'none' - +atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/tx1/JRA55' +atm_data_format = 'nc' +atm_data_type = 'JRA55_tx1' +year_init = 2005 +fyear_init = 2005 diff --git a/configuration/scripts/tests/cice_test_codecov.csh b/configuration/scripts/tests/cice_test_codecov.csh new file mode 100755 index 000000000..0b2d970ec --- /dev/null +++ b/configuration/scripts/tests/cice_test_codecov.csh @@ -0,0 +1,75 @@ +#!/bin/csh + +# This was a script on gordon +# This script should only be run on hardware with the gnu compiler with a +# modified Macros file to turn on the codecov flags +# This should be run interactively because git push will require login information + +#PBS -N cice_test +#PBS -q standard +#PBS -A NRLSS03755018 +#PBS -l application=Regional-Arctic-System-Model +#PBS -l select=1:ncpus=32:mpiprocs=32 +#PBS -l walltime=24:00:00 +#PBS -j oe +#PBS -M anthony.p.craig@gmail.com +#PBS -m be + +#set verbose +module load costinit git + +set scrdir = "~" +set testdir = "~/cice_testing" +set date0 = `date -u "+%y%m%d"` +set date = ${date0}cc + +mkdir -p ${testdir} + +cd ${testdir} + +# Check out current cice master +echo " " +echo "*** checkout current cice master ***" +git clone https://github.com/cice-consortium/cice cice.master.${date} --recursive +cd cice.master.${date} +set hash = `git rev-parse --short HEAD ` +cd ../ + +# Check out test_cice_icepack, remove all code and copy from cice master +# Need to be careful about dot files, particularly .git* files +# This copies in all files via standard file expansion (except dot files at root) +# This also copies in all dot file at the root that do not start with .g (ie. .git*) +echo " " +echo "*** checkout current test_cice_master ***" +git clone https://github.com/apcraig/test_cice_icepack test_cice_icepack.${date} +cd test_cice_icepack.${date} +echo " " +echo "*** remove current files and copy in cice master files ***" +set verbose +git rm -r * >& /dev/null +cp -p -r ../cice.master.${date}/* . +cp -p ../cice.master.${date}/.[a-f,h-z]* . + +# Clean up icepack .git stuff and commit +rm -r -f icepack/.git* +git add . +unset verbose +echo " " +echo "*** git status of changes ***" +git status +echo " " +echo "*** commit changes ***" +git commit -m "update test_cice_icepack master to ${hash}" + +# Push test_cice_icepack +echo " " +echo "*** push changes to test_cice_icepack ***" +git push origin master + +# Run test suite +echo " " +echo "*** run test suite ***" +./cice.setup --suite first_suite,base_suite,travis_suite,decomp_suite,reprosum_suite,quick_suite -m gordon -e gnu --testid T${date} --codecov --queue standard + +# The test suite will wait until all jobs are complete then run report_codecov.csh +# If that fails, you can run report_codecov.csh manually after all jobs are done diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst index b3088963d..033d4e8c2 100755 --- a/doc/source/user_guide/ug_case_settings.rst +++ b/doc/source/user_guide/ug_case_settings.rst @@ -356,6 +356,7 @@ Table of namelist options "","", "``LYq``", "COREII Large-Yeager (AOMIP) forcing data", ":cite:`Large09`" "","", "``JRA55_gx1``", "JRA55 forcing data for gx1 grid :cite:`Tsujino18`", "" "","", "``JRA55_gx3``", "JRA55 forcing data for gx3 grid :cite:`Tsujino18`", "" + "","", "``JRA55_tx1``", "JRA55 forcing data for tx1 grid :cite:`Tsujino18`", "" "","", "``monthly``", "monthly forcing data", "" "","", "``ncar``", "NCAR bulk forcing data", "" "","", "``box2001``", "forcing data for :cite:`Hunke01` box problem", "" diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 52621d612..04b82a401 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -668,32 +668,25 @@ Code Coverage Testing The ``--codecov`` feature in **cice.setup** provides a method to diagnose code coverage. This argument turns on special compiler flags including reduced optimization and then invokes the gcov tool. -This option is currently only available with the gnu compiler and on a few systems. +This option is currently only available with the gnu compiler and on a few systems +with modified Macros files. Because codecov.io does not support git submodule analysis right now, a customized repository has to be created to test CICE with Icepack integrated directly. The repository https://github.com/apcraig/Test_CICE_Icepack serves as the current default test repository. In general, to setup the code coverage test in CICE, the current CICE master has -to be copied into the Test_CICE_Icepack repository, then the code coverage tool can -be run on that repository. A sample script to do that would be:: +to be copied into the Test_CICE_Icepack repository, then the full test suite +can be run with the gnu compiler with the ``--codecov`` argument. - git clone https://github.com/cice-consortium/cice cice.master --recursive - - git clone https://github.com/apcraig/test_cice_icepack - cd test_cice_icepack - git rm -r * - cp -p -r ../cice.master/* . - git add . - git commit -m "update to current cice master" - git push origin master - - ./cice.setup --suite first_suite,base_suite,travis_suite,decomp_suite,reprosum_suite,quick_suite -m gordon -e gnu --codecov --testid cc01 - -To use, submit a full test suite using an updated Test_CICE_Icepack version -and the gnu compiler with the ``--codecov`` argument. The test suite will run and then a report will be generated and uploaded to the `codecov.io site `_ by the -**report_codecov.csh** script. +**report_codecov.csh** script. The env variable CODECOV_TOKEN needs to be defined +either in the environment or in a file named **~/.codecov_cice_token**. That +token provides write permission to the Test_CICE_Icepack codecov.io site and is available +by contacting the Consortium team directly. + +A script that carries out the end-to-end testing can be found in +**configuration/scripts/tests/cice_test_codecov.csh** This is a special diagnostic test and does not constitute proper model testing. General use is not recommended, this is mainly used as a diagnostic to periodically