Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update IO formats and add new IO namelist controls #928

Merged
merged 23 commits into from
Feb 22, 2024
Merged

Update IO formats and add new IO namelist controls #928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9804049
Update history_format, restart_format, add options cdf1, cdf2, cdf5,
apcraig Jan 24, 2024
f15d7c8
Add new IO namelist controls,
apcraig Jan 25, 2024
6c97fd1
Update documentation and testing
apcraig Jan 26, 2024
3fe2e6a
Update documentation
apcraig Jan 26, 2024
1caa8dd
Update documentation
apcraig Jan 26, 2024
46567e1
Update documentation
apcraig Jan 26, 2024
ebec9bc
Update documentation
apcraig Jan 26, 2024
aadaa4d
Update documentation
apcraig Jan 26, 2024
d2799fc
Update documentation
apcraig Jan 26, 2024
c66a119
Netcdf chunking and compression
anton-seaice Jan 29, 2024
4829247
Review comments and no-1d chunking
anton-seaice Jan 31, 2024
e2297a3
Merge pull request #115 from ACCESS-NRI/nc_chunk_compress
apcraig Jan 31, 2024
d199695
fix history chunking and comments / formatting
anton-seaice Feb 1, 2024
aac3a5e
Minor fixes
apcraig Feb 1, 2024
a1710e6
Merge pull request #116 from ACCESS-NRI/nc_chunk_compress
apcraig Feb 1, 2024
5b903f6
Minor code cleanup, update documentation
apcraig Feb 2, 2024
1619d6c
update documentation
apcraig Feb 2, 2024
e946aae
update set_nml.iobinary, use binary option for format namelist input
apcraig Feb 2, 2024
4f09d1f
Update the restart read to use default 'cdf1' format.
apcraig Feb 9, 2024
3cef8e5
Compare _format variables to the correct default (now 'cdf1') when co…
anton-seaice Feb 20, 2024
a4d37c8
Merge pull request #118 from ACCESS-NRI/nc_chunk_compress
apcraig Feb 20, 2024
2546726
Update pio iotask default after further testing. There is a problems
apcraig Feb 20, 2024
99071a2
Update documentation
apcraig Feb 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions cicecore/cicedyn/analysis/ice_history_shared.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,23 @@ module ice_history_shared
history_dir , & ! directory name for history file
incond_dir ! directory for snapshot initial conditions

character (len=char_len_long), public :: &
pointer_file ! input pointer file for restarts

character (len=char_len), public :: &
version_name

character (len=char_len), public :: &
history_format
history_format , & ! history format, cdf1, cdf2, cdf5, etc
history_rearranger ! history file rearranger, box or subset for pio

character (len=char_len), public :: &
hist_suffix(max_nstrm) ! appended to 'h' in filename when not 'x'

integer (kind=int_kind), public :: &
history_iotasks , & ! iotasks, root, stride defines io pes for pio
history_root , & ! iotasks, root, stride defines io pes for pio
history_stride , & ! iotasks, root, stride defines io pes for pio
history_deflate , & ! compression level for hdf5/netcdf4
history_chunksize(2) ! chunksize for hdf5/netcdf4

!---------------------------------------------------------------
! Instructions for adding a field: (search for 'example')
! Here or in ice_history_[process].F90:
Expand Down
303 changes: 247 additions & 56 deletions cicecore/cicedyn/general/ice_init.F90
View file
Open in desktop

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions cicecore/cicedyn/infrastructure/ice_read_write.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1241,7 +1241,7 @@ subroutine ice_read_nc_xy(fid, nrec, varname, work, diag, &
status = nf90_get_att(fid, varid, "_FillValue", missingvalue)
! call ice_check_nc(status, subname//' ERROR: Missing _FillValue', &
! file=__FILE__, line=__LINE__)
write(nu_diag,*) subname,' missingvalue= ',missingvalue
! write(nu_diag,*) subname,' missingvalue= ',missingvalue
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did not read the full discussion, but is there any reason why we are commenting these lines instead of removing them ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There a bit of useful debugging code that is part of the optional file read diagnostics that is turned off by default. It's sort of helpful to have this code around in case it's needed.

amin = minval(work_g1)
amax = maxval(work_g1, mask = work_g1 /= missingvalue)
asum = sum (work_g1, mask = work_g1 /= missingvalue)
Expand Down Expand Up @@ -1442,7 +1442,7 @@ subroutine ice_read_nc_xyz(fid, nrec, varname, work, diag, &
status = nf90_get_att(fid, varid, "_FillValue", missingvalue)
! call ice_check_nc(status, subname//' ERROR: Missing _FillValue', &
! file=__FILE__, line=__LINE__)
write(nu_diag,*) subname,' missingvalue= ',missingvalue
! write(nu_diag,*) subname,' missingvalue= ',missingvalue
do n=1,ncat
amin = minval(work_g1(:,:,n))
amax = maxval(work_g1(:,:,n), mask = work_g1(:,:,n) /= missingvalue)
Expand Down Expand Up @@ -1654,7 +1654,7 @@ subroutine ice_read_nc_xyf(fid, nrec, varname, work, diag, &
status = nf90_get_att(fid, varid, "_FillValue", missingvalue)
! call ice_check_nc(status, subname//' ERROR: Missing _FillValue', &
! file=__FILE__, line=__LINE__)
write(nu_diag,*) subname,' missingvalue= ',missingvalue
! write(nu_diag,*) subname,' missingvalue= ',missingvalue
do n = 1, nfreq
amin = minval(work_g1(:,:,n))
amax = maxval(work_g1(:,:,n), mask = work_g1(:,:,n) /= missingvalue)
Expand Down Expand Up @@ -2589,13 +2589,13 @@ subroutine ice_check_nc(status, abort_msg, file, line)
#ifdef USE_NETCDF
if (status /= nf90_noerr) then
if (present(file) .and. present(line)) then
call abort_ice(subname//trim(nf90_strerror(status))//', '//trim(abort_msg), &
call abort_ice(subname//' '//trim(nf90_strerror(status))//', '//trim(abort_msg), &
file=file, line=line)
elseif (present(file)) then
call abort_ice(subname//trim(nf90_strerror(status))//', '//trim(abort_msg), &
call abort_ice(subname//' '//trim(nf90_strerror(status))//', '//trim(abort_msg), &
file=file)
else
call abort_ice(subname//trim(nf90_strerror(status))//', '//trim(abort_msg))
call abort_ice(subname//' '//trim(nf90_strerror(status))//', '//trim(abort_msg))
endif
endif
#else
Expand Down
347 changes: 160 additions & 187 deletions cicecore/cicedyn/infrastructure/io/io_netcdf/ice_history_write.F90
View file
Open in desktop

Large diffs are not rendered by default.

52 changes: 41 additions & 11 deletions cicecore/cicedyn/infrastructure/io/io_netcdf/ice_restart.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ module ice_restart
use ice_read_write, only: ice_check_nc
use ice_restart_shared, only: &
restart_ext, restart_dir, restart_file, pointer_file, &
runid, use_restart_time, lcdf64, lenstr, restart_coszen
runid, use_restart_time, lenstr, restart_coszen, restart_format, &
restart_chunksize, restart_deflate
use ice_fileunits, only: nu_diag, nu_rst_pointer
use ice_exit, only: abort_ice
use icepack_intfc, only: icepack_query_parameters
Expand All @@ -29,10 +30,12 @@ module ice_restart
implicit none
private
public :: init_restart_write, init_restart_read, &
read_restart_field, write_restart_field, final_restart, &
query_field
read_restart_field, write_restart_field, final_restart, &
query_field

integer (kind=int_kind) :: ncid
integer (kind=int_kind) :: ncid , &
dimid_ni, & ! netCDF identifiers
dimid_nj

!=======================================================================

Expand Down Expand Up @@ -169,8 +172,7 @@ subroutine init_restart_write(filename_spec)
integer (kind=int_kind), allocatable :: dims(:)

integer (kind=int_kind) :: &
dimid_ni, & ! netCDF identifiers
dimid_nj, & !

dimid_ncat, & !
iflag, & ! netCDF creation flag
status ! status variable from netCDF routine
Expand Down Expand Up @@ -216,8 +218,18 @@ subroutine init_restart_write(filename_spec)
write(nu_rst_pointer,'(a)') filename
close(nu_rst_pointer)

iflag = 0
if (lcdf64) iflag = nf90_64bit_offset
if (restart_format == 'cdf1') then
iflag = nf90_clobber
elseif (restart_format == 'cdf2') then
iflag = ior(nf90_clobber,nf90_64bit_offset)
elseif (restart_format == 'cdf5') then
iflag = ior(nf90_clobber,nf90_64bit_data)
elseif (restart_format == 'hdf5') then
iflag = ior(nf90_clobber,nf90_netcdf4)
else
call abort_ice(subname//' ERROR: restart_format not allowed for '//trim(restart_format), &
file=__FILE__, line=__LINE__)
endif
status = nf90_create(trim(filename), iflag, ncid)
call ice_check_nc(status, subname//' ERROR: creating '//trim(filename), file=__FILE__, line=__LINE__)

Expand Down Expand Up @@ -873,14 +885,32 @@ subroutine define_rest_field(ncid, vname, dims)

integer (kind=int_kind) :: varid

integer (kind=int_kind) :: &
status ! status variable from netCDF routine
integer (kind=int_kind) :: chunks(size(dims)), status, i

character(len=*), parameter :: subname = '(define_rest_field)'

#ifdef USE_NETCDF

status = nf90_def_var(ncid,trim(vname),nf90_double,dims,varid)
call ice_check_nc(status, subname//' ERROR: def var '//trim(vname), file=__FILE__, line=__LINE__)

if (restart_format=='hdf5' .and. size(dims)>1) then
if (dims(1)==dimid_ni .and. dims(2)==dimid_nj) then
chunks(1)=restart_chunksize(1)
chunks(2)=restart_chunksize(2)
do i = 3, size(dims)
chunks(i) = 0
enddo
status = nf90_def_var_chunking(ncid, varid, NF90_CHUNKED, chunksizes=chunks)
call ice_check_nc(status, subname//' ERROR: chunking var '//trim(vname), file=__FILE__, line=__LINE__)
endif
endif

if (restart_format=='hdf5' .and. restart_deflate/=0) then
status=nf90_def_var_deflate(ncid, varid, shuffle=0, deflate=1, deflate_level=restart_deflate)
call ice_check_nc(status, subname//' ERROR deflating var '//trim(vname), file=__FILE__, line=__LINE__)
endif

#else
call abort_ice(subname//' ERROR: USE_NETCDF cpp not defined', &
file=__FILE__, line=__LINE__)
Expand Down Expand Up @@ -911,7 +941,7 @@ logical function query_field(nu,vname)
endif
call broadcast_scalar(query_field,master_task)
#else
call abort_ice(subname//' ERROR: USE_NETCDF cpp not defined for '//trim(ice_ic), &
call abort_ice(subname//' ERROR: USE_NETCDF cpp not defined', &
file=__FILE__, line=__LINE__)
#endif

Expand Down
Loading
Loading