From dc1269bcc1970bcc53bd1f95cd88cdac6e06fb68 Mon Sep 17 00:00:00 2001 From: apcraig Date: Wed, 17 Nov 2021 19:54:20 -0700 Subject: [PATCH 1/5] update parsing scripts to improve robustness, fix multi-pe advection=none --- cice.setup | 4 +++ cicecore/cicedynB/general/ice_init.F90 | 7 ++-- configuration/scripts/options/set_env.box2001 | 1 - configuration/scripts/parse_namelist.sh | 2 +- configuration/scripts/parse_settings.sh | 34 ++++++++++++------- 5 files changed, 31 insertions(+), 17 deletions(-) delete mode 100644 configuration/scripts/options/set_env.box2001 diff --git a/cice.setup b/cice.setup index be9266dd2..aae4319d4 100755 --- a/cice.setup +++ b/cice.setup @@ -1070,6 +1070,10 @@ EOF2 end ${casescr}/parse_settings.sh cice.settings ${fsmods} + if ($status != 0) then + echo "${0}: ERROR, parse_namelist.sh aborted" + exit -1 + endif ${casescr}/parse_namelist.sh ice_in ${fimods} if ($status != 0) then echo "${0}: ERROR, parse_namelist.sh aborted" diff --git a/cicecore/cicedynB/general/ice_init.F90 b/cicecore/cicedynB/general/ice_init.F90 index b299ef77f..7485cbe23 100644 --- a/cicecore/cicedynB/general/ice_init.F90 +++ b/cicecore/cicedynB/general/ice_init.F90 @@ -958,6 +958,10 @@ subroutine input_data abort_list = trim(abort_list)//":1" endif + if (ktransport <= 0) then + advection = 'none' + endif + if (ktransport > 0 .and. advection /= 'remap' .and. advection /= 'upwind') then if (my_task == master_task) write(nu_diag,*) subname//' ERROR: invalid advection=',trim(advection) abort_list = trim(abort_list)//":3" @@ -1467,9 +1471,6 @@ subroutine input_data endif write(nu_diag,1030) ' ssh_stress = ',trim(ssh_stress),trim(tmpstr2) - if (ktransport <= 0) then - advection = 'none' - endif if (trim(advection) == 'remap') then tmpstr2 = ' : linear remapping advection' elseif (trim(advection) == 'upwind') then diff --git a/configuration/scripts/options/set_env.box2001 b/configuration/scripts/options/set_env.box2001 deleted file mode 100644 index a3f7c10f5..000000000 --- a/configuration/scripts/options/set_env.box2001 +++ /dev/null @@ -1 +0,0 @@ -setenv NICELYR 1 diff --git a/configuration/scripts/parse_namelist.sh b/configuration/scripts/parse_namelist.sh index ea539a2d0..6dbad45d6 100755 --- a/configuration/scripts/parse_namelist.sh +++ b/configuration/scripts/parse_namelist.sh @@ -10,7 +10,7 @@ filename=$1 filemods=$2 #echo "$0 $1 $2" -echo "running parse_namelist.sh" +echo "running ${scriptname}" foundstring="FoundSTRING" vnamearray=() valuearray=() diff --git a/configuration/scripts/parse_settings.sh b/configuration/scripts/parse_settings.sh index d6ed31c15..fe7f87c1c 100755 --- a/configuration/scripts/parse_settings.sh +++ b/configuration/scripts/parse_settings.sh @@ -10,7 +10,7 @@ filename=$1 filemods=$2 #echo "$0 $1 $2" -echo "running parse_settings.sh" +echo "running ${scriptname}" foundstring="FoundSTRING" vnamearray=() valuearray=() @@ -23,8 +23,11 @@ do else #vname=`echo $line | sed "s|\(^\s*set\S*\)\s\{1,100\}\(\S*\)\s\{1,100\}\(\S*\).*$|\2|g"` #value=`echo $line | sed "s|\(^\s*set\S*\)\s\{1,100\}\(\S*\)\s\{1,100\}\(\S*\).*$|\3|g"` - vname=`echo $line | sed "s|\(^[[:space:]]*set[^[:space:]]*\)[[:space:]][[:space:]]*\([^[:space:]]*\)[[:space:]][[:space:]]*\([^[:space:]]*\).*$|\2|g"` + vname=`echo $line | sed "s|\(^[[:space:]]*set[^[:space:]]*\)[[:space:]][[:space:]]*\([^[:space:]]*\).*$|\2|g"` value=`echo $line | sed "s|\(^[[:space:]]*set[^[:space:]]*\)[[:space:]][[:space:]]*\([^[:space:]]*\)[[:space:]][[:space:]]*\([^[:space:]]*\).*$|\3|g"` + if [[ "${value}" == "${line}" ]]; then + value="" + fi # echo "$line $vname $value" found=${foundstring} @@ -44,18 +47,25 @@ do done #sed -i 's|\(^\s*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename - sed -i.sedbak -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename - - if [[ "${found}" == "${foundstring}" ]]; then - vnamearray+=($vname) - valuearray+=($value) + cp ${filename} ${filename}.check + sed -i.sedbak -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$foundstring"' |g' $filename.check + grep -q ${foundstring} ${filename}.check + if [ $? -eq 0 ]; then + sed -i.sedbak -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename + if [[ "${found}" == "${foundstring}" ]]; then + vnamearray+=($vname) + valuearray+=($value) + else + valuearray[$found]=${value} + fi + if [[ -e "${filename}.sedbak" ]]; then + rm ${filename}.sedbak + fi else - valuearray[$found]=${value} - fi - - if [[ -e "${filename}.sedbak" ]]; then - rm ${filename}.sedbak + echo "${scriptname} ERROR: parsing error for ${vname}" + exit -99 fi + rm ${filename}.check ${filename}.check.sedbak fi From 60816b1f6ad0e92a5e25f762b982556a3c37d977 Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 22 Nov 2021 20:00:48 -0700 Subject: [PATCH 2/5] Update cice script to improve performance including minor refactoring of parse_namelist and parse_settings to reduce cost and ability to use already setup ice_in file from a prior case in the suite. Added commented out timing ability in cice.setup. Change test default to PEND from FAIL. --- cice.setup | 71 ++++++++++++++++--- configuration/scripts/parse_namelist.sh | 14 ++-- .../scripts/parse_namelist_from_env.sh | 11 ++- configuration/scripts/parse_settings.sh | 14 ++-- 4 files changed, 72 insertions(+), 38 deletions(-) diff --git a/cice.setup b/cice.setup index aae4319d4..a5cbbd468 100755 --- a/cice.setup +++ b/cice.setup @@ -1,5 +1,7 @@ #!/bin/csh -f +#set pd0 = `date -u "+%s%N"` + set ICE_SANDBOX = `pwd` set ICE_VERSION = unknown if (-e cicecore/version.txt) then @@ -824,8 +826,8 @@ EOF # set default test output as failure if (${docase} == 0) then echo "#---" >! test_output - echo "FAIL ${testname_noid} build" >> test_output - echo "FAIL ${testname_noid} run" >> test_output + echo "PEND ${testname_noid} build" >> test_output + echo "PEND ${testname_noid} run" >> test_output endif # from basic script dir to case @@ -934,9 +936,21 @@ EOF if (-e ${fimods}) rm ${fimods} if (-e ${fsmods}) rm ${fsmods} + # Use an existing ice_in file from the suite if it exists + # to reduce time spend in parse_namelist + set iceinfn = ../ice_in_save_${grid}${soptions} + set skip_parse_namelist = spval + if (${dosuite} == 1) then + if (-e ${iceinfn}) then + echo "use ${iceinfn}" + cp ${iceinfn} ice_in + set skip_parse_namelist = true + endif + endif + + # Set decomp info in namelist cat >! ${fimods} << EOF1 # cice.setup settings - nprocs = ${task} nx_global = ${ICE_DECOMP_NXGLOB} ny_global = ${ICE_DECOMP_NYGLOB} @@ -965,7 +979,6 @@ EOF1 cat >! ${fsmods} << EOF1 # cice.setup settings - setenv ICE_SANDBOX ${ICE_SANDBOX} setenv ICE_SCRIPTS ${ICE_SCRIPTS} setenv ICE_CASENAME ${casename} @@ -1034,44 +1047,57 @@ EOF1 foreach name (${grid} $setsx) set found = 0 + if (-e ${ICE_SCRIPTS}/options/set_nml.${name}) then cat >> ${fimods} << EOF2 # set_nml.${name} - EOF2 - cat ${ICE_SCRIPTS}/options/set_nml.${name} >> ${fimods} - cat >> ${fimods} << EOF2 - + if ("${skip_parse_namelist}" == "true") then + # need to make sure the decomp info from the set_nml is picked up. each case + # has a slightly different decomp that is independent of the ice_in_save file. + # compute that then overwrite by set_nml as needed. + grep -i "distribution_type" ${ICE_SCRIPTS}/options/set_nml.${name} >> ${fimods} + grep -i "processor_shape" ${ICE_SCRIPTS}/options/set_nml.${name} >> ${fimods} + cat >> ${fimods} << EOF2 +# using saved ice_in EOF2 + else + cat ${ICE_SCRIPTS}/options/set_nml.${name} >> ${fimods} + cat >> ${fimods} << EOF2 +EOF2 + endif echo "adding namelist mods set_nml.${name}" echo "`date` ${0} adding namelist modes set_nml.${name}" >> ${casedir}/README.case set found = 1 endif + if (-e ${ICE_SCRIPTS}/options/set_env.${name}) then cat >> ${fsmods} << EOF2 # set_env.${name} - EOF2 cat ${ICE_SCRIPTS}/options/set_env.${name} >> ${fsmods} cat >> ${fsmods} << EOF2 - EOF2 echo "adding env mods set_env.${name}" echo "`date` ${0} adding namelist modes set_env.${name}" >> ${casedir}/README.case set found = 1 endif + if (${found} == 0) then echo "${0}: ERROR, ${ICE_SCRIPTS}/options/set_[nml,env].${name} not found" exit -1 endif end +#set pd1 = `date -u "+%s%N"` +#@ pdd = ( $pd1 - $pd0 ) / 1000000 +#echo "tcxp b4 parse $pdd" ${casescr}/parse_settings.sh cice.settings ${fsmods} if ($status != 0) then - echo "${0}: ERROR, parse_namelist.sh aborted" + echo "${0}: ERROR, parse_settings.sh aborted" exit -1 endif ${casescr}/parse_namelist.sh ice_in ${fimods} @@ -1082,6 +1108,20 @@ EOF2 source ./cice.settings source ./env.${machcomp} -nomodules || exit 2 ${casescr}/parse_namelist_from_env.sh ice_in + if ($status != 0) then + echo "${0}: ERROR, parse_namelist_from_env.sh aborted" + exit -1 + endif +#set pd1 = `date -u "+%s%N"` +#@ pdd = ( $pd1 - $pd0 ) / 1000000 +#echo "tcxp after parse $pdd" + + # Save ice_in in the suite to reduce time spend in parse_namelist + if (${dosuite} == 1) then + if !(-e ${iceinfn}) then + cp ice_in ${iceinfn} + endif + endif #------------------------------------------------------------ # Generate run script @@ -1166,6 +1206,10 @@ EOF echo "" endif +#set pd1 = `date -u "+%s%N"` +#@ pdd = ( $pd1 - $pd0 ) / 1000000 +#echo "tcxp case done $pdd" + # This is the foreach end for the testsuite end # This is the foreach end for the envnames @@ -1180,6 +1224,7 @@ if ( ${dosuite} == 1 ) then cat >> ${tsdir}/suite.submit << EOF0 set nonomatch && rm -f ciceexe.* && unset nonomatch +set nonomatch && rm -f ice_in_save* && unset nonomatch EOF0 @@ -1222,6 +1267,10 @@ endif #--------------------------------------------- +#set pd1 = `date -u "+%s%N"` +#@ pdd = ( $pd1 - $pd0 ) / 1000000 +#echo "tcxp done $pdd" + echo " " echo "${0} done" echo " " diff --git a/configuration/scripts/parse_namelist.sh b/configuration/scripts/parse_namelist.sh index 6dbad45d6..f5ae8a406 100755 --- a/configuration/scripts/parse_namelist.sh +++ b/configuration/scripts/parse_namelist.sh @@ -43,26 +43,20 @@ do fi done - #sed -i 's|\(^\s*'"$vname"'\s*\=\s*\)\(.*$\)|\1'"$value"'|g' $filename - cp ${filename} ${filename}.check - sed -i.sedbak -e 's|\(^[[:space:]]*'"$vname"'[[:space:]]*=[[:space:]]*\)\(.*$\)|\1'"$foundstring"'|g' ${filename}.check - grep -q ${foundstring} ${filename}.check - if [ $? -eq 0 ]; then - sed -i.sedbak -e 's|\(^[[:space:]]*'"$vname"'[[:space:]]*=[[:space:]]*\)\(.*$\)|\1'"$value"'|g' ${filename} + grep -q "^[[:space:]]*${vname}[[:space:]]*=" $filename + grepout=$? + if [ ${grepout} -eq 0 ]; then + sed -i -e 's|\(^[[:space:]]*'"$vname"'[[:space:]]*=[[:space:]]*\)\(.*$\)|\1'"$value"'|g' ${filename} if [[ "${found}" == "${foundstring}" ]]; then vnamearray+=($vname) valuearray+=($value) else valuearray[$found]=${value} fi - if [[ -e "${filename}.sedbak" ]]; then - rm ${filename}.sedbak - fi else echo "${scriptname} ERROR: parsing error for ${vname}" exit -99 fi - rm ${filename}.check ${filename}.check.sedbak fi diff --git a/configuration/scripts/parse_namelist_from_env.sh b/configuration/scripts/parse_namelist_from_env.sh index 4d829450f..08ca2154c 100755 --- a/configuration/scripts/parse_namelist_from_env.sh +++ b/configuration/scripts/parse_namelist_from_env.sh @@ -5,16 +5,13 @@ if [[ "$#" -ne 1 ]]; then exit -1 fi +scriptname=`basename "$0"` filename=$1 #echo "$0 $1" -echo "running parse_namelist_from_env.sh" +echo "running $scriptname" -sed -i.sedbak -e 's|ICE_SANDBOX|'"${ICE_SANDBOX}"'|g' $filename -sed -i.sedbak -e 's|ICE_MACHINE_INPUTDATA|'"${ICE_MACHINE_INPUTDATA}"'|g' $filename - -if [[ -e "${filename}.sedbak" ]]; then - rm ${filename}.sedbak -fi +sed -i -e 's|ICE_SANDBOX|'"${ICE_SANDBOX}"'|g' $filename +sed -i -e 's|ICE_MACHINE_INPUTDATA|'"${ICE_MACHINE_INPUTDATA}"'|g' $filename exit 0 diff --git a/configuration/scripts/parse_settings.sh b/configuration/scripts/parse_settings.sh index fe7f87c1c..a2aed53ce 100755 --- a/configuration/scripts/parse_settings.sh +++ b/configuration/scripts/parse_settings.sh @@ -46,26 +46,20 @@ do fi done - #sed -i 's|\(^\s*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename - cp ${filename} ${filename}.check - sed -i.sedbak -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$foundstring"' |g' $filename.check - grep -q ${foundstring} ${filename}.check - if [ $? -eq 0 ]; then - sed -i.sedbak -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename + grep -q "^[[:space:]]*set.* ${vname}[[:space:]]*" $filename + grepout=$? + if [ ${grepout} -eq 0 ]; then + sed -i -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename if [[ "${found}" == "${foundstring}" ]]; then vnamearray+=($vname) valuearray+=($value) else valuearray[$found]=${value} fi - if [[ -e "${filename}.sedbak" ]]; then - rm ${filename}.sedbak - fi else echo "${scriptname} ERROR: parsing error for ${vname}" exit -99 fi - rm ${filename}.check ${filename}.check.sedbak fi From 4c7987e42178762c1226590c0d28c02901dab148 Mon Sep 17 00:00:00 2001 From: apcraig Date: Mon, 22 Nov 2021 22:31:00 -0700 Subject: [PATCH 3/5] fix cice.setup for case --- cice.setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cice.setup b/cice.setup index a5cbbd468..ef08b3a7f 100755 --- a/cice.setup +++ b/cice.setup @@ -938,9 +938,9 @@ EOF # Use an existing ice_in file from the suite if it exists # to reduce time spend in parse_namelist - set iceinfn = ../ice_in_save_${grid}${soptions} set skip_parse_namelist = spval if (${dosuite} == 1) then + set iceinfn = ../ice_in_save_${grid}${soptions} if (-e ${iceinfn}) then echo "use ${iceinfn}" cp ${iceinfn} ice_in From 14c436ee4958d4b8dd35900c0308389d70f243af Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 23 Nov 2021 14:46:58 -0700 Subject: [PATCH 4/5] add sedbak implementation to support Mac sed --- configuration/scripts/parse_namelist.sh | 6 +++++- configuration/scripts/parse_namelist_from_env.sh | 8 ++++++-- configuration/scripts/parse_settings.sh | 6 +++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/configuration/scripts/parse_namelist.sh b/configuration/scripts/parse_namelist.sh index f5ae8a406..dcb0d1ccc 100755 --- a/configuration/scripts/parse_namelist.sh +++ b/configuration/scripts/parse_namelist.sh @@ -46,7 +46,7 @@ do grep -q "^[[:space:]]*${vname}[[:space:]]*=" $filename grepout=$? if [ ${grepout} -eq 0 ]; then - sed -i -e 's|\(^[[:space:]]*'"$vname"'[[:space:]]*=[[:space:]]*\)\(.*$\)|\1'"$value"'|g' ${filename} + sed -i.sedbak -e 's|\(^[[:space:]]*'"$vname"'[[:space:]]*=[[:space:]]*\)\(.*$\)|\1'"$value"'|g' ${filename} if [[ "${found}" == "${foundstring}" ]]; then vnamearray+=($vname) valuearray+=($value) @@ -62,4 +62,8 @@ do done < "$filemods" +if [[ -e "${filename}.sedbak" ]]; then + rm ${filename}.sedbak +fi + exit 0 diff --git a/configuration/scripts/parse_namelist_from_env.sh b/configuration/scripts/parse_namelist_from_env.sh index 08ca2154c..4c25d358d 100755 --- a/configuration/scripts/parse_namelist_from_env.sh +++ b/configuration/scripts/parse_namelist_from_env.sh @@ -11,7 +11,11 @@ filename=$1 #echo "$0 $1" echo "running $scriptname" -sed -i -e 's|ICE_SANDBOX|'"${ICE_SANDBOX}"'|g' $filename -sed -i -e 's|ICE_MACHINE_INPUTDATA|'"${ICE_MACHINE_INPUTDATA}"'|g' $filename +sed -i.sedbak -e 's|ICE_SANDBOX|'"${ICE_SANDBOX}"'|g' $filename +sed -i.sedbak -e 's|ICE_MACHINE_INPUTDATA|'"${ICE_MACHINE_INPUTDATA}"'|g' $filename + +if [[ -e "${filename}.sedbak" ]]; then + rm ${filename}.sedbak +fi exit 0 diff --git a/configuration/scripts/parse_settings.sh b/configuration/scripts/parse_settings.sh index a2aed53ce..a3f432801 100755 --- a/configuration/scripts/parse_settings.sh +++ b/configuration/scripts/parse_settings.sh @@ -49,7 +49,7 @@ do grep -q "^[[:space:]]*set.* ${vname}[[:space:]]*" $filename grepout=$? if [ ${grepout} -eq 0 ]; then - sed -i -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename + sed -i.sedbak -e 's|\(^[[:space:]]*set.* '"$vname"' \)[^#]*\(#*.*$\)|\1 '"$value"' \2|g' $filename if [[ "${found}" == "${foundstring}" ]]; then vnamearray+=($vname) valuearray+=($value) @@ -65,4 +65,8 @@ do done < "$filemods" +if [[ -e "${filename}.sedbak" ]]; then + rm ${filename}.sedbak +fi + exit 0 From 80c407aebb28706e11aca77d1d1be64fc7b761a1 Mon Sep 17 00:00:00 2001 From: apcraig Date: Wed, 24 Nov 2021 10:30:24 -0700 Subject: [PATCH 5/5] s/spend/spent --- cice.setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cice.setup b/cice.setup index ef08b3a7f..60c56e5c2 100755 --- a/cice.setup +++ b/cice.setup @@ -937,7 +937,7 @@ EOF if (-e ${fsmods}) rm ${fsmods} # Use an existing ice_in file from the suite if it exists - # to reduce time spend in parse_namelist + # to reduce time spent in parse_namelist set skip_parse_namelist = spval if (${dosuite} == 1) then set iceinfn = ../ice_in_save_${grid}${soptions} @@ -1116,7 +1116,7 @@ EOF2 #@ pdd = ( $pd1 - $pd0 ) / 1000000 #echo "tcxp after parse $pdd" - # Save ice_in in the suite to reduce time spend in parse_namelist + # Save ice_in in the suite to reduce time spent in parse_namelist if (${dosuite} == 1) then if !(-e ${iceinfn}) then cp ice_in ${iceinfn}