From ba1ddf7c335d642f6b30b6b220fd22ce84240d8e Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Tue, 1 Oct 2019 13:50:59 -0400 Subject: [PATCH] Remove Theia (#10) * Nightly tests on Hera * Remove FV3-MOM6-CICE5 tests from Theia (added on Hera) * Replace hera with hera.intel * Remove Theia and correct Hera directories. --- src/conf/module-setup.csh.inc | 6 - src/conf/module-setup.sh.inc | 6 - src/incmake/buildenv.mk | 2 +- src/incmake/confopt.mk | 15 +-- src/incmake/env/rdhpcs/detect.mk | 17 +-- src/incmake/env/rdhpcs/theia.gnu.mk | 7 -- src/incmake/env/rdhpcs/theia.intel.mk | 7 -- src/incmake/env/rdhpcs/theia.pgi.mk | 7 -- tests/apps.def | 19 +-- tests/commit.def | 2 - tests/multi-app-test.sh | 3 - tests/nightly/cron/hera-nemspara.cron | 4 + tests/nightly/cron/hera-user.cron | 3 + tests/nightly/cron/theia-nemspara.cron | 16 --- tests/nightly/cron/theia-user.cron | 6 - tests/nightly/nightly.def | 4 +- tests/nightly/run-from-hera-nemspara.sh | 5 + tests/nightly/run-from-hera-user.sh | 4 + tests/rtgen | 152 +----------------------- 19 files changed, 34 insertions(+), 251 deletions(-) delete mode 100644 src/incmake/env/rdhpcs/theia.gnu.mk delete mode 100644 src/incmake/env/rdhpcs/theia.intel.mk delete mode 100644 src/incmake/env/rdhpcs/theia.pgi.mk create mode 100644 tests/nightly/cron/hera-nemspara.cron create mode 100644 tests/nightly/cron/hera-user.cron delete mode 100644 tests/nightly/cron/theia-nemspara.cron delete mode 100644 tests/nightly/cron/theia-user.cron create mode 100755 tests/nightly/run-from-hera-nemspara.sh create mode 100755 tests/nightly/run-from-hera-user.sh diff --git a/src/conf/module-setup.csh.inc b/src/conf/module-setup.csh.inc index d3145e83..1d72b8ef 100644 --- a/src/conf/module-setup.csh.inc +++ b/src/conf/module-setup.csh.inc @@ -14,12 +14,6 @@ else if ( { test -d /scratch1 -a ! -d /scratch } ) then source /apps/lmod/lmod/init/$__ms_shell endif module purge -else if ( { test -d /scratch3 -a -d /scratch } ) then - # We are on NOAA Theia - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge else if ( { test -d /gpfs/hps -a -e /etc/SuSE-release } ) then # We are on NOAA Luna or Surge if ( ! { module help >& /dev/null } ) then diff --git a/src/conf/module-setup.sh.inc b/src/conf/module-setup.sh.inc index cf5f853c..b427faa9 100644 --- a/src/conf/module-setup.sh.inc +++ b/src/conf/module-setup.sh.inc @@ -28,12 +28,6 @@ elif [[ -d /scratch1 && ! -d /scratch ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi module purge -elif [[ -d /scratch3 && -d /scratch ]] ; then - # We are on NOAA Theia - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/$__ms_shell - fi - module purge elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/src/incmake/buildenv.mk b/src/incmake/buildenv.mk index 24c9ae7a..9c147782 100644 --- a/src/incmake/buildenv.mk +++ b/src/incmake/buildenv.mk @@ -2,7 +2,7 @@ # # This is the main driver for the build environment decision logic. # It walks throught the known build environments (WCOSS Phase 3, Linux -# GNU, Theia + Intel compiler, etc.) For each one, if the environment +# GNU, Hera + Intel compiler, etc.) For each one, if the environment # is detected on this machine, it is added to the known list. The # first detected is chosen as the default, unless the user chooses one # through the BUILD_ENV variable. diff --git a/src/incmake/confopt.mk b/src/incmake/confopt.mk index 31b981f8..9765365d 100644 --- a/src/incmake/confopt.mk +++ b/src/incmake/confopt.mk @@ -4,18 +4,10 @@ ifneq ($(CONFOPT),) $(info Automatically converting to new variables from CONFOPT="$(CONFOPT)") - ifeq ($(CONFOPT),nmmb_intel_theia) - CONFIGURE_NEMS_FILE ?= configure.nems.Theia.intel_nmmb - CHOSEN_MODULE ?= theia/ESMF_700_nmmb - - else ifeq ($(CONFOPT),nmmb_intel_wcoss) + ifeq ($(CONFOPT),nmmb_intel_wcoss) CONFIGURE_NEMS_FILE ?= configure.nems.Wcoss.intel_nmmb CHOSEN_MODULE ?= wcoss/ESMF_700_nmmb - else ifeq ($(CONFOPT),gsm_intel_theia) - CONFIGURE_NEMS_FILE ?= configure.nems.Theia.intel_gsm - CHOSEN_MODULE ?= theia/ESMF_700_gsm - else ifeq ($(CONFOPT),gsm_intel_wcoss) CONFIGURE_NEMS_FILE ?= configure.nems.Wcoss.intel_gsm CHOSEN_MODULE ?= wcoss.phase1/ESMF_700_gsm @@ -39,11 +31,6 @@ ifneq ($(CONFOPT),) EXTERNALS_NEMS_FILE ?= externals.nems.Gaea CHOSEN_MODULE ?= gaea/ESMF_NUOPC - else ifeq ($(CONFOPT),coupled_intel_theia) - CONFIGURE_NEMS_FILE ?= configure.nems.Theia.intel - EXTERNALS_NEMS_FILE ?= externals.nems.Theia - CHOSEN_MODULE ?= theia/ESMF_NUOPC - else ifeq ($(CONFOPT),coupled_intel_yellowstone) CONFIGURE_NEMS_FILE ?= configure.nems.Yellowstone.intel EXTERNALS_NEMS_FILE ?= externals.nems.Yellowstone diff --git a/src/incmake/env/rdhpcs/detect.mk b/src/incmake/env/rdhpcs/detect.mk index d7e6e7cc..9196128d 100644 --- a/src/incmake/env/rdhpcs/detect.mk +++ b/src/incmake/env/rdhpcs/detect.mk @@ -4,22 +4,17 @@ # ######################################################################## -ifneq (,$(and $(wildcard /scratch),$(wildcard /scratch3))) +ifneq (,$(and $(wildcard /scratch1),$(wildcard /scratch2))) NEMS_COMPILER?=intel - $(call add_build_env,theia.$(NEMS_COMPILER),env/rdhpcs/theia.$(NEMS_COMPILER).mk) + $(call add_build_env,hera.$(NEMS_COMPILER),env/rdhpcs/hera.$(NEMS_COMPILER).mk) else - ifneq (,$(and $(wildcard /scratch1),$(wildcard /scratch2))) + ifneq (,$(and $(wildcard /lfs1),$(wildcard /lfs3))) NEMS_COMPILER?=intel - $(call add_build_env,hera.$(NEMS_COMPILER),env/rdhpcs/hera.$(NEMS_COMPILER).mk) + $(call add_build_env,jet.$(NEMS_COMPILER),env/rdhpcs/jet.$(NEMS_COMPILER).mk) else - ifneq (,$(and $(wildcard /lfs1),$(wildcard /lfs3))) + ifneq (,$(shell hostname | grep -i gaea)) NEMS_COMPILER?=intel - $(call add_build_env,jet.$(NEMS_COMPILER),env/rdhpcs/jet.$(NEMS_COMPILER).mk) - else - ifneq (,$(shell hostname | grep -i gaea)) - NEMS_COMPILER?=intel - $(call add_build_env,gaea.$(NEMS_COMPILER),env/rdhpcs/gaea.$(NEMS_COMPILER).mk) - endif + $(call add_build_env,gaea.$(NEMS_COMPILER),env/rdhpcs/gaea.$(NEMS_COMPILER).mk) endif endif endif diff --git a/src/incmake/env/rdhpcs/theia.gnu.mk b/src/incmake/env/rdhpcs/theia.gnu.mk deleted file mode 100644 index 97976be8..00000000 --- a/src/incmake/env/rdhpcs/theia.gnu.mk +++ /dev/null @@ -1,7 +0,0 @@ -MACHINE_ID=theia -FULL_MACHINE_ID=theia -USE_MODULES=YES -DEFAULT_MODULE=$(FULL_MACHINE_ID)/ESMF_NUOPC -BUILD_TARGET=$(FULL_MACHINE_ID).$(NEMS_COMPILER) -NEMS_COMPILER=gnu -MODULE_LOGIC=$(call ULIMIT_MODULE_LOGIC,200000) diff --git a/src/incmake/env/rdhpcs/theia.intel.mk b/src/incmake/env/rdhpcs/theia.intel.mk deleted file mode 100644 index 18262967..00000000 --- a/src/incmake/env/rdhpcs/theia.intel.mk +++ /dev/null @@ -1,7 +0,0 @@ -MACHINE_ID=theia -FULL_MACHINE_ID=theia -USE_MODULES=YES -DEFAULT_MODULE=$(FULL_MACHINE_ID)/ESMF_NUOPC -BUILD_TARGET=$(FULL_MACHINE_ID).$(NEMS_COMPILER) -NEMS_COMPILER=intel -MODULE_LOGIC=$(call ULIMIT_MODULE_LOGIC,200000) diff --git a/src/incmake/env/rdhpcs/theia.pgi.mk b/src/incmake/env/rdhpcs/theia.pgi.mk deleted file mode 100644 index d7c62dcc..00000000 --- a/src/incmake/env/rdhpcs/theia.pgi.mk +++ /dev/null @@ -1,7 +0,0 @@ -MACHINE_ID=theia -FULL_MACHINE_ID=theia -USE_MODULES=YES -DEFAULT_MODULE=$(FULL_MACHINE_ID)/ESMF_NUOPC -BUILD_TARGET=$(FULL_MACHINE_ID).$(NEMS_COMPILER) -NEMS_COMPILER=pgi -MODULE_LOGIC=$(call ULIMIT_MODULE_LOGIC,200000) diff --git a/tests/apps.def b/tests/apps.def index f67a8874..4577152d 100644 --- a/tests/apps.def +++ b/tests/apps.def @@ -1,8 +1,5 @@ # List of valid platforms and the human-readable names of each. -PLATFORM theia NAME Theia -PLATFORM theia.slurm.intel NAME Theia (SLURM test) -PLATFORM hera NAME Hera -PLATFORM hera.intel NAME Hera (SLURM test) +PLATFORM hera.intel NAME Hera PLATFORM wcoss1 NAME WCOSS Phase 1 PLATFORM wcoss2 NAME WCOSS Phase 2 PLATFORM wcoss_cray NAME WCOSS Cray @@ -22,27 +19,19 @@ APP FV3GFS-GSDCHEM URL gerrit:EMC_FV3GFS-GSDCHEM # Shell expressions that generate scrub space for a given $username # on each platform. -ON theia SCRUB /scratch4/NCEPDEV/nems/scrub/$username -ON theia.slurm.intel SCRUB /scratch4/NCEPDEV/nems/scrub/$username -ON hera SCRUB /scratch1/NCEPDEV/nems/$username -ON hera.intel SCRUB /scratch1/NCEPDEV/nems/$username +ON hera.intel SCRUB /scratch1/NCEPDEV/nems/$username/scrub ON wcoss1 SCRUB /ptmpp1/$username ON wcoss2 SCRUB /ptmpd3/$username ON wcoss_cray SCRUB /gpfs/hps2/ptmp/$username ON wcoss_dell_p3 SCRUB /gpfs/dell2/ptmp/$username # List of apps to run on each platform. -ON theia APPS FV3-MOM6-CICE5 -ON theia.slurm.intel APPS NEMSfv3gfs WW3-FV3 -ON hera APPS FV3-MOM6-CICE5 -ON hera.intel APPS NEMSfv3gfs WW3-FV3 +#ON hera.intel APPS NEMSfv3gfs FV3-MOM6-CICE5 WW3-FV3 +ON hera.intel APPS NEMSfv3gfs FV3-MOM6-CICE5 ON wcoss1 APPS NEMSfv3gfs ON wcoss2 APPS NEMSfv3gfs WW3-FV3 ON wcoss_cray APPS NEMSfv3gfs FV3GFS-GSDCHEM ON wcoss_dell_p3 APPS NEMSfv3gfs # Extra arguments to rt.sh for each platform -ON theia EXTRA_ARGS --platform theia -ON theia.slurm.intel EXTRA_ARGS --platform theia.slurm.intel -ON hera EXTRA_ARGS --platform hera ON hera.intel EXTRA_ARGS --platform hera.intel diff --git a/tests/commit.def b/tests/commit.def index acd65c66..93af945d 100644 --- a/tests/commit.def +++ b/tests/commit.def @@ -1,6 +1,4 @@ # Directory to receive the webpage with regression test results -ON theia WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/ -ON theia.slurm.intel WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/ ON hera WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/ ON hera.intel WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/ ON wcoss1 WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/ diff --git a/tests/multi-app-test.sh b/tests/multi-app-test.sh index e0078f99..74eafdce 100755 --- a/tests/multi-app-test.sh +++ b/tests/multi-app-test.sh @@ -237,13 +237,11 @@ parse_control_file() { # Specifies the directory to use for this scripts work # space. Use the string "$username" in place of the user # name. - # ON theia SCRUB /scratch4/NCEPDEV/nems/scrub/$username #echo "Platform $key uses scrub space $value" set_global "scrub_space_for_$key" "$value" check_platform_in_key=YES elif [[ "$context" == ON && "$command" == APPS ]] ; then # Specifies which apps should be tested on each platform. - # ON theia APPS NEMSfv3gfs NEMSGSM HYCOM-GSM-CICE WW3-FV3 WW3-ATM FV3-MOM6-CICE5 #echo "Platform $key will run apps $value" set_global "app_list_for_$key" "$value" check_platform_in_key=YES @@ -282,7 +280,6 @@ parse_control_file() { elif [[ "$context" == ON && "$command" == WEBPAGE ]] ; then # Specifies the directory (local or remote) to receive the webpage that has test results. #ON tujet WEBPAGE /lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/dell-produtil/ - #ON theia WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/dell-produtil/ #echo "Platform $key sends to webpage $value" set_global "webpage_for_$key" "$value" diff --git a/tests/nightly/cron/hera-nemspara.cron b/tests/nightly/cron/hera-nemspara.cron new file mode 100644 index 00000000..b23fa28e --- /dev/null +++ b/tests/nightly/cron/hera-nemspara.cron @@ -0,0 +1,4 @@ +03 00 * * * mkdir -p /scratch1/NCEPDEV/nems/emc.nemspara/scrub /scratch1/NCEPDEV/stmp2/emc.nemspara/scrub + +40 22 * * * /scratch2/NCEPDEV/fv3-cam/Ratko.Vasic/multi-app-test/tests/nightly/run-from-hera-nemspara.sh ngt hera.intel test > /home/emc.nemspara/hera-nightly-test.log 2>&1 +47 * * * * /scratch2/NCEPDEV/fv3-cam/Ratko.Vasic/multi-app-test/tests/nightly/run-from-hera-nemspara.sh ngt hera.intel deliver > /home/emc.nemspara/hera-nightly-deliver.log 2>&1 diff --git a/tests/nightly/cron/hera-user.cron b/tests/nightly/cron/hera-user.cron new file mode 100644 index 00000000..91f07c4c --- /dev/null +++ b/tests/nightly/cron/hera-user.cron @@ -0,0 +1,3 @@ +01 00 * * * /home/Ratko.Vasic/bin/scrubber > /home/Ratko.Vasic/logs/hera_scrubber.log + +00 22 * * * /scratch2/NCEPDEV/fv3-cam/Ratko.Vasic/multi-app-test/tests/nightly/run-from-hera-user.sh ngt hera.intel checkout > /scratch2/NCEPDEV/fv3-cam/Ratko.Vasic/hera-nightly-checkout.log 2>&1 diff --git a/tests/nightly/cron/theia-nemspara.cron b/tests/nightly/cron/theia-nemspara.cron deleted file mode 100644 index 7d5307e5..00000000 --- a/tests/nightly/cron/theia-nemspara.cron +++ /dev/null @@ -1,16 +0,0 @@ -30 01 * * * /home/emc.nemspara/bin/scrubber 1 3 -25 13 * * * /home/emc.nemspara/bin/scrubber 2 3 -30 07 * * * /home/emc.nemspara/bin/scrubber 3 3 -25 19 * * * /home/emc.nemspara/bin/scrubber 4 3 - -# NEMS regression tests: -03 00 * * * mkdir -p /scratch4/NCEPDEV/nems/scrub/emc.nemspara - -58 18 * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-nemspara.sh ngt theia test > /scratch4/NCEPDEV/nems/scrub/emc.nemspara/nightly-test.log 2>&1 -47 * * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-nemspara.sh ngt theia deliver > /scratch4/NCEPDEV/nems/scrub/emc.nemspara/nightly-deliver.log 2>&1 - -58 18 * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-nemspara.sh ngt theia.intel test > /scratch4/NCEPDEV/nems/scrub/emc.nemspara/intel-nightly-test.log 2>&1 -47 * * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-nemspara.sh ngt theia.intel deliver > /scratch4/NCEPDEV/nems/scrub/emc.nemspara/intel-nightly-deliver.log 2>&1 - -58 18 * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-nemspara.sh -Mslurm ngt theia.slurm.intel test > /scratch4/NCEPDEV/nems/scrub/emc.nemspara/slurm-nightly-test.log 2>&1 -47 * * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-nemspara.sh ngt theia.slurm.intel deliver > /scratch4/NCEPDEV/nems/scrub/emc.nemspara/slurm-nightly-deliver.log 2>&1 diff --git a/tests/nightly/cron/theia-user.cron b/tests/nightly/cron/theia-user.cron deleted file mode 100644 index 951c0742..00000000 --- a/tests/nightly/cron/theia-user.cron +++ /dev/null @@ -1,6 +0,0 @@ -03 00 * * * mkdir -p /scratch4/NCEPDEV/meso/scrub/Ratko.Vasic - -28 18 * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-user.sh ngt theia checkout > /scratch4/NCEPDEV/meso/scrub/Ratko.Vasic/nightly-checkout.log 2>&1 -28 18 * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-user.sh ngt theia.intel checkout > /scratch4/NCEPDEV/meso/scrub/Ratko.Vasic/intel-nightly-checkout.log 2>&1 -28 18 * * * /scratch4/NCEPDEV/meso/noscrub/Ratko.Vasic/multi-app-test/tests/nightly/run-from-theia-user.sh ngt theia.slurm.intel checkout > /scratch4/NCEPDEV/meso/scrub/Ratko.Vasic/slurm-nightly-checkout.log 2>&1 - diff --git a/tests/nightly/nightly.def b/tests/nightly/nightly.def index 2f5d1968..5917a1dc 100644 --- a/tests/nightly/nightly.def +++ b/tests/nightly/nightly.def @@ -3,9 +3,7 @@ ON tujet WEBPAGE /lfs3/projects/hfv3gfs/emc.nemspara/web/rt ON ujet.slurm WEBPAGE /lfs3/projects/hfv3gfs/emc.nemspara/web/rt ON svjet WEBPAGE /lfs3/projects/hfv3gfs/emc.nemspara/web/rt ON xjet WEBPAGE /lfs3/projects/hfv3gfs/emc.nemspara/web/rt -ON theia WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/rt -ON theia.intel WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/rt -ON theia.slurm.intel WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/rt +ON hera.intel WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/rt ON wcoss1 WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/rt ON wcoss2 WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/rt ON wcoss_cray WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/rt diff --git a/tests/nightly/run-from-hera-nemspara.sh b/tests/nightly/run-from-hera-nemspara.sh new file mode 100755 index 00000000..bb4a356f --- /dev/null +++ b/tests/nightly/run-from-hera-nemspara.sh @@ -0,0 +1,5 @@ +#! /bin/bash --login +cd $( dirname "$0" ) +cd .. +source ../src/conf/module-setup.sh.inc +cat apps.def nightly/nightly.def | ./multi-app-test.sh "$@" diff --git a/tests/nightly/run-from-hera-user.sh b/tests/nightly/run-from-hera-user.sh new file mode 100755 index 00000000..f0ef5508 --- /dev/null +++ b/tests/nightly/run-from-hera-user.sh @@ -0,0 +1,4 @@ +#! /bin/sh +cd $( dirname "$0" ) +cd .. +cat apps.def nightly/nightly.def | ./multi-app-test.sh "$@" diff --git a/tests/rtgen b/tests/rtgen index afbbd932..f1b1b0d2 100755 --- a/tests/rtgen +++ b/tests/rtgen @@ -493,64 +493,6 @@ def parse_account_params(): return parsed_account_params ######################################################################## - -# Theia project selection - -def decide_project_theia(): - """!Chooses which project to use when submitting jobs on Theia. - - Uses the account_params program to scan the available core-hours - on Theia. Chooses the project with the most available core-hours. - If no projects have resources, or if some error happens, then - the "nems" project is used.""" - logger=logging.getLogger('rtgen') - try: - account_params=produtil.run.runstr( - batchexe('account_params'),logger=jlogger) - except(EnvironmentError,ExitStatusException) as ee: - logger.warning('Cannot run account_params: '+str(ee)) - logger.warning('Will use project "nems" for cpu hours.') - return 'nems' - default_project='nems' - projects=list() - projalloc=dict() - for line in account_params.splitlines(): - # Allocation: 6912 stmp 0.00 0.00 0.00 - m=re.match('^\s*Allocation:\s+(\d+)\s+(\S+)\s+([0-9.]+)',line) - if not m: - # skip other lines - logger.debug('Skip line: '+line.rstrip()) - continue - gid,name,alloc = m.groups() - try: - alloc=float(alloc) - if name=='nems': alloc/=2 - if not projects: - default_project=name - projects.append(name) - projalloc[name]=alloc - except (ValueError,TypeError) as vte: - logger.warning('Cannot parse: '+line.rstrip()) - continue - if not projects: - # Parse error or failure of account_params. - logger.warning('Could not parse account_params output. Will use default: '+default_project) - return default_project - projects.sort(lambda a,b: cmp(projalloc[a],projalloc[b])) - projchoose=projects[-1] - - if projalloc[projchoose]<1.0: - logger.warning('All projects passed core-hour limit; will use first project: '+default_project) - return default_project - - for proj in projects: - if proj==projchoose: - chose='<== chosen' - else: - chose='' - logger.info('%10s : %6d %s'%(proj,projalloc[proj],chose)) - return projchoose - # Hera project selection def decide_project_hera(): @@ -608,81 +550,7 @@ def decide_project_hera(): # logger.info('%10s : %6d %s'%(proj,projalloc[proj],chose)) # return projchoose -def decide_tmp_theia(): - """!Chooses a scratch space to use on Theia, based on how close - each space is to its quota. - - Uses the pan_df program to check the quota of stmp1 through stmp4. - Returns the one that is farthest from quota based on percent - usage. If this process fails, such as pan_df giving a non-zero - return status or unparseable output, then a random stmp is chosen. - - @returns path to a temporary directory, which may not yet exist.""" - logger=logging.getLogger('rtgen') - stmps=[ '/scratch3/NCEPDEV/stmp1', - '/scratch3/NCEPDEV/stmp2', - '/scratch4/NCEPDEV/stmp3', - '/scratch4/NCEPDEV/stmp4' ] - try: - args=['-B', '1G', '--' ] - args.extend(stmps) - pan_df=produtil.run.runstr(batchexe('pan_df')[args]) - storage=dict() - for m in re.finditer(r'''(?isx) - (?: - \s* (?P \S+ ) - [ \t\r\n]+ (?P \d+ ) - \s+ (?P \d+ ) - \s+ (?P \d+ ) - \s+ (?P [0-9.]+ ) % - \s+ (?P \S+ ) - | - (?P [^\r\n]*[\r\n] | [^\r\n]*\Z ) ) - ''',pan_df): - # Skip lines that do not have usage information (such as - # the header line). - if m.group('bad') or not m.group('mntpnt'): - logger.debug('pan_df: ignoring %s'%(repr(m.group(0).strip()),)) - continue - - mntpnt=m.group('mntpnt') - percent=m.group('percent') - percent=int(percent,10) - - # Skip lines with invalid mount points: - if not os.path.isdir(mntpnt): - logger.warning( - 'Ignoring invalid mount point from pan_df: %s'%( - mntpnt,)) - continue - - # Store all valid lines: - logger.debug('pan_df: %s at %d%% usage'%(mntpnt,percent)) - storage[mntpnt]=percent - - # Return the least used stmp if available. - if not storage: - logger.error( - 'No valid lines seen in pan_df output.') - else: - by_usage=storage.keys() - by_usage.sort( - lambda a,b: cmp(storage[a],storage[b])) - logger.info('%s: use this tmp (has lowest usage at %d%%)'%( - by_usage[0],storage[by_usage[0]])) - return os.path.join(by_usage[0],username()) - - except(EnvironmentError,ExitStatusException,KeyError,ValueError) as e: - # Log all likely errors before emergency fallback option: - logger.error(str(e),exc_info=True) - - use_me=random.choice(stmps) - logger.warning("Auto-detection of least used stmp failed.") - logger.warning("%s: randomly chosen stmp"%(use_me,)) - return os.path.join(use_me,username()) - ######################################################################## - def decide_tmp_hera(): """!Chooses a scratch space to use on Hera, based on how close each space is to its quota. @@ -698,7 +566,7 @@ def decide_tmp_hera(): # '/scratch1/NCEPDEV/stmp4', # '/scratch2/NCEPDEV/stmp1', # '/scratch2/NCEPDEV/stmp3' ] - return '/scratch1/NCEPDEV/stmp2/'+username() + return '/scratch1/NCEPDEV/stmp2/'+username()+'/scrub' # try: # args=['-B', '1G', '--' ] # args.extend(stmps) @@ -1068,12 +936,6 @@ class RTGen(TestGen): #if produtil.fileop.find_exe('sbatch',raise_missing=False): # out.write('module load slurm\n') out.write('have_qoutql=YES\n') - elif here.name == 'theia': - #if produtil.fileop.find_exe('sbatch',raise_missing=False): - # out.write('module load slurm\n') - out.write('module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles/\n') - out.write('module load rocoto/1.3.0rc2\n') - out.write('module load hpss emc-utils/1.1.0 ; have_qoutql=YES\n') elif here.name == 'hera': out.write('module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles/\n') out.write('module load rocoto/1.3.1\n') @@ -1482,9 +1344,7 @@ def rtgen(verbose,baseline_dir,dry_run,baseline,unique_id,temp, ## Decide the project: if project is None: - if produtil.cluster.name() == 'theia': - project=decide_project_theia() - elif produtil.cluster.name() == 'hera': + if produtil.cluster.name() == 'hera': project=decide_project_hera() elif produtil.cluster.name() in ['gyre','tide','luna','surge','venus','mars']: project=decide_project_wcoss() @@ -1495,7 +1355,7 @@ def rtgen(verbose,baseline_dir,dry_run,baseline,unique_id,temp, elif produtil.cluster.name() == 'gaea': project=decide_project_gaea() else: - fail('Unknown system "'+produtil.cluster.name()+'". Only Hera, Theia, Jet, WCOSS, and GAEA are supported.') + fail('Unknown system "'+produtil.cluster.name()+'". Only Hera, Jet, WCOSS, and GAEA are supported.') jlogger.info('Auto-chosen project for job submission is %s'%( repr(project),)) else: @@ -1504,9 +1364,7 @@ def rtgen(verbose,baseline_dir,dry_run,baseline,unique_id,temp, ## Decide the temp area if temp is None: - if produtil.cluster.name() == 'theia': - scratch_dir=decide_tmp_theia() - elif produtil.cluster.name() == 'hera': + if produtil.cluster.name() == 'hera': scratch_dir=decide_tmp_hera() elif produtil.cluster.name() in ['gyre','tide','luna','surge','mars','venus']: scratch_dir=decide_tmp_wcoss(produtil.cluster.where().wcoss_phase) @@ -1520,7 +1378,7 @@ def rtgen(verbose,baseline_dir,dry_run,baseline,unique_id,temp, assert('nceplibs' not in scratch_dir) assert('hwrfdata' not in scratch_dir) else: - fail('Unknown system "'+produtil.cluster.name()+'". Only Hera, Theia, Jet, WCOSS, and GAEA are supported.') + fail('Unknown system "'+produtil.cluster.name()+'". Only Hera, Jet, WCOSS, and GAEA are supported.') jlogger.info('Auto-chosen ptmp is %s'%(repr(scratch_dir),)) else: scratch_dir=temp