[develop] Fix issue 555 (restore default behavior of EXPT_BASEDIR opt…

…ion) (#562)

This PR restores the previous behavior of the variable EXPT_BASEDIR (Item 2 is the behavior that was previously broken), which has the following effect on the experiment directory EXPTDIR:

1) If EXPT_BASEDIR is not set or set to a null value, the default value (${HOMEdir}/../expt_dirs) will be used
2) If EXPT_BASEDIR is set to a relative path (i.e. the first character is not /), the user-specified path will be appended to the default value ${HOMEdir}/../expt_dirs (for example if the user specifies EXPT_BASEDIR=some/relative/path in their config.yaml, it will be updated to EXPT_BASEDIR=${HOMEdir}/../expt_dirs/some/relative/path in the workflow
3) If EXPT_BASEDIR is set to an absolute path, that path will be used as entered

After the above logic is applied, EXPTDIR will be created by joining the paths EXPT_BASEDIR and EXPT_SUBDIR as usual.

docs/UsersGuide/source/ConfigWorkflow.rst

@@ -215,7 +215,7 @@ Directory Parameters
 -----------------------
 
 ``EXPT_BASEDIR``: (Default: "")
-   The full path to the base directory in which the experiment directory (``EXPT_SUBDIR``) will be created. If this is not specified or if it is set to an empty string, it will default to ``${HOMEdir}/../expt_dirs``, where ``${HOMEdir}`` contains the full path to the ``ufs-srweather-app`` directory.
+   The full path to the base directory in which the experiment directory (``EXPT_SUBDIR``) will be created. If this is not specified or if it is set to an empty string, it will default to ``${HOMEdir}/../expt_dirs``, where ``${HOMEdir}`` contains the full path to the ``ufs-srweather-app`` directory. If set to a relative path, the provided path will be appended to the default value ``${HOMEdir}/../expt_dirs``. For example, if ``EXPT_BASEDIR=some/relative/path`` (i.e. a path that does not begin with ``/``), the value of ``EXPT_BASEDIR`` used by the workflow will be ``EXPT_BASEDIR=${HOMEdir}/../expt_dirs/some/relative/path``.
 
 ``EXPT_SUBDIR``: (Default: "")
    The user-designated name of the experiment directory (*not* its full path). The full path to the experiment directory, which will be contained in the variable ``EXPTDIR``, will be:

ush/config_defaults.yaml

@@ -455,7 +455,8 @@ workflow:
   # EXPT_BASEDIR:
   # The base directory in which the experiment directory will be created.  
   # If this is not specified or if it is set to an empty string, it will
-  # default to ${HOMEdir}/../expt_dirs.  
+  # default to ${HOMEdir}/../expt_dirs. If set to a relative path, the
+  # path will be appended to the default value ${HOMEdir}/../expt_dirs 
   #
   # EXPT_SUBDIR:
   # The name that the experiment directory (without the full path) will
@@ -472,7 +473,7 @@ workflow:
   # installed.
   #-----------------------------------------------------------------------
   #
-  EXPT_BASEDIR: '{{ workflow.EXPT_BASEDIR }}'
+  EXPT_BASEDIR: '' # This will be set in setup.py prior to extend_yaml() being called
   EXPT_SUBDIR: '{{ EXPT_SUBDIR }}'
   EXEC_SUBDIR: "exec"
   EXPTDIR: '{{ [EXPT_BASEDIR, EXPT_SUBDIR]|path_join }}'

ush/setup.py

@@ -142,6 +142,20 @@ def load_config_for_setup(ushdir, default_config, user_config):
     # User settings (take precedence over all others)
     update_dict(cfg_u, cfg_d)
 
+    # Set "Home" directory, the top-level ufs-srweather-app directory
+    homedir = os.path.abspath(os.path.dirname(__file__) + os.sep + os.pardir)
+    cfg_d["user"]["HOMEdir"] = homedir
+
+    # Special logic if EXPT_BASEDIR is a relative path; see config_defaults.yaml for explanation
+    expt_basedir = cfg_d["workflow"]["EXPT_BASEDIR"]
+    if (not expt_basedir) or (expt_basedir[0] != "/"):
+        expt_basedir = os.path.join(homedir, "..", "expt_dirs", expt_basedir)
+    try:
+        expt_basedir = os.path.realpath(expt_basedir)
+    except:
+        pass
+    cfg_d["workflow"]["EXPT_BASEDIR"] = os.path.abspath(expt_basedir)
+
     extend_yaml(cfg_d)
 
     # Do any conversions of data types
@@ -150,9 +164,6 @@ def load_config_for_setup(ushdir, default_config, user_config):
             if not (v is None or v == ""):
                 cfg_d[sect][k] = str_to_list(v)
 
-    for k, v in cfg_d["task_run_fcst"].items():
-        print(f"*** {k}: {v}")
-
     # Mandatory variables *must* be set in the user's config or the machine file; the default value is invalid
     mandatory = [
         "EXPT_SUBDIR",
@@ -214,7 +225,7 @@ def set_srw_paths(ushdir, expt_config):
     """
 
     # HOMEdir is the location of the SRW clone, one directory above ush/
-    homedir = os.path.abspath(os.path.dirname(__file__) + os.sep + os.pardir)
+    homedir = expt_config.get("user", {}).get("HOMEdir")
 
     # Read Externals.cfg
     mng_extrns_cfg_fn = os.path.join(homedir, "Externals.cfg")
@@ -253,7 +264,6 @@ def set_srw_paths(ushdir, expt_config):
         )
 
     return dict(
-        HOMEdir=homedir,
         USHdir=ushdir,
         UFS_WTHR_MDL_DIR=ufs_wthr_mdl_dir,
     )
@@ -339,38 +349,7 @@ def setup(USHdir, user_config_fn="config.yaml"):
               fcst_len_hrs_max = {fcst_len_hrs_max}"""
         )
 
-    #
-    # -----------------------------------------------------------------------
-    #
-    # If the base directory (EXPT_BASEDIR) in which the experiment subdirectory
-    # (EXPT_SUBDIR) will be located does not start with a "/", then it is
-    # either set to a null string or contains a relative directory.  In both
-    # cases, prepend to it the absolute path of the default directory under
-    # which the experiment directories are placed.  If EXPT_BASEDIR was set
-    # to a null string, it will get reset to this default experiment directory,
-    # and if it was set to a relative directory, it will get reset to an
-    # absolute directory that points to the relative directory under the
-    # default experiment directory.  Then create EXPT_BASEDIR if it doesn't
-    # already exist.
-    #
-    # -----------------------------------------------------------------------
-    #
-    expt_basedir = workflow_config.get("EXPT_BASEDIR")
-    homedir = expt_config["user"].get("HOMEdir")
-    if (not expt_basedir) or (expt_basedir[0] != "/"):
-        if not expt_basedir or "{{" in expt_basedir:
-            expt_basedir = ""
-        expt_basedir = os.path.join(homedir, "..", "expt_dirs", expt_basedir)
-    try:
-        expt_basedir = os.path.realpath(expt_basedir)
-    except:
-        pass
-    expt_basedir = os.path.abspath(expt_basedir)
-
-    workflow_config["EXPT_BASEDIR"] = expt_basedir
 
-    # Update some paths that include EXPT_BASEDIR
-    extend_yaml(expt_config)
     #
     # -----------------------------------------------------------------------
     #
@@ -381,7 +360,10 @@ def setup(USHdir, user_config_fn="config.yaml"):
     #
 
     expt_subdir = workflow_config.get("EXPT_SUBDIR", "")
-    exptdir = workflow_config["EXPTDIR"]
+    exptdir = workflow_config.get("EXPTDIR")
+
+    # Update some paths that include EXPTDIR and EXPT_BASEDIR
+    extend_yaml(expt_config)
     preexisting_dir_method = workflow_config.get("PREEXISTING_DIR_METHOD", "")
     try:
         check_for_preexist_dir_file(exptdir, preexisting_dir_method)