Include all python script in ush

ufs-community · Apr 30, 2022 · 89a2eaa · 89a2eaa
1 parent 2b36e92
commit 89a2eaa
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 83 deletions.
diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml
@@ -16,17 +16,12 @@ jobs:
           sudo apt-get update
           sudo apt-get install python3 python3-pip netcdf-bin
           sudo pip3 install pyyaml jinja2 f90nml
+          sudo pip3 install numpy matplotlib
 
       # Run python unittests
       - name: Run python unittests
         run: |
           cd ush
           python3 -m unittest -b python_utils/test_python_utils.py
-          python3 -m unittest -b check_ruc_lsm.py create_diag_table_file.py \
-                     create_model_configure_file.py link_fix.py set_cycle_dates.py \
-                     set_extrn_mdl_params.py set_FV3nml_sfc_climo_filenames.py \
-                     set_FV3nml_ens_stoch_seeds.py set_gridparams_ESGgrid.py \
-                     set_gridparams_GFDLgrid.py set_ozone_param.py \
-                     set_predef_grid_params.py set_thompson_mp_fix_files.py \
-                     get_crontab_contents.py
+          python3 -m unittest -b *.py
 
diff --git a/ush/mrms_pull_topofhour.py b/ush/mrms_pull_topofhour.py
@@ -4,79 +4,84 @@
 import bisect
 import numpy as np
 
-# Copy and unzip MRMS files that are closest to top of hour
-# Done every hour on a 20-minute lag             
-
-# Include option to define valid time on command line
-# Used to backfill verification
-#try:
-valid_time = str(sys.argv[1])
-
-YYYY = int(valid_time[0:4])
-MM   = int(valid_time[4:6])
-DD   = int(valid_time[6:8])
-HH   = int(valid_time[8:19])
-
-valid = datetime.datetime(YYYY,MM,DD,HH,0,0)
-
-#except IndexError:
-#    valid_time = None
-
-# Default to current hour if not defined on command line 
-#if valid_time is None:
-#    now  = datetime.datetime.utcnow()
-#    YYYY = int(now.strftime('%Y'))
-#    MM   = int(now.strftime('%m'))
-#    DD   = int(now.strftime('%d'))
-#    HH   = int(now.strftime('%H'))
-
-#    valid = datetime.datetime(YYYY,MM,DD,HH,0,0)
-#   valid_time = valid.strftime('%Y%m%d%H')
-
-print('Pulling '+valid_time+' MRMS data')
-
-# Set up working directory
-DATA_HEAD = str(sys.argv[2])
-MRMS_PROD_DIR = str(sys.argv[3])
-MRMS_PRODUCT = str(sys.argv[4])
-level = str(sys.argv[5])
-
-VALID_DIR = os.path.join(DATA_HEAD,valid.strftime('%Y%m%d'))
-if not os.path.exists(VALID_DIR):
-    os.makedirs(VALID_DIR)
-os.chdir(DATA_HEAD)
-
-# Sort list of files for each MRMS product
-print(valid.strftime('%Y%m%d'))
-if valid.strftime('%Y%m%d') < '20200303':
-    search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz'
-elif valid.strftime('%Y%m%d') >= '20200303':
-    search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz'
-file_list = [f for f in glob.glob(search_path)]
-time_list = [file_list[x][-24:-9] for x in range(len(file_list))]
-int_list = [int(time_list[x][0:8]+time_list[x][9:15]) for x in range(len(time_list))]
-int_list.sort()
-datetime_list = [datetime.datetime.strptime(str(x),"%Y%m%d%H%M%S") for x in int_list]
-
-# Find the MRMS file closest to the valid time
-i = bisect.bisect_left(datetime_list,valid)
-closest_timestamp = min(datetime_list[max(0, i-1): i+2], key=lambda date: abs(valid - date))
-
-# Check to make sure closest file is within +/- 15 mins of top of the hour
-# Copy and rename the file for future ease
-difference = abs(closest_timestamp - valid)
-if difference.total_seconds() <= 900:
-    filename1 = MRMS_PRODUCT+level+closest_timestamp.strftime('%Y%m%d-%H%M%S')+'.grib2.gz'
-    filename2 = MRMS_PRODUCT+level+valid.strftime('%Y%m%d-%H')+'0000.grib2.gz'
-
-    if valid.strftime('%Y%m%d') < '20200303':
-        print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
-
-        os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
-        os.system('gunzip '+VALID_DIR+'/'+filename2)
-    elif valid.strftime('%Y%m%d') >= '20200303':
-        print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
-
-        os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
-        os.system('gunzip '+VALID_DIR+'/'+filename2)
-
+if __name__ == '__main__':
+   # Copy and unzip MRMS files that are closest to top of hour
+   # Done every hour on a 20-minute lag             
+
+   # Include option to define valid time on command line
+   # Used to backfill verification
+   #try:
+   valid_time = str(sys.argv[1])
+
+   YYYY = int(valid_time[0:4])
+   MM   = int(valid_time[4:6])
+   DD   = int(valid_time[6:8])
+   HH   = int(valid_time[8:19])
+
+   valid = datetime.datetime(YYYY,MM,DD,HH,0,0)
+
+   #except IndexError:
+   #    valid_time = None
+
+   # Default to current hour if not defined on command line 
+   #if valid_time is None:
+   #    now  = datetime.datetime.utcnow()
+   #    YYYY = int(now.strftime('%Y'))
+   #    MM   = int(now.strftime('%m'))
+   #    DD   = int(now.strftime('%d'))
+   #    HH   = int(now.strftime('%H'))
+
+   #    valid = datetime.datetime(YYYY,MM,DD,HH,0,0)
+   #   valid_time = valid.strftime('%Y%m%d%H')
+
+   print('Pulling '+valid_time+' MRMS data')
+
+   # Set up working directory
+   DATA_HEAD = str(sys.argv[2])
+   MRMS_PROD_DIR = str(sys.argv[3])
+   MRMS_PRODUCT = str(sys.argv[4])
+   level = str(sys.argv[5])
+
+   VALID_DIR = os.path.join(DATA_HEAD,valid.strftime('%Y%m%d'))
+   if not os.path.exists(VALID_DIR):
+       os.makedirs(VALID_DIR)
+   os.chdir(DATA_HEAD)
+
+   # Sort list of files for each MRMS product
+   print(valid.strftime('%Y%m%d'))
+   if valid.strftime('%Y%m%d') < '20200303':
+       search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz'
+   elif valid.strftime('%Y%m%d') >= '20200303':
+       search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz'
+   file_list = [f for f in glob.glob(search_path)]
+   time_list = [file_list[x][-24:-9] for x in range(len(file_list))]
+   int_list = [int(time_list[x][0:8]+time_list[x][9:15]) for x in range(len(time_list))]
+   int_list.sort()
+   datetime_list = [datetime.datetime.strptime(str(x),"%Y%m%d%H%M%S") for x in int_list]
+
+   # Find the MRMS file closest to the valid time
+   i = bisect.bisect_left(datetime_list,valid)
+   closest_timestamp = min(datetime_list[max(0, i-1): i+2], key=lambda date: abs(valid - date))
+
+   # Check to make sure closest file is within +/- 15 mins of top of the hour
+   # Copy and rename the file for future ease
+   difference = abs(closest_timestamp - valid)
+   if difference.total_seconds() <= 900:
+       filename1 = MRMS_PRODUCT+level+closest_timestamp.strftime('%Y%m%d-%H%M%S')+'.grib2.gz'
+       filename2 = MRMS_PRODUCT+level+valid.strftime('%Y%m%d-%H')+'0000.grib2.gz'
+
+       if valid.strftime('%Y%m%d') < '20200303':
+           print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
+
+           os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
+           os.system('gunzip '+VALID_DIR+'/'+filename2)
+       elif valid.strftime('%Y%m%d') >= '20200303':
+           print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
+
+           os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2)
+           os.system('gunzip '+VALID_DIR+'/'+filename2)
+
+#dummy unittest
+class Testing(unittest.TestCase):
+   def test_mrms_pull_topfhour(self):
+     pass