Sb/cshore/tjh add notebooks

.cmtbSettings

@@ -3,6 +3,7 @@
 #
 ####################################
 #
+slack: None
 runFlag: true
 analyzeFlag: true
 plotFlag: false
@@ -14,6 +15,7 @@ hpcSettings:
    hpcCores: 36
    hpcNodes: 1
 testName: ''
+hotStartFlag: false
 logfileLoc: ./data/logs
 workingDirectory: ./data               # this is the directory where simulation files and QA/QC plots are made and existing architecture takes over
 netCDFdir: ./thredds_data             # this is the base directory where netCDF files are output

README.md

@@ -43,7 +43,7 @@ The test bed is established at the USACE CHL Field REsearch Facility in Duck, No
     - conda install <package>
 
       conda install -c conda-forge cartopy
-    * netCDF4, pyproj, utm, wavespectra, progressbar, opencv-python
+    * netCDF4, pyproj, utm (cf), wavespectra (cf), progressbar (cf), opencv (cf)
 
   - there's likely more, please add as you find!!  :-[]
 

frontback/frontBackNEW.py

@@ -4,7 +4,6 @@
 from getdatatestbed.getDataFRF import getDataTestBed
 import datetime as DT
 import os, glob, makenc, sys, shutil
-from subprocess import check_output
 import netCDF4 as nc
 import numpy as np
 import plotting.operationalPlots as oP
@@ -32,6 +31,8 @@ def ww3simSetup(startTimeString, inputDict, allWind , allWL, allWave, wrr):
 
     """
     # begin by setting up input parameters
+    import pdb
+    pdb.set_trace()
     simulationDuration = int(inputDict.get('simulationDuration', 24))
     plotFlag = inputDict.get('plotFlag', True)
 
@@ -69,7 +70,7 @@ def ww3simSetup(startTimeString, inputDict, allWind , allWL, allWave, wrr):
     assert rawspec is not None, "\n++++\nThere's No Wave data between %s and %s \n++++\n" % (startTime, endTime)
     # use generated time lists for these to provide accurate temporal values
     _, waveTimeList, wlTimeList, _, windTimeList = prepdata.createDifferentTimeLists(startTime, endTime, rawspec, rawWL,
-                                                                                     rawWind=rawwind)
+                                                                                     rawWind=rawwind,simTs=30*60)
     nFreq = np.size(rawspec['wavefreqbin'])
     # rotate and lower resolution of directional wave spectra
     wavepacket = prepdata.prep_spec(rawspec, version_prefix, datestr=dateString, plot=plotFlag, full=full, deltaangle=10,
@@ -208,8 +209,6 @@ class objects as listed below:
     # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
     # establishing the resolution of the input datetime
     # try:
-    projectStart = DT.datetime.strptime(startTime, '%Y-%m-%dT%H:%M:%SZ')
-    projectEnd = DT.datetime.strptime(inputDict['endTime'], '%Y-%m-%dT%H:%M:%SZ')
     dateString = wrr.dateString  # a string for file names
 
     # _____________________________________________________________________________
@@ -221,7 +220,11 @@ class objects as listed below:
     if postProcessingDict['phaseResolved'] == True:
         spatialPostProcessed = postData.processNLwave(spatialData, wrr=wrr, ncFilesOnly=False)
         pointPostProcessed = None
+    elif postProcessingDict['phaseAveraged'] == True:
+        spatialPostProcessed = postData.processPAwave(spatialData, wrr=wrr, ncFilesOnly=False)
+        pointPostProcessed = None  # this needs to be developed
     else:
+        # place holder for new development
         spatialPostProcessed, pointPostProcessed = someFunctionPostProcesing(spatialData, pointData)
 
     # __________________________2. make netCDF Files________________________________
@@ -254,10 +257,24 @@ class objects as listed below:
 
         elif postProcessingDict['phaseAveraged'] is True:
             variables = ncfile.variables.keys()
-            for var in variables:
-                if var in ['waveHs', 'waveDm', 'waveTm'] and plotFlag is True:
-                    plotOutFname = 'test.png'
-                    oP.unstructuredSpatialPlot(plotOutFname, fieldNc=ncfile, variable=var)
+            if postProcessingDict['gridType'] == 'unstructured':
+                for var in variables:
+                    if var in ['waveHs', 'waveDm', 'waveTm'] and plotFlag is True:
+                        plotOutFname = 'test.png'
+                        oP.unstructuredSpatialPlot(plotOutFname, fieldNc=ncfile, variable=var)
+            if postProcessingDict['gridType'] == '1D':
+                for var in ['aveN', 'aveE', 'waveHs']:
+                    for t, tt in enumerate(ncfile['time'][:]):
+                        plotOutFname = os.path.join(wrr.plottingDirectory, f"CMTB_{wrr.modelName}_{var}_" \
+                                       f"{nc.num2date(tt, ncfile['time'].units).strftime('%Y%m%dT%H%M%SZ')}.png")
+                        oP.plotWaveProfile(ncfile['xFRF'][:], ncfile[var][t], ncfile['elevation'][t], plotOutFname,
+                                           variableName=var)
+
+                    flist = glob.glob(os.path.join(wrr.plottingDirectory, f"CMTB_*{var}*.png"))
+                    sb.makegif(flist, ofname=os.path.join(wrr.plottingDirectory, f"CMTB_{wrr.modelName}_{var}.gif"))
+                    sb.myTarMaker(os.path.join(wrr.plottingDirectory, f'CMTB_{wrr.modelName}_{var}.tar.gz'),
+                                  fileList=flist)
+
         else:
             raise NotImplementedError("other model plotting types haven't been implemented yet")
     # __________________________2b. point output _________________________________
@@ -273,10 +290,7 @@ class objects as listed below:
     # __________________________2bb. point plotting ________________________________
         print("is this necessary??")
 
-
-
-
-
+
 
 def cshoreSimSetup(startTimeString, inputDict, allWave, allBathy, allWL, allWind, allCTD, wrr):
     """Author: David Young
@@ -305,34 +319,23 @@ def cshoreSimSetup(startTimeString, inputDict, allWave, allBathy, allWL, allWind
     # begin by setting up input parameters
     simulationDuration = int(inputDict.get('simulationDuration', 24))
     timeStep = (inputDict.get('timeStep',3600))
-    plotFlag = inputDict.get('plotFlag', True)
 
     print('TODO: rename these unpacked variables [frontbackNew.preprocess]')
-    try:
-        version_prefix = wrr.versionPrefix
-    except:
-        version_prefix = inputDict['modelSettings'].get('version_prefix', 'base').lower()
 
-    plotFlag = inputDict.get('plotFlag', True)
     pathPrefix = wrr.workingDirectory
-    model = wrr.modelName  # inputDict['modelSettings'].get('model', 'ww3').lower()
     rawspec = allWave
     rawwind = allWind
     rawWL = allWL
     rawCTD = allCTD
     rawBathy = allBathy
 
-
-    # ___________________define version parameters_________________________________
-    full = True
     # __________________set times _________________________________________________
     startTime = DT.datetime.strptime(startTimeString, '%Y-%m-%dT%H:%M:%SZ')
     endTime = startTime + DT.timedelta(0, simulationDuration * 3600, 0)
-    dateString = wrr.dateString  # startTime.strftime('%Y-%m-%dT%H%M%SZ')
     ftime = simulationDuration*3600.
-    reltime = np.arange(0,ftime+timeStep,timeStep)
+    reltime = np.arange(0, ftime+timeStep, timeStep)
     print("Model Time Start : %s  Model Time End:  %s" % (startTime, endTime))
-
+    model = wrr.modelName
     # __________________Make Diretories_____________________________________________
     fileHandling.makeCMTBfileStructure(pathPrefix)
 
@@ -351,29 +354,21 @@ def cshoreSimSetup(startTimeString, inputDict, allWave, allBathy, allWL, allWind
  #                                   outputPath=pathPrefix, model=model, waveTimeList=waveTimeList, ww3nFreq=nFreq)
  #   print('TODO: @Ty add values for nFreq here! [frontBackNew.line72]')
     print(reltime, startTime)
-    wavepacket = prepdata.prep_CSHOREwaves(rawspec, reltime , startTime)
+    wavepacket = prepdata.prep_CSHOREwaves(rawspec, reltime , startTime, pierAng=wrr.shoreNormal)
 
     wlTimeList = [startTime + DT.timedelta(seconds=tt) for tt in reltime]
-    WLpacket = prepdata.prep_WL(rawWL,wlTimeList)
+    WLpacket = prepdata.prep_WL(rawWL, wlTimeList)
     windTimeList= [startTime + DT.timedelta(seconds=tt) for tt in reltime]
-    windpacket = prepdata.prep_wind(rawwind, windTimeList, model=model)  # vector average, rotate winds, correct to 10m
+
+    # vector average, rotate winds,  correct to 10m
+    windpacket = prepdata.prep_wind(rawwind, windTimeList, model=model, shoreNormal=wrr.shoreNormal)
 
     # pull the stuff I need out of the dict
     timerun = inputDict['simulationDuration']
     version_prefix = inputDict['modelSettings']['version_prefix']
     profile_num = inputDict['profileNumber']
     bathy_loc = inputDict['bathyLoc']
-    workingDir = inputDict['workingDirectory']
-    if 'THREDDS' in inputDict:
-        server = inputDict['THREDDS']
-    else:
-        print('Chosing CHL thredds by Default, this may be slower!')
-        server = 'CHL'
-    # ____________________GENERAL ASSUMPTION VARIABLES__________
-
-    model = 'CSHORE'
-    path_prefix = os.path.join(workingDir, model,  '%s/' % version_prefix)
-    time_step = 1        # time step for model in hours
+
     dx = 1               # cross-shore grid spacing (FRF coord units - m)
     fric_fac = 0.015     # default friction factor
 
@@ -382,33 +377,12 @@ def cshoreSimSetup(startTimeString, inputDict, allWave, allBathy, allWL, allWind
     # Time Stuff!
     if type(timerun) == str:
         timerun = int(timerun)
-    #start_time = DT.datetime.strptime(startTime, '%Y-%m-%dT%H:%M:%SZ')
     start_time = startTime
-    bathy_loc_List = np.array(['integrated_bathy', 'survey'])
-
     assert start_time.minute == 0 and start_time.second == 0 and start_time.microsecond == 0, 'Your simulation must start on the hour!'
-
     end_time = start_time + DT.timedelta(days=0, hours=timerun) # removed for ilab=1 , minutes=1)
-    print(end_time)
-    date_str = start_time.strftime('%Y-%m-%dT%H%M%SZ')
-    # start making my metadata dict
-    #meta_dict = {'startTime': DT.datetime.strftime(startTime,'%Y-%m-%dT%H:%M:%SZ'),
-    #             'timerun': timerun,
-    #             'time_step': time_step,
-    #             'dx': dx,
-    #             'fric_fac': fric_fac,
-    #             'version': version_prefix}
-    #ftime = timerun * 3600  # [sec] final time, dictates model duration
-    #dt = time_step * 3600  # time interval (sec) for wave and water level conditions
-    #BC_dict = {'timebc_wave': np.arange(0, ftime + dt, dt)}
-
-    bathypacket = prepdata.prep_CSHOREbathy(rawBathy, bathy_loc, dx, wavepacket, profile_num=profile_num,fric_fac=fric_fac)
-    # ______________________________________________________________________________
-    # __________________Make Diretories_____________________________________________
-    #if not os.path.exists(path_prefix + date_str):  # if it doesn't exist
-    #    os.makedirs(path_prefix + date_str)  # make the directory
-    #if not os.path.exists(path_prefix + date_str + "/figures/"):
-    #    os.makedirs(path_prefix + date_str + "/figures/")
+
+    bathypacket = prepdata.prep_CSHOREbathy(rawBathy, bathy_loc, dx, wavepacket, profile_num=profile_num,
+                                            fric_fac=fric_fac)
 
     print("Model Time Start : %s  Model Time End:  %s" % (start_time, end_time))
     #print("Files will be placed in {0} folder".format(path_prefix + date_str))

genericWorkFlow.py

@@ -31,6 +31,8 @@ def Master_workFlow(inputDict):
     pbsFlag = inputDict['pbsFlag']
     analyzeFlag = inputDict['analyzeFlag']
     plotFlag = inputDict['plotFlag']
+    hotStartFlag = inputDict['hotStartFlag']
+    hotStartDirectory = inputDict.get('hotStartDirectory',None)
     modelName = inputDict['modelSettings'].get('modelName', None)
     log = inputDict.get('logging', True)
     updateBathy = inputDict.get('updateBathy', None)
@@ -94,9 +96,8 @@ def Master_workFlow(inputDict):
             if modelName in ['ww3']:
                 wrr = wrrClass.ww3io(workingDirectory=workingDirectory,testName=testName, versionPrefix=version_prefix,
                                      startTime=DT.datetime.strptime(time, '%Y-%m-%dT%H:%M:%SZ'),
-                                     endTime=DT.datetime.strptime(time, '%Y-%m-%dT%H:%M:%SZ') + DT.timedelta(
-                                     hours=inputDict['simulationDuration']), runFlag=runFlag,
-                                     generateFlag=generateFlag, readFlag=analyzeFlag, pbsFlag=pbsFlag)
+                                     endTime=DT.datetime.strptime(endTime, '%Y-%m-%dT%H:%M:%SZ'), runFlag=runFlag,
+                                     generateFlag=generateFlag, readFlag=analyzeFlag, pbsFlag=pbsFlag, hotStartFlag=hotStartFlag)
 
                 if generateFlag is True:
                     wavePacket, windPacket, wlPacket, bathyPacket, gridFname, wrr = frontBackNEW.ww3simSetup(time,
@@ -126,12 +127,13 @@ def Master_workFlow(inputDict):
                     wrr = pickle.load(open(os.path.join(workingDirectory,f"{dateString}_io.pickle"), 'rb'),
                                       protocol=pickle.HIGHEST_PROTOCOL)
             elif modelName in ['cshore']:
-                wrr = wrrClass.cshoreio(workingDirectory=workingDirectory,testName=testName, versionPrefix=version_prefix,
+                wrr = wrrClass.cshoreio(workingDirectory=workingDirectory, testName=testName, versionPrefix=version_prefix,
                                        startTime=DT.datetime.strptime(time, '%Y-%m-%dT%H:%M:%SZ'),
                                        simulatedRunTime=inputDict['simulationDuration'],
                                        endTime=DT.datetime.strptime(time, '%Y-%m-%dT%H:%M:%SZ') + DT.timedelta(
                                            hours=inputDict['simulationDuration']), runFlag=runFlag,
-                                       generateFlag=generateFlag, readFlag=analyzeFlag, pbsFlag=pbsFlag)
+                                       generateFlag=generateFlag, readFlag=analyzeFlag, pbsFlag=pbsFlag, shoreNormal=72,
+                                       xOrigin=rawspec['xFRF'])
                 if generateFlag is True:
                     rawbathy = go.getBathyTransectFromNC(profilenumbers=960)
                     rawctd = go.getCTD()
@@ -156,18 +158,18 @@ def Master_workFlow(inputDict):
                     print("windPacket has keys: {}".format(windPacket.keys()))
                 except AttributeError:
                     pass
-
-                if pbsFlag is True:
-                    wrr.hpcCores = inputDict['hpcSettings']['hpcCores']
-                    wrr.hpcNodes = inputDict['hpcSettings']['hpcNodes']
+                print(updateBathy)
                 # write simulation files (if assigned)
                 wrr.writeAllFiles(wavePacket=wavePacket, windPacket=windPacket, wlPacket=wlPacket,
                                   bathyPacket=bathyPacket, gridfname=gridFname,
-                                  ctdPacket=ctdPacket, updateBathy=updateBathy)
+                                  ctdPacket=ctdPacket, updateBathy=updateBathy,hotStartDirectory=hotStartDirectory)
 
             # run simulation (as appropriate)
             if runFlag is True:
-                wrr.runSimulation(modelExecutable=inputDict['modelExecutable'])
+                if pbsFlag is True:
+                    wrr.hpcCores = inputDict['hpcSettings']['hpcCores']
+                    wrr.hpcNodes = inputDict['hpcSettings']['hpcNodes']
+                wrr.runSimulation(modelExecutable=inputDict['modelExecutable'], hotStartDirectory=hotStartDirectory)
 
             # post process (as appropriate)
             if analyzeFlag == True:
@@ -183,7 +185,8 @@ def Master_workFlow(inputDict):
 
                 if inputDict['slack'] is not None:
                     myslack = cmtbSlack.slack('testbedutils/slackSettings.yml')  # initialize
-                    myslack.postMessageWithFiles(f"checkout {wrr.modelName} simulations from {wrr.dateString}",
+                    myslack.postMessageWithFiles(f"Dear Sir/Madam, please look at the plots generated by the "
+                                                 f"model {wrr.modelName} during the time of {wrr.dateString}",
                                                  moveFnames)
 
                 else:
@@ -202,7 +205,6 @@ def Master_workFlow(inputDict):
             logging.exception('\nERROR FOUND @ {}\n'.format(time, exc_info=True))
             os.chdir(cmtbRootDir)
 
-
 if __name__ == "__main__":
     opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
     print('___________________________________\n___________________________________\n___________________'