IFT_StepX_NotUsing_determine_IFT_epp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime as dt\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "from dateutil.relativedelta import relativedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Paths, input, and output file names\n",
    "\n",
    "outer_dir = os.path.abspath(os.path.join(os.getcwd() ,\"../..\"))\n",
    "refpth = os.path.join(outer_dir, 'IFT_files', 'Reference Files')\n",
    "sfe_char_csv = os.path.join(refpth, 'All SFE LOI Characteristics with MAF.csv')\n",
    "sfe_char_xls= os.path.join(refpth, 'All SFE LOI Characteristics.xlsx')\n",
    "subset_xls = os.path.join(refpth, 'Subset SFE LOIs.xlsx')\n",
    "\n",
    "unimpath = os.path.join(outer_dir, \"IFT_files\", 'Unimpaired Flow')\n",
    "comid_csv = os.path.join(refpth, 'SFER-POI-COMID-16Jun2020.csv')\n",
    "wytdir = os.path.join(unimpath, \"Water Year Types\")\n",
    "wmtdir = os.path.join(unimpath, \"Water Month Types\")\n",
    "\n",
    "startdir = os.path.join(outer_dir, 'IFT_files', 'IFT Results')\n",
    "wmtfile = os.path.join(wmtdir, \"LOI 9999 WMT.csv\") #file containing WMTs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original filename: \"00_add_ts_col.py\"\n",
    "def add_ts_col(tab):\n",
    "#adds column for TS as first column of DataFrame as formatted date\n",
    "    tab['TS'] = tab.index.strftime('%m/%d/%Y')\n",
    "    cols = tab.columns.tolist()\n",
    "    cols = cols[-1:] + cols[:-1]\n",
    "    tab = tab[cols]\n",
    "    return tab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original filename: \"00_get_all_sfe_lois.py\"\n",
    "# Read in SFE LOI characteristics table and calculate bankfull flow using cont. Area relation\n",
    "\n",
    "def get_all_sfe_lois():\n",
    "    sfelois = pd.read_csv(sfe_char_csv,index_col=0)\n",
    "    loi = [str(i) for i in sfelois['LOI']]\n",
    "    sfelois['Outlet LOI'] = sfelois['LOI']\n",
    "    sfelois['Contributing Area (mi^2)'] = sfelois['Contributing Area']\n",
    "    sfelois['MAF'] = sfelois['Mean Annual Flow (cfs)']\n",
    "    sfelois['Qbf'] = 71.5 * sfelois['Contributing Area (mi^2)'] #71.5 cfs/mi^2 according to Darren\n",
    "\n",
    "    subset = pd.read_excel(subset_xls)\n",
    "    sublois = sfelois.loc[subset['SWSID'],:]\n",
    "    return sublois, loi, sfelois"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original filename: \"00_read_loi_paradigm_flow_v3.py\"\n",
    "def read_loi_paradigm_flow(p):\n",
    "    unimpflowfile = os.path.join(unimpath, str(p) + '.csv')\n",
    "    unimp = pd.read_csv(unimpflowfile,index_col=0)\n",
    "    unimp.index = pd.DatetimeIndex(unimp.index)\n",
    "    return unimp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original filename: \"02_epp_flow_reqts_v4.py\"\n",
    "def get_wmt_month_data(wmt,m,timeseries,wmtf, boxd):\n",
    "    #process and save off values for month/WMT\n",
    "    aws = wmtf[((wmtf['Month']==m) & (wmtf['WMT']==wmt))] #filter by month & WMT\n",
    "    aws.index = range(len(aws))\n",
    "    boxd.append(aws['aws'].values) #save off set of AWS values to plot as boxplot later\n",
    "    timeseries = pd.concat([timeseries,aws['date'],aws['flow'],aws['aws']],ignore_index=True, axis=1) #save off flow and AWS\n",
    "    return timeseries, boxd, aws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original filename: \"02_epp_flow_reqts_v4.py\"\n",
    "def get_scen_data(aws, wmtstr, tempdf,ehf_q, ehf_a):\n",
    "    #This function gets IFTs for each month/WMT combo using EPF\n",
    "    ehf_a = np.array(ehf_a) #AWS from EPF\n",
    "    ehf_q = np.array(ehf_q) #Flwo from EPF\n",
    "    maxloiflow = ehf_q[max(np.where(ehf_a == max(ehf_a))[0])] #find flow where AWS is maximized\n",
    "    clipa = ehf_a[ehf_q <= maxloiflow] #only look at AWS Values below max AWS flow\n",
    "    clipq = ehf_q[ehf_q <= maxloiflow] #only look at Flow values below max AWS flow\n",
    "\n",
    "\n",
    "    awsdfsort = aws.sort_values(by=['aws','flow']) #sort values\n",
    "    sortflow = awsdfsort['flow'].values\n",
    "    sortaws = awsdfsort['aws'].values\n",
    "\n",
    "\n",
    "    quants = np.array([1, 0.9, 0.75, 0.5, 0.25, 0.1]) #list of percentile or quantiles\n",
    "    scendata = np.array([])\n",
    "    for i in range(len(quants)):\n",
    "        flwqnt = aws['flow'].quantile(quants[i]) #get quantile of flow\n",
    "        if flwqnt >= maxloiflow: #if the quantile flow is greater than the max AWS Flow, set requirement to max AWS flow\n",
    "            qntflow = maxloiflow\n",
    "        else: #otherwise, get quantile of AWS, then determine flow that gets this value (below max AWS flow)\n",
    "            awsqnt = aws['aws'].quantile(quants[i])\n",
    "            qntflow = np.interp(awsqnt, clipa, clipq)\n",
    "\n",
    "        scendata = np.append(scendata, qntflow) #add to table\n",
    "\n",
    "    scendata[scendata > maxloiflow] = maxloiflow #do not set IFT higher than max IFT flow\n",
    "    tempdf.loc[wmtstr] = np.transpose(scendata)\n",
    "    return tempdf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOI: 9999 processed.\n",
      "LOI: 1111 processed.\n",
      "LOI: 5555 processed.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Users\\kklausmeyer\\AppData\\Local\\Continuum\\anaconda3\\envs\\freshwater\\lib\\site-packages\\ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "# Original filename: \"02_epp_flow_reqts_v4.py\"\n",
    "#Calculates EPP IFTs\n",
    "\n",
    "\n",
    "wmttable = pd.read_csv(wmtfile,index_col=0)\n",
    "wmttable = wmttable.drop(['flow'],axis=1)\n",
    "\n",
    "loitab, loi, fultab = get_all_sfe_lois()\n",
    "wmts = ['Critically Dry','Dry','Below Median','Above Median','Wet','Extremely Wet'] #words to go with WMT numbers\n",
    "awspct = ['Max','90','75','50','25','10'] #Scenario percentiles for labelling\n",
    "awslabels = ['AWS_'+x for x in awspct]\n",
    "month = np.append(np.arange(10, 13), np.arange(1, 10)) #months in order of water year rather than calendar\n",
    "dates = [(dt.datetime(2020,1,1)+relativedelta(months=+x)).timetuple().tm_yday for x in range(12)] #first of each month day of year\n",
    "startdate = dt.datetime(1995,10,1) #first day of water year\n",
    "enddate = dt.datetime(1996,9,30) #last day of water year\n",
    "dates = pd.date_range(startdate,enddate, freq='MS') #date of first of each month\n",
    "tupleind = [] #create list of scenario percentile with each month\n",
    "for i in awslabels:\n",
    "    for j in dates:\n",
    "        tupleind.append(tuple((i, j)))\n",
    "awswmtindex = pd.MultiIndex.from_tuples(tupleind,names=['scenario','TS'])\n",
    "weaptable = pd.DataFrame(index = awswmtindex)\n",
    "for p in loi: #loop through LOIs\n",
    "    filename_ehf = os.path.join(refpth, 'LOI_' + p + '_EHF.xlsx')\n",
    "    if os.path.isfile(filename_ehf): #if we have an ecological performance function for this location, do calculation\n",
    "        EHF = pd.read_excel(filename_ehf, index_col=0)\n",
    "        EHF_a = list(EHF['AWS'])\n",
    "        EHF_q = list(EHF['Q'])\n",
    "        scenario_table = pd.DataFrame() #create dataframe for storing IFT for each scenario and month/WMT\n",
    "        for pct in awspct:\n",
    "            scenario_table['AWS_' + pct] = []\n",
    "        loi_results_fold = os.path.join(startdir, 'EPP Results by LOI', 'LOI ' + p + ' All Results')\n",
    "        if not os.path.exists(loi_results_fold):\n",
    "            os.makedirs(loi_results_fold)\n",
    "\n",
    "        unimp = read_loi_paradigm_flow(p) #read unimpaired flow\n",
    "        #add WMT to unimpaired flow table\n",
    "        wmtflow = unimp.reset_index().merge(wmttable, how=\"left\", on=['Month', 'Year']).set_index(unimp.index)\n",
    "        wmtflow['monthname'] = wmtflow.index.month_name()\n",
    "        wmtflow['date'] = wmtflow.index\n",
    "        wmtflow = wmtflow.dropna() #remove times that don't have WMTs (should be all, but just in case)\n",
    "\n",
    "\n",
    "        for m in month: # loop through months\n",
    "            mname = dt.datetime(2019,m,1).strftime(\"%B\") #get month name\n",
    "            wmtflow['aws'] = pd.DataFrame(np.interp(wmtflow['flow'], EHF_q, EHF_a),\n",
    "                 index=wmtflow.index, columns = ['AWS']) #use EPF to turn flow into AWS for each day in month\n",
    "            monthtimeseries = pd.DataFrame()\n",
    "            boxdata = []\n",
    "            for w in range(len(wmts)): #loop through WMTs, see functions above\n",
    "                wmt = wmts[w]\n",
    "                monthtimeseries, boxdata, aws_rel = get_wmt_month_data(wmt, m, monthtimeseries, wmtflow,boxdata)\n",
    "                wmtstr = wmt + ' ' + mname\n",
    "                scenario_table = get_scen_data(aws_rel,wmtstr,scenario_table, EHF_q, EHF_a)\n",
    "            #label columns of month time series by WMT for Date, Flow, and AWS\n",
    "            mtsfinal=monthtimeseries.rename(columns={0:\"Extr Dry Date\", 1:\"Extr Dry Flow\", 2:\"Extr Dry AWS\",\n",
    "                3:\"Dry Date\", 4:\"Dry Flow\", 5:\"Dry AWS\",\n",
    "                6:\"Below Med Date\", 7:\"Below Med Flow\", 8:\"Below Med AWS\",\n",
    "                9:\"Above Median Date\", 10:\"Above Median Flow\", 11:\"Above Median AWS\",\n",
    "                12:\"Wet Date\", 13:\"Wet Flow\", 14:\"Wet AWS\",\n",
    "                15:\"Extr Wet Date\", 16:\"Extr Wet Flow\", 17:\"Extr Wet AWS\"})\n",
    "            wmtlocid = \"LOI \" + p + ' ' + str(m) + '-' + mname\n",
    "            mtsfinal.to_csv(os.path.join(loi_results_fold, wmtlocid + '.csv'), encoding='utf-8', index=True) \n",
    "\n",
    "            #plot boxplots of each WMT AWS\n",
    "            fig = plt.figure()\n",
    "            ax = fig.add_subplot(1, 1, 1)\n",
    "            ax.boxplot(boxdata)\n",
    "            ax.set_xlabel(\"Water Month Type\")\n",
    "            ax.set_ylabel(\"AWS\")\n",
    "            ax.set_title(wmtlocid)\n",
    "            plt.xticks(np.arange(6) + 1,labels=wmts, rotation=30)\n",
    "            fig.tight_layout()\n",
    "            plt.savefig(os.path.join(loi_results_fold, wmtlocid + ' Box.png'))\n",
    "            plt.close(fig)\n",
    "\n",
    "        #format for table of IFTs for WEAP\n",
    "        for i in awslabels:\n",
    "            for w in range(len(wmts)):\n",
    "                wmt = wmts[w]\n",
    "                weaptable.loc[i, p+'_'+str(w+1)] = scenario_table.loc[\n",
    "                    scenario_table.index.str.find(wmt) == 0, i].get_values()\n",
    "\n",
    "        scenario_table.to_csv(os.path.join(startdir, 'EPP Results by LOI', 'LOI ' + p + ' AWS_Scenarios.csv'), encoding='utf-8', index=True)\n",
    "        print(\"LOI: \" + p + \" processed.\")\n",
    "    else: #don't have EPF so fill IFTs with 0\n",
    "        for i in awslabels:\n",
    "            for w in range(len(wmts)):\n",
    "                wmt = wmts[w]\n",
    "                weaptable.loc[i, p + '_' + str(w + 1)] = np.zeros_like(dates,dtype='int')\n",
    "        print(\"LOI: \" + p + \" not processed. Table populated with 0's.\")\n",
    "\n",
    "\n",
    "for i in awslabels: #finish formatting for WEAP\n",
    "    weaptablescen = weaptable.loc[i,:]\n",
    "    weaptablescen = add_ts_col(weaptablescen)\n",
    "    weaptablescen.to_csv(os.path.join(startdir, 'All LOI EPP ' + i + ' IFTs.csv'),  encoding='utf-8', index=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (freshwater)",
   "language": "python",
   "name": "freshwater"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}