read_calsim_and_collate_inputs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read inputs from dss files and collate into input and output csv files\n",
    "\n",
    "To prepate ANN data sets for training this notebook reads from the given dss files and builds the input and output csv files\n",
    "\n",
    "## ANN Inputs\n",
    "1.\tSac Flow\n",
    " * /CALSIM-SMOOTH/C_SAC048/FLOW/01JAN1922/1DAY/L2020A_DCP_EX/\n",
    " * /CALSIM/C_CSL004A/CHANNEL/01JAN1920/1MON/L2020A/\n",
    " * /CALSIM/C_CLV004/FLOW/01JAN1920/1MON/L2020A_DCP_EX/\n",
    " * /CALSIM/C_MOK019/CHANNEL/01JAN1920/1MON/L2020A/\n",
    "\n",
    "2.\tExports\n",
    " * /CALSIM/C_CAA003_TD/FLOW/01JAN1920/1MON/L2020A_DCP_EX/\n",
    " * /CALSIM/C_DMC000_TD/FLOW/01JAN1920/1MON/L2020A_DCP_EX/\n",
    " * /CALSIM/D408/FLOW/01JAN1920/1MON/L2020A_DCP_EX/\n",
    " * /CALSIM/D_SJR028_WTPDWS/FLOW/01JAN1920/1MON/L2020A_DCP_EX/\n",
    "\n",
    "3.\tDCC\n",
    " * /CALSIM/DXC/GATE-DAYS-OPEN/01JAN2010/1MON/L2020A/\n",
    "\n",
    "4.\tNet_DCD\n",
    " * /CALSIM/NET_DICU/DICU_FLOW/01JAN1920/1MON/L2020A/\n",
    "\n",
    "5.\tSJR flow\n",
    " * /CALSIM-SMOOTH/C_SJR070/FLOW/01JAN1935/1DAY/L2020A_DCP_EX/\n",
    "\n",
    "6.\tTide\n",
    " * /DWR/SAN_FRANCISCO/STAGE-MAX-MIN/01JAN1919/1DAY/ASTRO_NAVD_20170607/\n",
    "\n",
    "7.\tSMSCG\n",
    " * /MONTEZUMA/SMSCG/GATE-OPERATE/01JAN1922/1DAY/DCP_EX/\n",
    "\n",
    "\n",
    "## ANN Outputs\n",
    "\n",
    "Take all the EC and output to csv file\n",
    "```\n",
    "CVP_INTAKE :: //CVP_INTAKE/EC////\n",
    "MIDR_INTAKE :: //MIDR_INTAKE/EC////\n",
    "OLDR_CCF :: //OLDR_CCF/EC////\n",
    " * ROLD014 :: //ROLD014/EC////\n",
    "ROLD024 :: //ROLD024/EC////\n",
    "RSAC081 :: //ROLD081/EC////\n",
    "RSAC092 :: //RSAC092/EC////\n",
    "RSAN007 :: //RSAN007/EC////\n",
    "RSAN018 :: //RSAN018/EC////\n",
    "SLMZU003 :: //SLMZU003/EC////\n",
    "SLMZU011 :: //SLMZU011/EC////\n",
    "VICT_INTAKE :: //VICT_INTAKE/EC////\n",
    "```\n",
    "\n",
    "* means missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyhecdss\n",
    "import annutils\n",
    "import dssioutils\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hvplot.pandas\n",
    "import holoviews as hv\n",
    "import panel as pn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Display sample contents of a DSS File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "path_data = '.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyhecdss.DSSFile(os.path.join(path_data, 'SMSCG_operation_ON.dss')\n",
    "                 ).read_catalog().hvplot.table()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read inputs and outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfon = dssioutils.collate_calsim_inputs_for_ann(\n",
    "    os.path.join(path_data, 'SMSCG_operation_ON.dss'))\n",
    "dfoff = dssioutils.collate_calsim_inputs_for_ann(\n",
    "    os.path.join(path_data, 'SMSCG_operation_OFF.dss'))\n",
    "dfon.to_csv('smscg_input_on.csv')\n",
    "dfoff.to_csv('smscg_input_off.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bparts = ['CVP_INTAKE', 'MIDR_INTAKE', 'OLDR_CCF', 'ROLD024',\n",
    "          'RSAC081', 'RSAC092', 'RSAN007', 'RSAN018', 'SLMZU003', 'SLMZU011', 'VICT_INTAKE']\n",
    "\n",
    "\n",
    "def read_ec(dssfile, bpart):\n",
    "    df = list(pyhecdss.get_ts(dssfile, '//%s/EC////' % bpart))[0][0]\n",
    "    df.index = df.index.to_timestamp()\n",
    "    df.columns = ['%s_EC' % bpart]\n",
    "    return df\n",
    "\n",
    "\n",
    "dfon = pd.concat([read_ec(os.path.join(\n",
    "    path_data, 'SMSCG_operation_ON.dss'), bpart) for bpart in bparts], axis=1)\n",
    "dfon.to_csv('smscg_output_on.csv')\n",
    "dfoff = pd.concat([read_ec(os.path.join(\n",
    "    path_data, 'SMSCG_operation_OFF.dss'), bpart) for bpart in bparts], axis=1)\n",
    "dfoff.to_csv('smscg_output_off.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dms2]",
   "language": "python",
   "name": "conda-env-dms2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}