glow-jflvm-tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mlflow\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fix depth, vary $\\lambda$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2 [00:00<?, ?it/s]2020/03/09 18:34:47 INFO mlflow.projects: === Created directory /tmp/tmpybnhaogi for downloading remote URIs passed to arguments of type 'path' ===\n",
      "2020/03/09 18:34:47 INFO mlflow.projects: === Running command 'python run_experiment.py glow-jflvm erai/daily-1deg --scale 4 --layers 3 --depth 4 --min-filters 32 --max-filters 256 --lam 0.1 --alpha 0.0 --prior standard --dnet-layers 3 --dnet-filters 64 --splits 3 --batch-size 10 --var PRCP --epochs 50 --validate-freq 5 --region southeast_us --mode cv --auth gcs.secret.json' in run with ID '5f6bfaa315fe4f16aaa5062ae1eb3f05' === \n",
      "2020/03/09 21:11:49 INFO mlflow.projects: === Run (ID '5f6bfaa315fe4f16aaa5062ae1eb3f05') succeeded ===\n",
      " 50%|█████     | 1/2 [2:37:01<2:37:01, 9421.92s/it]2020/03/09 21:11:49 INFO mlflow.projects: === Created directory /tmp/tmpe3x7s412 for downloading remote URIs passed to arguments of type 'path' ===\n",
      "2020/03/09 21:11:49 INFO mlflow.projects: === Running command 'python run_experiment.py glow-jflvm erai/daily-1deg --scale 4 --layers 3 --depth 4 --min-filters 32 --max-filters 256 --lam 0.01 --alpha 0.0 --prior standard --dnet-layers 3 --dnet-filters 64 --splits 3 --batch-size 10 --var PRCP --epochs 50 --validate-freq 5 --region southeast_us --mode cv --auth gcs.secret.json' in run with ID '796fdd828cbc4cfcb3dc48b97c419f5b' === \n",
      "2020/03/09 23:35:24 ERROR mlflow.projects: === Run (ID '796fdd828cbc4cfcb3dc48b97c419f5b') interrupted, cancelling run ===\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-4d01cd8ee178>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m     mlflow.run('.', 'glow-jflvm', experiment_name='glow-jflvm-cv',\n\u001b[1;32m      9\u001b[0m                parameters={'scale': 4, 'layers': layers, 'depth': depth, 'lam': lam, 'epochs': epochs,\n\u001b[0;32m---> 10\u001b[0;31m                            'region': region, 'var': 'PRCP', 'data_lr': 'erai/daily-1deg', 'alpha': 0.0}, use_conda=False)\n\u001b[0m",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/mlflow/projects/__init__.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(uri, entry_point, version, parameters, experiment_name, experiment_id, backend, backend_config, use_conda, storage_dir, synchronous, run_id)\u001b[0m\n\u001b[1;32m    288\u001b[0m         use_conda=use_conda, storage_dir=storage_dir, synchronous=synchronous, run_id=run_id)\n\u001b[1;32m    289\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msynchronous\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 290\u001b[0;31m         \u001b[0m_wait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubmitted_run_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    291\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0msubmitted_run_obj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/mlflow/projects/__init__.py\u001b[0m in \u001b[0;36m_wait_for\u001b[0;34m(submitted_run_obj)\u001b[0m\n\u001b[1;32m    300\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m         \u001b[0mactive_run\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtracking\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMlflowClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_id\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_id\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 302\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0msubmitted_run_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m             \u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"=== Run (ID '%s') succeeded ===\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m             \u001b[0m_maybe_set_run_terminated\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactive_run\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"FINISHED\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/mlflow/projects/submitted_run.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommand_proc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcancel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1017\u001b[0m             \u001b[0mendtime\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1018\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1019\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1020\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m             \u001b[0;31m# https://bugs.python.org/issue25942\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36m_wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1651\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1652\u001b[0m                             \u001b[0;32mbreak\u001b[0m  \u001b[0;31m# Another thread waited.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1653\u001b[0;31m                         \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1654\u001b[0m                         \u001b[0;31m# Check the pid and loop as waitpid has been known to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1655\u001b[0m                         \u001b[0;31m# return 0 even without WNOHANG in odd situations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36m_try_wait\u001b[0;34m(self, wait_flags)\u001b[0m\n\u001b[1;32m   1609\u001b[0m             \u001b[0;34m\"\"\"All callers to this function MUST hold self._waitpid_lock.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1610\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1611\u001b[0;31m                 \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait_flags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1612\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mChildProcessError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1613\u001b[0m                 \u001b[0;31m# This happens if SIGCLD is set to be ignored or waiting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "layers = 3\n",
    "depth = 4\n",
    "epochs = 50\n",
    "region = 'southeast_us'\n",
    "lams = [0.1, 0.01] #[1.0, 0.1, 0.01]\n",
    "# run experiment\n",
    "for lam in tqdm(lams):\n",
    "    mlflow.run('.', 'glow-jflvm', experiment_name='glow-jflvm-cv',\n",
    "               parameters={'scale': 4, 'layers': layers, 'depth': depth, 'lam': lam, 'epochs': epochs,\n",
    "                           'region': region, 'var': 'PRCP', 'data_lr': 'erai/daily-1deg', 'alpha': 0.0}, use_conda=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fix $\\lambda$, vary depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/1 [00:00<?, ?it/s]2020/03/08 23:02:59 INFO mlflow.projects: === Created directory /tmp/tmprnkvrk6l for downloading remote URIs passed to arguments of type 'path' ===\n",
      "2020/03/08 23:02:59 INFO mlflow.projects: === Running command 'python run_experiment.py glow-jflvm erai/daily-1deg --scale 4 --layers 3 --depth 4 --min-filters 32 --max-filters 256 --lam 1.0 --alpha 0.0 --prior standard --dnet-layers 3 --dnet-filters 64 --splits 2 --batch-size 10 --var PRCP --epochs 30 --validate-freq 5 --region southeast_us --mode cv --auth gcs.secret.json' in run with ID '3a8ee6d40f2e449fb693fb7fec8c1b13' === \n",
      "2020/03/09 00:15:29 INFO mlflow.projects: === Run (ID '3a8ee6d40f2e449fb693fb7fec8c1b13') succeeded ===\n",
      "100%|██████████| 1/1 [1:12:30<00:00, 4350.44s/it]\n"
     ]
    }
   ],
   "source": [
    "layers = 3\n",
    "lam = 1.0\n",
    "epochs = [30]\n",
    "region = 'southeast_us'\n",
    "depths = [4]\n",
    "# run experiment\n",
    "for num_epochs, depth in tqdm(list(zip(epochs, depths))):\n",
    "    mlflow.run('.', 'glow-jflvm', experiment_name='glow-jflvm-cv',\n",
    "               parameters={'scale': 4, 'layers': layers, 'depth': depth, 'lam': lam, 'epochs': num_epochs,\n",
    "                           'region': region, 'splits': 2, 'data_lr': 'erai/daily-1deg', 'alpha': 0.0, 'var': 'PRCP'}, use_conda=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fix $\\lambda$ and depth, vary $\\alpha$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/3 [00:00<?, ?it/s]2020/03/08 15:50:20 INFO mlflow.projects: === Created directory /tmp/tmpd2cmk2_v for downloading remote URIs passed to arguments of type 'path' ===\n",
      "2020/03/08 15:50:20 INFO mlflow.projects: === Running command 'python run_experiment.py glow-jflvm erai/daily-1deg --scale 4 --layers 3 --depth 4 --min-filters 32 --max-filters 256 --lam 1.0 --alpha 0.0 --prior standard --dnet-layers 3 --dnet-filters 64 --splits 2 --batch-size 10 --var PRCP --epochs 20 --validate-freq 5 --region southeast_us --mode cv --auth gcs.secret.json' in run with ID 'c3cc62d18e17436b816a5068b41214d2' === \n",
      "2020/03/08 16:13:32 ERROR mlflow.projects: === Run (ID 'c3cc62d18e17436b816a5068b41214d2') interrupted, cancelling run ===\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-1d927079e448>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m     mlflow.run('.', 'glow-jflvm', experiment_name='glow-jflvm-cv',\n\u001b[1;32m     10\u001b[0m                parameters={'scale': 4, 'layers': layers, 'depth': depth, 'lam': lam, 'epochs': epochs, 'region': region, 'alpha': alpha,\n\u001b[0;32m---> 11\u001b[0;31m                            'splits': 2, 'data_lr': 'erai/daily-1deg', 'var': 'PRCP'}, use_conda=False)\n\u001b[0m",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/mlflow/projects/__init__.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(uri, entry_point, version, parameters, experiment_name, experiment_id, backend, backend_config, use_conda, storage_dir, synchronous, run_id)\u001b[0m\n\u001b[1;32m    288\u001b[0m         use_conda=use_conda, storage_dir=storage_dir, synchronous=synchronous, run_id=run_id)\n\u001b[1;32m    289\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msynchronous\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 290\u001b[0;31m         \u001b[0m_wait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubmitted_run_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    291\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0msubmitted_run_obj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/mlflow/projects/__init__.py\u001b[0m in \u001b[0;36m_wait_for\u001b[0;34m(submitted_run_obj)\u001b[0m\n\u001b[1;32m    300\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m         \u001b[0mactive_run\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtracking\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMlflowClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_id\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_id\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 302\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0msubmitted_run_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m             \u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"=== Run (ID '%s') succeeded ===\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m             \u001b[0m_maybe_set_run_terminated\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactive_run\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"FINISHED\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/mlflow/projects/submitted_run.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommand_proc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcancel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1017\u001b[0m             \u001b[0mendtime\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1018\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1019\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1020\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m             \u001b[0;31m# https://bugs.python.org/issue25942\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36m_wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1651\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1652\u001b[0m                             \u001b[0;32mbreak\u001b[0m  \u001b[0;31m# Another thread waited.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1653\u001b[0;31m                         \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1654\u001b[0m                         \u001b[0;31m# Check the pid and loop as waitpid has been known to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1655\u001b[0m                         \u001b[0;31m# return 0 even without WNOHANG in odd situations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36m_try_wait\u001b[0;34m(self, wait_flags)\u001b[0m\n\u001b[1;32m   1609\u001b[0m             \u001b[0;34m\"\"\"All callers to this function MUST hold self._waitpid_lock.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1610\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1611\u001b[0;31m                 \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait_flags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1612\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mChildProcessError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1613\u001b[0m                 \u001b[0;31m# This happens if SIGCLD is set to be ignored or waiting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "layers = 3\n",
    "lam = 1.0\n",
    "epochs = 20\n",
    "region = 'southeast_us'\n",
    "depth = 4\n",
    "alphas = [0.0, 1.0E-2, 1.0]\n",
    "# run experiment\n",
    "for alpha in tqdm(alphas):\n",
    "    mlflow.run('.', 'glow-jflvm', experiment_name='glow-jflvm-cv',\n",
    "               parameters={'scale': 4, 'layers': layers, 'depth': depth, 'lam': lam, 'epochs': epochs, 'region': region, 'alpha': alpha,\n",
    "                           'splits': 2, 'data_lr': 'erai/daily-1deg', 'var': 'PRCP'}, use_conda=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test informative vs non-informative prior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layers = 3\n",
    "lam = 1.0\n",
    "epochs = 20\n",
    "region = 'southeast_us'\n",
    "depth = 4\n",
    "alpha = 0.0\n",
    "priors = ['climatology', 'standard']\n",
    "# run experiment\n",
    "for prior in tqdm(priors):\n",
    "    mlflow.run('.', 'glow-jflvm', experiment_name='glow-jflvm-4x-cv',\n",
    "               parameters={'scale': 4, 'layers': layers, 'depth': depth, 'lam': lam, 'epochs': epochs,\n",
    "                           'region': region, 'alpha': alpha, 'prior': prior}, use_conda=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_lam = pd.read_csv('data/cv_lambda_maxt.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_lam.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lams = [0.01, 0.1, 1.0]\n",
    "avg_rmse = [df_lam.loc[df_lam['lam'] == lam]['rmse'].mean() for lam in lams]\n",
    "avg_bias = [df_lam.loc[df_lam['lam'] == lam]['bias'].mean() for lam in lams]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(2*6,4))\n",
    "plt.subplot(1,2,1)\n",
    "plt.plot(lams, avg_rmse)\n",
    "plt.xscale('log')\n",
    "plt.title('RMSE vs lambda')\n",
    "plt.tight_layout()\n",
    "plt.subplot(1,2,2)\n",
    "plt.plot(lams, avg_bias)\n",
    "plt.xscale('log')\n",
    "plt.title('bias vs lambda')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_alpha = pd.read_csv('data/cv_alpha_maxt.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alphas = [0.0001, 0.001, 0.01]\n",
    "avg_rmse = [df_alpha.loc[df_alpha['alpha'] == alpha]['rmse'].mean() for alpha in alphas]\n",
    "avg_bias = [df_alpha.loc[df_alpha['alpha'] == alpha]['bias'].mean() for alpha in alphas]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(2*6,4))\n",
    "plt.subplot(1,2,1)\n",
    "plt.plot(alphas, avg_rmse)\n",
    "plt.xscale('log')\n",
    "plt.title('RMSE vs alpha')\n",
    "plt.tight_layout()\n",
    "plt.subplot(1,2,2)\n",
    "plt.plot(alphas, avg_bias)\n",
    "plt.xscale('log')\n",
    "plt.title('bias vs alpha')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_depth = pd.read_csv('data/cv_depth_maxt.csv')\n",
    "df_depth.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "depths = [4,8,16]\n",
    "avg_rmse = [df_depth.loc[df_depth['depth'] == depth]['rmse'].mean() for depth in depths]\n",
    "avg_bias = [df_depth.loc[df_depth['depth'] == depth]['bias'].mean() for depth in depths]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(2*6,4))\n",
    "plt.subplot(1,2,1)\n",
    "plt.plot(depths, avg_rmse)\n",
    "plt.title('RMSE vs depth')\n",
    "plt.tight_layout()\n",
    "plt.subplot(1,2,2)\n",
    "plt.plot(depths, avg_bias)\n",
    "plt.title('bias vs depth')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}