multimodal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle \n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import librosa\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm import tqdm\n",
    "import torch.nn.functional as F\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import librosa\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import noisereduce as nr\n",
    "import IPython\n",
    "from collections import Counter\n",
    "from sklearn.metrics import mean_squared_error, accuracy_score\n",
    "\n",
    "torch.manual_seed(10)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "np.random.seed(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device=torch.device('cuda:0')\n",
    "else:\n",
    "    device=torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>emotion</th>\n",
       "      <th>val</th>\n",
       "      <th>act</th>\n",
       "      <th>dom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6.2901</td>\n",
       "      <td>8.2357</td>\n",
       "      <td>Ses01F_impro01_F000</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10.0100</td>\n",
       "      <td>11.3925</td>\n",
       "      <td>Ses01F_impro01_F001</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>14.8872</td>\n",
       "      <td>18.0175</td>\n",
       "      <td>Ses01F_impro01_F002</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>27.4600</td>\n",
       "      <td>31.4900</td>\n",
       "      <td>Ses01F_impro01_F005</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>85.2700</td>\n",
       "      <td>88.0200</td>\n",
       "      <td>Ses01F_impro01_F012</td>\n",
       "      <td>ang</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  start_time  end_time             wav_file emotion  val  act  dom\n",
       "0      0      6.2901    8.2357  Ses01F_impro01_F000     neu  2.5  2.5  2.5\n",
       "1      1     10.0100   11.3925  Ses01F_impro01_F001     neu  2.5  2.5  2.5\n",
       "2      2     14.8872   18.0175  Ses01F_impro01_F002     neu  2.5  2.5  2.5\n",
       "3      3     27.4600   31.4900  Ses01F_impro01_F005     neu  2.5  3.5  2.0\n",
       "4      4     85.2700   88.0200  Ses01F_impro01_F012     ang  2.0  3.5  3.5"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "audio = pd.read_csv('audio_df_improvised.csv')\n",
    "\n",
    "audio.reset_index(inplace=True)\n",
    "audio.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio.emotion = pd.Categorical(pd.factorize(audio.emotion)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>emotion</th>\n",
       "      <th>val</th>\n",
       "      <th>act</th>\n",
       "      <th>dom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6.2901</td>\n",
       "      <td>8.2357</td>\n",
       "      <td>Ses01F_impro01_F000</td>\n",
       "      <td>0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10.0100</td>\n",
       "      <td>11.3925</td>\n",
       "      <td>Ses01F_impro01_F001</td>\n",
       "      <td>0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>14.8872</td>\n",
       "      <td>18.0175</td>\n",
       "      <td>Ses01F_impro01_F002</td>\n",
       "      <td>0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>27.4600</td>\n",
       "      <td>31.4900</td>\n",
       "      <td>Ses01F_impro01_F005</td>\n",
       "      <td>0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>85.2700</td>\n",
       "      <td>88.0200</td>\n",
       "      <td>Ses01F_impro01_F012</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  start_time  end_time             wav_file emotion  val  act  dom\n",
       "0      0      6.2901    8.2357  Ses01F_impro01_F000       0  2.5  2.5  2.5\n",
       "1      1     10.0100   11.3925  Ses01F_impro01_F001       0  2.5  2.5  2.5\n",
       "2      2     14.8872   18.0175  Ses01F_impro01_F002       0  2.5  2.5  2.5\n",
       "3      3     27.4600   31.4900  Ses01F_impro01_F005       0  2.5  3.5  2.0\n",
       "4      4     85.2700   88.0200  Ses01F_impro01_F012       1  2.0  3.5  3.5"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audio.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wav_file</th>\n",
       "      <th>emotion</th>\n",
       "      <th>transcription</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ses01F_impro01_F000</td>\n",
       "      <td>0</td>\n",
       "      <td>excuse me .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ses01F_impro01_F001</td>\n",
       "      <td>0</td>\n",
       "      <td>yeah .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ses01F_impro01_F002</td>\n",
       "      <td>0</td>\n",
       "      <td>is there a problem ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ses01F_impro01_F005</td>\n",
       "      <td>0</td>\n",
       "      <td>well what s the problem ? let me change it .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ses01F_impro01_F012</td>\n",
       "      <td>1</td>\n",
       "      <td>that s out of control .</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              wav_file emotion                                 transcription\n",
       "0  Ses01F_impro01_F000       0                                   excuse me .\n",
       "1  Ses01F_impro01_F001       0                                        yeah .\n",
       "2  Ses01F_impro01_F002       0                          is there a problem ?\n",
       "3  Ses01F_impro01_F005       0  well what s the problem ? let me change it .\n",
       "4  Ses01F_impro01_F012       1                       that s out of control ."
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "old_text = pd.read_csv('text_df.csv')\n",
    "old_text.emotion = audio.emotion\n",
    "old_text.emotion = pd.Categorical(pd.factorize(old_text.emotion)[0])\n",
    "old_text.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>excuse me .</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yeah .</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>is there a problem ?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>well what s the problem ? let me change it .</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>that s out of control .</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           data label\n",
       "0                                   excuse me .     0\n",
       "1                                        yeah .     0\n",
       "2                          is there a problem ?     0\n",
       "3  well what s the problem ? let me change it .     0\n",
       "4                       that s out of control .     1"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = pd.DataFrame()\n",
    "text['data'] = old_text.transcription\n",
    "text['label'] = old_text.emotion\n",
    "\n",
    "text.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenization\n",
    "import spacy\n",
    "tok = spacy.load('en')\n",
    "def tokenize (text):\n",
    "    text = re.sub(r\"[^\\x00-\\x7F]+\", \" \", text)\n",
    "    regex = re.compile('[' + re.escape(string.punctuation) + '0-9\\\\r\\\\t\\\\n]') # remove punctuation and numbers\n",
    "    nopunct = regex.sub(\" \", text.lower())\n",
    "    return [token.text for token in tok.tokenizer(nopunct)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "#count number of occurences of each word\n",
    "counts = Counter()\n",
    "for index, row in text.iterrows():\n",
    "    counts.update(tokenize(row['data']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating vocabulary\n",
    "vocab2index = {\"\":0, \"UNK\":1}\n",
    "words = [\"\", \"UNK\"]\n",
    "for word in counts:\n",
    "    vocab2index[word] = len(words)\n",
    "    words.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_sentence(text, vocab2index, N=70):\n",
    "    tokenized = tokenize(text)\n",
    "    encoded = np.zeros(N, dtype=int)\n",
    "    enc1 = np.array([vocab2index.get(word, vocab2index[\"UNK\"]) for word in tokenized])\n",
    "    length = min(N, len(enc1))\n",
    "    encoded[:length] = enc1[:length]\n",
    "    return encoded, length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "      <th>label</th>\n",
       "      <th>encoded_data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>excuse me .</td>\n",
       "      <td>0</td>\n",
       "      <td>[[2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yeah .</td>\n",
       "      <td>0</td>\n",
       "      <td>[[5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>is there a problem ?</td>\n",
       "      <td>0</td>\n",
       "      <td>[[6, 7, 8, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>well what s the problem ? let me change it .</td>\n",
       "      <td>0</td>\n",
       "      <td>[[10, 11, 12, 13, 9, 14, 15, 3, 16, 17, 4, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>that s out of control .</td>\n",
       "      <td>1</td>\n",
       "      <td>[[18, 12, 19, 20, 21, 4, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           data label  \\\n",
       "0                                   excuse me .     0   \n",
       "1                                        yeah .     0   \n",
       "2                          is there a problem ?     0   \n",
       "3  well what s the problem ? let me change it .     0   \n",
       "4                       that s out of control .     1   \n",
       "\n",
       "                                        encoded_data  \n",
       "0  [[2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...  \n",
       "1  [[5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...  \n",
       "2  [[6, 7, 8, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...  \n",
       "3  [[10, 11, 12, 13, 9, 14, 15, 3, 16, 17, 4, 0, ...  \n",
       "4  [[18, 12, 19, 20, 21, 4, 0, 0, 0, 0, 0, 0, 0, ...  "
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text['encoded_data'] = text['data'].apply(lambda x: np.array(encode_sentence(x,vocab2index )))\n",
    "text.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 1099, 1: 289, 2: 608, 3: 947})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(text['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = list(text['encoded_data'])\n",
    "y = list(text['label'])\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train = X[:2214]\n",
    "X_valid = X[2214:]\n",
    "y_train = y[:2214]\n",
    "y_valid = y[2214:]\n",
    "# X_train = X[243:]\n",
    "# X_valid = X[:243]\n",
    "# y_train = y[243:]\n",
    "# y_valid = y[:243]\n",
    "# X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Text(Dataset):\n",
    "    def __init__(self, X, Y):\n",
    "        self.X = X\n",
    "        self.y = Y\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return torch.from_numpy(self.X[idx][0].astype(np.int32)), self.y[idx], self.X[idx][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text = Text(X_train, y_train)\n",
    "valid_text = Text(X_valid, y_valid)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "vocab_size = len(words)\n",
    "text_train_loader = DataLoader(train_text, batch_size=batch_size, shuffle=False)\n",
    "text_valid_loader = DataLoader(valid_text, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_text_model(model, loss_fn, train_loader, valid_loader, epochs, learning_rate, optimizer, train_losses, valid_losses, comment, change_lr=None):\n",
    "    print(\"Training Text Model\")\n",
    "#     tb = SummaryWriter(comment=comment)\n",
    "    for epoch in range(1, epochs+1):\n",
    "        model.train()\n",
    "        batch_losses=[]\n",
    "        if change_lr:\n",
    "            optimizer = change_lr(optimizer, epoch, learning_rate)\n",
    "        for x, y, l in text_train_loader:\n",
    "            x = x.to(device, dtype=torch.long)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_pred = model(x, l)\n",
    "            optimizer.zero_grad()\n",
    "            loss = loss_fn(y_pred, y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            batch_losses.append(loss.item())\n",
    "            y = y.cpu().detach()\n",
    "        train_losses.append(batch_losses)\n",
    "            \n",
    "        model.eval()\n",
    "        batch_losses=[]\n",
    "        trace_y = []\n",
    "        trace_yhat = []\n",
    "        correct = 0\n",
    "        for x, y, l in text_valid_loader:\n",
    "            x = x.to(device, dtype=torch.long)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_hat = model(x, l)\n",
    "            loss = loss_fn(y_hat, y)\n",
    "            \n",
    "            pred = torch.max(y_hat, 1)[1]\n",
    "            correct += (pred == y).float().sum()\n",
    "            \n",
    "            trace_y.append(y.cpu().detach().numpy())\n",
    "            trace_yhat.append(y_hat.cpu().detach().numpy())      \n",
    "            batch_losses.append(loss.item())\n",
    "            \n",
    "\n",
    "        valid_losses.append(batch_losses)\n",
    "        trace_y = np.concatenate(trace_y)\n",
    "        trace_yhat = np.concatenate(trace_yhat)\n",
    "        accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)\n",
    "        unweighted_accuracy = accuracy_score( trace_yhat.argmax(axis=1), trace_y )\n",
    "        weighted_accuracy = balanced_accuracy_score( trace_yhat.argmax(axis=1), trace_y )\n",
    "        \n",
    "#         tb.add_scalar(\n",
    "#             'train_loss', np.mean(train_losses[-1]), epoch\n",
    "#         )\n",
    "#         tb.add_scalar(\n",
    "#             'valid_loss', np.mean(valid_losses[-1]), epoch\n",
    "#         )\n",
    "#         tb.add_scalar(\n",
    "#             'Unweighted Accuracy', unweighted_accuracy, epoch\n",
    "#         )     \n",
    "#         tb.add_scalar(\n",
    "#             'Weighted Accuracy', weighted_accuracy, epoch\n",
    "#         )   \n",
    "        if epoch%5 == 0:\n",
    "            print(\"Epoch - {} Train-Loss : {} Valid-Loss : {} Correct : {}\".format(epoch, np.mean(train_losses[-1]), np.mean(valid_losses[-1]), correct))\n",
    "            print(\"unweighted_accuracy : {} weighted_accuracy : {}\".format(unweighted_accuracy, weighted_accuracy ))\n",
    "#         print(\"train loss %.3f, val loss %.3f, val accuracy %.3f, and val rmse %.3f\" % (sum_loss/total, val_loss, val_acc, val_rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, epochs=10, lr=0.001):\n",
    "    parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "    optimizer = torch.optim.Adam(parameters, lr=lr)\n",
    "    for i in range(epochs):\n",
    "        model.train()\n",
    "        sum_loss = 0.0\n",
    "        total = 0\n",
    "        for x, y, l in text_train_loader:\n",
    "            x = x.to(device, dtype=torch.long)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_pred = model(x, l)\n",
    "            optimizer.zero_grad()\n",
    "            loss = F.cross_entropy(y_pred, y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            sum_loss += loss.item()*y.shape[0]\n",
    "            y = y.cpu().detach()\n",
    "            total += y.shape[0]\n",
    "        val_loss, val_acc, val_rmse = validation_metrics(model, text_valid_loader)\n",
    "        if i % 5 == 1:\n",
    "            print(\"train loss %.3f, val loss %.3f, val accuracy %.3f, and val rmse %.3f\" % (sum_loss/total, val_loss, val_acc, val_rmse))\n",
    "\n",
    "def validation_metrics (model, text_valid_loader):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    sum_loss = 0.0\n",
    "    sum_rmse = 0.0\n",
    "    for x, y, l in text_valid_loader:\n",
    "        x = x.to(device, dtype=torch.long)\n",
    "        y = y.to(device, dtype=torch.long)\n",
    "        y_hat = model(x, l)\n",
    "        loss = F.cross_entropy(y_hat, y)\n",
    "        y_hat = y_hat.cpu().detach()\n",
    "        y = y.cpu().detach()\n",
    "        pred = torch.max(y_hat, 1)[1]\n",
    "        correct += (pred == y).float().sum()\n",
    "        total += y.shape[0]\n",
    "        sum_loss += loss.item()*y.shape[0]\n",
    "        sum_rmse += np.sqrt(mean_squared_error(pred, y.unsqueeze(-1)))*y.shape[0]\n",
    "    return sum_loss/total, correct/total, sum_rmse/total"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fixed Length module with random embeddings "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM_fixed_len(nn.Module) :\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim) :\n",
    "        super().__init__()\n",
    "        self.embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_dim, 4)\n",
    "        self.dropout = nn.Dropout(0.2)\n",
    "        self.activation = nn.Softmax(dim=1)\n",
    "        \n",
    "    def forward(self, x, l):\n",
    "        x = self.embeddings(x)\n",
    "        x = self.dropout(x)\n",
    "        lstm_out, (ht, ct) = self.lstm(x)\n",
    "        return self.activation(self.linear(ht[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_fixed =  LSTM_fixed_len(vocab_size, 56, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device=torch.device('cuda:0')\n",
    "else:\n",
    "    device=torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTM_fixed_len(\n",
       "  (embeddings): Embedding(2255, 56, padding_idx=0)\n",
       "  (lstm): LSTM(56, 64, batch_first=True)\n",
       "  (linear): Linear(in_features=64, out_features=4, bias=True)\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       "  (activation): Softmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_fixed.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_model(model_fixed, epochs=500, lr=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model with glove embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_glove_vectors(glove_file=\".vector_cache/glove.6B.50d.txt\"):\n",
    "    \"\"\"Load the glove word vectors\"\"\"\n",
    "    word_vectors = {}\n",
    "    with open(glove_file) as f:\n",
    "        for line in f:\n",
    "            split = line.split()\n",
    "            word_vectors[split[0]] = np.array([float(x) for x in split[1:]])\n",
    "    return word_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_emb_matrix(pretrained, word_counts, emb_size = 50):\n",
    "    \"\"\" Creates embedding matrix from word vectors\"\"\"\n",
    "    vocab_size = len(word_counts) + 2\n",
    "    vocab_to_idx = {}\n",
    "    vocab = [\"\", \"UNK\"]\n",
    "    W = np.zeros((vocab_size, emb_size), dtype=\"float32\")\n",
    "    W[0] = np.zeros(emb_size, dtype='float32') # adding a vector for padding\n",
    "    W[1] = np.random.uniform(-0.25, 0.25, emb_size) # adding a vector for unknown words \n",
    "    vocab_to_idx[\"UNK\"] = 1\n",
    "    i = 2\n",
    "    for word in word_counts:\n",
    "        if word in word_vecs:\n",
    "            W[i] = word_vecs[word]\n",
    "        else:\n",
    "            W[i] = np.random.uniform(-0.25,0.25, emb_size)\n",
    "        vocab_to_idx[word] = i\n",
    "        vocab.append(word)\n",
    "        i += 1   \n",
    "    return W, np.array(vocab), vocab_to_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vecs = load_glove_vectors()\n",
    "pretrained_weights, vocab, vocab2index = get_emb_matrix(word_vecs, counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM_glove_vecs(torch.nn.Module) :\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim, glove_weights) :\n",
    "        super().__init__()\n",
    "        self.embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)\n",
    "        self.embeddings.weight.data.copy_(torch.from_numpy(glove_weights))\n",
    "        self.embeddings.weight.requires_grad = True \n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_dim, 4)\n",
    "        self.dropout = nn.Dropout(0.2)\n",
    "        \n",
    "    def forward(self, x, l):\n",
    "        x = self.embeddings(x)\n",
    "        x = self.dropout(x)\n",
    "        lstm_out, (ht, ct) = self.lstm(x)\n",
    "        return self.linear(ht[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm = LSTM_glove_vecs(vocab_size, 50, 64, pretrained_weights)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTM_glove_vecs(\n",
       "  (embeddings): Embedding(2255, 50, padding_idx=0)\n",
       "  (lstm): LSTM(50, 64, batch_first=True)\n",
       "  (linear): Linear(in_features=64, out_features=4, bias=True)\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "optimizer = optim.Adam(lstm.parameters(), lr=learning_rate)\n",
    "epochs = 300\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "train_losses=[]\n",
    "valid_losses=[]\n",
    "comment = \"None\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Text Model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py36/lib/python3.6/site-packages/sklearn/metrics/classification.py:1437: UserWarning: y_pred contains classes not in y_true\n",
      "  warnings.warn('y_pred contains classes not in y_true')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 5 Train-Loss : 1.3003970781962078 Valid-Loss : 1.2070510387420654 Correct : 284.0\n",
      "unweighted_accuracy : 0.3895747599451303 weighted_accuracy : 0.19505494505494506\n",
      "Epoch - 10 Train-Loss : 1.2977413336435955 Valid-Loss : 1.2229628562927246 Correct : 285.0\n",
      "unweighted_accuracy : 0.39094650205761317 weighted_accuracy : 0.3622589531680441\n",
      "Epoch - 15 Train-Loss : 1.295873721440633 Valid-Loss : 1.221449613571167 Correct : 284.0\n",
      "unweighted_accuracy : 0.3895747599451303 weighted_accuracy : 0.4308437067773167\n",
      "Epoch - 20 Train-Loss : 1.295275052388509 Valid-Loss : 1.221219539642334 Correct : 282.0\n",
      "unweighted_accuracy : 0.3868312757201646 weighted_accuracy : 0.13001383125864455\n",
      "Epoch - 25 Train-Loss : 1.2936944166819255 Valid-Loss : 1.225310206413269 Correct : 283.0\n",
      "unweighted_accuracy : 0.38820301783264743 weighted_accuracy : 0.21370781322237634\n",
      "Epoch - 30 Train-Loss : 1.2941944201787312 Valid-Loss : 1.2197266817092896 Correct : 283.0\n",
      "unweighted_accuracy : 0.38820301783264743 weighted_accuracy : 0.22551162328631036\n",
      "Epoch - 35 Train-Loss : 1.2931732734044392 Valid-Loss : 1.2189528942108154 Correct : 286.0\n",
      "unweighted_accuracy : 0.39231824417009603 weighted_accuracy : 0.2972222222222222\n",
      "Epoch - 40 Train-Loss : 1.291282097498576 Valid-Loss : 1.224862813949585 Correct : 282.0\n",
      "unweighted_accuracy : 0.3868312757201646 weighted_accuracy : 0.36148148148148146\n",
      "Epoch - 45 Train-Loss : 1.2933183113733928 Valid-Loss : 1.2199152708053589 Correct : 285.0\n",
      "unweighted_accuracy : 0.39094650205761317 weighted_accuracy : 0.29694019471488176\n",
      "Epoch - 50 Train-Loss : 1.288619597752889 Valid-Loss : 1.2373557090759277 Correct : 288.0\n",
      "unweighted_accuracy : 0.3950617283950617 weighted_accuracy : 0.26827852998065765\n",
      "Epoch - 55 Train-Loss : 1.3435309330622356 Valid-Loss : 1.2956844568252563 Correct : 287.0\n",
      "unweighted_accuracy : 0.3936899862825789 weighted_accuracy : 0.31002331002331\n",
      "Epoch - 60 Train-Loss : 1.2905773321787517 Valid-Loss : 1.2156376838684082 Correct : 286.0\n",
      "unweighted_accuracy : 0.39231824417009603 weighted_accuracy : 0.27903327055869426\n",
      "Epoch - 65 Train-Loss : 1.271421233812968 Valid-Loss : 1.2353345155715942 Correct : 281.0\n",
      "unweighted_accuracy : 0.38545953360768176 weighted_accuracy : 0.32744588744588743\n",
      "Epoch - 70 Train-Loss : 1.2821391026178997 Valid-Loss : 1.2376813888549805 Correct : 287.0\n",
      "unweighted_accuracy : 0.3936899862825789 weighted_accuracy : 0.49017764134332625\n",
      "Epoch - 75 Train-Loss : 1.2730555534362793 Valid-Loss : 1.2409515380859375 Correct : 286.0\n",
      "unweighted_accuracy : 0.39231824417009603 weighted_accuracy : 0.20810823397913564\n",
      "Epoch - 80 Train-Loss : 1.2571808099746704 Valid-Loss : 1.2984154224395752 Correct : 268.0\n",
      "unweighted_accuracy : 0.3676268861454047 weighted_accuracy : 0.24574475077653044\n",
      "Epoch - 85 Train-Loss : 1.2349772453308105 Valid-Loss : 1.2687242031097412 Correct : 261.0\n",
      "unweighted_accuracy : 0.35802469135802467 weighted_accuracy : 0.28741926776948523\n",
      "Epoch - 90 Train-Loss : 1.1471736431121826 Valid-Loss : 1.2130664587020874 Correct : 313.0\n",
      "unweighted_accuracy : 0.42935528120713307 weighted_accuracy : 0.37123585345045224\n",
      "Epoch - 95 Train-Loss : 1.0275567173957825 Valid-Loss : 1.212453842163086 Correct : 335.0\n",
      "unweighted_accuracy : 0.45953360768175583 weighted_accuracy : 0.38960132664176783\n",
      "Epoch - 100 Train-Loss : 0.9266239404678345 Valid-Loss : 1.2722232341766357 Correct : 351.0\n",
      "unweighted_accuracy : 0.48148148148148145 weighted_accuracy : 0.4027533327629899\n",
      "Epoch - 105 Train-Loss : 0.835670272509257 Valid-Loss : 1.311612606048584 Correct : 374.0\n",
      "unweighted_accuracy : 0.5130315500685871 weighted_accuracy : 0.44549387771414006\n",
      "Epoch - 110 Train-Loss : 0.7147568861643473 Valid-Loss : 1.3711533546447754 Correct : 375.0\n",
      "unweighted_accuracy : 0.51440329218107 weighted_accuracy : 0.43397324463500936\n",
      "Epoch - 115 Train-Loss : 0.613359401623408 Valid-Loss : 1.3029561042785645 Correct : 397.0\n",
      "unweighted_accuracy : 0.5445816186556928 weighted_accuracy : 0.4609592154474704\n",
      "Epoch - 120 Train-Loss : 0.5665003259976705 Valid-Loss : 1.3798367977142334 Correct : 395.0\n",
      "unweighted_accuracy : 0.541838134430727 weighted_accuracy : 0.4478177557209816\n",
      "Epoch - 125 Train-Loss : 0.525043378273646 Valid-Loss : 1.6044343709945679 Correct : 386.0\n",
      "unweighted_accuracy : 0.5294924554183813 weighted_accuracy : 0.4419365390211806\n",
      "Epoch - 130 Train-Loss : 0.48124365011850995 Valid-Loss : 1.5575125217437744 Correct : 383.0\n",
      "unweighted_accuracy : 0.5253772290809328 weighted_accuracy : 0.4432509197963309\n",
      "Epoch - 135 Train-Loss : 0.468195378780365 Valid-Loss : 1.6035962104797363 Correct : 380.0\n",
      "unweighted_accuracy : 0.5212620027434842 weighted_accuracy : 0.4346463294211017\n",
      "Epoch - 140 Train-Loss : 0.5750943620999655 Valid-Loss : 1.6230485439300537 Correct : 393.0\n",
      "unweighted_accuracy : 0.5390946502057613 weighted_accuracy : 0.44046125353915366\n",
      "Epoch - 145 Train-Loss : 0.6064356664816538 Valid-Loss : 1.3543168306350708 Correct : 393.0\n",
      "unweighted_accuracy : 0.5390946502057613 weighted_accuracy : 0.4520133803482261\n",
      "Epoch - 150 Train-Loss : 0.48488863309224445 Valid-Loss : 1.418359398841858 Correct : 376.0\n",
      "unweighted_accuracy : 0.5157750342935528 weighted_accuracy : 0.4225007405066059\n",
      "Epoch - 155 Train-Loss : 0.41052958369255066 Valid-Loss : 1.4241666793823242 Correct : 398.0\n",
      "unweighted_accuracy : 0.5459533607681756 weighted_accuracy : 0.4429847033012343\n",
      "Epoch - 160 Train-Loss : 0.35586227973302204 Valid-Loss : 1.6411278247833252 Correct : 383.0\n",
      "unweighted_accuracy : 0.5253772290809328 weighted_accuracy : 0.43207126332158885\n",
      "Epoch - 165 Train-Loss : 0.33207155267397565 Valid-Loss : 1.6326780319213867 Correct : 391.0\n",
      "unweighted_accuracy : 0.5363511659807956 weighted_accuracy : 0.4453377831912121\n",
      "Epoch - 170 Train-Loss : 0.31112226843833923 Valid-Loss : 1.65627121925354 Correct : 371.0\n",
      "unweighted_accuracy : 0.5089163237311386 weighted_accuracy : 0.41526906707518824\n",
      "Epoch - 175 Train-Loss : 0.2937558591365814 Valid-Loss : 1.6677618026733398 Correct : 391.0\n",
      "unweighted_accuracy : 0.5363511659807956 weighted_accuracy : 0.44336203122719975\n",
      "Epoch - 180 Train-Loss : 0.26895207663377124 Valid-Loss : 1.7403464317321777 Correct : 371.0\n",
      "unweighted_accuracy : 0.5089163237311386 weighted_accuracy : 0.4183255912155084\n",
      "Epoch - 185 Train-Loss : 0.2509895861148834 Valid-Loss : 1.7450438737869263 Correct : 388.0\n",
      "unweighted_accuracy : 0.532235939643347 weighted_accuracy : 0.43836530273776997\n",
      "Epoch - 190 Train-Loss : 0.250136395295461 Valid-Loss : 1.8096051216125488 Correct : 394.0\n",
      "unweighted_accuracy : 0.5404663923182441 weighted_accuracy : 0.43941745463650284\n",
      "Epoch - 195 Train-Loss : 0.24144467214743295 Valid-Loss : 1.7747124433517456 Correct : 389.0\n",
      "unweighted_accuracy : 0.53360768175583 weighted_accuracy : 0.44072799983587435\n",
      "Epoch - 200 Train-Loss : 0.23189943532148996 Valid-Loss : 1.8241678476333618 Correct : 391.0\n",
      "unweighted_accuracy : 0.5363511659807956 weighted_accuracy : 0.4424880506902299\n",
      "Epoch - 205 Train-Loss : 0.22061670819918314 Valid-Loss : 1.9229662418365479 Correct : 387.0\n",
      "unweighted_accuracy : 0.5308641975308642 weighted_accuracy : 0.4383019849410318\n",
      "Epoch - 210 Train-Loss : 0.22671495378017426 Valid-Loss : 1.8394718170166016 Correct : 388.0\n",
      "unweighted_accuracy : 0.532235939643347 weighted_accuracy : 0.42839573496698535\n",
      "Epoch - 215 Train-Loss : 0.22332443296909332 Valid-Loss : 1.9120912551879883 Correct : 380.0\n",
      "unweighted_accuracy : 0.5212620027434842 weighted_accuracy : 0.43119942204433365\n",
      "Epoch - 220 Train-Loss : 0.257028063138326 Valid-Loss : 1.9071704149246216 Correct : 372.0\n",
      "unweighted_accuracy : 0.5102880658436214 weighted_accuracy : 0.4375851940422656\n",
      "Epoch - 225 Train-Loss : 0.23092393577098846 Valid-Loss : 1.779615879058838 Correct : 384.0\n",
      "unweighted_accuracy : 0.5267489711934157 weighted_accuracy : 0.43690727661240336\n",
      "Epoch - 230 Train-Loss : 0.2673032035430272 Valid-Loss : 1.905436396598816 Correct : 399.0\n",
      "unweighted_accuracy : 0.5473251028806584 weighted_accuracy : 0.45568527312930296\n",
      "Epoch - 235 Train-Loss : 0.23826960225900015 Valid-Loss : 1.722952127456665 Correct : 390.0\n",
      "unweighted_accuracy : 0.5349794238683128 weighted_accuracy : 0.4402264588597907\n",
      "Epoch - 240 Train-Loss : 0.25326654811700183 Valid-Loss : 2.0259063243865967 Correct : 398.0\n",
      "unweighted_accuracy : 0.5459533607681756 weighted_accuracy : 0.4581500735887576\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 245 Train-Loss : 0.2530027727286021 Valid-Loss : 1.6727993488311768 Correct : 405.0\n",
      "unweighted_accuracy : 0.5555555555555556 weighted_accuracy : 0.45577122898669364\n",
      "Epoch - 250 Train-Loss : 0.23220261434713999 Valid-Loss : 1.8455308675765991 Correct : 396.0\n",
      "unweighted_accuracy : 0.5432098765432098 weighted_accuracy : 0.449298220650362\n",
      "Epoch - 255 Train-Loss : 0.20733735958735147 Valid-Loss : 1.908883810043335 Correct : 404.0\n",
      "unweighted_accuracy : 0.5541838134430727 weighted_accuracy : 0.4530366627800955\n",
      "Epoch - 260 Train-Loss : 0.25146592656771344 Valid-Loss : 2.0423483848571777 Correct : 393.0\n",
      "unweighted_accuracy : 0.5390946502057613 weighted_accuracy : 0.4461297777827926\n",
      "Epoch - 265 Train-Loss : 0.2869443396727244 Valid-Loss : 1.8457614183425903 Correct : 404.0\n",
      "unweighted_accuracy : 0.5541838134430727 weighted_accuracy : 0.45916997512122615\n",
      "Epoch - 270 Train-Loss : 0.2292718142271042 Valid-Loss : 1.8411818742752075 Correct : 402.0\n",
      "unweighted_accuracy : 0.551440329218107 weighted_accuracy : 0.4531530994197872\n",
      "Epoch - 275 Train-Loss : 0.20225339382886887 Valid-Loss : 1.7740027904510498 Correct : 395.0\n",
      "unweighted_accuracy : 0.541838134430727 weighted_accuracy : 0.44962523840671764\n",
      "Epoch - 280 Train-Loss : 0.20021666089693704 Valid-Loss : 1.911937952041626 Correct : 409.0\n",
      "unweighted_accuracy : 0.5610425240054869 weighted_accuracy : 0.4641188701812596\n",
      "Epoch - 285 Train-Loss : 0.17751801013946533 Valid-Loss : 1.8781564235687256 Correct : 401.0\n",
      "unweighted_accuracy : 0.5500685871056241 weighted_accuracy : 0.45846520528216\n",
      "Epoch - 290 Train-Loss : 0.16963385542233786 Valid-Loss : 2.026921033859253 Correct : 394.0\n",
      "unweighted_accuracy : 0.5404663923182441 weighted_accuracy : 0.453277525135737\n",
      "Epoch - 295 Train-Loss : 0.16912210981051126 Valid-Loss : 2.0512168407440186 Correct : 394.0\n",
      "unweighted_accuracy : 0.5404663923182441 weighted_accuracy : 0.45847067206207337\n"
     ]
    }
   ],
   "source": [
    "# train_model(lstm, epochs=300, lr=0.01)\n",
    "train_text_model(lstm, loss_fn, text_train_loader, text_valid_loader, epochs, learning_rate, optimizer, train_losses, valid_losses, comment, change_lr=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py36/lib/python3.6/site-packages/torch/serialization.py:256: UserWarning: Couldn't retrieve source code for container of type LSTM_fixed_len. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "torch.save(model_fixed, 'fixed_text_model.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_melspectrogram_db(file_path, sr=None, n_fft=2048, hop_length=512, n_mels=128,top_db=80):\n",
    "    wav,sr = librosa.load(file_path,sr=sr)\n",
    "    \n",
    "    if wav.shape[0]<5*sr:\n",
    "        wav=np.pad(wav,int(np.ceil((5*sr-wav.shape[0])/2)),mode='reflect')\n",
    "    else:\n",
    "        wav=wav[:5*sr]\n",
    "    spec=librosa.feature.melspectrogram(wav, sr=sr, n_fft=n_fft,\n",
    "              hop_length=hop_length,n_mels=n_mels)\n",
    "    spec_db=librosa.power_to_db(spec,top_db=top_db)\n",
    "    return spec_db\n",
    "\n",
    "def normalize(spec):\n",
    "    return (spec - spec.mean())/(spec.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Audio(Dataset):\n",
    "    def __init__(self, base, df, in_col, out_col):\n",
    "        self.df = df\n",
    "        self.data = []\n",
    "        self.labels = []\n",
    "        for ind in tqdm(range(len(df))):\n",
    "            row = df.iloc[ind]\n",
    "            file_path = base + '/' + row[in_col] + '.wav'\n",
    "            self.data.append(normalize(get_melspectrogram_db(file_path))[np.newaxis,...])\n",
    "            self.labels.append(row[out_col])\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    def __getitem__(self, idx):\n",
    "        return self.data[idx], self.labels[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "train = audio[:2214]\n",
    "valid = audio[2214:]\n",
    "# train  = audio[243:]\n",
    "# valid = audio[:243]\n",
    "# train, valid = train_test_split(audio, test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2214/2214 [00:21<00:00, 102.64it/s]\n",
      "100%|██████████| 729/729 [00:06<00:00, 105.10it/s]\n"
     ]
    }
   ],
   "source": [
    "train_audio = Audio('/data/home/advaitmb/datasets/sentences', train, 'wav_file', 'emotion')\n",
    "valid_audio = Audio('/data/home/advaitmb/datasets/sentences', valid, 'wav_file', 'emotion')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_train_loader = DataLoader(train_audio, batch_size=64, shuffle=False)\n",
    "audio_valid_loader = DataLoader(valid_audio, batch_size=64, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN(\n",
      "  (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "  (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (conv2): Conv2d(32, 64, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "  (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (conv3): Conv2d(64, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "  (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (conv4): Conv2d(128, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "  (fc1): Linear(in_features=7168, out_features=512, bias=True)\n",
      "  (fc2): Linear(in_features=512, out_features=4, bias=True)\n",
      "  (activation): Softmax(dim=1)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__() # just run the init of parent class (nn.Module)\n",
    "        self.conv1 = nn.Conv2d(1, 32, 5, 2, padding=1) # input is 1 image, 32 output channels, 5x5 kernel / window\n",
    "        self.bn1 = nn.BatchNorm2d(32)\n",
    "        self.conv2 = nn.Conv2d(32, 64, 5, 2, padding=1) # input is 32, bc the first layer output 32. Then we say the output will be 64 channels, 5x5 kernel / window\n",
    "        self.bn2 = nn.BatchNorm2d(64)\n",
    "        self.conv3 = nn.Conv2d(64, 128, 5, 2, padding=1)\n",
    "        self.bn3 = nn.BatchNorm2d(128)\n",
    "        self.conv4 = nn.Conv2d(128, 128, 5, 2, padding=1)\n",
    "        \n",
    "        self.fc1 = nn.Linear((7*8*128), 512) #flattening.\n",
    "        \n",
    "        \n",
    "        self.fc2 = nn.Linear(512, 4) # 512 in, 2 out bc we're doing 2 classes (dog vs cat).\n",
    "        self.activation = nn.Softmax(dim=1)\n",
    "\n",
    "    def convs(self, x):\n",
    "        # max pooling over 2x2\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = self.bn1(x)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.bn2(x)\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = self.bn3(x)\n",
    "        x = F.relu(self.conv4(x))\n",
    "\n",
    "        return x\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.convs(x)\n",
    "        x = x.reshape(x.size(0), -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x) # bc this is our output layer. No activation here.\n",
    "        return self.activation(x)\n",
    "\n",
    "\n",
    "cnn = CNN()\n",
    "print(cnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN(\n",
       "  (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "  (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv2): Conv2d(32, 64, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "  (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv3): Conv2d(64, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "  (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv4): Conv2d(128, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "  (fc1): Linear(in_features=7168, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=4, bias=True)\n",
       "  (activation): Softmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setlr(optimizer, lr):\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "    return optimizer\n",
    "\n",
    "def lr_decay(optimizer, epoch, learning_rate):\n",
    "    if epoch%10==0:\n",
    "        new_lr = learning_rate / (10**(epoch//10))\n",
    "        optimizer = setlr(optimizer, new_lr)\n",
    "        print(\"Changed learning rate to {}\".format(new_lr))\n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def train_model(model, loss_fn, train_loader, valid_loader, epochs, optimizer, train_losses, valid_losses, change_lr=None):\n",
    "    \n",
    "    for epoch in range(1,epochs+1):\n",
    "        \n",
    "        model.train()\n",
    "        batch_losses=[]\n",
    "        if change_lr:\n",
    "            optimizer = change_lr(optimizer, epoch, learning_rate)\n",
    "        for i, data in tqdm(enumerate(train_loader)):\n",
    "            x, y = data\n",
    "            optimizer.zero_grad()\n",
    "            x = x.to(device, dtype=torch.float32)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_hat = model(x)\n",
    "            loss = loss_fn(y_hat, y)\n",
    "            loss.backward()\n",
    "            batch_losses.append(loss.item())\n",
    "            optimizer.step()\n",
    "        train_losses.append(batch_losses)\n",
    "\n",
    "\n",
    "#         print(\"Epoch - {} Train-Loss : {}\".format(epoch, np.mean(train_losses[-1])))\n",
    "        model.eval()\n",
    "        batch_losses=[]\n",
    "        trace_y = []\n",
    "        trace_yhat = []\n",
    "        for i, data in enumerate(valid_loader):\n",
    "            x, y = data\n",
    "            x = x.to(device, dtype=torch.float32)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_hat = model(x)\n",
    "            loss = loss_fn(y_hat, y)\n",
    "            trace_y.append(y.cpu().detach().numpy())\n",
    "            trace_yhat.append(y_hat.cpu().detach().numpy())      \n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "        valid_losses.append(batch_losses)\n",
    "        trace_y = np.concatenate(trace_y)\n",
    "        trace_yhat = np.concatenate(trace_yhat)\n",
    "        accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)\n",
    "        unweighted_accuracy = accuracy_score( trace_yhat.argmax(axis=1), trace_y )\n",
    "        weighted_accuracy = balanced_accuracy_score( trace_yhat.argmax(axis=1), trace_y )\n",
    "        print(\"Epoch - {} Train-Loss : {} Valid-Loss : {}\".format(epoch, np.mean(train_losses[-1]), np.mean(valid_losses[-1])))\n",
    "        print(\"unweighted_accuracy : {} weighted_accuracy : {}\".format(unweighted_accuracy, weighted_accuracy ))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def lr_decay_exp(optimizer, epoch, learning_rate):\n",
    "#     learning_rate = (2e-1)/(1 + epoch*)\n",
    "#     optimizer = setlr(optimizer, new_lr)\n",
    "#     print(\"Changed learning rate to {}\".format(new_lr))\n",
    "#     return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 2e-5\n",
    "optimizer = optim.Adam(cnn.parameters(), lr=learning_rate)\n",
    "epochs = 15\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "train_losses=[]\n",
    "valid_losses=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.88it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Train-Loss : 1.342157656805856 Valid-Loss : 1.2985920310020447\n",
      "unweighted_accuracy : 0.3991769547325103 weighted_accuracy : 0.5474965229485396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.80it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 2 Train-Loss : 1.2856719936643328 Valid-Loss : 1.2494985858599346\n",
      "unweighted_accuracy : 0.5034293552812071 weighted_accuracy : 0.5708199549359244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.82it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 3 Train-Loss : 1.227051840509687 Valid-Loss : 1.226865440607071\n",
      "unweighted_accuracy : 0.522633744855967 weighted_accuracy : 0.5244472740631875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.82it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 4 Train-Loss : 1.188248782498496 Valid-Loss : 1.212343047062556\n",
      "unweighted_accuracy : 0.50480109739369 weighted_accuracy : 0.5158692353048849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.82it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 5 Train-Loss : 1.1604122349194117 Valid-Loss : 1.1991311808427174\n",
      "unweighted_accuracy : 0.5089163237311386 weighted_accuracy : 0.5284088772330916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.84it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 6 Train-Loss : 1.1377275909696307 Valid-Loss : 1.1903707285722096\n",
      "unweighted_accuracy : 0.522633744855967 weighted_accuracy : 0.5542925290069974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.79it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 7 Train-Loss : 1.118407917022705 Valid-Loss : 1.1838748653729756\n",
      "unweighted_accuracy : 0.5390946502057613 weighted_accuracy : 0.5768348666899391\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.83it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 8 Train-Loss : 1.1003415516444615 Valid-Loss : 1.1779148280620575\n",
      "unweighted_accuracy : 0.5500685871056241 weighted_accuracy : 0.5892454041632874\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.83it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 9 Train-Loss : 1.0821815882410322 Valid-Loss : 1.1718366593122482\n",
      "unweighted_accuracy : 0.5500685871056241 weighted_accuracy : 0.5933934152458404\n",
      "Changed learning rate to 2.0000000000000003e-06\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.84it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 10 Train-Loss : 1.0617310081209455 Valid-Loss : 1.1630895485480626\n",
      "unweighted_accuracy : 0.5651577503429356 weighted_accuracy : 0.6184124134606593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.82it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 11 Train-Loss : 1.0551531655447823 Valid-Loss : 1.1586727102597554\n",
      "unweighted_accuracy : 0.5706447187928669 weighted_accuracy : 0.6244586453564781\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.86it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 12 Train-Loss : 1.0523176414625985 Valid-Loss : 1.1578205525875092\n",
      "unweighted_accuracy : 0.5720164609053497 weighted_accuracy : 0.6279733779733779\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:09,  3.58it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 13 Train-Loss : 1.0502966557230269 Valid-Loss : 1.1573929190635681\n",
      "unweighted_accuracy : 0.5747599451303155 weighted_accuracy : 0.6329283238291464\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:12,  2.86it/s]\n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 14 Train-Loss : 1.0484359008925301 Valid-Loss : 1.1569918741782506\n",
      "unweighted_accuracy : 0.5761316872427984 weighted_accuracy : 0.6339598342563262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:13,  2.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 15 Train-Loss : 1.0466408457074847 Valid-Loss : 1.1566048661867778\n",
      "unweighted_accuracy : 0.5775034293552812 weighted_accuracy : 0.6349844815760505\n"
     ]
    }
   ],
   "source": [
    "train_model(cnn, loss_fn, audio_train_loader, audio_valid_loader, epochs, optimizer, train_losses, valid_losses, lr_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CRNN(\n",
      "  (audioModel): CNN(\n",
      "    (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "    (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (conv2): Conv2d(32, 64, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "    (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (conv3): Conv2d(64, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "    (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (conv4): Conv2d(128, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
      "    (fc1): Linear(in_features=7168, out_features=512, bias=True)\n",
      "    (fc2): Linear(in_features=512, out_features=4, bias=True)\n",
      "    (activation): Softmax(dim=1)\n",
      "  )\n",
      "  (textModel): LSTM_glove_vecs(\n",
      "    (embeddings): Embedding(2255, 50, padding_idx=0)\n",
      "    (lstm): LSTM(50, 64, batch_first=True)\n",
      "    (linear): Linear(in_features=64, out_features=4, bias=True)\n",
      "    (dropout): Dropout(p=0.2, inplace=False)\n",
      "  )\n",
      "  (linear): Linear(in_features=8, out_features=4, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class CRNN(nn.Module):\n",
    "    def __init__(self, audioModel, textModel):\n",
    "        \n",
    "        # CNN for Audio\n",
    "        super().__init__()\n",
    "\n",
    "        self.audioModel = audioModel.eval()\n",
    "        self.textModel = textModel.eval()\n",
    "        \n",
    "        self.linear = nn.Linear(8, 4)\n",
    "\n",
    "\n",
    "    def forward(self, ax, tx, l):\n",
    "        #Audio \n",
    "        ax = self.audioModel(ax)\n",
    "        tx = self.textModel(tx, l)\n",
    "        \n",
    "        x = torch.cat((ax, tx), dim=1)\n",
    "        x = self.linear(x)\n",
    "        return F.softmax(x, dim=1)\n",
    "\n",
    "crnn = CRNN(cnn, lstm)\n",
    "print(crnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "for param in crnn.audioModel.parameters():\n",
    "    param.requires_grad=False\n",
    "    \n",
    "for param in crnn.textModel.parameters():\n",
    "    param.requires_grad=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# import torch\n",
    "# import torch.nn as nn\n",
    "# import torch.nn.functional as F\n",
    "\n",
    "# class CRNN(nn.Module):\n",
    "#     def __init__(self, vocab_size, embedding_dim, hidden_dim):\n",
    "        \n",
    "#         # CNN for Audio\n",
    "#         super().__init__()\n",
    "#         self.conv1 = nn.Conv2d(1, 32, 5, 2, padding=1) \n",
    "#         self.bn1 = nn.BatchNorm2d(32)\n",
    "#         self.conv2 = nn.Conv2d(32, 64, 5, 2, padding=1) \n",
    "#         self.bn2 = nn.BatchNorm2d(64)\n",
    "#         self.conv3 = nn.Conv2d(64, 128, 5, 2, padding=1)\n",
    "#         self.bn3 = nn.BatchNorm2d(128)\n",
    "#         self.conv4 = nn.Conv2d(128, 128, 5, 2, padding=1)\n",
    "#         self.afc1 = nn.Linear((7*8*128), 512)\n",
    "#         self.afc2 = nn.Linear(512, 128) \n",
    "        \n",
    "#         # RNN for Text\n",
    "#         self.embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)\n",
    "#         self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "#         self.tfc1 = nn.Linear(hidden_dim, 32)\n",
    "#         self.dropout = nn.Dropout(0.2)\n",
    "\n",
    "#         self.linear = nn.Linear(160, 4)\n",
    "#     def convs(self, x):\n",
    "#         x = F.relu(self.conv1(x))\n",
    "#         x = self.bn1(x)\n",
    "#         x = F.relu(self.conv2(x))\n",
    "#         x = self.bn2(x)\n",
    "#         x = F.relu(self.conv3(x))\n",
    "#         x = self.bn3(x)\n",
    "#         x = F.relu(self.conv4(x))\n",
    "\n",
    "#         return x\n",
    "\n",
    "#     def forward(self, ax, tx):\n",
    "#         #Audio \n",
    "#         ax = self.convs(ax)\n",
    "#         ax = ax.reshape(ax.size(0), -1)\n",
    "#         ax = F.relu(self.afc1(ax))\n",
    "#         ax = F.relu(self.afc2(ax)) # bc this is our output layer. No activation here.\n",
    "        \n",
    "#         #Text\n",
    "#         tx = self.embeddings(tx)\n",
    "# #         tx = self.dropout(tx)\n",
    "#         lstm_out, (ht, ct) = self.lstm(tx)\n",
    "#         tx = F.relu(self.tfc1(ht[-1]))\n",
    "        \n",
    "#         x = torch.cat((ax, tx), dim=1)\n",
    "#         x = self.linear(x)\n",
    "#         return F.softmax(x, dim=1)\n",
    "\n",
    "# crnn = CRNN(vocab_size, 50, 64)\n",
    "# print(crnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CRNN(\n",
       "  (audioModel): CNN(\n",
       "    (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "    (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (conv2): Conv2d(32, 64, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "    (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (conv3): Conv2d(64, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "    (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (conv4): Conv2d(128, 128, kernel_size=(5, 5), stride=(2, 2), padding=(1, 1))\n",
       "    (fc1): Linear(in_features=7168, out_features=512, bias=True)\n",
       "    (fc2): Linear(in_features=512, out_features=4, bias=True)\n",
       "    (activation): Softmax(dim=1)\n",
       "  )\n",
       "  (textModel): LSTM_glove_vecs(\n",
       "    (embeddings): Embedding(2255, 50, padding_idx=0)\n",
       "    (lstm): LSTM(50, 64, batch_first=True)\n",
       "    (linear): Linear(in_features=64, out_features=4, bias=True)\n",
       "    (dropout): Dropout(p=0.2, inplace=False)\n",
       "  )\n",
       "  (linear): Linear(in_features=8, out_features=4, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crnn.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "def train_model(model, loss_fn, audio_train_loader, text_train_loader, audio_valid_loader, text_valid_loader, epochs, optimizer, train_losses, valid_losses, change_lr=None): \n",
    "    for epoch in range(1,epochs+1):\n",
    "        \n",
    "        model.train()\n",
    "        batch_losses=[]\n",
    "        if change_lr:\n",
    "            optimizer = change_lr(optimizer, epoch, learning_rate)\n",
    "        for a_data, t_data in tqdm(zip(audio_train_loader, text_train_loader)):\n",
    "            ax, ay = a_data\n",
    "            tx, ty, l = t_data\n",
    "            ax = ax.to(device, dtype=torch.float32)\n",
    "            ay = ay.to(device, dtype=torch.long)\n",
    "            tx = tx.to(device, dtype=torch.long)\n",
    "            y_hat = model(ax, tx, l)\n",
    "            loss = loss_fn(y_hat, ay)\n",
    "            loss.backward()\n",
    "            batch_losses.append(loss.item())\n",
    "            optimizer.step()\n",
    "        train_losses.append(batch_losses)\n",
    "\n",
    "        model.eval()\n",
    "        batch_losses=[]\n",
    "        trace_y = []\n",
    "        trace_yhat = []\n",
    "        for a_data, t_data in zip(audio_train_loader, text_train_loader):\n",
    "            ax, ay = a_data\n",
    "            tx, ty, l = t_data\n",
    "            ax = ax.to(device, dtype=torch.float32)\n",
    "            ay = ay.to(device, dtype=torch.long)\n",
    "            tx = tx.to(device, dtype=torch.long)\n",
    "            y_hat = model(ax, tx, l)\n",
    "            loss = loss_fn(y_hat, ay)\n",
    "            trace_y.append(ay.cpu().detach().numpy())\n",
    "            trace_yhat.append(y_hat.cpu().detach().numpy())      \n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "        valid_losses.append(batch_losses)\n",
    "        trace_y = np.concatenate(trace_y)\n",
    "        trace_yhat = np.concatenate(trace_yhat)\n",
    "        accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)\n",
    "        unweighted_accuracy = accuracy_score( trace_yhat.argmax(axis=1), trace_y )\n",
    "        weighted_accuracy = balanced_accuracy_score( trace_yhat.argmax(axis=1), trace_y )\n",
    "        print(\"Epoch - {} Train-Loss : {} Valid-Loss : {}\".format(epoch, 0, np.mean(valid_losses[-1])))\n",
    "        print(\"unweighted_accuracy : {} weighted_accuracy : {}\".format(unweighted_accuracy, weighted_accuracy ))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "vocab_size = len(words)\n",
    "text_train_loader = DataLoader(train_text, batch_size=batch_size, shuffle=False)\n",
    "text_valid_loader = DataLoader(valid_text, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "optimizer = optim.SGD(crnn.parameters(), lr=learning_rate, momentum=0.9)\n",
    "epochs = 1\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "train_losses=[]\n",
    "valid_losses=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n",
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "print(audio_train_loader.dataset.labels[100:120])\n",
    "print(text_train_loader.dataset.y[100:120])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [00:03, 11.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Train-Loss : 0 Valid-Loss : 0.9993273683956692\n",
      "unweighted_accuracy : 0.7461607949412827 weighted_accuracy : 0.8384779944910277\n"
     ]
    }
   ],
   "source": [
    "train_model(crnn, loss_fn, audio_train_loader, text_train_loader, audio_valid_loader, text_valid_loader, epochs, optimizer, train_losses, valid_losses, lr_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "god = torch.load('god_model.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "audioModel.conv1.weight\n",
      "audioModel.conv1.bias\n",
      "audioModel.bn1.weight\n",
      "audioModel.bn1.bias\n",
      "audioModel.conv2.weight\n",
      "audioModel.conv2.bias\n",
      "audioModel.bn2.weight\n",
      "audioModel.bn2.bias\n",
      "audioModel.conv3.weight\n",
      "audioModel.conv3.bias\n",
      "audioModel.bn3.weight\n",
      "audioModel.bn3.bias\n",
      "audioModel.conv4.weight\n",
      "audioModel.conv4.bias\n",
      "audioModel.fc1.weight\n",
      "audioModel.fc1.bias\n",
      "audioModel.fc2.weight\n",
      "audioModel.fc2.bias\n",
      "textModel.embeddings.weight\n",
      "textModel.lstm.weight_ih_l0\n",
      "textModel.lstm.weight_hh_l0\n",
      "textModel.lstm.bias_ih_l0\n",
      "textModel.lstm.bias_hh_l0\n",
      "textModel.linear.weight\n",
      "textModel.linear.bias\n",
      "linear.weight\n",
      "linear.bias\n"
     ]
    }
   ],
   "source": [
    "for name, param in crnn.named_parameters():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "audioModel.conv1.weight\n",
      "audioModel.conv1.bias\n",
      "audioModel.bn1.weight\n",
      "audioModel.bn1.bias\n",
      "audioModel.conv2.weight\n",
      "audioModel.conv2.bias\n",
      "audioModel.bn2.weight\n",
      "audioModel.bn2.bias\n",
      "audioModel.conv3.weight\n",
      "audioModel.conv3.bias\n",
      "audioModel.bn3.weight\n",
      "audioModel.bn3.bias\n",
      "audioModel.conv4.weight\n",
      "audioModel.conv4.bias\n",
      "audioModel.fc1.weight\n",
      "audioModel.fc1.bias\n",
      "audioModel.fc2.weight\n",
      "audioModel.fc2.bias\n",
      "textModel.embeddings.weight\n",
      "textModel.lstm.weight_ih_l0\n",
      "textModel.lstm.weight_hh_l0\n",
      "textModel.lstm.bias_ih_l0\n",
      "textModel.lstm.bias_hh_l0\n",
      "textModel.linear.weight\n",
      "textModel.linear.bias\n",
      "linear.weight\n",
      "linear.bias\n"
     ]
    }
   ],
   "source": [
    "for name, param in god.named_parameters():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}