Test-Supervised_Timbre_All.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adult-submission",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import csv\n",
    "import sys, os.path\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import torch\n",
    "from torch import optim, nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms\n",
    "#from torch.utils.tensorboard import SummaryWriter\n",
    "#from torchsummary import summary\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import matplotlib as mpl\n",
    "mpl.rcParams['figure.dpi'] = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "renewable-penguin",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our modules\n",
    "import sys\n",
    "sys.path.append('.')\n",
    "sys.path.append('..')\n",
    "\n",
    "from vae import configs, train, plot_utils, models\n",
    "from vae.data import build_dataloader\n",
    "from vae.latent_spaces import dimensionality_reduction, plot_spaces\n",
    "from vae.reconstructions import plot_reconstructions\n",
    "from vae.models import model_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "comparative-company",
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_acc(y_pred, y_test):\n",
    "    y_pred_softmax = torch.log_softmax(y_pred, dim = 1)\n",
    "    _, y_pred_tags = torch.max(y_pred_softmax, dim = 1)    \n",
    "    \n",
    "    correct_pred = (y_pred_tags == y_test).float()\n",
    "    acc = correct_pred.sum() / len(correct_pred)\n",
    "    \n",
    "    acc = torch.round(acc * 100)\n",
    "    \n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "verbal-scholarship",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['violin', 'viola', 'cello', 'double-bass',\n",
    "                'clarinet', 'bass-clarinet', 'saxophone', 'flute', 'oboe', 'bassoon', 'contrabassoon',\n",
    "                'french-horn', 'trombone', 'trumpet', 'tuba', 'english-horn',\n",
    "                'guitar', 'mandolin', 'banjo', 'chromatic-percussion']\n",
    "\n",
    "chromatic_perc = ['agogo-bells', 'banana-shaker', 'bass-drum', 'bell-tree', 'cabasa', 'Chinese-hand-cymbals',\n",
    "                        'castanets', 'Chinese-cymbal', 'clash-cymbals', 'cowbell', 'djembe', 'djundjun', 'flexatone', 'guiro',\n",
    "                        'lemon-shaker',  'motor-horn',  'ratchet', 'sheeps-toenails', 'sizzle-cymbal', 'sleigh-bells', 'snare-drum',\n",
    "                        'spring-coil', 'squeaker', 'strawberry-shaker', 'surdo', 'suspended-cymbal', 'swanee-whistle',\n",
    "                        'tambourine', 'tam-tam', 'tenor-drum', 'Thai-gong', 'tom-toms', 'train-whistle', 'triangle',\n",
    "                        'vibraslap', 'washboard', 'whip', 'wind-chimes', 'woodblock', 'cor-anglais']\n",
    "\n",
    "labels_list = [i for i in range(len(classes))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dominican-portfolio",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dict of classes\n",
    "classes_dict = {classes[i]: classes.index(classes[i]) for i in range(len(classes))}\n",
    "\n",
    "\n",
    "df = pd.DataFrame.from_dict(classes_dict, orient='index', columns=['class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "prescribed-peoples",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>violin</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>viola</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cello</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>double-bass</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clarinet</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bass-clarinet</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>saxophone</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flute</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oboe</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bassoon</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>contrabassoon</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>french-horn</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trombone</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trumpet</th>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tuba</th>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>english-horn</th>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guitar</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mandolin</th>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>banjo</th>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chromatic-percussion</th>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      class\n",
       "violin                    0\n",
       "viola                     1\n",
       "cello                     2\n",
       "double-bass               3\n",
       "clarinet                  4\n",
       "bass-clarinet             5\n",
       "saxophone                 6\n",
       "flute                     7\n",
       "oboe                      8\n",
       "bassoon                   9\n",
       "contrabassoon            10\n",
       "french-horn              11\n",
       "trombone                 12\n",
       "trumpet                  13\n",
       "tuba                     14\n",
       "english-horn             15\n",
       "guitar                   16\n",
       "mandolin                 17\n",
       "banjo                    18\n",
       "chromatic-percussion     19"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "smooth-finding",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of files in the training dataset: 1357\n"
     ]
    }
   ],
   "source": [
    "model_name = 'supervised_timbre'\n",
    "input = 'mel_cut'\n",
    "trained_epochs = 320\n",
    "\n",
    "test_dataset, test_dataloader = build_dataloader.build_testset(input, model_name)\n",
    "print('Number of files in the training dataset:', len(test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "painful-wisconsin",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = model_utils.import_model(model_name, input)\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model.to(device)\n",
    "checkpoint = torch.load(os.path.join(configs.ParamsConfig.TRAINED_MODELS_PATH, 'saved_model_' + str(trained_epochs) + \"epochs.pth\"))\n",
    "model.load_state_dict(checkpoint['model'])\n",
    "with torch.no_grad():\n",
    "    model.eval()\n",
    "    \n",
    "    y_pred_list = []\n",
    "    y_test_list = []\n",
    "    for sample_batch, file, y in test_dataloader:\n",
    "            sample_batch = sample_batch.to(device, dtype=torch.float)\n",
    "            y_test_list.append(y)\n",
    "            y_test_pred, w = model(sample_batch)\n",
    "            _, y_pred_tags = torch.max(y_test_pred, dim = 1)\n",
    "            y_pred_list.append(y_pred_tags.cpu().numpy())\n",
    "\n",
    "y_pred_list = [a.squeeze().tolist() for a in y_pred_list]\n",
    "y_test = [a.squeeze().tolist() for a in y_test_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "established-things",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of parameters: 122388\n",
      "AttentionTimbreEncoder(\n",
      "  (multihead_attn): MultiheadAttention(\n",
      "    (out_proj): _LinearWithBias(in_features=128, out_features=128, bias=True)\n",
      "  )\n",
      "  (fc_1): Linear(in_features=2816, out_features=20, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model_utils.show_total_params(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "convertible-arbitration",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert y_pred_list != y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "behind-burning",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.43      0.65      0.51       150\n",
      "           1       0.78      0.70      0.74        97\n",
      "           2       0.56      0.46      0.51        89\n",
      "           3       0.79      0.59      0.68        85\n",
      "           4       0.63      0.56      0.60        85\n",
      "           5       0.81      0.49      0.61        94\n",
      "           6       0.46      0.33      0.38        73\n",
      "           7       0.66      0.76      0.71        88\n",
      "           8       0.68      0.87      0.76        60\n",
      "           9       0.88      0.53      0.66        72\n",
      "          10       0.90      0.51      0.65        71\n",
      "          11       0.62      0.69      0.65        65\n",
      "          12       0.35      0.64      0.45        83\n",
      "          13       0.82      0.77      0.80        48\n",
      "          14       0.76      0.81      0.79        97\n",
      "          15       0.69      0.59      0.64        69\n",
      "          16       1.00      0.73      0.84        11\n",
      "          17       0.67      0.50      0.57         8\n",
      "          18       0.33      0.14      0.20         7\n",
      "          19       0.17      0.40      0.24         5\n",
      "\n",
      "    accuracy                           0.62      1357\n",
      "   macro avg       0.65      0.59      0.60      1357\n",
      "weighted avg       0.66      0.62      0.62      1357\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import  classification_report\n",
    "print(classification_report(y_test, y_pred_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "rubber-champagne",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "southern-hurricane",
   "metadata": {},
   "outputs": [],
   "source": [
    "h8_report = classification_report(y_test, y_pred_list, output_dict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "meaning-finance",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5132275132275133"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h8_report['0']['f1-score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "surgical-lighting",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "for key in h8_report.keys():\n",
    "    try:\n",
    "        key = int(key)\n",
    "        sns.pointplot(x=h8_report[key]['f1-score'], y=key)\n",
    "        plt.show()\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "seasonal-mattress",
   "metadata": {},
   "source": [
    "## References\n",
    "\n",
    "* https://towardsdatascience.com/pytorch-tabular-multiclass-classification-9f8211a123ab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "removable-quarter",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}