mlp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from matplotlib import pyplot as plt\n",
    "# scikit learn logistic regression (NOT DOWNSAMPLED)\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "data_dir = \"data/\"\n",
    "\n",
    "college_players_stats = pd.read_csv(f'{data_dir}college_players_stats.csv', index_col=0)\n",
    "drafted_players = pd.read_csv(f'{data_dir}drafted_players.csv', index_col=0)\n",
    "not_drafted = pd.read_csv(f'{data_dir}not_drafted.csv', index_col=0)\n",
    "players_stats = pd.read_csv(f'{data_dir}players_stats.csv', index_col=0)\n",
    "\n",
    "# add label and normalize relevant values\n",
    "\n",
    "players_stats['drafted'] = players_stats['name'].isin(drafted_players['name'])\n",
    "players_stats['drafted'] = players_stats['drafted'].astype(int)\n",
    "\n",
    "\n",
    "# add label and normalize relevant values\n",
    "\n",
    "players_stats['drafted'] = players_stats['name'].isin(drafted_players['name'])\n",
    "players_stats['drafted'] = players_stats['drafted'].astype(int)\n",
    "players_stats = players_stats.drop(columns=['name'])\n",
    "\n",
    "# discretize height\n",
    "def convert_height(height_str):\n",
    "    feet, inches = map(int, height_str.split('-'))\n",
    "    return feet + inches / 12  \n",
    "\n",
    "# Apply the conversion function to the height column\n",
    "players_stats['height'] = players_stats['height'].apply(convert_height)\n",
    "\n",
    "# add binary columns for each position label\n",
    "def str_to_int(maybe_bool):\n",
    "    if isinstance(maybe_bool, bool):\n",
    "        return int(maybe_bool) \n",
    "    return maybe_bool\n",
    "\n",
    "def weight_to_int(input):\n",
    "    if(input == \"22l\"):\n",
    "        return 221\n",
    "    return(int(input)) \n",
    "players_stats['weight'] = players_stats['weight'].apply(weight_to_int)\n",
    "\n",
    "players_stats = pd.get_dummies(players_stats, columns=['position'])\n",
    "players_stats = players_stats.applymap(str_to_int)\n",
    "\n",
    "players_stats.info()\n",
    "non_numeric_columns = players_stats.select_dtypes(exclude=[\"number\"]).columns\n",
    "print(\"Non-numeric columns:\", non_numeric_columns)\n",
    "\n",
    "# TODO: add other metrics for training\n",
    "df = players_stats\n",
    "\n",
    "df.dropna(inplace=True)\n",
    "pd.set_option('display.max_columns', None)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(df['drafted'], return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42\n",
    "epochs = 250\n",
    "lr = 0.01\n",
    "batch_size = 32\n",
    "\n",
    "# get data\n",
    "x = df.drop(columns=['drafted']).values\n",
    "y = df['drafted'].values\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=seed)\n",
    "\n",
    "# downsampling\n",
    "#separate drafted and non-drafted \n",
    "df_drafted = df[df['drafted'] == 1]\n",
    "df_non_drafted = df[df['drafted'] == 0]\n",
    "\n",
    "\n",
    "# Drop 20000 random samples from non-drafted\n",
    "df_non_drafted_sampled = df_non_drafted.sample(len(df_non_drafted) - 20000, random_state=seed)\n",
    "\n",
    "df_downsampled = pd.concat([df_drafted, df_non_drafted_sampled])\n",
    "x = df_downsampled.drop(columns=['drafted']).values\n",
    "y = df_downsampled['drafted'].values\n",
    "\n",
    "xd_train, xd_test, yd_train, yd_test = train_test_split(x, y, test_size=0.2, random_state=seed)\n",
    "\n",
    "\n",
    "\n",
    "count = 0\n",
    "count_tot = 0\n",
    "for y1,y2 in zip(y_train, y_test):\n",
    "    if(y1 == 1):\n",
    "        count+=1\n",
    "    if(y2 == 1):\n",
    "        count+=1\n",
    "    count_tot +=1\n",
    "\n",
    "print(\"Count drafted:\")\n",
    "print(count)\n",
    "print(\"Count not drafted:\")\n",
    "print(count_tot)\n",
    "\n",
    "np.unique(y_test, return_counts=True)\n",
    "\n",
    "count = 0\n",
    "count_tot = 0\n",
    "for y1,y2 in zip(yd_train, yd_test):\n",
    "    if(y1 == 1):\n",
    "        count+=1\n",
    "    if(y2 == 1):\n",
    "        count+=1\n",
    "    count_tot +=1\n",
    "\n",
    "print(\"Count downsample drafted:\")\n",
    "print(count)\n",
    "print(\"Count downsample not drafted:\")\n",
    "print(count_tot)\n",
    "\n",
    "np.unique(y_test, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score\n",
    "\n",
    "def explain(y_test, y_pred, average='weighted'):\n",
    "    \"\"\"Look at precision, recall etc. for the model\"\"\"\n",
    "    precision = precision_score(y_test, y_pred, average=average, zero_division=0)\n",
    "    recall = recall_score(y_test, y_pred, average=average, zero_division=0)\n",
    "    f1 = f1_score(y_test, y_pred, average=average, zero_division=0)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "\n",
    "    print(\"Precision: \", precision)\n",
    "    print(\"Recall: \", recall)    \n",
    "    print(\"F1: \", f1)\n",
    "    print(\"Accuracy: \", accuracy)\n",
    "    print(\"Distribution\", np.unique(y_pred, return_counts=True))\n",
    "\n",
    "    return {\n",
    "        \"precision\": precision,\n",
    "        \"recall\": recall,\n",
    "        \"f1\": f1,\n",
    "        \"accuracy\": accuracy\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#NEURAL NET\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mlp\n",
    "import tqdm\n",
    "\n",
    "class MLP(torch.nn.Module):\n",
    "    def __init__(self, hidden=16):\n",
    "        super(MLP, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(44, hidden)\n",
    "        self.relu = torch.nn.ReLU()\n",
    "        self.fc2 = torch.nn.Linear(hidden, 1)\n",
    "        self.sigmoid = torch.nn.Sigmoid()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.fc2(x)\n",
    "        x = self.sigmoid(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "grid_search_results = {}\n",
    "\n",
    "for hidden in [16, 32, 64]:\n",
    "    for lr in [0.01, 0.001]:\n",
    "        for batch_size in [32, 64]:\n",
    "\n",
    "            k_folds = 5\n",
    "            kfold = KFold(n_splits=k_folds, shuffle=True, random_state=10)\n",
    "\n",
    "            fold_results = []\n",
    "\n",
    "            xdfold = np.concatenate((xd_train, xd_test), axis = 0)\n",
    "            ydfold = np.concatenate((yd_train, yd_test), axis = 0)\n",
    "            for fold, (train_idx, test_idx) in enumerate(kfold.split(xdfold)):\n",
    "                x_t, y_t = xdfold[train_idx], ydfold[train_idx]\n",
    "                x_v, y_v = xdfold[test_idx], ydfold[test_idx]\n",
    "\n",
    "                train_dataset = torch.utils.data.TensorDataset(\n",
    "                    torch.tensor(x_t, dtype=torch.float32),\n",
    "                    torch.tensor(y_t, dtype=torch.float32).reshape(-1, 1)\n",
    "                )\n",
    "                train_dl = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "\n",
    "                model = MLP()\n",
    "                criterion = torch.nn.BCELoss()\n",
    "                optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "                for epoch in range(epochs):\n",
    "                    for x_batch, y_batch in train_dl:\n",
    "                        optimizer.zero_grad()\n",
    "                        output = model(x_batch)\n",
    "                        loss = criterion(output, y_batch)\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                \n",
    "                val_outputs = model(torch.tensor(x_v, dtype=torch.float32))\n",
    "                predictions = [1 if x>0.5 else 0 for x in val_outputs]\n",
    "\n",
    "                results = explain(predictions, y_v, average='binary')\n",
    "                fold_results.append(results)\n",
    "\n",
    "            #avg explain results across all folds\n",
    "\n",
    "            print(\"AVERAGE RESULTS\")\n",
    "            print(\"\\n\\n\")\n",
    "\n",
    "            #make a new dict with the avgs\n",
    "            countdict = defaultdict(lambda: {'sum': 0, 'count': 0})\n",
    "            for d in fold_results:\n",
    "                for key, value in d.items():\n",
    "                    countdict[key]['sum'] += value\n",
    "                    countdict[key]['count'] += 1\n",
    "            averaged_dict_mlp = {key: acc['sum'] / acc['count'] for key, acc in countdict.items()}\n",
    "\n",
    "\n",
    "\n",
    "            print(averaged_dict_mlp)\n",
    "\n",
    "            grid_search_results[(hidden, lr, batch_size)] = averaged_dict_mlp\n",
    "\n",
    "pd.DataFrame(grid_search_results).T.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "qrs = pd.DataFrame(grid_search_results).T\n",
    "\n",
    "# remove precision and recall columns\n",
    "qrs = qrs.drop(columns=['precision', 'recall'])\n",
    "\n",
    "qrs.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DOWNSAMPLED "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# K FOLD EVALUATION ON DOWNSAMPLED\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_folds = 5\n",
    "kfold = KFold(n_splits=k_folds, shuffle=True, random_state=10)\n",
    "\n",
    "fold_results = []\n",
    "\n",
    "xdfold = np.concatenate((xd_train, xd_test), axis = 0)\n",
    "ydfold = np.concatenate((yd_train, yd_test), axis = 0)\n",
    "for fold, (train_idx, test_idx) in enumerate(kfold.split(xdfold)):\n",
    "    x_t, y_t = xdfold[train_idx], ydfold[train_idx]\n",
    "    x_v, y_v = xdfold[test_idx], ydfold[test_idx]\n",
    "\n",
    "    train_dataset = torch.utils.data.TensorDataset(\n",
    "        torch.tensor(x_t, dtype=torch.float32),\n",
    "        torch.tensor(y_t, dtype=torch.float32).reshape(-1, 1)\n",
    "    )\n",
    "    train_dl = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "\n",
    "    model = MLP()\n",
    "    criterion = torch.nn.BCELoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        for x_batch, y_batch in train_dl:\n",
    "            optimizer.zero_grad()\n",
    "            output = model(x_batch)\n",
    "            loss = criterion(output, y_batch)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "    \n",
    "    val_outputs = model(torch.tensor(x_v, dtype=torch.float32))\n",
    "    predictions = [1 if x>0.5 else 0 for x in val_outputs]\n",
    "\n",
    "    results = explain(predictions, y_v, average='binary')\n",
    "    fold_results.append(results)\n",
    "\n",
    "#avg explain results across all folds\n",
    "\n",
    "print(\"AVERAGE RESULTS\")\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "#make a new dict with the avgs\n",
    "countdict = defaultdict(lambda: {'sum': 0, 'count': 0})\n",
    "for d in fold_results:\n",
    "    for key, value in d.items():\n",
    "        countdict[key]['sum'] += value\n",
    "        countdict[key]['count'] += 1\n",
    "averaged_dict_mlp = {key: acc['sum'] / acc['count'] for key, acc in countdict.items()}\n",
    "\n",
    "\n",
    "\n",
    "print(averaged_dict_mlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TOTAL DOWNSAMPLED RESULTS\n",
    "down_models = {\n",
    "    \"Logistic Regression\": LogisticRegression(random_state=seed, max_iter=250),\n",
    "    \"Decision Tree\": DecisionTreeClassifier(random_state=seed),\n",
    "    \"Random Forest\": RandomForestClassifier(random_state=seed),\n",
    "    \"Naive Bayes\": GaussianNB(),\n",
    "    \"SVM\": SVC(random_state=seed)\n",
    "}\n",
    "\n",
    "results = {}\n",
    "for name, model in down_models.items():\n",
    "    print(name)\n",
    "\n",
    "    model.fit(xd_train, yd_train)\n",
    "    yd_pred = model.predict(xd_test)\n",
    "    \n",
    "    results[name] = explain(yd_test, yd_pred, average='binary')\n",
    "    print(\"\\n\\n\")\n",
    "# add the MLP\n",
    "results[\"2 layer MLP\"] = averaged_dict_mlp\n",
    "# plot the results as a hist\n",
    "results_df = pd.DataFrame(results)\n",
    "results_df.plot(kind='bar', figsize=(10, 6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#K FOLD CROSS WITHOUT DOWNSAMPLING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "k_folds = 5\n",
    "kfold = KFold(n_splits=k_folds, shuffle=True, random_state=20)\n",
    "\n",
    "fold_results = []\n",
    "\n",
    "for fold, (train_idx, test_idx) in enumerate(kfold.split(x_train)):\n",
    "    x_t, y_t = x_train[train_idx], y_train[train_idx]\n",
    "    x_v, y_v = x_train[test_idx], y_train[test_idx]\n",
    "\n",
    "    train_dataset = torch.utils.data.TensorDataset(\n",
    "        torch.tensor(x_t, dtype=torch.float32),\n",
    "        torch.tensor(y_t, dtype=torch.float32).reshape(-1, 1)\n",
    "    )\n",
    "    train_dl = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "\n",
    "    model = MLP()\n",
    "    criterion = torch.nn.BCELoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        for x_batch, y_batch in train_dl:\n",
    "            optimizer.zero_grad()\n",
    "            output = model(x_batch)\n",
    "            loss = criterion(output, y_batch)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "    \n",
    "    val_outputs = model(torch.tensor(x_v, dtype=torch.float32))\n",
    "    predictions = [1 if x>0.5 else 0 for x in val_outputs]\n",
    "\n",
    "    results = explain(predictions, y_v, average='macro')\n",
    "    fold_results.append(results)\n",
    "\n",
    "#avg explain results across all folds\n",
    "\n",
    "print(\"AVERAGE RESULTS\")\n",
    "print(\"\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#make a new dict with the avgs\n",
    "countdict = defaultdict(lambda: {'sum': 0, 'count': 0})\n",
    "for d in fold_results:\n",
    "    for key, value in d.items():\n",
    "        countdict[key]['sum'] += value\n",
    "        countdict[key]['count'] += 1\n",
    "averaged_dict_mlp_nd = {key: acc['sum'] / acc['count'] for key, acc in countdict.items()}\n",
    "\n",
    "print(averaged_dict_mlp_nd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# TOTAL NON-DOWNSAMPLED RESULTS\n",
    "models = {\n",
    "    \"Logistic Regression\": LogisticRegression(random_state=seed, max_iter=250),\n",
    "    \"Decision Tree\": DecisionTreeClassifier(random_state=seed),\n",
    "    \"Random Forest\": RandomForestClassifier(random_state=seed),\n",
    "    \"Naive Bayes\": GaussianNB(),\n",
    "    \"SVM\": SVC(random_state=seed)\n",
    "}\n",
    "\n",
    "results = {}\n",
    "for name, model in models.items():\n",
    "    print(name)\n",
    "\n",
    "    model.fit(x_train, y_train)\n",
    "    yd_pred = model.predict(x_test)\n",
    "    \n",
    "    results[name] = explain(y_test, yd_pred, average='macro')\n",
    "    print(\"\\n\\n\")\n",
    "# add the MLP\n",
    "results[\"2 layer MLP\"] = averaged_dict_mlp_nd\n",
    "# plot the results as a hist\n",
    "results_df = pd.DataFrame(results)\n",
    "results_df.plot(kind='bar', figsize=(10, 6))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "csci360_lab1",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}