baselineModels_word2Vec_CV_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Daniel\\Anaconda3\\envs\\py27\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import *\n",
    "from sklearn.cross_validation import KFold, train_test_split\n",
    "\n",
    "from sklearn.datasets import load_svmlight_file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Baseline models based on Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "without_time = True\n",
    "\n",
    "if without_time:\n",
    "    svmlight_file = \"./cleaned_data/features_svmlight_w2v.train\"\n",
    "else:\n",
    "    svmlight_file = \"./cleaned_data/features_svmlight_w2v_wt.train\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_data_from_svmlight(svmlight_file):\n",
    "    data_train = load_svmlight_file(svmlight_file,n_features=4908)\n",
    "    X_train = data_train[0]\n",
    "    Y_train = data_train[1]\n",
    "    return X_train, Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "RANDOM_STATE = 6250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, Y = get_data_from_svmlight(svmlight_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Models\n",
    "\n",
    "Logistic, SVM, Decision Tree, Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#input: X_train, Y_train\n",
    "#output: Y_pred\n",
    "def logistic_regression_pred(X_train, Y_train, X_test):\n",
    "    #train a logistic regression classifier using X_train and Y_train. Use this to predict labels of X_train\n",
    "    #use default params for the classifier\n",
    "    log_reg = LogisticRegression(random_state=RANDOM_STATE)\n",
    "    log_reg.fit(X_train, Y_train)\n",
    "    Y_pred = log_reg.predict(X_test)\n",
    "\n",
    "    return Y_pred\n",
    "\n",
    "\n",
    "#input: X_train, Y_train\n",
    "#output: Y_pred\n",
    "def svm_pred(X_train, Y_train, X_test):\n",
    "    #train a SVM classifier using X_train and Y_train. Use this to predict labels of X_train\n",
    "    #use default params for the classifier\n",
    "    lin_svc = LinearSVC(random_state=RANDOM_STATE)\n",
    "    lin_svc.fit(X_train, Y_train)\n",
    "    Y_pred = lin_svc.predict(X_test)\n",
    "\n",
    "    return Y_pred\n",
    "\n",
    "\n",
    "#input: X_train, Y_train\n",
    "#output: Y_pred\n",
    "def decisionTree_pred(X_train, Y_train, X_test):\n",
    "    #train a logistic regression classifier using X_train and Y_train. Use this to predict labels of X_train\n",
    "    #use max_depth as 5\n",
    "    dec_tree = DecisionTreeClassifier(max_depth=5, random_state=RANDOM_STATE)\n",
    "    dec_tree.fit(X_train, Y_train)\n",
    "    Y_pred = dec_tree.predict(X_test)\n",
    "\n",
    "    return Y_pred\n",
    "\n",
    "\n",
    "#input: X_train, Y_train\n",
    "#output: Y_pred\n",
    "def randomForest_pred(X_train, Y_train, X_test):\n",
    "    #train a logistic regression classifier using X_train and Y_train. Use this to predict labels of X_train\n",
    "    #use max_depth as 5\n",
    "    rand_forest = RandomForestClassifier(n_estimators=100, random_state=RANDOM_STATE)\n",
    "    rand_forest.fit(X_train, Y_train)\n",
    "    Y_pred = rand_forest.predict(X_test)\n",
    "\n",
    "    return Y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def classification_metrics(Y_pred, Y_true):\n",
    "    #NOTE: It is important to provide the output in the same order\n",
    "    accuracy = accuracy_score(Y_true, Y_pred)\n",
    "    auc = roc_auc_score(Y_true, Y_pred)\n",
    "    precision = precision_score(Y_true, Y_pred)\n",
    "    recall = recall_score(Y_true, Y_pred)\n",
    "    fscore = f1_score(Y_true, Y_pred)\n",
    "\n",
    "    return accuracy, auc, precision, recall, fscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = KFold(np.shape(X_train)[0], 10, random_state=RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cv_logistic(X,Y,kfold):\n",
    "    \n",
    "    acc_list, auc_list = [], []\n",
    "    \n",
    "    for train, test in kfold:\n",
    "        X_train, Y_train, X_fold, Y_true = X[train], Y[train], X[test], Y[test]\n",
    "\n",
    "        Y_pred = logistic_regression_pred(X_train, Y_train, X_fold)\n",
    "\n",
    "        metrics = classification_metrics(Y_pred, Y_true)\n",
    "        acc_list.append(metrics[0])\n",
    "        auc_list.append(metrics[1])\n",
    "\n",
    "    acc, auc = np.mean(acc_list), np.mean(auc_list)\n",
    "\n",
    "    return acc, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Daniel\\Anaconda3\\envs\\py27\\lib\\site-packages\\sklearn\\metrics\\classification.py:1113: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\Daniel\\Anaconda3\\envs\\py27\\lib\\site-packages\\sklearn\\metrics\\classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.66356984478935699, 0.5)\n"
     ]
    }
   ],
   "source": [
    "print cv_logistic(X_train,Y_train,kfold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cv_svm(X,Y,kfold):\n",
    "\n",
    "    acc_list, auc_list = [], []\n",
    "    \n",
    "    for train, test in kfold:\n",
    "        X_train, Y_train, X_fold, Y_true = X[train], Y[train], X[test], Y[test]\n",
    "\n",
    "        Y_pred = svm_pred(X_train, Y_train, X_fold)\n",
    "\n",
    "        metrics = classification_metrics(Y_pred, Y_true)\n",
    "        acc_list.append(metrics[0])\n",
    "        auc_list.append(metrics[1])\n",
    "\n",
    "    acc, auc = np.mean(acc_list), np.mean(auc_list)\n",
    "\n",
    "    return acc, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.73597560975609766, 0.63102986220404123)\n"
     ]
    }
   ],
   "source": [
    "print cv_svm(X_train,Y_train,kfold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cv_decision_tree(X,Y,kfold):\n",
    "    \n",
    "    acc_list, auc_list = [], []\n",
    "    \n",
    "    for train, test in kfold:\n",
    "        X_train, Y_train, X_fold, Y_true = X[train], Y[train], X[test], Y[test]\n",
    "\n",
    "        Y_pred = decisionTree_pred(X_train, Y_train, X_fold)\n",
    "\n",
    "        metrics = classification_metrics(Y_pred, Y_true)\n",
    "        acc_list.append(metrics[0])\n",
    "        auc_list.append(metrics[1])\n",
    "\n",
    "    acc, auc = np.mean(acc_list), np.mean(auc_list)\n",
    "\n",
    "    return acc, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.72990022172948998, 0.64917739872839508)\n"
     ]
    }
   ],
   "source": [
    "print cv_decision_tree(X_train,Y_train,kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cv_random_forest(X,Y,kfold):\n",
    "    \n",
    "    acc_list, auc_list = [], []\n",
    "    \n",
    "    for train, test in kfold:\n",
    "        X_train, Y_train, X_fold, Y_true = X[train], Y[train], X[test], Y[test]\n",
    "\n",
    "        Y_pred = randomForest_pred(X_train, Y_train, X_fold)\n",
    "\n",
    "        metrics = classification_metrics(Y_pred, Y_true)\n",
    "        acc_list.append(metrics[0])\n",
    "        auc_list.append(metrics[1])\n",
    "\n",
    "    acc, auc = np.mean(acc_list), np.mean(auc_list)\n",
    "\n",
    "    return acc, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.76759793052475978, 0.67500876683684763)\n"
     ]
    }
   ],
   "source": [
    "print cv_random_forest(X_train,Y_train,kfold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the test data on the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#input: Name of classifier, predicted labels, actual labels\n",
    "def display_metrics(classifierName,Y_pred,Y_true):\n",
    "    print \"______________________________________________\"\n",
    "    print \"Classifier: \"+classifierName\n",
    "    acc, auc_, precision, recall, f1score = classification_metrics(Y_pred,Y_true)\n",
    "    print \"Accuracy: \"+str(acc)\n",
    "    print \"AUC: \"+str(auc_)\n",
    "    print \"Precision: \"+str(precision)\n",
    "    print \"Recall: \"+str(recall)\n",
    "    print \"F1-score: \"+str(f1score)\n",
    "    print \"______________________________________________\"\n",
    "    print \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "______________________________________________\n",
      "Classifier: Logistic Regression\n",
      "Accuracy: 0.678832116788\n",
      "AUC: 0.5\n",
      "Precision: 0.0\n",
      "Recall: 0.0\n",
      "F1-score: 0.0\n",
      "______________________________________________\n",
      "\n",
      "______________________________________________\n",
      "Classifier: SVM\n",
      "Accuracy: 0.725060827251\n",
      "AUC: 0.605897686543\n",
      "Precision: 0.679245283019\n",
      "Recall: 0.272727272727\n",
      "F1-score: 0.389189189189\n",
      "______________________________________________\n",
      "\n",
      "______________________________________________\n",
      "Classifier: Decision Tree\n",
      "Accuracy: 0.720194647202\n",
      "AUC: 0.596326164875\n",
      "Precision: 0.673469387755\n",
      "Recall: 0.25\n",
      "F1-score: 0.364640883978\n",
      "______________________________________________\n",
      "\n",
      "______________________________________________\n",
      "Classifier: Random Forest\n",
      "Accuracy: 0.749391727494\n",
      "AUC: 0.639784946237\n",
      "Precision: 0.745762711864\n",
      "Recall: 0.333333333333\n",
      "F1-score: 0.460732984293\n",
      "______________________________________________\n",
      "\n"
     ]
    }
   ],
   "source": [
    "display_metrics(\"Logistic Regression\",logistic_regression_pred(X_train,Y_train,X_test),Y_test)\n",
    "\n",
    "display_metrics(\"SVM\",svm_pred(X_train,Y_train,X_test),Y_test)\n",
    "\n",
    "display_metrics(\"Decision Tree\",decisionTree_pred(X_train,Y_train,X_test),Y_test)\n",
    "\n",
    "display_metrics(\"Random Forest\",randomForest_pred(X_train,Y_train,X_test),Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py27]",
   "language": "python",
   "name": "conda-env-py27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}