features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['figure.figsize'] = 16, 12\n",
    "import pandas as pd\n",
    "import gensim as gs\n",
    "import nltk.data\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import *\n",
    "import string\n",
    "import re\n",
    "from collections import Counter\n",
    "import scipy as scp\n",
    "import matplotlib.cm as cm\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.decomposition import PCA, IncrementalPCA, KernelPCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor, RandomForestRegressor, AdaBoostRegressor, RandomForestClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import SVC, SVR\n",
    "from sklearn import decomposition, pipeline, metrics, grid_search\n",
    "from sklearn import cross_validation\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "import itertools\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.linear_model import Ridge, ElasticNet \n",
    "import sys\n",
    "import os\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics.pairwise import pairwise_distances\n",
    "from HTMLParser import HTMLParser\n",
    "import pickle\n",
    "from collections import Counter\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class RegressionClassifier:\n",
    "\n",
    "    regression_model = None\n",
    "    n = None\n",
    "    values = None\n",
    "    quantiles = None\n",
    "\n",
    "    def __init__(self, regression_model):\n",
    "        self.regression_model = regression_model\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.values = sorted(set(y.tolist()))\n",
    "        self.n = len(set(y.tolist()))\n",
    "        self.quantiles = (np.bincount(y)/float(y.shape[0]))[self.values]\n",
    "\n",
    "    def gaussian(self, x, mu, sig):\n",
    "        return np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))\n",
    "\n",
    "    def predict(self, X, mean_pow=2, gaus_pow=1):\n",
    "        y_pred = self.regression_model.predict(X)\n",
    "        t = np.percentile(y_pred, np.cumsum(self.quantiles)*100)\n",
    "\n",
    "        def q_order(x):\n",
    "            for i in range(t.shape[0]):\n",
    "                if x <= t[i]:\n",
    "                    return i\n",
    "\n",
    "        return np.array(map(q_order, y_pred.tolist()), dtype=np.int) + 1\n",
    "\n",
    "\n",
    "def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):\n",
    "    \"\"\"\n",
    "    Returns the confusion matrix between rater's ratings\n",
    "    \"\"\"\n",
    "    assert(len(rater_a) == len(rater_b))\n",
    "    if min_rating is None:\n",
    "        min_rating = min(rater_a + rater_b)\n",
    "    if max_rating is None:\n",
    "        max_rating = max(rater_a + rater_b)\n",
    "    num_ratings = int(max_rating - min_rating + 1)\n",
    "    conf_mat = [[0 for i in range(num_ratings)]\n",
    "                for j in range(num_ratings)]\n",
    "    for a, b in zip(rater_a, rater_b):\n",
    "        conf_mat[a - min_rating][b - min_rating] += 1\n",
    "    return conf_mat\n",
    "\n",
    "\n",
    "def histogram(ratings, min_rating=None, max_rating=None):\n",
    "    \"\"\"\n",
    "    Returns the counts of each type of rating that a rater made\n",
    "    \"\"\"\n",
    "    if min_rating is None:\n",
    "        min_rating = min(ratings)\n",
    "    if max_rating is None:\n",
    "        max_rating = max(ratings)\n",
    "    num_ratings = int(max_rating - min_rating + 1)\n",
    "    hist_ratings = [0 for x in range(num_ratings)]\n",
    "    for r in ratings:\n",
    "        hist_ratings[r - min_rating] += 1\n",
    "    return hist_ratings\n",
    "\n",
    "\n",
    "def quadratic_weighted_kappa(y, y_pred):\n",
    "    \"\"\"\n",
    "    Calculates the quadratic weighted kappa\n",
    "    axquadratic_weighted_kappa calculates the quadratic weighted kappa\n",
    "    value, which is a measure of inter-rater agreement between two raters\n",
    "    that provide discrete numeric ratings.  Potential values range from -1\n",
    "    (representing complete disagreement) to 1 (representing complete\n",
    "    agreement).  A kappa value of 0 is expected if all agreement is due to\n",
    "    chance.\n",
    "    quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b\n",
    "    each correspond to a list of integer ratings.  These lists must have the\n",
    "    same length.\n",
    "    The ratings should be integers, and it is assumed that they contain\n",
    "    the complete range of possible ratings.\n",
    "    quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating\n",
    "    is the minimum possible rating, and max_rating is the maximum possible\n",
    "    rating\n",
    "    \"\"\"\n",
    "    rater_a = y\n",
    "    rater_b = y_pred\n",
    "    min_rating=None\n",
    "    max_rating=None\n",
    "    rater_a = np.array(rater_a, dtype=int)\n",
    "    rater_b = np.array(rater_b, dtype=int)\n",
    "    assert(len(rater_a) == len(rater_b))\n",
    "    if min_rating is None:\n",
    "        min_rating = min(min(rater_a), min(rater_b))\n",
    "    if max_rating is None:\n",
    "        max_rating = max(max(rater_a), max(rater_b))\n",
    "    conf_mat = confusion_matrix(rater_a, rater_b,\n",
    "                                min_rating, max_rating)\n",
    "    num_ratings = len(conf_mat)\n",
    "    num_scored_items = float(len(rater_a))\n",
    "\n",
    "    hist_rater_a = histogram(rater_a, min_rating, max_rating)\n",
    "    hist_rater_b = histogram(rater_b, min_rating, max_rating)\n",
    "\n",
    "    numerator = 0.0\n",
    "    denominator = 0.0\n",
    "\n",
    "    for i in range(num_ratings):\n",
    "        for j in range(num_ratings):\n",
    "            expected_count = (hist_rater_a[i] * hist_rater_b[j]\n",
    "                              / num_scored_items)\n",
    "            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)\n",
    "            numerator += d * conf_mat[i][j] / num_scored_items\n",
    "            denominator += d * expected_count / num_scored_items\n",
    "\n",
    "    return (1.0 - numerator / denominator)\n",
    "\n",
    "\n",
    "df = pd.read_pickle('./../data/df_basfe.pkl')\n",
    "\n",
    "# kappa_scorer = metrics.make_scorer(quadratic_weighted_kappa, greater_is_better = True)\n",
    "\n",
    "# idx_row_train = np.where(np.array((~df.median_relevance.isnull()).tolist()))[0]\n",
    "# idx_row_test = np.where(np.array((df.median_relevance.isnull()).tolist()))[0]\n",
    "\n",
    "# y_train_all = df['median_relevance'].values[idx_row_train].astype(int)\n",
    "\n",
    "# rel_var = df['relevance_variance'].values[idx_row_train]\n",
    "\n",
    "# data = np.load('./../data/tfidf_svd.npz')\n",
    "# data = dict(map(lambda k: ((int(k.split(' ')[0]), int(k.split(' ')[1])), data[k]), data.keys()))\n",
    "\n",
    "# path = './../data/w2v_models/'\n",
    "# w2v_models = {}\n",
    "# for name in os.listdir(path):\n",
    "#     w2v_models[name] = Word2Vec.load(os.path.join(path, name))\n",
    "    \n",
    "\n",
    "# # number of Q in T\n",
    "# df['nqt'] = map(lambda t: len(t[0].intersection(t[1])), \n",
    "#                 zip(map(lambda s: set(s.split()), df['qcnss'].values), \n",
    "#                     map(lambda s: set(s.split()), df['tcnss'].values)))\n",
    "\n",
    "# predictors = df.columns[list(set(range(5, 10)).union([17, 18]).union(range(27, len(df.columns))))]\n",
    "# X_base = df[predictors].as_matrix()\n",
    "\n",
    "idx_row_train = np.where(np.array((~df.median_relevance.isnull()).tolist()))[0]\n",
    "idx_row_test = np.where(np.array((df.median_relevance.isnull()).tolist()))[0]\n",
    "\n",
    "y_train_all = df['median_relevance'].values[idx_row_train].astype(int)\n",
    "\n",
    "\n",
    "# def qdist(q):\n",
    "#     v = df.loc[(df['qcnss'] == q) & (~df['median_relevance'].isnull()), \n",
    "#                'median_relevance'].values.astype(int).tolist()\n",
    "#     v.extend([1, 2, 3, 4])\n",
    "#     v = np.bincount(v)[1:] - 1\n",
    "#     return v / float(v.sum())\n",
    "\n",
    "# qdist_dict = dict(map(lambda q: (q, qdist(q)), set(df['qcnss'].values)))\n",
    "# q_rel_prior = np.array(map(lambda q: qdist_dict[q], df['qcnss'].values.tolist()))\n",
    "# X_base = np.hstack((X_base, q_rel_prior))\n",
    "\n",
    "\n",
    "# X_emp = X_base\n",
    "# np.savez('./../data/features/X_emp.npz', data=X_emp)\n",
    "\n",
    "# X_emp_pca = PCA(n_components=100).fit_transform(X_base)\n",
    "# np.savez('./../data/features/X_emp_pca.npz', data=X_emp_pca)\n",
    "\n",
    "# data = np.load('./../data/tfidf_svd.npz')\n",
    "# data = dict(map(lambda k: ((int(k.split(' ')[0]), int(k.split(' ')[1])), data[k]), data.keys()))\n",
    "\n",
    "# X_lsa_word = data[(3, 300)]\n",
    "\n",
    "# np.savez('./../data/features/X_lsa_word.npz', data=X_lsa_word)\n",
    "\n",
    "# w2v = w2v_models['w2v_size100_mincount5_window7.bin']\n",
    "\n",
    "# Q = np.array(map(lambda s: \n",
    "#              reduce(lambda x, y: x + y, \n",
    "#                     map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split()), \n",
    "#                     np.zeros(w2v['foot'].shape))/len(s.split()), \n",
    "#              df['qcnss'].values))   \n",
    "# T = np.array(map(lambda s: \n",
    "#              reduce(lambda x, y: x + y, \n",
    "#                     map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split()), \n",
    "#                     np.zeros(w2v['foot'].shape))/len(s.split()), \n",
    "#              df['tcnss'].values))\n",
    "# X_w2v_local = Q - T\n",
    "\n",
    "# np.savez('./../data/features/X_w2v_local.npz', data=X_w2v_local)\n",
    "\n",
    "\n",
    "# txt = map(lambda p: '. '.join(p), zip(df['qcnss'].values, df['tcnss'].values))\n",
    "# tfv = TfidfVectorizer(analyzer='char', ngram_range=(1, 5))\n",
    "# X_tfv = tfv.fit_transform(txt)\n",
    "# svd = TruncatedSVD(n_components=300)\n",
    "# X_lsa_char = svd.fit_transform(X_tfv)  \n",
    "# np.savez('./../data/features/X_lsa_char.npz', data=X_lsa_char)\n",
    "\n",
    "X_emp = np.load('./../data/features/X_emp.npz')['data']\n",
    "X_emp_pca = np.load('./../data/features/X_emp_pca.npz')['data']\n",
    "X_lsa_char = np.load('./../data/features/X_lsa_char.npz')['data']\n",
    "X_lsa_word = np.load('./../data/features/X_lsa_word.npz')['data']\n",
    "X_w2v_local = np.load('./../data/features/X_w2v_local.npz')['data']\n",
    "\n",
    "good_predictors = ['p1', 'p2', 'p3', 'p4', \n",
    "                   'cos_qt_cnss_tfidf2_svd200',\n",
    "                   't_len',\n",
    "                   'q_len',\n",
    "                   'obm25_3_qt_cnss',\n",
    "                   'cos_qt_cnss_w2v_size52_mincount1_window3',\n",
    "                   'cos_min_qd_cnss_tfidf1_svd300',\n",
    "                   'min_mean_cos_qt_cnss_w2v_size52_mincount1_window3',\n",
    "                   'd_loglen',\n",
    "                   'cos_max_qd_cnss_tfidf2_svd200',\n",
    "                   'cos_mean_qd_cnss_tfidf2_svd100',\n",
    "                   'cos_qd_cnss_tfidf1_svd100',\n",
    "                   'cos_max_qd_cnss_w2v_size100_mincount3_window3',\n",
    "                   'sop_qt_cnss',\n",
    "                   'jsc_qd_cnss']\n",
    "good_predictors_idx  = [386, 387, 388, 389, 55, 1, 0, 91, 102, 33, 279, 3, 58, 53, 20, 189, 7, 14]\n",
    "\n",
    "X_emp_good = X_emp[:, good_predictors_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp RegressionClassifier GradientBoostingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.837473277332 0.665697511434\n",
      "0.836344402653 0.672226702709\n",
      "0.836799475568 0.678343778241\n",
      "0.832590051097 0.691597441896\n",
      "0.83509058069 0.683261783554\n",
      "0.833725813082 0.667910981464\n",
      "0.836983927716 0.673236880933\n",
      "0.838709812198 0.645988716893\n",
      "0.835184964949 0.674678488027\n",
      "0.830864184451 0.700294355112\n",
      "---------------\n",
      "[0.835376648974, 0.675323664026, 0.00228082125494, 0.0140819671066]\n"
     ]
    }
   ],
   "source": [
    "X = X_emp\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = GradientBoostingRegressor(loss='ls',\n",
    "                                      learning_rate=0.1,\n",
    "                                      n_estimators=500,\n",
    "                                      max_depth=3,\n",
    "                                      min_samples_split=2,\n",
    "                                      min_samples_leaf=1,\n",
    "                                      subsample=0.9)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = GradientBoostingRegressor(loss='ls',\n",
    "                                  learning_rate=0.1,\n",
    "                                  n_estimators=500,\n",
    "                                  max_depth=3,\n",
    "                                  min_samples_split=2,\n",
    "                                  min_samples_leaf=1,\n",
    "                                  subsample=0.9)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/_emp_rc_gbr.npz', \n",
    "         score_lb=0.63868191,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp RegressionClassifier RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = X_emp\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = RandomForestRegressor(n_estimators=1000,\n",
    "                                  max_depth=45,\n",
    "                                  max_features='auto',\n",
    "                                  n_jobs=-1) \n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = RandomForestRegressor(n_estimators=1000,\n",
    "                              max_depth=45,\n",
    "                              max_features='auto',\n",
    "                              n_jobs=-1) \n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/_emp_rc_rfr.npz', \n",
    "         score_lb=0.66350144,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_lsa_word SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.871723078232 0.616944799676\n",
      "0.873743805447 0.623747926625\n",
      "0.875316944802 0.57777326624\n",
      "0.877744742199 0.647443292631\n",
      "0.874125130925 0.585120635672\n",
      "0.873536391418 0.584860737542\n",
      "0.875451830652 0.578920304679\n",
      "0.872174415725 0.635270903103\n",
      "0.873628295375 0.604883962332\n",
      "0.87180896328 0.565992560791\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(X_lsa_word)\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/lsaword_svc.npz', \n",
    "         score_lb=0.58504,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_lsa_char SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.955189024571 0.586374989803\n",
      "0.953555276725 0.619300027646\n",
      "0.952973546919 0.542499158567\n",
      "0.95181587709 0.569604116169\n",
      "0.951827503002 0.613250464269\n",
      "0.954589565137 0.659811335206\n",
      "0.952579128057 0.617933990488\n",
      "0.952056684826 0.622869682577\n",
      "0.95316306651 0.644803931736\n",
      "0.950640112684 0.660023888854\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(X_lsa_char)\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/lsachar_svc.npz', \n",
    "         score_lb=0.58178,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_w2v_local SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(X_w2v_local)\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/w2vlocal_svc.npz', \n",
    "         score_lb=0.62522,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_w2v_local RC on SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(X_w2v_local)\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/w2vlocal_rc_svr.npz', \n",
    "         score_lb=0.63773,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_w2v_local RC on GBR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(X_w2v_local)\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = GradientBoostingRegressor(loss='ls',\n",
    "                                      learning_rate=0.1,\n",
    "                                      n_estimators=500,\n",
    "                                      max_depth=3,\n",
    "                                      min_samples_split=2,\n",
    "                                      min_samples_leaf=1,\n",
    "                                      subsample=0.9)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = GradientBoostingRegressor(loss='ls',\n",
    "                                  learning_rate=0.1,\n",
    "                                  n_estimators=500,\n",
    "                                  max_depth=3,\n",
    "                                  min_samples_split=2,\n",
    "                                  min_samples_leaf=1,\n",
    "                                  subsample=0.9)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/w2vlocal_rc_gbr.npz', \n",
    "         score_lb=0.58084,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_w2v_local RC on KNR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.742593807511 0.592799216922\n",
      "0.733967275356 0.627791879049\n",
      "0.739380637897 0.612196809104\n",
      "0.738824334249 0.634972199232\n",
      "0.73803660758 0.638545516333\n",
      "0.738832524535 0.629495268768\n",
      "0.737556833168 0.618567089965\n",
      "0.736522948781 0.639245022396\n",
      "0.738189549509 0.612738046256\n",
      "0.734050523711 0.650682645483\n",
      "---------------\n",
      "[ 0.7377955   0.62570337]\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(X_w2v_local)\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = KNeighborsRegressor(n_neighbors=7)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = KNeighborsRegressor(n_neighbors=7)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/w2vlocal_rc_knr.npz', \n",
    "         score_lb=0.58902,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_pca + X_lsa_word SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.964067891713 0.619497114331\n",
      "0.962306907762 0.626698723703\n",
      "0.963252651189 0.60236654134\n",
      "0.962028937444 0.678650423059\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-487420804acb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi_cv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mStratifiedKFold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train_all\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSVC\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rbf'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdegree\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4196\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_all\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0my_train_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mscore_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquadratic_weighted_kappa\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train_all\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m         \u001b[0mseed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrnd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miinfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'i'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msolver_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_seed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m         \u001b[0;31m# see comment on the other call to np.iinfo in this file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36m_dense_fit\u001b[0;34m(self, X, y, sample_weight, solver_type, kernel, random_seed)\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mcache_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoef0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoef0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m                 \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gamma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 236\u001b[0;31m                 max_iter=self.max_iter, random_seed=random_seed)\n\u001b[0m\u001b[1;32m    237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_warn_from_fit_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_pca, X_lsa_word)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-7d2eb445502c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSVC\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rbf'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdegree\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4196\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx_row_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_all\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m         \u001b[0mseed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrnd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miinfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'i'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msolver_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_seed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m         \u001b[0;31m# see comment on the other call to np.iinfo in this file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36m_dense_fit\u001b[0;34m(self, X, y, sample_weight, solver_type, kernel, random_seed)\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mcache_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoef0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoef0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m                 \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gamma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 236\u001b[0;31m                 max_iter=self.max_iter, random_seed=random_seed)\n\u001b[0m\u001b[1;32m    237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_warn_from_fit_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/emppca_lsaword_svc.npz', \n",
    "         score_lb=0.62990,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_pca + X_lsa_char SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.981074213634 0.624642084559\n",
      "0.981652012889 0.668066599169\n",
      "0.979891710834 0.637541871735\n",
      "0.981768261712 0.615696410248\n",
      "0.980008877268 0.65216056139\n",
      "0.982081323697 0.629610380685\n",
      "0.981330340547 0.669160725852\n",
      "0.981885216747 0.667098355447\n",
      "0.9807528129 0.680382587349\n",
      "0.981541924328 0.62895560082\n",
      "---------------\n",
      "[0.981198669456, 0.647331517725, 0.000724814879247, 0.0216241052085]\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp[:, 386:], X_emp_pca, X_lsa_char)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/_emppca_lsachar_svc.npz', \n",
    "         score_lb=0.62212771,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_pca + X_lsa_char RC on SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.987769252582 0.671812800859\n",
      "0.987656147159 0.684460853774\n",
      "0.98754237893 0.692616954484\n",
      "0.988793829448 0.626348636214\n",
      "0.986693515821 0.68428517036\n",
      "0.987773692341 0.692472264807\n",
      "0.988116372826 0.662993522968\n",
      "0.987549329881 0.677752392077\n",
      "0.987208215631 0.699269720428\n",
      "0.987435625131 0.66340750651\n",
      "---------------\n",
      "[0.987653835975, 0.675541982248, 0.000522778136415, 0.0201014219458]\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp[:, 386:], X_emp_pca, X_lsa_char)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/_emppca_lsachar_rc_svr.npz', \n",
    "         score_lb=0.65015604,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_pca + X_w2v_local SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp[:, 386:], X_emp_pca, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/_emppca_w2vlocal_svc.npz', \n",
    "         score_lb=0.61530543,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_pca + X_w2v_local RC on SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.978439752226 0.663659081625\n",
      "0.977644543011 0.658973039055\n",
      "0.976961933637 0.649797425756\n",
      "0.97775831124 0.677324265653\n",
      "0.977936257003 0.682238396748\n",
      "0.978277448905 0.645396471733\n",
      "0.979473734882 0.664017858765\n",
      "0.979021473634 0.679801661444\n",
      "0.976974788135 0.66955531461\n",
      "0.976747378635 0.682875565494\n",
      "---------------\n",
      "[0.977923562131, 0.667363908088, 0.000855205416522, 0.012676942266]\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp[:, 386:], X_emp_pca, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/_emppca_w2vlocal_rc_svr.npz', \n",
    "         score_lb=0.63769467,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# X_emp_pca + X_lsa_char + X_w2v_local SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.981757195937 0.646793950146\n",
      "0.981025241363 0.660580550508\n",
      "0.98044990284 0.650290883573\n",
      "0.981637281596 0.644923266841\n",
      "0.983374632313 0.634579470842\n",
      "0.981644645326 0.669248853592\n",
      "0.980397525141 0.620609570457\n",
      "0.981088017361 0.625696763108\n",
      "0.980677319839 0.652064988218\n",
      "0.980027483598 0.660918586624\n",
      "---------------\n",
      "[0.981207924531, 0.646570688391, 0.00091244020579, 0.0148886477657]\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_pca, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/emppca_lsachar_w2vlocal_svc.npz', \n",
    "         score_lb=0.62600,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# X_emp_pca + X_lsa_char + X_w2v_local RC on SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_pca, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/emppca_lsachar_w2vlocal_rc_svr.npz', \n",
    "         score_lb=0.65211,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# X_emp_pca + X_lsa_char + X_w2v_local RC on SVR - C 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.803113410167 0.635121064309\n",
      "0.79971103293 0.67936329083\n",
      "0.799824801159 0.678343778241\n",
      "0.79971103293 0.669168164943\n",
      "0.799379161612 0.683261783554\n",
      "0.80188123556 0.652560179375\n",
      "0.799229246168 0.662993522968\n",
      "0.797093873716 0.695171181695\n",
      "0.799595378215 0.685949469544\n",
      "0.799595378215 0.696195816378\n",
      "---------------\n",
      "[0.799913455067, 0.673812825184, 0.0015201209736, 0.0182550084236]\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_pca, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=0.5, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=0.5, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/emppca_lsachar_w2vlocal_rc_svr_c05.npz', \n",
    "         score_lb=0.64532,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_pca + X_lsa_char + X_lsa_char + X_w2v_local RC on SVR - C 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp[:, 386:], X_emp_pca, X_lsa_word, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=3, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=3, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/emppca_lsaword_lsachar_w2vlocal_rc_svr_c3.npz', \n",
    "         score_lb=0.67260,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_good + X_lsa_word + X_lsa_char + X_w2v_local RC on SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_good, X_lsa_word, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=3, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train] - 4.5)\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=3, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all - 4.5)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/empgood_lsaword_lsachar_w2vlocal_rc_svr_c3.npz', \n",
    "         score_lb=0.67534,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# X_emp_good poly2 + X_lsa_word + X_lsa_char + X_w2v_local RC on SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((PolynomialFeatures(degree=2).fit_transform(X_emp_good), \n",
    "                                              X_lsa_word, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=3, kernel='rbf', degree=1, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train] - 4.5)\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SVR(C=3, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "model = model.fit(X[idx_row_train, :], y_train_all - 4.5)\n",
    "rc = RegressionClassifier(model)\n",
    "rc.fit(X, y_train_all)\n",
    "y_pred = rc.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"id\": df.index[df['median_relevance'].isnull()], \n",
    "                           \"prediction\": y_pred[idx_row_test].astype(int)})\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('./../data/features/results/_empgoodpoly2_lsaword_lsachar_w2vlocal_rc_svr_c3.npz', \n",
    "         score_lb=0.67611959,\n",
    "         scores_train_cv=np.array(scores), \n",
    "         scores_train_cv_mean=mean, \n",
    "         scores_train_cv_std=std, \n",
    "         y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# TESTING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.960379254037 0.624965693126\n",
      "0.958438945943 0.679194548656\n",
      "0.95823798052 0.695371473622\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-cf7131408a19>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi_cv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mStratifiedKFold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train_all\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSVC\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rbf'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdegree\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4196\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_all\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0my_train_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mscore_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquadratic_weighted_kappa\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train_all\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m         \u001b[0mseed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrnd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miinfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'i'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msolver_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_seed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m         \u001b[0;31m# see comment on the other call to np.iinfo in this file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36m_dense_fit\u001b[0;34m(self, X, y, sample_weight, solver_type, kernel, random_seed)\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mcache_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoef0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoef0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m                 \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gamma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepsilon\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 236\u001b[0;31m                 max_iter=self.max_iter, random_seed=random_seed)\n\u001b[0m\u001b[1;32m    237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_warn_from_fit_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_good, X_lsa_char, X_lsa_word, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVC(C=10, kernel='rbf', degree=2, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train])   \n",
    "    y_train_pred = model.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = model.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.917115353543 0.689139454229\n",
      "0.917574918124 0.684460853774\n",
      "0.916323467606 0.714026718849\n",
      "0.917233613437 0.724221844736\n",
      "0.917090367809 0.685308557166\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-56-ad21fdaae4b7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mrc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRegressionClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_all\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0my_train_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0mscore_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquadratic_weighted_kappa\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train_all\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_train\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0my_cv_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_cv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-89cd7cda01fe>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X, mean_pow, gaus_pow)\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmean_pow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgaus_pow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregression_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m         \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpercentile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcumsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquantiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    290\u001b[0m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_for_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0mpredict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sparse_predict\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sparse\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dense_predict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_dense_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/sklearn/svm/base.pyc\u001b[0m in \u001b[0;36m_dense_predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    313\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprobA_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprobB_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msvm_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msvm_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkernel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m             \u001b[0mdegree\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdegree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoef0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoef0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gamma\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 315\u001b[0;31m             cache_size=self.cache_size)\n\u001b[0m\u001b[1;32m    316\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_sparse_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((PolynomialFeatures(degree=2).fit_transform(X_emp_good), \n",
    "                                              X_lsa_word, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    model = SVR(C=3, kernel='rbf', degree=1, cache_size=4196*4)\n",
    "    model = model.fit(X[i_train, :], y_train_all[i_train] - 4.5)\n",
    "    rc = RegressionClassifier(model)\n",
    "    rc.fit(X, y_train_all)\n",
    "    y_train_pred = rc.predict(X[i_train, :])\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(X[i_cv, :])\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.146998\n",
      "[1]\teval-rmse:1.589428\n",
      "[2]\teval-rmse:1.226997\n",
      "[3]\teval-rmse:1.000726\n",
      "[4]\teval-rmse:0.873514\n",
      "[5]\teval-rmse:0.800046\n",
      "[6]\teval-rmse:0.758873\n",
      "[7]\teval-rmse:0.738359\n",
      "[8]\teval-rmse:0.730486\n",
      "[9]\teval-rmse:0.722334\n",
      "[10]\teval-rmse:0.720635\n",
      "[11]\teval-rmse:0.718853\n",
      "[12]\teval-rmse:0.717779\n",
      "[13]\teval-rmse:0.718563\n",
      "[14]\teval-rmse:0.718861\n",
      "[15]\teval-rmse:0.717563\n",
      "[16]\teval-rmse:0.717160\n",
      "[17]\teval-rmse:0.715467\n",
      "[18]\teval-rmse:0.714808\n",
      "[19]\teval-rmse:0.715038\n",
      "[20]\teval-rmse:0.717068\n",
      "[21]\teval-rmse:0.716013\n",
      "[22]\teval-rmse:0.715515\n",
      "[23]\teval-rmse:0.717944\n",
      "Stopping. Best iteration:\n",
      "[18]\teval-rmse:0.714808\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.147975\n",
      "[1]\teval-rmse:1.595110\n",
      "[2]\teval-rmse:1.229753\n",
      "[3]\teval-rmse:1.003925\n",
      "[4]\teval-rmse:0.870494\n",
      "[5]\teval-rmse:0.797326\n",
      "[6]\teval-rmse:0.757405\n",
      "[7]\teval-rmse:0.739400\n",
      "[8]\teval-rmse:0.731534\n",
      "[9]\teval-rmse:0.726585\n",
      "[10]\teval-rmse:0.721193\n",
      "[11]\teval-rmse:0.720567\n",
      "[12]\teval-rmse:0.722070\n",
      "[13]\teval-rmse:0.721881\n",
      "[14]\teval-rmse:0.724252\n",
      "[15]\teval-rmse:0.727322\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.865234229612 0.661620651817\n",
      "0.838733535461"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[16]\teval-rmse:0.725432\n",
      "Stopping. Best iteration:\n",
      "[11]\teval-rmse:0.720567\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.153580\n",
      "[1]\teval-rmse:1.598267\n",
      "[2]\teval-rmse:1.230134\n",
      "[3]\teval-rmse:0.996911\n",
      "[4]\teval-rmse:0.856710\n",
      "[5]\teval-rmse:0.779129\n",
      "[6]\teval-rmse:0.734178\n",
      "[7]\teval-rmse:0.710256\n",
      "[8]\teval-rmse:0.702134\n",
      "[9]\teval-rmse:0.698365\n",
      "[10]\teval-rmse:0.695617\n",
      "[11]\teval-rmse:0.694606\n",
      "[12]\teval-rmse:0.693922\n",
      "[13]\teval-rmse:0.691066\n",
      "[14]\teval-rmse:0.690955\n",
      "[15]\teval-rmse:0.690659\n",
      "[16]\teval-rmse:0.690516\n",
      "[17]\teval-rmse:0.691145\n",
      "[18]\teval-rmse:0.691725\n",
      "[19]\teval-rmse:0.692683\n",
      "[20]\teval-rmse:0.692345\n",
      "[21]\teval-rmse:0.693454\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.651836450933\n",
      "0.863990082288"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stopping. Best iteration:\n",
      "[16]\teval-rmse:0.690516\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.147801\n",
      "[1]\teval-rmse:1.587214\n",
      "[2]\teval-rmse:1.222159\n",
      "[3]\teval-rmse:0.989749\n",
      "[4]\teval-rmse:0.850875\n",
      "[5]\teval-rmse:0.772732\n",
      "[6]\teval-rmse:0.727401\n",
      "[7]\teval-rmse:0.704215\n",
      "[8]\teval-rmse:0.691692\n",
      "[9]\teval-rmse:0.686941\n",
      "[10]\teval-rmse:0.681840\n",
      "[11]\teval-rmse:0.681311\n",
      "[12]\teval-rmse:0.681716\n",
      "[13]\teval-rmse:0.682588\n",
      "[14]\teval-rmse:0.682499\n",
      "[15]\teval-rmse:0.681181\n",
      "[16]\teval-rmse:0.681578\n",
      "[17]\teval-rmse:0.679843\n",
      "[18]\teval-rmse:0.678927\n",
      "[19]\teval-rmse:0.679389\n",
      "[20]\teval-rmse:0.682441\n",
      "[21]\teval-rmse:0.683538\n",
      "[22]\teval-rmse:0.681429\n",
      "[23]\teval-rmse:0.684007\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.695675492251\n",
      "0.87240893123"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stopping. Best iteration:\n",
      "[18]\teval-rmse:0.678927\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.149564\n",
      "[1]\teval-rmse:1.585657\n",
      "[2]\teval-rmse:1.219775\n",
      "[3]\teval-rmse:0.987315\n",
      "[4]\teval-rmse:0.847293\n",
      "[5]\teval-rmse:0.771469\n",
      "[6]\teval-rmse:0.727897\n",
      "[7]\teval-rmse:0.707323\n",
      "[8]\teval-rmse:0.696801\n",
      "[9]\teval-rmse:0.692224\n",
      "[10]\teval-rmse:0.683828\n",
      "[11]\teval-rmse:0.681292\n",
      "[12]\teval-rmse:0.677694\n",
      "[13]\teval-rmse:0.677402\n",
      "[14]\teval-rmse:0.672975\n",
      "[15]\teval-rmse:0.671741\n",
      "[16]\teval-rmse:0.672012\n",
      "[17]\teval-rmse:0.672898\n",
      "[18]\teval-rmse:0.673722\n",
      "[19]\teval-rmse:0.673768\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.699753542606\n",
      "0.847487219797"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[20]\teval-rmse:0.673981\n",
      "Stopping. Best iteration:\n",
      "[15]\teval-rmse:0.671741\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.146099\n",
      "[1]\teval-rmse:1.585786\n",
      "[2]\teval-rmse:1.221456\n",
      "[3]\teval-rmse:0.992957\n",
      "[4]\teval-rmse:0.858073\n",
      "[5]\teval-rmse:0.783617\n",
      "[6]\teval-rmse:0.745351\n",
      "[7]\teval-rmse:0.723579\n",
      "[8]\teval-rmse:0.716324\n",
      "[9]\teval-rmse:0.712024\n",
      "[10]\teval-rmse:0.710242\n",
      "[11]\teval-rmse:0.709734\n",
      "[12]\teval-rmse:0.708289\n",
      "[13]\teval-rmse:0.710409\n",
      "[14]\teval-rmse:0.711043\n",
      "[15]\teval-rmse:0.710196\n",
      "[16]\teval-rmse:0.709699\n",
      "[17]\teval-rmse:0.711493\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.697589198837\n",
      "0.846236182823"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stopping. Best iteration:\n",
      "[12]\teval-rmse:0.708289\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.146595\n",
      "[1]\teval-rmse:1.587469\n",
      "[2]\teval-rmse:1.220443\n",
      "[3]\teval-rmse:0.995420\n",
      "[4]\teval-rmse:0.862449\n",
      "[5]\teval-rmse:0.788518\n",
      "[6]\teval-rmse:0.751396\n",
      "[7]\teval-rmse:0.735260\n",
      "[8]\teval-rmse:0.724894\n",
      "[9]\teval-rmse:0.717431\n",
      "[10]\teval-rmse:0.712904\n",
      "[11]\teval-rmse:0.710503\n",
      "[12]\teval-rmse:0.709010\n",
      "[13]\teval-rmse:0.707714\n",
      "[14]\teval-rmse:0.710081\n",
      "[15]\teval-rmse:0.711482\n",
      "[16]\teval-rmse:0.713057\n",
      "[17]\teval-rmse:0.711895\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.6740513023\n",
      "0.849606727872"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[18]\teval-rmse:0.708857\n",
      "Stopping. Best iteration:\n",
      "[13]\teval-rmse:0.707714\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.147793\n",
      "[1]\teval-rmse:1.589640\n",
      "[2]\teval-rmse:1.218672\n",
      "[3]\teval-rmse:0.980351\n",
      "[4]\teval-rmse:0.842444\n",
      "[5]\teval-rmse:0.762984\n",
      "[6]\teval-rmse:0.721587\n",
      "[7]\teval-rmse:0.697971\n",
      "[8]\teval-rmse:0.683724\n",
      "[9]\teval-rmse:0.679107\n",
      "[10]\teval-rmse:0.674153\n",
      "[11]\teval-rmse:0.671958\n",
      "[12]\teval-rmse:0.672803\n",
      "[13]\teval-rmse:0.673538\n",
      "[14]\teval-rmse:0.671645\n",
      "[15]\teval-rmse:0.670293\n",
      "[16]\teval-rmse:0.670802\n",
      "[17]\teval-rmse:0.672419\n",
      "[18]\teval-rmse:0.673989\n",
      "[19]\teval-rmse:0.674412\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.674261216729\n",
      "0.850876220442"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[20]\teval-rmse:0.675350\n",
      "Stopping. Best iteration:\n",
      "[15]\teval-rmse:0.670293\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.154510\n",
      "[1]\teval-rmse:1.601201\n",
      "[2]\teval-rmse:1.235402\n",
      "[3]\teval-rmse:1.011742\n",
      "[4]\teval-rmse:0.878774\n",
      "[5]\teval-rmse:0.804458\n",
      "[6]\teval-rmse:0.765781\n",
      "[7]\teval-rmse:0.747751\n",
      "[8]\teval-rmse:0.735395\n",
      "[9]\teval-rmse:0.729205\n",
      "[10]\teval-rmse:0.725937\n",
      "[11]\teval-rmse:0.725609\n",
      "[12]\teval-rmse:0.725798\n",
      "[13]\teval-rmse:0.723233\n",
      "[14]\teval-rmse:0.726639\n",
      "[15]\teval-rmse:0.727668\n",
      "[16]\teval-rmse:0.725667\n",
      "[17]\teval-rmse:0.723917\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.700294355112\n",
      "0.853377724941"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[18]\teval-rmse:0.724932\n",
      "Stopping. Best iteration:\n",
      "[13]\teval-rmse:0.723233\n",
      "\n",
      "Will train until eval error hasn't decreased in 5 rounds.\n",
      "[0]\teval-rmse:2.147774\n",
      "[1]\teval-rmse:1.586430\n",
      "[2]\teval-rmse:1.219696\n",
      "[3]\teval-rmse:0.991563\n",
      "[4]\teval-rmse:0.858700\n",
      "[5]\teval-rmse:0.783849\n",
      "[6]\teval-rmse:0.739475\n",
      "[7]\teval-rmse:0.716691\n",
      "[8]\teval-rmse:0.707886\n",
      "[9]\teval-rmse:0.705403\n",
      "[10]\teval-rmse:0.703345\n",
      "[11]\teval-rmse:0.701050\n",
      "[12]\teval-rmse:0.701219\n",
      "[13]\teval-rmse:0.701890\n",
      "[14]\teval-rmse:0.704360\n",
      "[15]\teval-rmse:0.703719\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.652136524993\n",
      "0.843258002196 0.689023373595\n",
      "[0.853120885666, 0.679624210917, 0.010187073537, 0.0184739091085]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[16]\teval-rmse:0.704601\n",
      "Stopping. Best iteration:\n",
      "[11]\teval-rmse:0.701050\n",
      "\n"
     ]
    }
   ],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_good, X_lsa_word, X_lsa_char, X_w2v_local)))\n",
    "scores = []\n",
    "\n",
    "num_round = 100\n",
    "param = {'objective': 'reg:linear',\n",
    "         'nthread': 8,\n",
    "         'eval_metric': 'rmse',\n",
    "         'subsample': 0.8,\n",
    "         'silent': 1}\n",
    "plst = param.items()\n",
    "\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    \n",
    "    dtrain = xgb.DMatrix(X[i_train, :], label=y_train_all[i_train])\n",
    "    dcv = xgb.DMatrix(X[i_cv, :], label=y_train_all[i_cv])\n",
    "    evallist = [(dcv, 'eval')]\n",
    "    \n",
    "    bst = xgb.train(plst, dtrain, num_round, evallist, early_stopping_rounds=5)\n",
    "    \n",
    "    rc = RegressionClassifier(bst)\n",
    "    rc.fit(None, y_train_all)\n",
    "    y_train_pred = rc.predict(dtrain)\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(dcv)\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '[%s]' % ', '.join(map(str, np.hstack((mean, std)).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_pca, X_lsa_word)))\n",
    "scores = []\n",
    "\n",
    "num_round = 100\n",
    "param = {'objective': 'multi:softmax',\n",
    "         'nthread': 8,\n",
    "         'eval_metric': 'merror',\n",
    "         'subsample': 0.8,\n",
    "         'num_class': 4,\n",
    "         'silent': 1}\n",
    "plst = param.items()\n",
    "\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    \n",
    "    dtrain = xgb.DMatrix(X[i_train, :], label=y_train_all[i_train] - 1)\n",
    "    dcv = xgb.DMatrix(X[i_cv, :], label=y_train_all[i_cv] - 1)\n",
    "    evallist = [(dcv, 'eval')]\n",
    "    \n",
    "    bst = xgb.train(plst, dtrain, num_round, evallist, early_stopping_rounds=5)\n",
    "    \n",
    "    y_train_pred = bst.predict(dtrain) + 1\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = bst.predict(dcv) + 1\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = StandardScaler().fit_transform(np.hstack((X_emp_pca, X_lsa_word)))\n",
    "scores = []\n",
    "\n",
    "num_round = 100\n",
    "param = {'objective': 'reg:linear',\n",
    "         'nthread': 8,\n",
    "         'eval_metric': 'rmse',\n",
    "         'subsample': 0.8,\n",
    "         'silent': 1}\n",
    "plst = param.items()\n",
    "\n",
    "rc_best = None\n",
    "score_cv_best = -1\n",
    "\n",
    "for i_train, i_cv in StratifiedKFold(y_train_all, 10, shuffle=True):\n",
    "    \n",
    "    dtrain = xgb.DMatrix(X[i_train, :], label=y_train_all[i_train])\n",
    "    dcv = xgb.DMatrix(X[i_cv, :], label=y_train_all[i_cv])\n",
    "    evallist = [(dcv, 'eval')]\n",
    "    \n",
    "    bst = xgb.train(plst, dtrain, num_round, evallist, early_stopping_rounds=5)\n",
    "    \n",
    "    rc = RegressionClassifier(bst)\n",
    "    rc.fit(None, y_train_all)\n",
    "    y_train_pred = rc.predict(dtrain)\n",
    "    score_train = quadratic_weighted_kappa(y_train_all[i_train], y_train_pred)\n",
    "    y_cv_pred = rc.predict(dcv)\n",
    "    score_cv = quadratic_weighted_kappa(y_train_all[i_cv], y_cv_pred)\n",
    "    print score_train, score_cv\n",
    "    scores.append((score_train, score_cv))\n",
    "    \n",
    "    if score_cv > score_cv_best:        \n",
    "        score_cv_best = score_cv\n",
    "        rc_best = rc\n",
    "\n",
    "mean = np.array(scores).mean(axis=0)\n",
    "std = np.array(scores).std(axis=0)\n",
    "print '---------------'\n",
    "print mean\n",
    "\n",
    "y_pred = rc_best.predict(xgb.DMatrix(X, label=y_train_all))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}