basic_and_similarities_feature_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import gensim as gs\n",
    "import nltk.data\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import *\n",
    "import string\n",
    "import re\n",
    "from collections import Counter\n",
    "import scipy as scp\n",
    "import matplotlib.cm as cm\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.decomposition import PCA, IncrementalPCA, KernelPCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor, RandomForestRegressor, AdaBoostRegressor, RandomForestClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import decomposition, pipeline, metrics, grid_search\n",
    "from sklearn import cross_validation\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "import itertools\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.linear_model import Ridge, ElasticNet \n",
    "import sys\n",
    "import os\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics.pairwise import pairwise_distances\n",
    "from HTMLParser import HTMLParser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# basic preprocessing\n",
    "col_types = {u'id': np.int, u'query': np.str, u'product_title': np.str, \n",
    "             u'product_description': np.str, u'median_relevance': np.int, u'relevance_variance': np.float}\n",
    "srt_lowerizer = lambda s: s.lower()\n",
    "convertors = {u'query': srt_lowerizer, u'product_title': srt_lowerizer, u'product_description': srt_lowerizer}\n",
    "df = pd.concat([pd.read_csv('./../data/raw/train.csv', dtype=col_types, converters=convertors, index_col=u'id'), \\\n",
    "                pd.read_csv('./../data/raw/test.csv', dtype=col_types, converters=convertors, index_col=u'id')])\n",
    "\n",
    "idx_row_train = np.where(np.array((~df.median_relevance.isnull()).tolist()))[0]\n",
    "idx_row_test = np.where(np.array((df.median_relevance.isnull()).tolist()))[0]\n",
    "\n",
    "y_train = df['median_relevance'].values[idx_row_train].astype(int)\n",
    "\n",
    "df.product_description = df.product_description.fillna('')\n",
    "\n",
    "df['q_len'] = df['query'].apply(lambda s: len(s))\n",
    "df['t_len'] = df['product_title'].apply(lambda s: len(s))\n",
    "df['d_len'] = df['product_description'].apply(lambda s: len(s))\n",
    "df['d_loglen'] = df['product_description'].apply(lambda s: np.log(len(s) + 1))\n",
    "df['d_exist'] = df['product_description'].apply(lambda s: 1.0 if len(s) > 0 else 0.0)\n",
    "\n",
    "\n",
    "# remove html and non-ascii\n",
    "class MLStripper(HTMLParser):\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "        self.fed = []\n",
    "    def handle_data(self, d):\n",
    "        self.fed.append(d)\n",
    "    def get_data(self):\n",
    "        return ' '.join(self.fed)\n",
    "\n",
    "def strip_tags(html):\n",
    "    s = MLStripper()\n",
    "    s.feed(html)\n",
    "    return s.get_data()\n",
    "\n",
    "df['q'] = df['query'].apply(\n",
    "    lambda s: ''.join([w for w in strip_tags(s.decode('ascii', 'ignore'))]))\n",
    "df['t'] = df['product_title'].apply(\n",
    "    lambda s: ''.join([w for w in strip_tags(s.decode('ascii', 'ignore'))]))\n",
    "df['d'] = df['product_description'].apply(\n",
    "    lambda s: ''.join([w for w in strip_tags(s.decode('ascii', 'ignore'))]))\n",
    "\n",
    "\n",
    "# remove punctuation\n",
    "# c = clean\n",
    "puncts = ''.join(set(string.punctuation + '\\t\\n'))\n",
    "r = re.compile(r'[\\s{}]+'.format(re.escape(puncts)))\n",
    "\n",
    "def clean_string(s):\n",
    "    return ' '.join([w for w in r.split(s)]).strip()\n",
    "\n",
    "df['qc'] = df['q'].apply(clean_string)\n",
    "df['tc'] = df['t'].apply(clean_string)\n",
    "df['dc'] = df['d'].apply(clean_string)\n",
    "\n",
    "# s = sentences\n",
    "df['dcs'] = df['d'].apply(lambda d: list([s for s in map(clean_string, re.split('\\n.!?', d)) if len(s) > 0]))\n",
    "df['d_numsent'] = df['dcs'].apply(lambda p: len(p))\n",
    "df['d_lognumsent'] = df['d_numsent'].apply(lambda x: np.log(x + 1))\n",
    "\n",
    "\n",
    "# remove stop words\n",
    "# ns = no stop words\n",
    "stopword_set = set(stopwords.words('english'))\n",
    "\n",
    "def remove_stop_words(s):\n",
    "    return ' '.join([w for w in s.split(' ') if w not in stopword_set])\n",
    "\n",
    "df['qcns'] = df['qc'].apply(remove_stop_words)\n",
    "df['tcns'] = df['tc'].apply(remove_stop_words)\n",
    "df['dcns'] = df['dc'].apply(remove_stop_words)\n",
    "\n",
    "df['dcsns'] = df['dcs'].apply(lambda p: map(remove_stop_words, p))\n",
    "\n",
    "\n",
    "# stemming\n",
    "# nss = no top words and stemmed\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "def stem(s):\n",
    "    return ' '.join([stemmer.stem(w) for w in s.split(' ')])\n",
    "\n",
    "df['qcnss'] = df['qcns'].apply(stem)\n",
    "df['tcnss'] = df['tcns'].apply(stem)\n",
    "df['dcnss'] = df['dcns'].apply(stem)\n",
    "\n",
    "df['dcsnss'] = df['dcsns'].apply(lambda p: map(stem, p))\n",
    "\n",
    "\n",
    "#sop - Simpson Overlap coefficient: f(A, B) = n(A ∩ B) / min( n(A), n(B))\n",
    "\n",
    "def overlap_coefficient(A, B):\n",
    "    if len(A) == 0 or len(B) == 0:\n",
    "        return 0\n",
    "    return len(A.intersection(B))/float(min(len(A), len(B)))\n",
    "\n",
    "df['sop_qt_cnss'] = \\\n",
    "    df.apply(lambda r: overlap_coefficient(set(r['qcnss'].split(' ')), set(r['tcnss'].split(' '))), axis=1)\n",
    "\n",
    "df['sop_qd_cnss'] = \\\n",
    "    df.apply(lambda r: overlap_coefficient(set(r['qcnss'].split(' ')), set(r['dcnss'].split(' '))), axis=1)\n",
    "\n",
    "df['tmp_col'] = \\\n",
    "    df.apply(lambda r: \\\n",
    "             map(lambda s: overlap_coefficient(set(r['qcnss'].split(' ')), set(s.split(' '))), r['dcsnss']), axis=1)\n",
    "\n",
    "df['sop_min_qds_cnss'] = df['tmp_col'].apply(lambda v: np.min(v) if len(v) > 0 else 0)\n",
    "df['sop_max_qds_cnss'] = df['tmp_col'].apply(lambda v: np.max(v) if len(v) > 0 else 0)\n",
    "df['sop_mean_qds_cnss'] = df['tmp_col'].apply(lambda v: np.mean(v) if len(v) > 0 else 0)\n",
    "df['sop_median_qds_cnss'] = df['tmp_col'].apply(lambda v: np.median(v) if len(v) > 0 else 0)\n",
    "\n",
    "\n",
    "#jsc - Jaccard similarity coefficient: f(A, B) = n(A ∩ B) / n(A ∪ B)\n",
    "\n",
    "def jaccard_coefficient(A, B):\n",
    "    if len(A) == 0 and len(B) == 0:\n",
    "        return 0\n",
    "    return len(A.intersection(B))/float(len(A.union(B)))\n",
    "\n",
    "df['jsc_qt_cnss'] = \\\n",
    "    df.apply(lambda r: jaccard_coefficient(set(r['qcnss'].split(' ')), set(r['tcnss'].split(' '))), axis=1)\n",
    "\n",
    "df['jsc_qd_cnss'] = \\\n",
    "    df.apply(lambda r: jaccard_coefficient(set(r['qcnss'].split(' ')), set(r['dcnss'].split(' '))), axis=1)\n",
    "\n",
    "df['tmp_col'] = \\\n",
    "    df.apply(lambda r: \\\n",
    "             map(lambda s: jaccard_coefficient(set(r['qcnss'].split(' ')), set(s.split(' '))), r['dcsnss']), axis=1)\n",
    "\n",
    "df['jsc_min_qds_cnss'] = df['tmp_col'].apply(lambda v: np.min(v) if len(v) > 0 else 0)\n",
    "df['jsc_max_qds_cnss'] = df['tmp_col'].apply(lambda v: np.max(v) if len(v) > 0 else 0)\n",
    "df['jsc_mean_qds_cnss'] = df['tmp_col'].apply(lambda v: np.mean(v) if len(v) > 0 else 0)\n",
    "df['jsc_median_qds_cnss'] = df['tmp_col'].apply(lambda v: np.median(v) if len(v) > 0 else 0)\n",
    "\n",
    "df = df.drop('tmp_col', 1)\n",
    "\n",
    "# LSA\n",
    "def cos_sim(A, B):\n",
    "    v = (A*B).sum(axis=1)/np.sqrt((A**2).sum(axis=1) * (B**2).sum(axis=1))\n",
    "    v[np.isnan(v)] = 0\n",
    "    return v\n",
    "\n",
    "ngram_range_max_grid = [1, 2]\n",
    "svd_n_components_grid = [100, 200, 300, 400, 500]\n",
    "\n",
    "tfidf_svd_models = {}\n",
    "for ngram_range_max in ngram_range_max_grid:\n",
    "    tfidf_svd_models[ngram_range_max] = {}\n",
    "    for svd_n_components in svd_n_components_grid:\n",
    "        tfidf_svd_models[ngram_range_max][svd_n_components] = {}\n",
    "        print '%i - %i' % (ngram_range_max, svd_n_components)\n",
    "        tfv = TfidfVectorizer(analyzer='word', ngram_range=(1, ngram_range_max), \n",
    "                              sublinear_tf=1, use_idf=1, smooth_idf=1,\n",
    "                              token_pattern=r'\\w{1,}', min_df=3)\n",
    "        all_texts = list([t for t in df['qcnss'].values.tolist() if len(t) > 0])\n",
    "        all_texts.extend([t for t in df['tcnss'].values.tolist() if len(t) > 0])\n",
    "        tfv = tfv.fit(all_texts)\n",
    "        tf_idf_all_texts = tfv.transform(all_texts)\n",
    "        svd = TruncatedSVD(n_components=svd_n_components)\n",
    "        svd = svd.fit(tf_idf_all_texts)\n",
    "        tfidf_svd_models[ngram_range_max][svd_n_components]['qt'] = (tfv, svd)\n",
    "        q_nss_tfidf_svd = svd.transform(tfv.transform(df['qcnss'].values.tolist()))\n",
    "        t_nss_tfidf_svd = svd.transform(tfv.transform(df['tcnss'].values.tolist()))\n",
    "\n",
    "        df['cos_qt_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = cos_sim(q_nss_tfidf_svd, t_nss_tfidf_svd)\n",
    "        df['cos_qt_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = \\\n",
    "            df['cos_qt_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)].fillna(0)\n",
    "        \n",
    "        tfv = TfidfVectorizer(analyzer='word', ngram_range=(1, ngram_range_max), \n",
    "                              sublinear_tf=1, use_idf=1, smooth_idf=1,\n",
    "                              token_pattern=r'\\w{1,}', min_df=3)\n",
    "        all_texts = list([t for t in df['qcnss'].values.tolist() if len(t) > 0])\n",
    "        all_texts.extend([t for t in df['dcnss'].values.tolist() if len(t) > 0])\n",
    "        tfv = tfv.fit(all_texts)\n",
    "        tf_idf_all_texts = tfv.transform(all_texts)\n",
    "        svd = TruncatedSVD(n_components=svd_n_components)\n",
    "        svd = svd.fit(tf_idf_all_texts)\n",
    "        tfidf_svd_models[ngram_range_max][svd_n_components]['qd'] = (tfv, svd)\n",
    "        q_nss_tfidf_svd = svd.transform(tfv.transform(df['qcnss'].values.tolist()))\n",
    "        d_nss_tfidf_svd = svd.transform(tfv.transform(df['dcnss'].values.tolist()))\n",
    "\n",
    "        df['cos_qd_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = cos_sim(q_nss_tfidf_svd, d_nss_tfidf_svd)\n",
    "        df['cos_qd_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = \\\n",
    "            df['cos_qd_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)].fillna(0)\n",
    "\n",
    "        ds_nss_tfidf_svd_cos = []\n",
    "        for i in range(df.shape[0]):\n",
    "            if len(df['dcsnss'].values[i]) == 0:\n",
    "                ds_nss_tfidf_svd_cos.append(np.array([0]))\n",
    "                continue\n",
    "            tmp_q = np.repeat(q_nss_tfidf_svd[i, :], len(df['dcsnss'].values[i])).reshape(\n",
    "                (len(df['dcsnss'].values[i]), q_nss_tfidf_svd.shape[1]), order='F')\n",
    "            tmp_d = svd.transform(tfv.transform(df['dcsnss'].values[i]))\n",
    "            ds_nss_tfidf_svd_cos.append(cos_sim(tmp_q, tmp_d))\n",
    "\n",
    "        df['cos_min_qd_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = \\\n",
    "            map(lambda v: np.min(v), ds_nss_tfidf_svd_cos)\n",
    "        df['cos_max_qd_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = \\\n",
    "            map(lambda v: np.max(v), ds_nss_tfidf_svd_cos)\n",
    "        df['cos_mean_qd_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = \\\n",
    "            map(lambda v: np.mean(v), ds_nss_tfidf_svd_cos)\n",
    "        df['cos_median_qd_cnss_tfidf%i_svd%i' % (ngram_range_max, svd_n_components)] = \\\n",
    "            map(lambda v: np.median(v), ds_nss_tfidf_svd_cos)\n",
    "\n",
    "\n",
    "# Okapi BM25\n",
    "\n",
    "class OkapiBM25:\n",
    "    \n",
    "    cv = None\n",
    "    N = 1\n",
    "    word_doc_freq = {}\n",
    "    k = 1.5\n",
    "    b = 0.75\n",
    "    avgdl = 0\n",
    "    \n",
    "    def __init__(self, max_n_gram=2, k=1.5, b=0.75):\n",
    "        self.cv = CountVectorizer(analyzer='word', ngram_range=(1, max_n_gram), stop_words=None)\n",
    "        self.k = k        \n",
    "        self.b = b\n",
    "    \n",
    "    def fit(self, X):\n",
    "        self.N = X.shape[0]\n",
    "        X = self.cv.fit_transform(X)\n",
    "        self.avgdl = X.sum()/float(X.shape[0])\n",
    "        self.word_doc_freq = dict(\n",
    "            zip(map(lambda p: p[0], sorted(self.cv.vocabulary_.items(), key=lambda p: p[1])), \n",
    "                np.array(X.sum(axis=0), dtype=np.int).flatten().tolist()))\n",
    "            \n",
    "    def idf(self, w):\n",
    "        n = self.word_doc_freq[w] if w in self.word_doc_freq else 0\n",
    "        return np.log((self.N - n + 0.5)/(n + 0.5))\n",
    "    \n",
    "    def f(self, w, d):\n",
    "        return sum(map(lambda x: 1 if x == w else 0, d.split()))\n",
    "    \n",
    "    def score_word(self, w, d):\n",
    "        fqd = self.f(w, d)\n",
    "        return self.idf(w) * (fqd * (self.k + 1))/(fqd + self.k*(1 - self.b + self.b*(len(d.split()))/self.avgdl))\n",
    "    \n",
    "    def score_query(self, q, d):\n",
    "        return sum(map(lambda w: self.score_word(w, d), q.split()))\n",
    "\n",
    "ngram_range_max_grid = range(1, 4)\n",
    "for ngram_range_max in ngram_range_max_grid:\n",
    "    print ngram_range_max\n",
    "    \n",
    "    obm25 = OkapiBM25(max_n_gram=ngram_range_max)\n",
    "    obm25.fit(df['tcnss'].values)\n",
    "    df['obm25_%i_qt_cnss' % ngram_range_max] = \\\n",
    "        map(lambda p: obm25.score_query(p[0], p[1]), zip(df['qcnss'].values, df['tcnss'].values))\n",
    "    \n",
    "    obm25 = OkapiBM25(max_n_gram=ngram_range_max)\n",
    "    obm25.fit(df['dcnss'].values)\n",
    "    df['obm25_%i_qd_cnss' % ngram_range_max] = \\\n",
    "        map(lambda p: obm25.score_query(p[0], p[1]), zip(df['qcnss'].values, df['dcnss'].values))\n",
    "    \n",
    "    sent_scores = map(lambda p: np.array(map(lambda d: obm25.score_query(p[0], d), p[1])), \n",
    "                      zip(df['qcnss'].values, df['dcsnss'].values))\n",
    "    \n",
    "    df['obm25_%i_min_qd_cnss' % ngram_range_max] = map(lambda v: np.min(v) if len(v) > 0 else 0, sent_scores)\n",
    "    df['obm25_%i_max_qd_cnss' % ngram_range_max] = map(lambda v: np.max(v) if len(v) > 0 else 0, sent_scores)\n",
    "    df['obm25_%i_mean_qd_cnss' % ngram_range_max] = map(lambda v: np.mean(v) if len(v) > 0 else 0, sent_scores)\n",
    "    df['obm25_%i_median_qd_cnss' % ngram_range_max] = map(lambda v: np.median(v) if len(v) > 0 else 0, sent_scores)\n",
    "\n",
    "\n",
    "# word2vec similarities\n",
    "path = './../data/w2v_models/'\n",
    "w2v_models = {}\n",
    "for name in os.listdir(path):\n",
    "    w2v_models[name] = Word2Vec.load(os.path.join(path, name))\n",
    "\n",
    "# w2v: cos similarities between mean vectors\n",
    "\n",
    "for w2v_model_key in w2v_models.keys():\n",
    "    print w2v_model_key\n",
    "    \n",
    "    w2v = w2v_models[w2v_model_key]\n",
    "    Q_mean = np.array(map(lambda s: \n",
    "                      reduce(lambda x, y: x + y, \n",
    "                             map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split()), \n",
    "                             np.zeros(w2v['foot'].shape))/len(s.split()), \n",
    "                      df['qcnss'].values))\n",
    "    T_mean = np.array(map(lambda s: \n",
    "                      reduce(lambda x, y: x + y, \n",
    "                             map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split()), \n",
    "                             np.zeros(w2v['foot'].shape))/len(s.split()), \n",
    "                      df['tcnss'].values))\n",
    "    # D_mean = np.array(map(lambda s: \n",
    "    #                   reduce(lambda x, y: x + y, \n",
    "    #                          map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split()), \n",
    "    #                          np.zeros(w2v['foot'].shape))/len(s.split()), \n",
    "    #                   df['dcnss'].values))\n",
    "    D_list_mean = []\n",
    "    for d in df['dcsnss'].values:\n",
    "        D_list_mean.append(\n",
    "            np.array(map(lambda s: \n",
    "                     reduce(lambda x, y: x + y, \n",
    "                            map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split()), \n",
    "                            np.zeros(w2v['foot'].shape))/len(s.split()), \n",
    "                     d)))\n",
    "\n",
    "    df['cos_qt_cnss_%s' % w2v_model_key.replace('.bin', '')] = cos_sim(Q_mean, T_mean)\n",
    "\n",
    "    ds_nss_w2v_cos = []\n",
    "    for i in range(df.shape[0]):\n",
    "        if len(df['dcsnss'].values[i]) == 0:\n",
    "            ds_nss_w2v_cos.append(np.array([0]))\n",
    "            continue\n",
    "        tmp_q = np.repeat(Q_mean[i, :], len(df['dcsnss'].values[i])).reshape(\n",
    "            (len(df['dcsnss'].values[i]), Q_mean.shape[1]), order='F')\n",
    "        tmp_d = D_list_mean[i]\n",
    "        ds_nss_w2v_cos.append(cos_sim(tmp_q, tmp_d))\n",
    "\n",
    "    df['cos_min_qd_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda v: np.min(v), ds_nss_w2v_cos)\n",
    "    df['cos_max_qd_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda v: np.max(v), ds_nss_w2v_cos)\n",
    "    df['cos_mean_qd_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda v: np.mean(v), ds_nss_w2v_cos)\n",
    "    df['cos_median_qd_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda v: np.median(v), ds_nss_w2v_cos)\n",
    "\n",
    "\n",
    "# w2v: min mean cos\n",
    "\n",
    "for w2v_model_key in w2v_models.keys():\n",
    "    print w2v_model_key\n",
    "    \n",
    "    w2v = w2v_models[w2v_model_key]\n",
    "\n",
    "    Q_list_words = map(lambda s: \n",
    "                       np.array(map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split())), \n",
    "                       df['qcnss'].values)\n",
    "    T_list_words = map(lambda s: \n",
    "                       np.array(map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split())), \n",
    "                       df['tcnss'].values)\n",
    "    D_list_words = map(lambda s: \n",
    "                       np.array(map(lambda w: w2v[w] if w in w2v else np.zeros(w2v['foot'].shape), s.split())), \n",
    "                       df['dcnss'].values)\n",
    "\n",
    "    df['min_mean_cos_qt_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda m: m.min(axis=1).mean(), \n",
    "            map(lambda t: pairwise_distances(t[0], t[1], metric='cosine'), zip(Q_list_words, T_list_words)))\n",
    "\n",
    "    df['min_mean_cos_qd_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda m: m.min(axis=1).mean() if type(m) is not int else 0, \n",
    "            map(lambda t: pairwise_distances(t[0], t[1], metric='cosine') if np.prod(t[1].shape) > 0 else 0, \n",
    "                zip(Q_list_words, D_list_words)))\n",
    "        \n",
    "\n",
    "# w2v: max mean sim\n",
    "\n",
    "def sent_sim_matrix(s1, s2, w2v):\n",
    "    s1 = s1.split()\n",
    "    s2 = s2.split()\n",
    "    if len(s1) == 0 or len(s2) == 0:\n",
    "        return 0\n",
    "    m = np.zeros((len(s1), len(s2)))\n",
    "    for i in range(len(s1)):\n",
    "        for j in range(len(s2)):\n",
    "            m[i, j] = w2v.similarity(s1[i], s2[j]) if s1[i] in w2v and s2[j] in w2v else 0\n",
    "    return m\n",
    "\n",
    "for w2v_model_key in w2v_models.keys():\n",
    "    print w2v_model_key\n",
    "    \n",
    "    w2v = w2v_models[w2v_model_key]\n",
    "\n",
    "    df['max_mean_sim_qt_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda m: m.max(axis=1).mean(), \n",
    "            map(lambda t: sent_sim_matrix(t[0], t[1], w2v), zip(df['qcnss'].values, df['tcnss'].values)))\n",
    "    \n",
    "    df['max_mean_sim_qt_cnss_%s' % w2v_model_key.replace('.bin', '')] = \\\n",
    "        map(lambda m: m.max(axis=1).mean() if type(m) is not int else 0, \n",
    "            map(lambda t: sent_sim_matrix(t[0], t[1], w2v), zip(df['qcnss'].values, df['dcnss'].values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}