evaluate_blends-phonemes-best-length-blind.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import Levenshtein"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from seqtools import *                                                                                                                                                           \n",
    "import configargparse                                                                                                                                                            \n",
    "import pandas as pd                                                                                                                                                              \n",
    "import torch                                                                                                                                                                     \n",
    "import torch.autograd as autograd                                                                                                                                                \n",
    "import torch.nn as nn                                                                                                                                                            \n",
    "import torch.nn.functional as F                                                                                                                                                  \n",
    "import torch.optim as optim                                                                                                                                                      \n",
    "import pandas as pd                                                                                                                                                              \n",
    "torch.manual_seed(1)                                                                                                                                                             \n",
    "from torch.autograd import Variable                                                                                                                                              \n",
    "import numpy as np                                                                                                                                                               \n",
    "import sys                                                                                                                                                                       \n",
    "# Minimum count                                                                                                                                                                  \n",
    "MIN_COUNT = 1                                                                                                                                                                    \n",
    "# Min length of input and output sequence                                                                                                                                        \n",
    "MIN_LENGTH = 1                                                                                                                                                                   \n",
    "# Max length of input and output sequence                                                                                                                                        \n",
    "MAX_LENGTH = 30   \n",
    "import Levenshtein"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import kenlm\n",
    "lmodel = kenlm.Model('./data/wordlist_english_filtered_threshold100-kenlm.arpa')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from lstm_model import LSTMTagger, filter_pairs, prepare_data, read_langs, get_validation_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./seq2seq_output/global_candidates.phonemes', sep='\\t', skiprows=3, names=['word','phonemes'])\n",
    "df=df.dropna()\n",
    "\n",
    "import kenlm\n",
    "\n",
    "phoneme_model = kenlm.Model('./data/cmudict-ud.arpa')\n",
    "\n",
    "def phoneme_score(s):\n",
    "    return phoneme_model.score(s)\n",
    "\n",
    "def phoneme_len(s):\n",
    "    return len(s.split(' '))\n",
    "\n",
    "df[\"score\"]=df[\"phonemes\"].apply(phoneme_score)\n",
    "df[\"len\"]=df[\"phonemes\"].apply(phoneme_len)\n",
    "df[\"nscore\"]=df[\"score\"]/(df[\"len\"]**0.4)\n",
    "\n",
    "df.sort_values(by='nscore', ascending=True)\n",
    "\n",
    "phoneme_score_dict=dict(zip(df.word.values, df.nscore.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def encode_word(examples):\n",
    "    seqs=[]\n",
    "    for example in examples:\n",
    "        seqs.append(indexes_from_sentence(input_lang, example))\n",
    "    seqs_padded = [pad_seq(seq, MAX_LENGTH) for seq in seqs]\n",
    "        # Turn padded arrays into (batch_size x max_len) tensors, transpose into (max_len x batch_size)                                                                              \n",
    "    input_var = Variable(torch.LongTensor(seqs_padded)).transpose(0, 1)                                                                                                         \n",
    "    if USE_CUDA:                                                                                                                                                                 \n",
    "        input_var = input_var.cuda()                                                                                                                                                                                                                                                                                                                            \n",
    "    return input_var                                                                                                                                                                                           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_model(embedding_dim, hidden_dim, vocab_size, tagset_size, batch_size, model_filename, n_layers=1):\n",
    "    t=torch.load(model_filename)\n",
    "    m=LSTMTagger(embedding_dim=embedding_dim,hidden_dim=hidden_dim,\n",
    "                 vocab_size=vocab_size,tagset_size=tagset_size,batch_size=batch_size, n_layers=n_layers)\n",
    "    m.load_state_dict(t)\n",
    "    m.hidden=m.init_hidden(batch_size=m.batch_size)\n",
    "    if USE_CUDA:\n",
    "        m.cuda()\n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_from_copy_edit(example, copyedits):\n",
    "    output_str=[]\n",
    "    for i,s in enumerate(list(example)):\n",
    "        if copyedits[i] == 'C':\n",
    "            output_str.append(s)\n",
    "    return u''.join(output_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_predict_raw(m, example):  \n",
    "    import itertools\n",
    "    outputs=[]\n",
    "    m.hidden=m.init_hidden(batch_size=m.batch_size)\n",
    "    t = encode_word(list(itertools.repeat(example, m.batch_size)))\n",
    "    scores = m(t)\n",
    "    scores=F.log_softmax(scores[0,:]) #do a softmax since LSTM onlt returns linear output out.\n",
    "    return scores.cpu().data.numpy()\n",
    "    #return scores[0,:,:].cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_predict(m, examples):  \n",
    "    import itertools\n",
    "    outputs=[]\n",
    "    for example in examples:\n",
    "        m.hidden=m.init_hidden(batch_size=m.batch_size)\n",
    "        t = encode_word(list(itertools.repeat(example, m.batch_size)))\n",
    "        scores = m(t)\n",
    "        preds=[output_lang.index2word[ix] for ix in np.argmax(scores[0,:,:].cpu().data.numpy(), axis=1)]\n",
    "        output_str=get_from_copy_edit(example, preds)\n",
    "        outputs.append(u''.join(output_str))\n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score_candidate_length(c, example,length_model):\n",
    "    import scipy.stats\n",
    "    needed_length, needed_length_std=length_model.predict(np.array([[len(example)]]), return_std=True)\n",
    "    needed_length=needed_length[0]\n",
    "    needed_length_std=needed_length_std[0]\n",
    "    clength=len(c)\n",
    "    #print \"NL:{} CL:{}\".format(needed_length, clength)\n",
    "    return scipy.stats.norm.logpdf(clength, loc=needed_length, scale=needed_length_std)-scipy.stats.norm.logpdf(needed_length, loc=needed_length, scale=needed_length_std)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_predict_lm(m, examples, lmodel, length_model, alpha=0.5, use_lm=False, use_pm=False, use_length=False):  \n",
    "    import itertools\n",
    "    outputs=[]\n",
    "    preds=[]\n",
    "    odflist=[]\n",
    "    for example in examples:\n",
    "        m.hidden=m.init_hidden(batch_size=m.batch_size)\n",
    "        t = encode_word(list(itertools.repeat(example, m.batch_size)))\n",
    "        scores = m(t)\n",
    "        scores=F.log_softmax(scores[0,:]).cpu().data.numpy()\n",
    "        candidates, lstm_scores=zip(*get_k_best_blends(scores, example, k=5))\n",
    "        lm_scores=[lmodel.score(' '.join(list(c)))/(float(len(c))**alpha) for c in candidates]\n",
    "        pm_scores=[]\n",
    "        for c in candidates:\n",
    "            if c in phoneme_score_dict:\n",
    "                pm_scores.append(phoneme_score_dict[c])\n",
    "            else:\n",
    "                pm_scores.append(0.0)\n",
    "        length_scores=[score_candidate_length(c, example, length_model) for c in candidates]\n",
    "        odf=pd.DataFrame().from_records(zip(candidates, lstm_scores,lm_scores, pm_scores, length_scores), columns=['candidate','lstm','lm', 'pm','lengthscore'])\n",
    "        odf[\"total\"]=odf[\"lstm\"]\n",
    "        if use_lm:\n",
    "            odf[\"total\"]=odf[\"total\"]+odf[\"lm\"]\n",
    "        if use_pm:\n",
    "            odf[\"total\"]=odf[\"total\"]+odf[\"pm\"]\n",
    "        if use_length:\n",
    "            odf[\"total\"]=odf[\"total\"]+odf[\"lengthscore\"]\n",
    "        odf=odf.sort_values(by=['total'], ascending=False)\n",
    "        odf=odf.reset_index()\n",
    "        odflist.append(odf)\n",
    "        preds.append(odf.ix[0].candidate)\n",
    "    return preds, pd.concat(odflist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "class Pmf(Counter):\n",
    "    \"\"\"A Counter with probabilities.\"\"\"\n",
    "\n",
    "    def normalize(self):\n",
    "        \"\"\"Normalizes the PMF so the probabilities add to 1.\"\"\"\n",
    "        total = float(sum(self.values()))\n",
    "        for key in self:\n",
    "            self[key] /= total\n",
    "\n",
    "    def __add__(self, other):\n",
    "        \"\"\"Adds two distributions.\n",
    "\n",
    "        The result is the distribution of sums of values from the\n",
    "        two distributions.\n",
    "\n",
    "        other: Pmf\n",
    "\n",
    "        returns: new Pmf\n",
    "        \"\"\"\n",
    "        pmf = Pmf()\n",
    "        for key1, prob1 in self.items():\n",
    "            for key2, prob2 in other.items():\n",
    "                pmf[key1 + key2] += prob1 * prob2\n",
    "        return pmf\n",
    "\n",
    "    def __hash__(self):\n",
    "        \"\"\"Returns an integer hash value.\"\"\"\n",
    "        return id(self)\n",
    "    \n",
    "    def __eq__(self, other):\n",
    "        return self is other\n",
    "\n",
    "    def render(self):\n",
    "        \"\"\"Returns values and their probabilities, suitable for plotting.\"\"\"\n",
    "        return zip(*sorted(self.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def only_length(split_file_name, length_model_name):\n",
    "    from collections import Counter\n",
    "    _, _,train_pairs,val_pairs,t_pairs=pickle.load(open(split_file_name,'rb'))\n",
    "    length_model = pickle.load(open(length_model_name,'rb'))\n",
    "    prefixes=[t.find('D') for t in zip(*train_pairs)[1]]\n",
    "    cprefixes=Counter(prefixes)\n",
    "    cpmf=Pmf(cprefixes)\n",
    "    cpmf.normalize()\n",
    "    cpmf_dist_keys, cpmf_dist_values=zip(*sorted(cpmf.items()))    \n",
    "    #print cpmf_dist_keys, cpmf_dist_values\n",
    "    preds=[]\n",
    "    test_pairs=zip(*t_pairs)[0]\n",
    "    for t in test_pairs:\n",
    "        tlen, tstd=length_model.predict(np.array([len(t)]).reshape(-1,1), return_std=True)\n",
    "        tlen_sample = int(np.random.normal(loc=tlen, scale=tstd))\n",
    "        if tlen_sample > len(t):\n",
    "            tlen_sample = len(t)\n",
    "        len_prefix=np.random.choice(len(cpmf_dist_values), p=cpmf_dist_values)\n",
    "        #print \"Length\", len_prefix\n",
    "        pb, sb = t.split('}')\n",
    "        p1blend=pb[:len_prefix]\n",
    "        if len(p1blend) < tlen_sample:\n",
    "            s1blend=sb[-(tlen_sample-len_prefix):]\n",
    "        else:\n",
    "            s1blend=''\n",
    "        preds.append(p1blend+s1blend)\n",
    "    gold_preds=[get_from_copy_edit(val_pair[0], val_pair[1]) for val_pair in t_pairs]\n",
    "    edit_scores=[Levenshtein.distance(p,g) for p, g in zip(preds, gold_preds)]\n",
    "\n",
    "    return np.mean(edit_scores), preds, gold_preds, edit_scores, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def only_length_blind(split_file_name, length_model_name):\n",
    "    from collections import Counter\n",
    "    _, _,train_pairs,val_pairs,t_pairs=pickle.load(open(split_file_name,'rb'))\n",
    "    t_pairs=blind_pairs\n",
    "    length_model = pickle.load(open(length_model_name,'rb'))\n",
    "    prefixes=[t.find('D') for t in zip(*train_pairs)[1]]\n",
    "    cprefixes=Counter(prefixes)\n",
    "    cpmf=Pmf(cprefixes)\n",
    "    cpmf.normalize()\n",
    "    cpmf_dist_keys, cpmf_dist_values=zip(*sorted(cpmf.items()))    \n",
    "    #print cpmf_dist_keys, cpmf_dist_values\n",
    "    preds=[]\n",
    "    test_pairs=zip(*t_pairs)[0]\n",
    "    for t in test_pairs:\n",
    "        tlen, tstd=length_model.predict(np.array([len(t)]).reshape(-1,1), return_std=True)\n",
    "        tlen_sample = int(np.random.normal(loc=tlen, scale=tstd))\n",
    "        if tlen_sample > len(t):\n",
    "            tlen_sample = len(t)\n",
    "        len_prefix=np.random.choice(len(cpmf_dist_values), p=cpmf_dist_values)\n",
    "        #print \"Length\", len_prefix\n",
    "        pb, sb = t.split('}')\n",
    "        p1blend=pb[:len_prefix]\n",
    "        if len(p1blend) < tlen_sample:\n",
    "            s1blend=sb[-(tlen_sample-len_prefix):]\n",
    "        else:\n",
    "            s1blend=''\n",
    "        preds.append(p1blend+s1blend)\n",
    "    gold_preds=[get_from_copy_edit(val_pair[0], val_pair[1]) for val_pair in t_pairs]\n",
    "    edit_scores=[Levenshtein.distance(unicode(p),unicode(g)) for p, g in zip(preds, gold_preds)]\n",
    "\n",
    "    return np.mean(edit_scores), preds, gold_preds, edit_scores, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_edit_distance(embedding_dim, hidden_dim, vocab_size, tagset_size, batch_size, model_filename, split_file_name):\n",
    "    m=load_model(embedding_dim=embedding_dim,hidden_dim=hidden_dim,vocab_size=vocab_size,tagset_size=tagset_size, batch_size=batch_size, model_filename=model_filename)\n",
    "    _, _, _,val_pairs,t_pairs=pickle.load(open(split_file_name,'rb'))\n",
    "    test_pairs=list(zip(*t_pairs)[0])\n",
    "    preds=get_predict(m,test_pairs)\n",
    "    gold_preds=[get_from_copy_edit(val_pair[0], val_pair[1]) for val_pair in t_pairs]\n",
    "    edit_scores=[Levenshtein.distance(p,g) for p, g in zip(preds, gold_preds)]\n",
    "    return np.mean(edit_scores), preds, gold_preds, edit_scores, m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import numpy as np\n",
    "from itertools import product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def k_shortest_paths(G, source, target, k, weight=None):\n",
    "    from itertools import islice\n",
    "    return list(islice(nx.shortest_simple_paths(G, source, target, weight=weight), k))\n",
    "\n",
    "def get_k_best(M, k=5):\n",
    "    # We negate all things to find the best (longest). This only works because all entries in M are negative.\n",
    "    #So making them positive is fine.\n",
    "    G=nx.DiGraph()\n",
    "    for r in np.arange(0, M.shape[0]-1):\n",
    "        tuples=list(product([(r, c) for c in np.arange(0, M.shape[1])], [(r+1, c) for c in np.arange(0, M.shape[1])]))\n",
    "        for t in tuples:\n",
    "            G.add_edge(t[0], t[1], weight=-M[t[1][0], t[1][1]])\n",
    "\n",
    "    for c in np.arange(0, M.shape[1]):\n",
    "        G.add_edge(-1, (0,c), weight=-M[0][c])\n",
    "        G.add_edge((M.shape[0]-1, c), M.shape[0], weight=0)\n",
    "    \n",
    "    tags=k_shortest_paths(G, -1, M.shape[0], k=k, weight='weight')\n",
    "    return tags\n",
    "\n",
    "def print_tags(M , tags):\n",
    "    for tag in tags:\n",
    "        print tag[1:-1], np.sum(M[cell[0],cell[1]] for cell in tag[1:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_k_best_blends(M, s, k=5):\n",
    "    N=M[:len(s)+1, :]\n",
    "    hyps=get_k_best(N,k=k)\n",
    "    #print_tags(N, hyps)\n",
    "    candidates=[]\n",
    "    for hyp in hyps:\n",
    "        candidates.append((get_from_copy_edit(s, [output_lang.index2word[t] for t in zip(*hyp[1:-1])[1]]), np.sum(M[cell[0],cell[1]] for cell in hyp[1:-1])))\n",
    "    return candidates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_edit_distance_lm(embedding_dim, hidden_dim, vocab_size, tagset_size, batch_size, model_filename, split_file_name, length_model_name, use_lm=False, use_pm=False, use_length=False):\n",
    "    m=load_model(embedding_dim=embedding_dim,hidden_dim=hidden_dim,vocab_size=vocab_size,tagset_size=tagset_size, batch_size=batch_size, model_filename=model_filename)\n",
    "    _, _, _,val_pairs,t_pairs=pickle.load(open(split_file_name,'rb'))\n",
    "    length_model = pickle.load(open(length_model_name,'rb'))\n",
    "    test_pairs=list(zip(*t_pairs)[0])\n",
    "    preds, _=get_predict_lm(m,test_pairs, lmodel, length_model, use_lm=use_lm, use_pm=use_pm, use_length=use_length)\n",
    "    gold_preds=[get_from_copy_edit(val_pair[0], val_pair[1]) for val_pair in t_pairs]\n",
    "    edit_scores=[Levenshtein.distance(p,g) for p, g in zip(preds, gold_preds)]\n",
    "    return np.mean(edit_scores), preds, gold_preds, edit_scores, m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_scores_fold(fno):\n",
    "    m=load_model(50,50,input_lang.n_words, output_lang.n_words, 20, './seq2seq_output/test_knight//lstm_{}_best'.format(fno))\n",
    "    _, _, _,val_pairs,t_pairs=pickle.load(open('./seq2seq_output/test_knight//split_lstm_{}'.format(fno),'rb'))\n",
    "    length_model = pickle.load(open('./seq2seq_output/test_knight_length/length_{}'.format(fno),'rb'))\n",
    "    test_pairs=list(zip(*t_pairs)[0])\n",
    "    preds, odf=get_predict_lm(m,test_pairs, lmodel, length_model=length_model)\n",
    "    gold_preds=[get_from_copy_edit(val_pair[0], val_pair[1]) for val_pair in t_pairs]\n",
    "    edit_scores=[Levenshtein.distance(p,g) for p, g in zip(preds, gold_preds)]\n",
    "    fdf=pd.DataFrame().from_records(zip(gold_preds, preds, edit_scores),columns=['g','p','s'])\n",
    "    return odf, fdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_score_string(s, M, output_lang):\n",
    "    scores=[]\n",
    "    for i, a in enumerate(s):\n",
    "        scores.append(M[i][output_lang.word2index[a]])\n",
    "    scores.append(M[len(s)][EOS_token])\n",
    "    return np.sum(scores)\n",
    "\n",
    "def get_edit_distance_repr(components, blend):\n",
    "    import difflib\n",
    "    add=False\n",
    "    output=[]\n",
    "    for c in difflib.ndiff(components, blend):\n",
    "        if c[0]=='-':\n",
    "            output.append('D')\n",
    "        elif c[0]=='+':\n",
    "            add = True\n",
    "            output.append(c[1:].strip())\n",
    "        else:\n",
    "            output.append('C')\n",
    "    return ''.join(output), add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_predict_lm_ensemble(ensemble, examples, lmodel, length_model, alpha=0.5):  \n",
    "    import itertools\n",
    "    outputs=[]\n",
    "    preds=[]\n",
    "    for example in examples:\n",
    "        ensemble_scores=[]\n",
    "        for m in ensemble:\n",
    "            m.hidden=m.init_hidden(batch_size=m.batch_size)\n",
    "            t = encode_word(list(itertools.repeat(example, m.batch_size)))\n",
    "            scores = m(t)\n",
    "            scores=F.log_softmax(scores[0,:]).cpu().data.numpy()\n",
    "            ensemble_scores.append(scores)\n",
    "        scores = np.mean(ensemble_scores, axis=0) # Average all scores\n",
    "        candidates, lstm_scores=zip(*get_k_best_blends(scores, example, k=20))\n",
    "        pm_scores=[]\n",
    "        for c in candidates:\n",
    "            if c in phoneme_score_dict:\n",
    "                pm_scores.append(phoneme_score_dict[c])\n",
    "            else:\n",
    "                pm_scores.append(0.0)\n",
    "        length_scores=[score_candidate_length(c, example, length_model) for c in candidates]\n",
    "        lm_scores=[lmodel.score(' '.join(list(c)))/(float(len(c))**alpha) for c in candidates]\n",
    "        odf=pd.DataFrame().from_records(zip(candidates, lstm_scores,lm_scores, pm_scores, length_scores), columns=['candidate','lstm','lm', 'pm','lengthscore'])\n",
    "        odf[\"total\"]=odf[\"lstm\"]+odf[\"lm\"]+odf[\"pm\"]+odf[\"lengthscore\"]\n",
    "        odf=odf.sort_values(by=['total'], ascending=False)\n",
    "        odf=odf.reset_index()\n",
    "        preds.append(odf.ix[0].candidate)\n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bdf=pd.read_csv('./data/components-blends-blind.csv', sep='\\t', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "blind_pairs=zip(bdf.source.values, bdf.target.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1078\n"
     ]
    }
   ],
   "source": [
    "print len(blind_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_edit_distance_blind_lm_ensemble(embedding_dim, hidden_dim, vocab_size, tagset_size, batch_size, ensemble_size, model_filename, split_file_name, length_model):\n",
    "    ensemble=[]\n",
    "    t_pairs=blind_pairs\n",
    "    test_pairs=zip(*blind_pairs)[0]\n",
    "    for e in np.arange(0, ensemble_size):\n",
    "        model_filename = model_filename.format(e)\n",
    "        m=load_model(embedding_dim=embedding_dim,hidden_dim=hidden_dim,vocab_size=vocab_size,tagset_size=tagset_size, batch_size=batch_size, model_filename=model_filename)\n",
    "        ensemble.append(m)\n",
    "    preds=get_predict_lm_ensemble(ensemble, test_pairs, lmodel, length_model)\n",
    "    gold_preds=[get_from_copy_edit(val_pair[0], val_pair[1]) for val_pair in t_pairs]\n",
    "    edit_scores=[Levenshtein.distance(p,g) for p, g in zip(preds, gold_preds)]\n",
    "    return np.mean(edit_scores), preds, gold_preds, edit_scores, ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_edit_distance_lm_blind(embedding_dim, hidden_dim, vocab_size, tagset_size, batch_size, model_filename, split_file_name, length_model_name, use_lm=False, use_pm=False, use_length=False):\n",
    "    m=load_model(embedding_dim=embedding_dim,hidden_dim=hidden_dim,vocab_size=vocab_size,tagset_size=tagset_size, batch_size=batch_size, model_filename=model_filename)\n",
    "    _, _, _,val_pairs,_=pickle.load(open(split_file_name,'rb'))\n",
    "    length_model = pickle.load(open(length_model_name,'rb'))\n",
    "    t_pairs=blind_pairs\n",
    "    test_pairs=list(zip(*t_pairs)[0])\n",
    "    preds, _=get_predict_lm(m,test_pairs, lmodel, length_model, use_lm=use_lm, use_pm=use_pm, use_length=use_length)\n",
    "    gold_preds=[get_from_copy_edit(val_pair[0], val_pair[1]) for val_pair in t_pairs]\n",
    "    edit_scores=[Levenshtein.distance(p,g) for p, g in zip(preds, gold_preds)]\n",
    "    return np.mean(edit_scores), preds, gold_preds, edit_scores, m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "Filtered to 397 pairs\n",
      "Indexing words...\n",
      "Indexed 30 words in input language, 5 words in output\n",
      "Trimming languages\n",
      "keep_words 27 / 27 = 1.0000\n",
      "keep_words 2 / 2 = 1.0000\n",
      "Read 397 sentence pairs\n",
      "Trimmed from 397 pairs to 397, 1.0000 of total\n",
      "Testing we can get a random batch.\n",
      "Testing models\n",
      "{u'C': 3, u'D': 4}\n"
     ]
    }
   ],
   "source": [
    "source_lang='components'\n",
    "target_lang='blends-knight'\n",
    "input_lang, output_lang, pairs = prepare_data(source_lang, target_lang, split_by_char_tokenizer, join_by_char_tokenizer, split_by_char_tokenizer, join_by_char_tokenizer, False)                                                                                                                                                                              \n",
    "print output_lang.word2index                                                            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Only Length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vvkulkarni/postdoc/lib/python2.7/site-packages/ipykernel_launcher.py:31: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.271057513914657\n"
     ]
    }
   ],
   "source": [
    "escores=[]\n",
    "for fno in np.arange(0, 10):\n",
    "    e,preds,gold_preds,edit_scores, m=get_edit_distance_lm_blind(50,50,input_lang.n_words, output_lang.n_words, 20, './seq2seq_output/test_knight//lstm_{}_best'.format(fno),'./seq2seq_output/test_knight//split_lstm_{}'.format(fno), './seq2seq_output/test_knight_length/length_{}'.format(fno))\n",
    "    escores.append(e)\n",
    "print np.mean(escores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vvkulkarni/postdoc/lib/python2.7/site-packages/ipykernel_launcher.py:31: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1398886827458257\n"
     ]
    }
   ],
   "source": [
    "escores=[]\n",
    "for fno in np.arange(0, 10):\n",
    "    e,preds,gold_preds,edit_scores, m=get_edit_distance_lm_blind(50,50,input_lang.n_words, output_lang.n_words, 20, './seq2seq_output/test_knight//lstm_{}_best'.format(fno),'./seq2seq_output/test_knight//split_lstm_{}'.format(fno), './seq2seq_output/test_knight_length/length_{}'.format(fno), use_lm=True, use_pm=True)\n",
    "    escores.append(e)\n",
    "print np.mean(escores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LM + LEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vvkulkarni/postdoc/lib/python2.7/site-packages/ipykernel_launcher.py:31: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.9825602968460114\n"
     ]
    }
   ],
   "source": [
    "escores=[]\n",
    "for fno in np.arange(0, 10):\n",
    "    e,preds,gold_preds,edit_scores, m=get_edit_distance_lm_blind(50,50,input_lang.n_words, output_lang.n_words, 20, './seq2seq_output/test_knight//lstm_{}_best'.format(fno),'./seq2seq_output/test_knight//split_lstm_{}'.format(fno), './seq2seq_output/test_knight_length/length_{}'.format(fno), use_lm=True, use_pm=True, use_length=True)\n",
    "    escores.append(e)\n",
    "print np.mean(escores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM + LEN+ Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vvkulkarni/postdoc/lib/python2.7/site-packages/ipykernel_launcher.py:27: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.9167903525046381 1.920686456400742\n"
     ]
    }
   ],
   "source": [
    "embedding_dim=50\n",
    "hidden_dim=50\n",
    "vocab_size=input_lang.n_words\n",
    "tagset_size=output_lang.n_words\n",
    "batch_size=20\n",
    "escores=[]\n",
    "for fno in np.arange(0,10):\n",
    "    model_filename=\"./seq2seq_output/test_knight_ensemble///lstm_{}_{}_best\".format(fno,'{}')\n",
    "    split_file_name = \"./seq2seq_output/test_knight_ensemble///split_lstm_{}_{}\".format(fno, 0)\n",
    "    length_model = pickle.load(open('./seq2seq_output/test_knight_length/length_{}'.format(fno),'rb'))\n",
    "    e,preds,gold_preds,edit_scores,m=get_edit_distance_blind_lm_ensemble(embedding_dim, hidden_dim, vocab_size, tagset_size, batch_size, ensemble_size=50, model_filename=model_filename, split_file_name=split_file_name, length_model=length_model)\n",
    "    escores.append(e)\n",
    "print np.mean(escores), np.median(escores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}