Computing SEARs for sentiment analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Path to sears repository\n",
    "sys.path.append('sears') # noqa\n",
    "import paraphrase_scorer\n",
    "import onmt_model\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU ID 0\n",
      "Loading model parameters.\n",
      "Loading model parameters.\n",
      "Loading model parameters.\n",
      "Loading model parameters.\n"
     ]
    }
   ],
   "source": [
    "ps = paraphrase_scorer.ParaphraseScorer(gpu_id=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "def load_polarity(path='/home/marcotcr/phd/datasets/sentiment-sentences/'):\n",
    "    data = []\n",
    "    labels = []\n",
    "    f_names = ['rt-polarity.neg', 'rt-polarity.pos']\n",
    "    for (l, f) in enumerate(f_names):\n",
    "        for line in open(os.path.join(path, f), 'rb'):\n",
    "            try:\n",
    "                line.decode('utf8')\n",
    "            except:\n",
    "                continue\n",
    "            data.append(line.decode('utf-8').strip().replace('. . .', '...'))\n",
    "            labels.append(l)\n",
    "    label_names = ['negative', 'positive']\n",
    "    return data, labels, label_names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_polarity_imdb(path='/home/marcotcr/phd/datasets/sentiment-sentences-other/'):\n",
    "    data = []\n",
    "    labels = []\n",
    "    # f_names = ['amazon_cells_labelled.txt', 'imdb_labelled.txt', 'yelp_labelled.txt']\n",
    "    f_names = ['imdb_labelled.txt']\n",
    "    for (l, f) in enumerate(f_names):\n",
    "        for line in open(os.path.join(path, f), 'rb'):\n",
    "            try:\n",
    "                line.decode('utf8')\n",
    "            except:\n",
    "                continue\n",
    "            sentence, label = line.decode('utf-8').split('\\t')\n",
    "            label = int(label)\n",
    "            data.append(sentence.strip())\n",
    "            labels.append(label)\n",
    "    label_names = ['negative', 'positive']\n",
    "    return data, labels, label_names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en')\n",
    "import replace_rules\n",
    "tokenizer = replace_rules.Tokenizer(nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = pickle.load(open('polarity.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = all_data['data']\n",
    "labels = all_data['labels']\n",
    "label_names = all_data['label_names']\n",
    "val = all_data['imdb']\n",
    "val_labels = all_data['imdb_labels']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# pickle.dump({'data': data, 'labels': labels, 'label_names': label_names, 'imdb': val, 'imdb_labels': val_labels}, open('/tmp/polarity.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = tokenizer.clean_for_model(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# val, val_labels, _ = load_polarity_imdb()\n",
    "clean_val = tokenizer.clean_for_model(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/marcotcr/phd/sears/ENV/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average train sequence length: 37\n",
      "Epoch 1/10\n",
      " - 4s - loss: 0.6882 - acc: 0.5868\n",
      "Epoch 2/10\n",
      " - 3s - loss: 0.6378 - acc: 0.7744\n",
      "Epoch 3/10\n",
      " - 3s - loss: 0.5177 - acc: 0.8703\n",
      "Epoch 4/10\n",
      " - 3s - loss: 0.3858 - acc: 0.9220\n",
      "Epoch 5/10\n",
      " - 3s - loss: 0.2802 - acc: 0.9564\n",
      "Epoch 6/10\n",
      " - 3s - loss: 0.2042 - acc: 0.9776\n",
      "Epoch 7/10\n",
      " - 3s - loss: 0.1514 - acc: 0.9865\n",
      "Epoch 8/10\n",
      " - 3s - loss: 0.1141 - acc: 0.9924\n",
      "Epoch 9/10\n",
      " - 3s - loss: 0.0880 - acc: 0.9945\n",
      "Epoch 10/10\n",
      " - 3s - loss: 0.0680 - acc: 0.9965\n"
     ]
    }
   ],
   "source": [
    "model = fasttext.FastTextClassifier()\n",
    "model.fit(data, labels, ngram_range=2, epochs=10, maxlen=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.772"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(model.predict(clean_val) == val_labels).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_for_onmt = [' '.join([a.text for a in x]) for x in nlp.tokenizer.pipe(val)]\n",
    "val_for_onmt = [onmt_model.clean_text(x, only_upper=False) for x in val_for_onmt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "right = np.where(model.predict(clean_val) == val_labels)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "right_preds = np.array([val_labels[i] for i in right])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def find_flips(instance, model, topk=10, threshold=-10, ):\n",
    "    orig_pred = model.predict([instance])[0]\n",
    "    instance_for_onmt = onmt_model.clean_text(' '.join([x.text for x in nlp.tokenizer(instance)]), only_upper=False)\n",
    "    paraphrases = ps.generate_paraphrases(instance_for_onmt, topk=topk, edit_distance_cutoff=4, threshold=threshold)\n",
    "    texts = tokenizer.clean_for_model(tokenizer.clean_for_humans([x[0] for x in paraphrases]))\n",
    "    preds = model.predict(texts)\n",
    "    fs = [(texts[i], paraphrases[i][1]) for i in np.where(preds != orig_pred)[0]]\n",
    "    return fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "orig_scores = {}\n",
    "flips = collections.defaultdict(lambda: [])\n",
    "for i, idx in enumerate(right):\n",
    "    if i % 100 == 0:\n",
    "        print(i)\n",
    "    if val[idx] in flips:\n",
    "        continue\n",
    "    fs = find_flips(val[idx], model, topk=100, threshold=-10)\n",
    "    flips[val[idx]].extend([x[0] for x in fs])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "right_val = [clean_val[i] for i in right]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tr2 = replace_rules.TextToReplaceRules(nlp, right_val, [], min_freq=0.005, min_flip=0.00, ngram_size=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "frequent_rules = []\n",
    "rule_idx = {}\n",
    "rule_flips = {}\n",
    "for z, f in enumerate(flips):\n",
    "    rules = tr2.compute_rules(f, flips[f], use_pos=True, use_tags=True)\n",
    "    for rs in rules:\n",
    "        for r in rs:\n",
    "            if r.hash() not in rule_idx:\n",
    "                i = len(rule_idx)\n",
    "                rule_idx[r.hash()] = i\n",
    "                rule_flips[i] = []\n",
    "                frequent_rules.append(r)\n",
    "            i = rule_idx[r.hash()]\n",
    "            rule_flips[i].append(z)\n",
    "    if z % 500 == 0:\n",
    "        print (z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "token_right = tokenizer.tokenize(right_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_preds = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "234706"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(frequent_rules)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "5000\n",
      "10000\n",
      "15000\n",
      "20000\n",
      "25000\n",
      "30000\n",
      "35000\n",
      "40000\n",
      "45000\n",
      "50000\n",
      "55000\n",
      "60000\n",
      "65000\n",
      "70000\n",
      "75000\n",
      "80000\n",
      "85000\n",
      "90000\n",
      "95000\n",
      "100000\n",
      "105000\n",
      "110000\n",
      "115000\n",
      "120000\n",
      "125000\n",
      "130000\n",
      "135000\n",
      "140000\n",
      "145000\n",
      "150000\n",
      "155000\n",
      "160000\n",
      "165000\n",
      "170000\n",
      "175000\n",
      "180000\n",
      "185000\n",
      "190000\n",
      "195000\n",
      "200000\n",
      "205000\n",
      "210000\n",
      "215000\n",
      "220000\n",
      "225000\n",
      "230000\n",
      "364.8736500740051\n"
     ]
    }
   ],
   "source": [
    "a = time.time()\n",
    "rule_flips = {}\n",
    "rule_other_texts = {}\n",
    "rule_other_flips = {}\n",
    "rule_applies = {}\n",
    "for i, r in enumerate(frequent_rules):\n",
    "    idxs = list(tr2.get_rule_idxs(r))\n",
    "    to_apply = [token_right[x] for x in idxs]\n",
    "    applies, nt = r.apply_to_texts(to_apply, fix_apostrophe=False)\n",
    "    applies = [idxs[x] for x in applies]\n",
    "    old_texts = [right_val[i] for i in applies]\n",
    "    old_labels = right_preds[applies]\n",
    "    to_compute = [x for x in nt if x not in model_preds]\n",
    "    if to_compute:\n",
    "        preds = model.predict(to_compute)\n",
    "        for x, y in zip(to_compute, preds):\n",
    "            model_preds[x] = y\n",
    "    new_labels = np.array([model_preds[x] for x in nt])\n",
    "    where_flipped = np.where(new_labels != old_labels)[0]\n",
    "    flips = sorted([applies[x] for x in where_flipped])\n",
    "    rule_flips[i] = flips\n",
    "    rule_other_texts[i] = nt\n",
    "    rule_other_flips[i] = where_flipped\n",
    "    rule_applies[i] = applies\n",
    "    if i % 5000 == 0:\n",
    "        print(i)\n",
    "print(time.time() - a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "really_frequent_rules = [i for i in range(len(rule_flips)) if len(rule_flips[i]) > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# to_compute_score = collections.defaultdict(lambda: set())\n",
    "# for i in really_frequent_rules:\n",
    "#     orig_texts =  [right_val[z] for z in rule_applies[i]]\n",
    "#     new_texts = rule_other_texts[i]\n",
    "#     for o, n in zip(orig_texts, new_texts):\n",
    "#         to_compute_score[o].add(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "threshold = -7.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orig_scores = {}\n",
    "for i, t in enumerate(right_val):\n",
    "    orig_scores[i] = ps.score_sentences(t, [t])[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I want rules s.t. the decile > -7.15. The current bottom 10% of a rule is always a lower bound on the decile, so if I see applies / 10 with score < -7.15 I can stop computing scores for that rule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ps_scores = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ps.last = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "rule_scores = []\n",
    "rejected = set()\n",
    "for idx, i in enumerate(really_frequent_rules):\n",
    "    orig_texts =  [right_val[z] for z in rule_applies[i]]\n",
    "    orig_scor = [orig_scores[z] for z in rule_applies[i]]\n",
    "    scores = np.ones(len(orig_texts)) * -50\n",
    "#     if idx in rejected:\n",
    "#         rule_scores.append(scores)\n",
    "#         continue\n",
    "    decile = np.ceil(.1 * len(orig_texts))\n",
    "    new_texts = rule_other_texts[i]\n",
    "    bad_scores = 0\n",
    "    for j, (o, n, orig) in enumerate(zip(orig_texts, new_texts, orig_scor)):\n",
    "        if o not in ps_scores:\n",
    "            ps_scores[o] = {}\n",
    "        if n not in ps_scores[o]:\n",
    "            if n == '':\n",
    "                score = -40\n",
    "            else:\n",
    "                score = ps.score_sentences(o, [n])[0]\n",
    "            ps_scores[o][n] = min(0, score - orig)\n",
    "        scores[j] = ps_scores[o][n]\n",
    "        if ps_scores[o][n] < threshold:\n",
    "            bad_scores += 1\n",
    "        if bad_scores >= decile:\n",
    "            rejected.add(idx)\n",
    "            break\n",
    "    rule_scores.append(scores)\n",
    "            \n",
    "    if i % 100 == 0:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# pickle.dump({'ps_scores': ps_scores, 'orig_scores': orig_scores}, open('/home/marcotcr/tmp/polarity_scoresz.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "760"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rule_scores) - len(rejected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rule_flip_scores = [rule_scores[i][rule_other_flips[really_frequent_rules[i]]] for i in range(len(rule_scores))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "frequent_flips = [np.array(rule_applies[i])[rule_other_flips[i]] for i in really_frequent_rules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rule_precsupports = [len(rule_applies[i]) for i in really_frequent_rules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from rule_picking import disqualify_rules\n",
    "threshold=-7.15\n",
    "# x = choose_rules_coverage(fake_scores, frequent_flips, frequent_supports,\n",
    "disqualified = disqualify_rules(rule_scores, frequent_flips,\n",
    "                          rule_precsupports, \n",
    "                      min_precision=0.0, min_flips=6, \n",
    "                         min_bad_score=threshold, max_bad_proportion=.10,\n",
    "                          max_bad_sum=999999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1215,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# [(i, x.hash()) for (i, x) in enumerate(frequent_rules) if 'text_movie -> text_film' in x.hash()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2581441402435303\n",
      "Instances flipped: 81 (0.10)\n",
      "0     204   249   movie -> film                       f:11 avg_s:0.82 bad_s:0.06 bad_sum:7 Prec:0.09 Supp:0.16\n",
      "1     2282  3895  film -> movie                       f:16 avg_s:0.50 bad_s:0.03 bad_sum:4 Prec:0.14 Supp:0.15\n",
      "2     8732  12537 this -> that                        f:6 avg_s:0.42 bad_s:0.05 bad_sum:7 Prec:0.04 Supp:0.19\n",
      "3     123033 171222 film VERB -> movie VERB             f:6 avg_s:0.52 bad_s:0.09 bad_sum:3 Prec:0.17 Supp:0.05\n",
      "4     8839  12672 DET NN -> the NN                    f:12 avg_s:0.04 bad_s:0.08 bad_sum:36 Prec:0.03 Supp:0.58\n",
      "5     25301 36366 is -> was                           f:28 avg_s:0.02 bad_s:0.04 bad_sum:9 Prec:0.12 Supp:0.31\n",
      "6     226   305   this movie -> that film             f:7 avg_s:0.05 bad_s:0.03 bad_sum:1 Prec:0.21 Supp:0.04\n",
      "7     97147 135986 It 's -> It was                     f:6 avg_s:0.02 bad_s:0.04 bad_sum:1 Prec:0.23 Supp:0.03\n",
      "8     64161 91161 VERB PRON . -> VERB him .           f:6 avg_s:0.03 bad_s:0.04 bad_sum:1 Prec:0.24 Supp:0.03\n",
      "9     16359 21075 DET movie -> the film               f:9 avg_s:0.56 bad_s:0.07 bad_sum:6 Prec:0.10 Supp:0.12\n"
     ]
    }
   ],
   "source": [
    "from rule_picking import choose_rules_coverage\n",
    "threshold=-7.15\n",
    "a = time.time()\n",
    "x = choose_rules_coverage(rule_flip_scores, frequent_flips, None,\n",
    "                          None, len(right_preds),\n",
    "                                frequent_scores_on_all=None, k=10, metric='max',\n",
    "                      min_precision=0.0, min_flips=0, exp=True,\n",
    "                         min_bad_score=threshold, max_bad_proportion=.1,\n",
    "                          max_bad_sum=999999,\n",
    "                         disqualified=disqualified,\n",
    "                         start_from=[])\n",
    "print(time.time() -a)\n",
    "support_denominator = float(len(right_preds))\n",
    "soup = lambda x: len(rule_applies[really_frequent_rules[x]]) / support_denominator \n",
    "prec = lambda x: frequent_flips[x].shape[0] / float(len(rule_scores[x]))\n",
    "fl = len(set([a for r in x for a in frequent_flips[r]]))\n",
    "print('Instances flipped: %d (%.2f)' % (fl, fl / float(len(right_preds))))\n",
    "print('\\n'.join(['%-5d %-5d %-5d %-35s f:%d avg_s:%.2f bad_s:%.2f bad_sum:%d Prec:%.2f Supp:%.2f' % (\n",
    "                i, x[i], really_frequent_rules[r],\n",
    "                frequent_rules[really_frequent_rules[r]].hash().replace('text_', '').replace('pos_', '').replace('tag_', ''),\n",
    "                frequent_flips[r].shape[0],\n",
    "                np.exp(rule_flip_scores[r]).mean(), (rule_scores[r] < threshold).mean(),\n",
    "                (rule_scores[r] < threshold).sum(), prec(r), soup(r)) for i, r in enumerate(x)]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a couple of examples from the first rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rule: text_movie -> text_film\n",
      "\n",
      "A very , very , very slow - moving , aimless movie about a distressed , drifting young man .\n",
      "P(positive):0.38\n",
      "\n",
      "A very , very , very slow - moving , aimless film about a distressed , drifting young man .\n",
      "P(positive):0.51\n",
      "\n",
      "\n",
      "Yeah , the movie pretty much sucked .\n",
      "P(positive):0.29\n",
      "\n",
      "Yeah , the film pretty much sucked .\n",
      "P(positive):0.52\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: text_film -> text_movie\n",
      "\n",
      "Yes , this film does require a rather significant amount of puzzle - solving , but the pieces fit together to create a beautiful picture .\n",
      "P(positive):0.53\n",
      "\n",
      "Yes , this movie does require a rather significant amount of puzzle - solving , but the pieces fit together to create a beautiful picture .\n",
      "P(positive):0.32\n",
      "\n",
      "\n",
      "The film succeeds despite , or perhaps because of , an obviously meagre budget .\n",
      "P(positive):0.50\n",
      "\n",
      "The movie succeeds despite , or perhaps because of , an obviously meagre budget .\n",
      "P(positive):0.22\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: text_this -> text_that\n",
      "\n",
      "If there was ever a movie that needed word - of - mouth to promote , this is it .\n",
      "P(positive):0.57\n",
      "\n",
      "If there was ever a movie that needed word - of - mouth to promote , that is it .\n",
      "P(positive):0.49\n",
      "\n",
      "\n",
      "I 'll even say it again  this is torture .\n",
      "P(positive):0.44\n",
      "\n",
      "I'll even say it again  that is torture .\n",
      "P(positive):0.50\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: text_film pos_VERB -> text_movie pos_VERB\n",
      "\n",
      "Yes , this film does require a rather significant amount of puzzle - solving , but the pieces fit together to create a beautiful picture .\n",
      "P(positive):0.53\n",
      "\n",
      "Yes , this movie does require a rather significant amount of puzzle - solving , but the pieces fit together to create a beautiful picture .\n",
      "P(positive):0.32\n",
      "\n",
      "\n",
      "The film succeeds despite , or perhaps because of , an obviously meagre budget .\n",
      "P(positive):0.50\n",
      "\n",
      "The movie succeeds despite , or perhaps because of , an obviously meagre budget .\n",
      "P(positive):0.22\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: pos_DET tag_NN -> text_the tag_NN\n",
      "\n",
      "It 's hard not to fall head - over - heels in love with that girl .\n",
      "P(positive):0.52\n",
      "\n",
      "It's hard not to fall head - over - heels in love with the girl .\n",
      "P(positive):0.45\n",
      "\n",
      "\n",
      "When I first watched this movie , in the 80s , I loved it .\n",
      "P(positive):0.52\n",
      "\n",
      "When I first watched the movie , in the 80s , I loved it .\n",
      "P(positive):0.40\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: text_is -> text_was\n",
      "\n",
      "It is an insane game .\n",
      "P(positive):0.76\n",
      "\n",
      "It was an insane game .\n",
      "P(positive):0.38\n",
      "\n",
      "\n",
      "I do think Tom Hanks is a good actor .\n",
      "P(positive):0.63\n",
      "\n",
      "I do think Tom Hanks was a good actor .\n",
      "P(positive):0.47\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: text_this text_movie -> text_that text_film\n",
      "\n",
      "But this movie is definitely a below average rent .\n",
      "P(positive):0.32\n",
      "\n",
      "But that film is definitely a below average rent .\n",
      "P(positive):0.69\n",
      "\n",
      "\n",
      "No actress has been worse used that June Allison in this movie .\n",
      "P(positive):0.37\n",
      "\n",
      "No actress has been worse used that June Allison in that film .\n",
      "P(positive):0.52\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: text_It text_'s -> text_It text_was\n",
      "\n",
      "It 's PURE BRILLIANCE .\n",
      "P(positive):0.54\n",
      "\n",
      "It was PURE BRILLIANCE .\n",
      "P(positive):0.34\n",
      "\n",
      "\n",
      "It 's hard not to fall head - over - heels in love with that girl .\n",
      "P(positive):0.52\n",
      "\n",
      "It was hard not to fall head - over - heels in love with that girl .\n",
      "P(positive):0.41\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: pos_VERB pos_PRON tag_. -> pos_VERB text_him tag_.\n",
      "\n",
      "When I first watched this movie , in the 80s , I loved it .\n",
      "P(positive):0.52\n",
      "\n",
      "When I first watched this movie , in the 80s , I loved him .\n",
      "P(positive):0.40\n",
      "\n",
      "\n",
      "I rather enjoyed it .\n",
      "P(positive):0.56\n",
      "\n",
      "I rather enjoyed him .\n",
      "P(positive):0.43\n",
      "\n",
      "\n",
      "---------------\n",
      "Rule: pos_DET text_movie -> text_the text_film\n",
      "\n",
      "Yeah , the movie pretty much sucked .\n",
      "P(positive):0.29\n",
      "\n",
      "Yeah , the film pretty much sucked .\n",
      "P(positive):0.52\n",
      "\n",
      "\n",
      "This movie suffered because of the writing , it needed more suspense .\n",
      "P(positive):0.42\n",
      "\n",
      "the film suffered because of the writing , it needed more suspense .\n",
      "P(positive):0.59\n",
      "\n",
      "\n",
      "---------------\n"
     ]
    }
   ],
   "source": [
    "for r in x:\n",
    "    rid = really_frequent_rules[r]\n",
    "    rule =  frequent_rules[rid]\n",
    "    print('Rule: %s' % rule.hash())\n",
    "    print()\n",
    "    for f in rule_flips[rid][:2]:\n",
    "        print('%s\\nP(positive):%.2f' % (right_val[f], model.predict_proba([right_val[f]])[0, 1]))\n",
    "        print()\n",
    "        new = rule.apply(token_right[f])[1]\n",
    "        print('%s\\nP(positive):%.2f' % (new, model.predict_proba([new])[0, 1]))\n",
    "        print()\n",
    "        print()\n",
    "    print('---------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sears",
   "language": "python",
   "name": "sears"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}