data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from surprise import SVD\n",
    "from surprise import Dataset\n",
    "from surprise import Reader\n",
    "from surprise import evaluate, print_perf\n",
    "from surprise import KNNBasic\n",
    "import gzip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load & Sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt = pd.read_csv('ratings_Grocery_and_Gourmet_Food.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1297155 entries, 0 to 1297154\n",
      "Data columns (total 4 columns):\n",
      "A1ZQZ8RJS1XVTX    1297155 non-null object\n",
      "0657745316        1297155 non-null object\n",
      "5.0               1297155 non-null float64\n",
      "1381449600        1297155 non-null int64\n",
      "dtypes: float64(1), int64(1), object(2)\n",
      "memory usage: 39.6+ MB\n"
     ]
    }
   ],
   "source": [
    "dt.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt.columns = ['reviewerID','productID','rating','unixTime']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>productID</th>\n",
       "      <th>rating</th>\n",
       "      <th>unixTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A31W38VGZAUUM4</td>\n",
       "      <td>0700026444</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1354752000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A3I0AV0UJX5OH0</td>\n",
       "      <td>1403796890</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1385942400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A3QAAOLIXKV383</td>\n",
       "      <td>1403796890</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1307836800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AB1A5EGHHVA9M</td>\n",
       "      <td>141278509X</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1332547200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A3DTB6RVENLQ9Q</td>\n",
       "      <td>1453060375</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1362268800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       reviewerID   productID  rating    unixTime\n",
       "0  A31W38VGZAUUM4  0700026444     5.0  1354752000\n",
       "1  A3I0AV0UJX5OH0  1403796890     1.0  1385942400\n",
       "2  A3QAAOLIXKV383  1403796890     3.0  1307836800\n",
       "3   AB1A5EGHHVA9M  141278509X     5.0  1332547200\n",
       "4  A3DTB6RVENLQ9Q  1453060375     1.0  1362268800"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt['unixTime'] = pd.to_datetime(dt['unixTime'],unit='s')\n",
    "dt = dt.rename(columns={'unixTime':'date'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timestamp('2014-07-23 00:00:00')"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Only take recent reviews\n",
    "dt = dt.loc[dt['date']>'2011-01-01']\n",
    "#Last Date\n",
    "dt['date'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFfBJREFUeJzt3X+snuV93/H3JxhSloSYwIEh25kz\n1cpK0ELAclwhRVnojCFVjLQgEW3BQVTeGOkSZVJL+seskkWi/zQdW0rFgofdJSGMNsNLTVyPJKom\nBcIhoRBCMp9SGo5g2MGE0LEmIv3uj+dy8+TwnHOu4+Bzn+D3S3r03Pf3vu77us5lDp9z/3jOSVUh\nSVKPVw09AEnSzw9DQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSt1VDD+DlduaZ\nZ9b69euHHoYk/Vx54IEHvldVU4u1e8WFxvr165menh56GJL0cyXJX/W08/KUJKmboSFJ6mZoSJK6\nGRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqdsr7hPhkjSk9df/ySD9Pn7ju5elH880JEndDA1J\nUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1G3R0Ejy5iQPjr1+kOTDSd6Q5ECSg+399NY+SW5KMpPk\noSQXjB1re2t/MMn2sfqFSR5u+9yUJK0+sQ9J0jAWDY2q+k5VnV9V5wMXAi8AnweuB+6pqg3APW0d\n4FJgQ3vtAG6GUQAAO4G3A5uAnWMhcHNre3S/ra0+Xx+SpAEs9fLUxcBfVNVfAduA3a2+G7i8LW8D\n9tTIvcDqJOcAlwAHqupIVT0LHAC2tm2nVdVXq6qAPXOONakPSdIAlhoaVwKfbctnV9VTAO39rFZf\nAzwxts9sqy1Un51QX6iPn5JkR5LpJNOHDx9e4pckSerVHRpJTgHeA/y3xZpOqNUx1LtV1S1VtbGq\nNk5NTS1lV0nSEizlTONS4OtV9XRbf7pdWqK9H2r1WWDd2H5rgScXqa+dUF+oD0nSAJYSGu/jJ5em\nAPYCR5+A2g7cNVa/qj1FtRl4rl1a2g9sSXJ6uwG+Bdjftj2fZHN7auqqOcea1IckaQBdvxo9yd8D\n/inwL8fKNwJ3JLkG+C5wRavvAy4DZhg9aXU1QFUdSfIx4P7W7oaqOtKWrwVuA04F7m6vhfqQJA2g\nKzSq6gXgjDm1Zxg9TTW3bQHXzXOcXcCuCfVp4LwJ9Yl9SJKG4SfCJUndDA1JUjdDQ5LUzdCQJHUz\nNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUz\nNCRJ3QwNSVK3rtBIsjrJnUm+neTRJL+c5A1JDiQ52N5Pb22T5KYkM0keSnLB2HG2t/YHk2wfq1+Y\n5OG2z01J0uoT+5AkDaP3TOM/AF+sqn8EvBV4FLgeuKeqNgD3tHWAS4EN7bUDuBlGAQDsBN4ObAJ2\njoXAza3t0f22tvp8fUiSBrBoaCQ5DXgHcCtAVf2oqr4PbAN2t2a7gcvb8jZgT43cC6xOcg5wCXCg\nqo5U1bPAAWBr23ZaVX21qgrYM+dYk/qQJA2g50zjHwKHgf+S5BtJPpXkNcDZVfUUQHs/q7VfAzwx\ntv9sqy1Un51QZ4E+JEkD6AmNVcAFwM1V9Tbg/7LwZaJMqNUx1Lsl2ZFkOsn04cOHl7KrJGkJekJj\nFpitqvva+p2MQuTpdmmJ9n5orP26sf3XAk8uUl87oc4CffyUqrqlqjZW1capqamOL0mSdCwWDY2q\n+j/AE0ne3EoXA98C9gJHn4DaDtzVlvcCV7WnqDYDz7VLS/uBLUlObzfAtwD727bnk2xuT01dNedY\nk/qQJA1gVWe7Xwc+neQU4DHgakaBc0eSa4DvAle0tvuAy4AZ4IXWlqo6kuRjwP2t3Q1VdaQtXwvc\nBpwK3N1eADfO04ckaQBdoVFVDwIbJ2y6eELbAq6b5zi7gF0T6tPAeRPqz0zqQ5I0DD8RLknqZmhI\nkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhI\nkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG5doZHk8SQPJ3kwyXSrvSHJgSQH2/vprZ4kNyWZSfJQ\nkgvGjrO9tT+YZPtY/cJ2/Jm2bxbqQ5I0jKWcafyTqjq/qja29euBe6pqA3BPWwe4FNjQXjuAm2EU\nAMBO4O3AJmDnWAjc3Noe3W/rIn1Ikgbws1ye2gbsbsu7gcvH6ntq5F5gdZJzgEuAA1V1pKqeBQ4A\nW9u206rqq1VVwJ45x5rUhyRpAL2hUcCfJnkgyY5WO7uqngJo72e1+hrgibF9Z1ttofrshPpCfUiS\nBrCqs91FVfVkkrOAA0m+vUDbTKjVMdS7tSDbAfDGN75xKbtKkpag60yjqp5s74eAzzO6J/F0u7RE\nez/Ums8C68Z2Xws8uUh97YQ6C/Qxd3y3VNXGqto4NTXV8yVJko7BoqGR5DVJXnd0GdgCfBPYCxx9\nAmo7cFdb3gtc1Z6i2gw81y4t7Qe2JDm93QDfAuxv255Psrk9NXXVnGNN6kOSNICey1NnA59vT8Gu\nAj5TVV9Mcj9wR5JrgO8CV7T2+4DLgBngBeBqgKo6kuRjwP2t3Q1VdaQtXwvcBpwK3N1eADfO04ck\naQCLhkZVPQa8dUL9GeDiCfUCrpvnWLuAXRPq08B5vX1IkobhJ8IlSd0MDUlSN0NDktTN0JAkdTM0\nJEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0\nJEndDA1JUjdDQ5LUrTs0kpyU5BtJvtDW35TkviQHk3wuySmt/uq2PtO2rx87xkdb/TtJLhmrb221\nmSTXj9Un9iFJGsZSzjQ+BDw6tv47wCeqagPwLHBNq18DPFtVvwh8orUjybnAlcBbgK3A77cgOgn4\nJHApcC7wvtZ2oT4kSQPoCo0ka4F3A59q6wHeBdzZmuwGLm/L29o6bfvFrf024Paq+mFV/SUwA2xq\nr5mqeqyqfgTcDmxbpA9J0gB6zzR+D/gN4G/b+hnA96vqxbY+C6xpy2uAJwDa9uda+7+rz9lnvvpC\nfUiSBrBoaCT5VeBQVT0wXp7QtBbZ9nLVJ41xR5LpJNOHDx+e1ESS9DLoOdO4CHhPkscZXTp6F6Mz\nj9VJVrU2a4En2/IssA6gbX89cGS8Pmef+erfW6CPn1JVt1TVxqraODU11fElSZKOxaKhUVUfraq1\nVbWe0Y3sL1XVPwe+DLy3NdsO3NWW97Z12vYvVVW1+pXt6ao3ARuArwH3Axvak1KntD72tn3m60OS\nNICf5XMavwl8JMkMo/sPt7b6rcAZrf4R4HqAqnoEuAP4FvBF4Lqq+nG7Z/FBYD+jp7PuaG0X6kOS\nNIBVizf5iar6CvCVtvwYoyef5rb5G+CKefb/OPDxCfV9wL4J9Yl9SJKG4SfCJUndDA1JUjdDQ5LU\nzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LU\nzdCQJHUzNCRJ3QwNSVI3Q0OS1G3R0EjyC0m+luTPkzyS5Ldb/U1J7ktyMMnnkpzS6q9u6zNt+/qx\nY3201b+T5JKx+tZWm0ly/Vh9Yh+SpGH0nGn8EHhXVb0VOB/YmmQz8DvAJ6pqA/AscE1rfw3wbFX9\nIvCJ1o4k5wJXAm8BtgK/n+SkJCcBnwQuBc4F3tfaskAfkqQBLBoaNfLXbfXk9irgXcCdrb4buLwt\nb2vrtO0XJ0mr315VP6yqvwRmgE3tNVNVj1XVj4DbgW1tn/n6kCQNoOueRjsjeBA4BBwA/gL4flW9\n2JrMAmva8hrgCYC2/TngjPH6nH3mq5+xQB+SpAF0hUZV/biqzgfWMjoz+KVJzdp75tn2ctVfIsmO\nJNNJpg8fPjypiSTpZbCkp6eq6vvAV4DNwOokq9qmtcCTbXkWWAfQtr8eODJen7PPfPXvLdDH3HHd\nUlUbq2rj1NTUUr4kSdIS9Dw9NZVkdVs+FfgV4FHgy8B7W7PtwF1teW9bp23/UlVVq1/Znq56E7AB\n+BpwP7ChPSl1CqOb5XvbPvP1IUkawKrFm3AOsLs95fQq4I6q+kKSbwG3J/n3wDeAW1v7W4E/TDLD\n6AzjSoCqeiTJHcC3gBeB66rqxwBJPgjsB04CdlXVI+1YvzlPH5KkASwaGlX1EPC2CfXHGN3fmFv/\nG+CKeY71ceDjE+r7gH29fUiShuEnwiVJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAk\ndTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktRt0dBI\nsi7Jl5M8muSRJB9q9TckOZDkYHs/vdWT5KYkM0keSnLB2LG2t/YHk2wfq1+Y5OG2z01JslAfkqRh\n9JxpvAj826r6JWAzcF2Sc4HrgXuqagNwT1sHuBTY0F47gJthFADATuDtwCZg51gI3NzaHt1va6vP\n14ckaQCLhkZVPVVVX2/LzwOPAmuAbcDu1mw3cHlb3gbsqZF7gdVJzgEuAQ5U1ZGqehY4AGxt206r\nqq9WVQF75hxrUh+SpAEs6Z5GkvXA24D7gLOr6ikYBQtwVmu2BnhibLfZVluoPjuhzgJ9SJIG0B0a\nSV4L/BHw4ar6wUJNJ9TqGOrdkuxIMp1k+vDhw0vZVZK0BF2hkeRkRoHx6ar641Z+ul1aor0favVZ\nYN3Y7muBJxepr51QX6iPn1JVt1TVxqraODU11fMlSZKOQc/TUwFuBR6tqt8d27QXOPoE1HbgrrH6\nVe0pqs3Ac+3S0n5gS5LT2w3wLcD+tu35JJtbX1fNOdakPiRJA1jV0eYi4P3Aw0kebLXfAm4E7khy\nDfBd4Iq2bR9wGTADvABcDVBVR5J8DLi/tbuhqo605WuB24BTgbvbiwX6kCQNYNHQqKr/xeT7DgAX\nT2hfwHXzHGsXsGtCfRo4b0L9mUl9SJKG4SfCJUndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS\n1M3QkCR1MzQkSd16fo2IJB2T9df/yWB9P37juwfr+5XMMw1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3\nQ0OS1M3QkCR1MzQkSd0MDUlSt0VDI8muJIeSfHOs9oYkB5IcbO+nt3qS3JRkJslDSS4Y22d7a38w\nyfax+oVJHm773JQkC/UhSRpOz5nGbcDWObXrgXuqagNwT1sHuBTY0F47gJthFADATuDtwCZg51gI\n3NzaHt1v6yJ9SJIGsmhoVNWfAUfmlLcBu9vybuDysfqeGrkXWJ3kHOAS4EBVHamqZ4EDwNa27bSq\n+mpVFbBnzrEm9SFJGsix3tM4u6qeAmjvZ7X6GuCJsXazrbZQfXZCfaE+JEkDeblvhGdCrY6hvrRO\nkx1JppNMHz58eKm7S5I6HWtoPN0uLdHeD7X6LLBurN1a4MlF6msn1Bfq4yWq6paq2lhVG6empo7x\nS5IkLeZYQ2MvcPQJqO3AXWP1q9pTVJuB59qlpf3AliSntxvgW4D9bdvzSTa3p6aumnOsSX1Ikgay\n6B9hSvJZ4J3AmUlmGT0FdSNwR5JrgO8CV7Tm+4DLgBngBeBqgKo6kuRjwP2t3Q1VdfTm+rWMntA6\nFbi7vVigj+NmqD8Y4x+LkfTzYtHQqKr3zbPp4gltC7hunuPsAnZNqE8D502oPzOpD+nnlX/FTq8E\nfiJcktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVK3RT/cp1c2PwUvaSk805AkdTM0\nJEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd1WfGgk2ZrkO0lmklw/9Hgk\n6US2okMjyUnAJ4FLgXOB9yU5d9hRSdKJa0WHBrAJmKmqx6rqR8DtwLaBxyRJJ6yVHhprgCfG1mdb\nTZI0gFTV0GOYV5IrgEuq6tfa+vuBTVX163Pa7QB2tNU3A985xi7PBL53jPseT45raRzX0jiupXml\njusfVNXUYo1W+t/TmAXWja2vBZ6c26iqbgFu+Vk7SzJdVRt/1uO83BzX0jiupXFcS3Oij2ulX566\nH9iQ5E1JTgGuBPYOPCZJOmGt6DONqnoxyQeB/cBJwK6qemTgYUnSCWtFhwZAVe0D9i1Tdz/zJa7j\nxHEtjeNaGse1NCf0uFb0jXBJ0sqy0u9pSJJWkBMuNJLsSnIoyTfn2Z4kN7VfW/JQkgtWyLjemeS5\nJA+2179bpnGtS/LlJI8meSTJhya0WfY56xzXss9Zkl9I8rUkf97G9dsT2rw6yefafN2XZP0KGdcH\nkhwem69fO97jGuv7pCTfSPKFCduWfb46xzXIfCV5PMnDrc/pCduP7/djVZ1QL+AdwAXAN+fZfhlw\nNxBgM3DfChnXO4EvDDBf5wAXtOXXAf8bOHfoOesc17LPWZuD17blk4H7gM1z2vxr4A/a8pXA51bI\nuD4A/Kfl/m+s9f0R4DOT/r2GmK/OcQ0yX8DjwJkLbD+u348n3JlGVf0ZcGSBJtuAPTVyL7A6yTkr\nYFyDqKqnqurrbfl54FFe+qn8ZZ+zznEtuzYHf91WT26vuTcOtwG72/KdwMVJsgLGNYgka4F3A5+a\np8myz1fnuFaq4/r9eMKFRoeV/KtLfrldXrg7yVuWu/N2WeBtjH5KHTfonC0wLhhgztoljQeBQ8CB\nqpp3vqrqReA54IwVMC6Af9YuadyZZN2E7cfD7wG/AfztPNsHma+OccEw81XAnyZ5IKPfhjHXcf1+\nNDReatJPMCvhJ7KvM/qY/1uB/wj89+XsPMlrgT8CPlxVP5i7ecIuyzJni4xrkDmrqh9X1fmMfoPB\npiTnzWkyyHx1jOt/AOur6h8D/5Of/HR/3CT5VeBQVT2wULMJteM6X53jWvb5ai6qqgsY/fbv65K8\nY8724zpfhsZLdf3qkuVWVT84enmhRp9dOTnJmcvRd5KTGf2P+dNV9ccTmgwyZ4uNa8g5a31+H/gK\nsHXOpr+brySrgNezjJcm5xtXVT1TVT9sq/8ZuHAZhnMR8J4kjzP6LdbvSvJf57QZYr4WHddA80VV\nPdneDwGfZ/TbwMcd1+9HQ+Ol9gJXtScQNgPPVdVTQw8qyd8/eh03ySZG/3bPLEO/AW4FHq2q352n\n2bLPWc+4hpizJFNJVrflU4FfAb49p9leYHtbfi/wpWp3MIcc15zr3u9hdJ/ouKqqj1bV2qpaz+gm\n95eq6l/Mabbs89UzriHmK8lrkrzu6DKwBZj7xOVx/X5c8Z8If7kl+Syjp2rOTDIL7GR0U5Cq+gNG\nnz6/DJgBXgCuXiHjei9wbZIXgf8HXHm8v3Gai4D3Aw+36+EAvwW8cWxsQ8xZz7iGmLNzgN0Z/QGx\nVwF3VNUXktwATFfVXkZh94dJZhj9xHzlcR5T77j+TZL3AC+2cX1gGcY10QqYr55xDTFfZwOfbz8L\nrQI+U1VfTPKvYHm+H/1EuCSpm5enJEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1\n+/+XFCVGAOVoKAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ffa82b0d9e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Histogram of ratings\n",
    "rating_hist = plt.hist(dt['rating'],align='mid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Group new data by product\n",
    "user_groupby = dt.groupby('reviewerID',as_index=False)\n",
    "user_review_count = user_groupby['productID'].count()\n",
    "user_review_count.columns = ['reviewerID','count_review']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Take top users who rated the most\n",
    "user_most_ratings = user_review_count.sort_values(by='count_review',ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Find products rated by these top users\n",
    "product_rated = dt.loc[dt['reviewerID'].isin(user_most_ratings['reviewerID'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Take top products among top users (i.e top products most rated by top users)\n",
    "product_to_sample = product_rated.pivot_table('rating',index='productID',aggfunc=len).sort_values(by='rating',ascending=False).head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Filter original data for these products only\n",
    "data = dt.loc[dt['productID'].isin(product_to_sample.index)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Drop users who have few ratings\n",
    "ureview_count = data.pivot_table('rating',index='reviewerID',aggfunc=len)\n",
    "user_interest = ureview_count[ureview_count['rating']>=4]\n",
    "data = data.loc[data['reviewerID'].isin(user_interest.index)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#data.to_csv('sample_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>9989.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.840625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.083991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            rating\n",
       "count  9989.000000\n",
       "mean      3.840625\n",
       "std       1.083991\n",
       "min       1.000000\n",
       "25%       3.000000\n",
       "50%       4.000000\n",
       "75%       5.000000\n",
       "max       5.000000"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1050"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['reviewerID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09513333333333333"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)/(data['reviewerID'].nunique()*data['productID'].nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import metadata of products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read meta data of products into data frame\n",
    "import gzip\n",
    "\n",
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield eval(l)\n",
    "\n",
    "def getDF(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')\n",
    "\n",
    "df_meta = getDF('meta_Grocery_and_Gourmet_Food.json.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_meta.rename(columns={'asin':'productID'},inplace='True')\n",
    "df_meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Acquire list of products to filter from all products\n",
    "product_list = list(set(data['productID']))\n",
    "df_meta = df_meta.loc[df_meta['productID'].isin(product_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df_meta.to_csv('sample_data_meta.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}