pernir-show-effect.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d2c4451",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from caboose_nbr.pernir import Pernir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fdc9103c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_baskets = pd.read_csv('data/instacart_30k/train_baskets.csv.gz')\n",
    "test_baskets = pd.read_csv('data/instacart_30k/test_baskets.csv')\n",
    "valid_baskets = pd.read_csv('data/instacart_30k/valid_baskets.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7da77935",
   "metadata": {},
   "outputs": [],
   "source": [
    "aisles = pd.read_csv('data/instacart_30k/aisles.csv')\n",
    "products = pd.read_csv('data/instacart_30k/products.csv')\n",
    "products_with_aisles = products.merge(aisles, on='aisle_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e280852f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_baskets_with_aisles = train_baskets.merge(products_with_aisles, left_on=\"item_id\", right_on=\"product_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8ebe3f90",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Works for seed=42 and sample_size=100,250,1000\n",
    "\n",
    "seed = 1312\n",
    "sample_size = 100\n",
    "sensitive_aisles = [82, 92, 102, 56]\n",
    "\n",
    "np.random.seed(seed)\n",
    "\n",
    "baby_baskets = train_baskets_with_aisles[train_baskets_with_aisles.aisle_id.isin(sensitive_aisles)]\n",
    "all_baby_users = baby_baskets.user_id.unique()\n",
    "baby_users = np.array(np.random.choice(all_baby_users, sample_size))\n",
    "baby_user_baskets = train_baskets_with_aisles[train_baskets_with_aisles.user_id.isin(baby_users)]\n",
    "\n",
    "other_aisles = [aisle for aisle in baby_user_baskets.aisle_id.unique() if aisle not in sensitive_aisles]\n",
    "\n",
    "all_nonbaby_users  = train_baskets_with_aisles[(train_baskets_with_aisles.aisle_id.isin(other_aisles)) \\\n",
    "                          & (~train_baskets_with_aisles.user_id.isin(all_baby_users))].user_id.unique()\n",
    "\n",
    "\n",
    "nonbaby_users = np.array(np.random.choice(all_nonbaby_users, sample_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd35664a",
   "metadata": {},
   "outputs": [],
   "source": [
    "users = np.concatenate((baby_users, nonbaby_users))\n",
    "sampled_train_baskets = train_baskets[train_baskets['user_id'].isin(users)]\n",
    "sampled_test_baskets = test_baskets[test_baskets['user_id'].isin(users)]\n",
    "sampled_valid_baskets = valid_baskets[valid_baskets['user_id'].isin(users)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f78ee573",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15486, 2)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_train_baskets[['user_id','item_id']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4532e902",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start of knn\n",
      "knn finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "--Creating transpose of R...\n",
      "--Computing row norms...\n",
      "--Configuring for top-k -- num_threads: 8; pinning? false;\n",
      "--Scheduling parallel top-k computation...\n"
     ]
    }
   ],
   "source": [
    "pernir_caboose = Pernir(sampled_train_baskets, sampled_test_baskets, 'caboose')\n",
    "pernir_caboose.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "563a0e21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TB before 40107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ssc/projects/caboose/caboose_nbr/pernir.py:113: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.train_baskets['user_item'] = self.train_baskets.apply(lambda x: (x['user_id'], x['item_id']), axis = 1)\n",
      "/Users/ssc/projects/caboose/caboose_nbr/pernir.py:115: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.train_baskets.drop('user_item', axis=1, inplace=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TB after 40102\n",
      "User 206056: (1,4) --> 0\n",
      "---\n",
      "TB before 40102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ssc/projects/caboose/caboose_nbr/pernir.py:113: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.train_baskets['user_item'] = self.train_baskets.apply(lambda x: (x['user_id'], x['item_id']), axis = 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TB after 40100\n",
      "User 168374: (1,1) --> 0\n",
      "---\n",
      "TB before 40100\n",
      "TB after 39989\n",
      "User 160617: (1,29) --> 0\n",
      "---\n",
      "TB before 39989\n",
      "TB after 39988\n",
      "User 194289: (1,1) --> 0\n",
      "---\n",
      "TB before 39988\n",
      "TB after 39986\n",
      "User 61832: (1,2) --> 0\n",
      "---\n",
      "TB before 39986\n",
      "TB after 39984\n",
      "User 160640: (1,1) --> 0\n",
      "---\n",
      "TB before 39984\n",
      "TB after 39966\n",
      "User 33263: (1,2) --> 0\n",
      "---\n",
      "TB before 39966\n",
      "TB after 39961\n",
      "User 84199: (1,1) --> 0\n",
      "---\n",
      "TB before 39961\n",
      "TB after 39958\n",
      "User 44924: (1,2) --> 0\n",
      "---\n",
      "TB before 39958\n",
      "TB after 39921\n",
      "User 175385: (1,31) --> 0\n",
      "---\n",
      "TB before 39921\n",
      "TB after 39918\n",
      "User 100195: (1,1) --> 0\n",
      "---\n",
      "TB before 39918\n",
      "TB after 39916\n",
      "User 205905: (1,1) --> 0\n",
      "---\n",
      "TB before 39916\n",
      "TB after 39912\n",
      "User 105379: (1,3) --> 0\n",
      "---\n",
      "TB before 39912\n",
      "TB after 39788\n",
      "User 165126: (3,27) --> 0\n",
      "---\n",
      "TB before 39788\n",
      "TB after 39750\n",
      "User 59123: (1,15) --> 0\n",
      "---\n",
      "TB before 39750\n",
      "TB after 39748\n",
      "User 140342: (1,1) --> 0\n",
      "---\n",
      "TB before 39748\n",
      "TB after 39587\n",
      "User 140883: (1,32) --> 0\n",
      "---\n",
      "TB before 39587\n",
      "TB after 39536\n",
      "User 168208: (1,20) --> 0\n",
      "---\n",
      "TB before 39536\n",
      "TB after 39535\n",
      "User 197982: (1,1) --> 0\n",
      "---\n",
      "TB before 39535\n",
      "TB after 39530\n",
      "User 21491: (3,4) --> 0\n",
      "---\n",
      "TB before 39530\n",
      "TB after 39529\n",
      "User 159945: (1,1) --> 0\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "baby_items = set(baby_baskets.item_id.unique())\n",
    "\n",
    "for user in baby_users:\n",
    "    \n",
    "    predictions, scores = pernir_caboose.predict_for_user(user, 10)\n",
    "    nb = pernir_caboose.user_neighbors[user]\n",
    "    predicted_baby_items = set(predictions) & baby_items\n",
    "    has_baby_items = len(predicted_baby_items) > 0\n",
    "    \n",
    "    if has_baby_items:\n",
    "        chosen_users_items = sampled_train_baskets[sampled_train_baskets.user_id==user].item_id.unique()\n",
    "        chosen_users_baby_items = set(chosen_users_items) & baby_items\n",
    "\n",
    "        to_forget = [(user, item) for item in chosen_users_baby_items]\n",
    "        pernir_caboose.forget_interactions(to_forget)\n",
    "        #predictions_after_forget = pernir_caboose.predict_for_user(user, 10)\n",
    "        predictions_after_forget, scores_after = pernir_caboose.predict_for_user(user, 10)\n",
    "        na = pernir_caboose.user_neighbors[user]\n",
    "        remaining_baby_items = set(predictions_after_forget) & baby_items\n",
    "        #print('Before', predictions)\n",
    "        #print('Before', nb)        \n",
    "        #print('After', predictions_after_forget)\n",
    "        #print('After', na)        \n",
    "        print(f'User {user}: ({len(predicted_baby_items)},{len(to_forget)}) --> {len(remaining_baby_items)}')\n",
    "        print('---')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c254081",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}