preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "foreign-oriental",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import datetime\n",
    "import sklearn\n",
    "import tqdm\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "headed-spokesman",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_usage = pd.read_csv('data/App_usage_trace.txt', sep=' ', names=['user', 'time', 'location', 'app', 'traffic'])\n",
    "df_usage = df_usage[['user', 'time', 'app']]\n",
    "\n",
    "# merging the consecutive usage records of the same app in one minute\n",
    "df_usage['time'] = df_usage['time'].apply(lambda x: str(x)[:-2])\n",
    "df_usage.drop_duplicates(inplace=True)\n",
    "\n",
    "# delete apps used less than 10 times for all users\n",
    "df_usage = df_usage[df_usage.groupby('app')['app'].transform('count').ge(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "universal-samba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>time</th>\n",
       "      <th>app</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200813</td>\n",
       "      <td>361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200816</td>\n",
       "      <td>361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200816</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200816</td>\n",
       "      <td>360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200816</td>\n",
       "      <td>612</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user          time  app\n",
       "0     0  201604200813  361\n",
       "5     0  201604200816  361\n",
       "6     0  201604200816   31\n",
       "7     0  201604200816  360\n",
       "8     0  201604200816  612"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_usage.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "rational-annex",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1123955/1123955 [05:38<00:00, 3322.00it/s]\n"
     ]
    }
   ],
   "source": [
    "prev_user = -1\n",
    "prev_time = -1\n",
    "app_seq = []\n",
    "time_seq = []\n",
    "all_app_seq= []\n",
    "all_time_seq = []\n",
    "\n",
    "seq_length = 4\n",
    "\n",
    "# df_usage is already sorted based on user and time sequence\n",
    "for i in tqdm.tqdm(range(len(df_usage))):\n",
    "    user = df_usage.iloc[i]['user']\n",
    "    app = df_usage.iloc[i]['app']\n",
    "    time = df_usage.iloc[i]['time']\n",
    "    time = datetime.datetime.strptime(time, '%Y%m%d%H%M')\n",
    "    \n",
    "    if prev_user != user:\n",
    "        app_seq = [app]\n",
    "        time_seq = [time]\n",
    "        all_app_seq.append([])\n",
    "        all_time_seq.append([])\n",
    "        \n",
    "    else:\n",
    "        # same sequence if the time gap between them is equal to or less than 7 mins\n",
    "        if (time - prev_time).total_seconds()//60 <= 7:\n",
    "            if len(app_seq) == seq_length:\n",
    "                all_app_seq.append(app_seq)\n",
    "                all_time_seq.append([(prev_time - x).total_seconds()//60 for x in time_seq])\n",
    "                app_seq = app_seq[1:] + [app]\n",
    "                time_seq = time_seq[1:] + [time]\n",
    "            else:\n",
    "                app_seq.append(app)\n",
    "                time_seq.append(time)\n",
    "                all_app_seq.append([])\n",
    "                all_time_seq.append([])\n",
    "        else:\n",
    "            app_seq = [app]\n",
    "            time_seq = [time]\n",
    "            all_app_seq.append([])\n",
    "            all_time_seq.append([])\n",
    "    \n",
    "    prev_user = user\n",
    "    prev_time = time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "unlimited-storm",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>time</th>\n",
       "      <th>app</th>\n",
       "      <th>app_seq</th>\n",
       "      <th>time_seq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200816</td>\n",
       "      <td>612</td>\n",
       "      <td>[361, 361, 31, 360]</td>\n",
       "      <td>[3.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200817</td>\n",
       "      <td>31</td>\n",
       "      <td>[361, 31, 360, 612]</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200817</td>\n",
       "      <td>360</td>\n",
       "      <td>[31, 360, 612, 31]</td>\n",
       "      <td>[1.0, 1.0, 1.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200817</td>\n",
       "      <td>361</td>\n",
       "      <td>[360, 612, 31, 360]</td>\n",
       "      <td>[1.0, 1.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200824</td>\n",
       "      <td>1</td>\n",
       "      <td>[612, 31, 360, 361]</td>\n",
       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200829</td>\n",
       "      <td>31</td>\n",
       "      <td>[31, 360, 361, 1]</td>\n",
       "      <td>[7.0, 7.0, 7.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200829</td>\n",
       "      <td>612</td>\n",
       "      <td>[360, 361, 1, 31]</td>\n",
       "      <td>[12.0, 12.0, 5.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200829</td>\n",
       "      <td>360</td>\n",
       "      <td>[361, 1, 31, 612]</td>\n",
       "      <td>[12.0, 5.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200829</td>\n",
       "      <td>4</td>\n",
       "      <td>[1, 31, 612, 360]</td>\n",
       "      <td>[5.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>201604200831</td>\n",
       "      <td>31</td>\n",
       "      <td>[31, 612, 360, 4]</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    user          time  app              app_seq                time_seq\n",
       "8      0  201604200816  612  [361, 361, 31, 360]    [3.0, 0.0, 0.0, 0.0]\n",
       "10     0  201604200817   31  [361, 31, 360, 612]    [0.0, 0.0, 0.0, 0.0]\n",
       "13     0  201604200817  360   [31, 360, 612, 31]    [1.0, 1.0, 1.0, 0.0]\n",
       "14     0  201604200817  361  [360, 612, 31, 360]    [1.0, 1.0, 0.0, 0.0]\n",
       "16     0  201604200824    1  [612, 31, 360, 361]    [1.0, 0.0, 0.0, 0.0]\n",
       "17     0  201604200829   31    [31, 360, 361, 1]    [7.0, 7.0, 7.0, 0.0]\n",
       "19     0  201604200829  612    [360, 361, 1, 31]  [12.0, 12.0, 5.0, 0.0]\n",
       "20     0  201604200829  360    [361, 1, 31, 612]   [12.0, 5.0, 0.0, 0.0]\n",
       "22     0  201604200829    4    [1, 31, 612, 360]    [5.0, 0.0, 0.0, 0.0]\n",
       "23     0  201604200831   31    [31, 612, 360, 4]    [0.0, 0.0, 0.0, 0.0]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_usage['app_seq'] = all_app_seq\n",
    "df_usage['time_seq'] = all_time_seq\n",
    "\n",
    "# only filled sequences are treated as data\n",
    "df_usage = df_usage[df_usage['app_seq'].map(len) != 0]\n",
    "# delete users who have sequences less than 50\n",
    "df_usage = df_usage[df_usage.groupby('user')['user'].transform('count').ge(50)]\n",
    "\n",
    "df_usage[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "creative-assurance",
   "metadata": {},
   "outputs": [],
   "source": [
    "# represent time as weekday_time\n",
    "def prep_time(t):\n",
    "    t = t[:-2]\n",
    "    weekday = datetime.datetime.strptime(t[:-2], '%Y%m%d').weekday()\n",
    "    return '{}_{}'.format(weekday, t[-2:])\n",
    "\n",
    "df_usage['time'] = df_usage['time'].apply(lambda x: prep_time(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "powerful-buffalo",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>time</th>\n",
       "      <th>app</th>\n",
       "      <th>app_seq</th>\n",
       "      <th>time_seq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>2_08</td>\n",
       "      <td>612</td>\n",
       "      <td>[361, 361, 31, 360]</td>\n",
       "      <td>[3.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>2_08</td>\n",
       "      <td>31</td>\n",
       "      <td>[361, 31, 360, 612]</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>2_08</td>\n",
       "      <td>360</td>\n",
       "      <td>[31, 360, 612, 31]</td>\n",
       "      <td>[1.0, 1.0, 1.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>2_08</td>\n",
       "      <td>361</td>\n",
       "      <td>[360, 612, 31, 360]</td>\n",
       "      <td>[1.0, 1.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>2_08</td>\n",
       "      <td>1</td>\n",
       "      <td>[612, 31, 360, 361]</td>\n",
       "      <td>[1.0, 0.0, 0.0, 0.0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    user  time  app              app_seq              time_seq\n",
       "8      0  2_08  612  [361, 361, 31, 360]  [3.0, 0.0, 0.0, 0.0]\n",
       "10     0  2_08   31  [361, 31, 360, 612]  [0.0, 0.0, 0.0, 0.0]\n",
       "13     0  2_08  360   [31, 360, 612, 31]  [1.0, 1.0, 1.0, 0.0]\n",
       "14     0  2_08  361  [360, 612, 31, 360]  [1.0, 1.0, 0.0, 0.0]\n",
       "16     0  2_08    1  [612, 31, 360, 361]  [1.0, 0.0, 0.0, 0.0]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_usage.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "genuine-transformation",
   "metadata": {},
   "outputs": [],
   "source": [
    "user2id = {u: i for i, u in enumerate(sorted(df_usage['user'].unique()))}\n",
    "app_set = set()\n",
    "for s in df_usage['app_seq'].values:\n",
    "    app_set.update(s)\n",
    "app2id = {a: i for i, a in enumerate(sorted(app_set))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "entertaining-turtle",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dict2file(dic, filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        for k, v in dic.items():\n",
    "            f.write(\"{}\\t{}\\n\".format(k, v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "geographic-mountain",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict2file(user2id, \"data/user2id.txt\")\n",
    "dict2file(app2id, \"data/app2id.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "parliamentary-bruce",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dataset = pd.DataFrame()\n",
    "df_dataset['user'] = df_usage['user'].apply(lambda x: user2id[x])\n",
    "df_dataset['time'] = df_usage['time']\n",
    "df_dataset['app_seq'] = df_usage['app_seq'].apply(lambda x: [app2id[c] for c in x])\n",
    "df_dataset['time_seq'] = df_usage['time_seq']\n",
    "df_dataset['app'] = df_usage['app'].apply(lambda x: app2id[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "inclusive-slide",
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = train_test_split(df_dataset, test_size=0.2, random_state=2021, stratify=df_dataset['user'])\n",
    "train.to_csv('data/train.txt', sep='\\t', index=False)\n",
    "test.to_csv('data/test.txt', sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}