Route_Augmentation.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.Load Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from geopy.distance import geodesic\n",
    "from math import sqrt\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50253, 28)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>userId</th>\n",
       "      <th>gender</th>\n",
       "      <th>sport</th>\n",
       "      <th>duration</th>\n",
       "      <th>calories</th>\n",
       "      <th>distance</th>\n",
       "      <th>avg_heart_rate</th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>...</th>\n",
       "      <th>validate</th>\n",
       "      <th>avg_alti</th>\n",
       "      <th>change_alti</th>\n",
       "      <th>max_alti</th>\n",
       "      <th>min_alti</th>\n",
       "      <th>diff_alti</th>\n",
       "      <th>avg_speed</th>\n",
       "      <th>Cluster</th>\n",
       "      <th>Route</th>\n",
       "      <th>Route_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>611012078</td>\n",
       "      <td>2568526</td>\n",
       "      <td>male</td>\n",
       "      <td>run</td>\n",
       "      <td>3158</td>\n",
       "      <td>830.588</td>\n",
       "      <td>10.02</td>\n",
       "      <td>154.914</td>\n",
       "      <td>[7.099486151710153, 7.0994688011705875, 7.0993...</td>\n",
       "      <td>[43.68301374837756, 43.683006623759866, 43.682...</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>87.8552</td>\n",
       "      <td>756.8</td>\n",
       "      <td>139.4</td>\n",
       "      <td>76.0</td>\n",
       "      <td>63.4</td>\n",
       "      <td>11.574343</td>\n",
       "      <td>1</td>\n",
       "      <td>('run', 1)</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id   userId gender sport  duration  calories  distance  \\\n",
       "0  611012078  2568526   male   run      3158   830.588     10.02   \n",
       "\n",
       "   avg_heart_rate                                          longitude  \\\n",
       "0         154.914  [7.099486151710153, 7.0994688011705875, 7.0993...   \n",
       "\n",
       "                                            latitude  ... validate avg_alti  \\\n",
       "0  [43.68301374837756, 43.683006623759866, 43.682...  ...     True  87.8552   \n",
       "\n",
       "  change_alti max_alti min_alti diff_alti  avg_speed Cluster       Route  \\\n",
       "0       756.8    139.4     76.0      63.4  11.574343       1  ('run', 1)   \n",
       "\n",
       "   Route_id  \n",
       "0         9  \n",
       "\n",
       "[1 rows x 28 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://drive.google.com/file/d/1-RXYMh61x9asaVU_w5tDHtq0CMmp_Rdn/view?usp=sharing\n",
    "file_path_train = 'D:\\MiniProject_P3_fitrec\\TrainData.csv'\n",
    "# https://drive.google.com/file/d/1--znLUhuYqoMMqQo1Gnf6WafrzzniIFL/view?usp=sharing\n",
    "file_path_test = 'D:\\MiniProject_P3_fitrec\\TestData.csv'\n",
    "\n",
    "\n",
    "train_df = pd.read_csv(file_path_train)\n",
    "test_df = pd.read_csv(file_path_test)\n",
    "\n",
    "print(train_df.shape)\n",
    "display(train_df.head(1))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.Find workout records that return to start point at the end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define function to compute distance based on Latitude, Longitude, Altitude\n",
    "def geodis(lat_0, lon_0, alt_0, lat_1, lon_1, alt_1):\n",
    "    dis = geodesic((lat_0, lon_0), (lat_1, lon_1)).km\n",
    "    dis = sqrt(dis**2 + (alt_0/1000-alt_1/1000)**2)\n",
    "    return dis\n",
    "\n",
    "# Define function to check if a workout record has returned to start point at the end\n",
    "\n",
    "\n",
    "def isback(df_row, num_to_check):\n",
    "\n",
    "\n",
    "    '''\n",
    "    df_row: a row of dataframe\n",
    "    num_to_check: number of points to check\n",
    "\n",
    "    1. We take num_to_check points at the beginning of workout route and num_to_check points\n",
    "    at the end of the workout route\n",
    "\n",
    "    2. We compute the distances between each point at the beginning with all points at the end respectively\n",
    "\n",
    "    3. If one distance is smaller than threshold, then we return 1 else we return 0\n",
    "\n",
    "    '''\n",
    "\n",
    "    if df_row.sport == 'run':\n",
    "        thres = 0.02\n",
    "    else:\n",
    "        thres = 0.04\n",
    "\n",
    "    lat_head = eval(df_row.latitude)[0:num_to_check]\n",
    "    lon_head = eval(df_row.longitude)[0:num_to_check]\n",
    "    alt_head = eval(df_row.altitude)[0:num_to_check]\n",
    "    lat_tail = eval(df_row.latitude)[-num_to_check:]\n",
    "    lon_tail = eval(df_row.longitude)[-num_to_check:]\n",
    "    alt_tail = eval(df_row.altitude)[-num_to_check:]\n",
    "\n",
    "    dis_list = []\n",
    "\n",
    "    for i in range(0, num_to_check):\n",
    "        dis = [geodis(lat_head[i], lon_head[i], alt_head[i], lat_tail[j],\n",
    "                      lon_tail[j], alt_tail[j]) for j in range(0, num_to_check)]\n",
    "        dis_list.extend(dis)\n",
    "\n",
    "    if min(dis_list) < thres:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    29470\n",
      "1    20783\n",
      "Name: isback, dtype: int64\n",
      "Wall time: 9min 42s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_df['isback'] = train_df.apply(lambda x: isback(x, 5), axis=1)\n",
    "test_df['isback'] = test_df.apply(lambda x: isback(x, 5), axis=1)\n",
    "\n",
    "print(train_df.isback.value_counts())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4.EXtend workout routes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract rows where workout route has returned to starting point at the end\n",
    "\n",
    "adjust_train_df = train_df[train_df.isback==1].copy()\n",
    "adjust_test_df = test_df[test_df.isback==1].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define function to create new sequential features\n",
    "\n",
    "def update_sequence(df_row, max_extend_point):\n",
    "\n",
    "\n",
    "    '''\n",
    "    df_row: a row of dataframe\n",
    "    max_extend_point: maximum number of points to extend\n",
    "\n",
    "\n",
    "    1. We randomly draw a number as the number of points to extend:\n",
    "    ext_len\n",
    "\n",
    "    2. We extract the number of points from start of workout sequence:\n",
    "    lat_head, lon_head, alt_head, distance_head\n",
    "\n",
    "    3. We generate Gaussian noise and add to the latitude and longitude of the\n",
    "    extracted sequence:\n",
    "    lat_head_noise, lon_head_noise\n",
    "\n",
    "    4. We re-calculate distance sequence due to added noise to latitude and longitude\n",
    "\n",
    "    5. Because we will extend a number of points to sequence data, we also randomly\n",
    "    remove same number of points from original sequence so that the sequence length\n",
    "    is unchanged:\n",
    "    tmp_alt, tmp_distance\n",
    "\n",
    "    6. For dropped points, we re-calculate the distance and speed of neighbouring point\n",
    "\n",
    "    7. For altitude and distance, we concatenate extracted points from beginning of\n",
    "    sequence and original sequence with points randomly dropped:\n",
    "    alt_head+tmp_alt, distance_head+tmp_distance\n",
    "\n",
    "    8. For speed, heart rate, we keep original sequence with points randomly dropped:\n",
    "    tmp_heart, tmp_speed\n",
    "\n",
    "    9. We also return the index of the last point from the original sequence:\n",
    "    complete_idx\n",
    "    '''\n",
    "\n",
    "    # Randonly draw the number of points to extend\n",
    "    ext_len = random.randrange(30, max_extend_point)\n",
    "\n",
    "    # Extract number of points from start of workout sequence\n",
    "    lat_head = eval(df_row.latitude)[0:ext_len]\n",
    "    lon_head = eval(df_row.longitude)[0:ext_len]\n",
    "    alt_head = eval(df_row.altitude)[0:ext_len]\n",
    "    distance_head = eval(df_row.derived_distance)[0:ext_len]\n",
    "\n",
    "    # Generate Gaussian noise\n",
    "    max_noise_lat = np.absolute(np.array(lat_head).mean()/100000000.)\n",
    "    max_noise_lon = np.absolute(np.array(lat_head).mean()/100000000.)\n",
    "\n",
    "    noise_lat = np.random.normal(0, max_noise_lat, ext_len)\n",
    "    noise_lon = np.random.normal(0, max_noise_lon, ext_len)\n",
    "\n",
    "    # Add Gaussian noise to latitude and longitude of extended route\n",
    "    lat_head_noise = np.add(lat_head, noise_lat)\n",
    "    lon_head_noise = np.add(lon_head, noise_lon)\n",
    "\n",
    "    # Update distance array based on new latitude and longitude with noise\n",
    "    dis_tail = distance_head[-1]\n",
    "    distance_head = np.array([geodis(lat_head_noise[idx], lon_head_noise[idx], alt_head[idx],\n",
    "                                    lat_head_noise[idx+1], lon_head_noise[idx+1], alt_head[idx+1]) for idx in range(len(distance_head)-1)])\n",
    "    distance_head = np.append(distance_head, dis_tail)\n",
    "\n",
    "\n",
    "    # Sample indices to drop from original route\n",
    "    # we don't want to touch the head and tail point\n",
    "    drop_indices = random.sample(range(1, 498), ext_len)\n",
    "\n",
    "    # Adjust distance and speed due to dropped points\n",
    "\n",
    "    # Get value from each cell for each feature\n",
    "    tmp_lat = eval(df_row.latitude)\n",
    "    tmp_lon = eval(df_row.longitude)\n",
    "    tmp_alt = eval(df_row.altitude)\n",
    "    tmp_heart = eval(df_row.heart_rate)\n",
    "    tmp_speed = eval(df_row.derived_speed)\n",
    "    tmp_distance = eval(df_row.derived_distance)\n",
    "    tmp_timestamp = eval(df_row.timestamp)\n",
    "\n",
    "    tmp_df = pd.DataFrame(data=[tmp_lat[:499],\n",
    "                                tmp_lon[:499],\n",
    "                                tmp_alt[:499],\n",
    "                                tmp_heart[:499],\n",
    "                                tmp_speed[:499],\n",
    "                                tmp_distance[:499],\n",
    "                                tmp_timestamp[:499]]).T\n",
    "\n",
    "    tmp_df.rename(columns={0: 'latitude',\n",
    "                            1: 'longitude',\n",
    "                            2: 'altitude',\n",
    "                            3: 'heart_rate',\n",
    "                            4: 'derived_speed',\n",
    "                            5: 'derived_distance',\n",
    "                            6: 'timestamp'}, inplace=True)\n",
    "\n",
    "    # Adjust distance and speed due to dropped points\n",
    "    for idx in drop_indices:\n",
    "\n",
    "        # Find idx of previous row in case the row is already deleted\n",
    "        prev_idx = idx-1\n",
    "        while prev_idx not in tmp_df.index:\n",
    "            prev_idx -= 1\n",
    "\n",
    "        # Find idx of next row in case the row is already deleted\n",
    "        next_idx = idx+1\n",
    "        while next_idx not in tmp_df.index:\n",
    "            next_idx += 1\n",
    "\n",
    "        # idx point will be deleted, we add idx point distance to the distance at previous point\n",
    "        tmp_df.loc[prev_idx, 'derived_distance'] += tmp_df.loc[idx,\n",
    "                                                                'derived_distance']\n",
    "        # Re-calculate speed based on new distance for previous point\n",
    "        tmp_df.loc[prev_idx, 'derived_speed'] = tmp_df.loc[prev_idx, 'derived_distance'] / \\\n",
    "            ((tmp_df.loc[next_idx, 'timestamp'] -\n",
    "                tmp_df.loc[prev_idx, 'timestamp'])/3600)\n",
    "        # Drop row at idx point\n",
    "        tmp_df.drop([idx], inplace=True)\n",
    "\n",
    "    # Get reduced feature arrays\n",
    "    tmp_lat = tmp_df.latitude.to_numpy()\n",
    "    tmp_lon = tmp_df.longitude.to_numpy()\n",
    "    tmp_alt = tmp_df.altitude.to_numpy()\n",
    "    tmp_heart = tmp_df.heart_rate.to_numpy()\n",
    "    tmp_speed = tmp_df.derived_speed.to_numpy()\n",
    "    tmp_distance = tmp_df.derived_distance.to_numpy()\n",
    "\n",
    "    # Store idx where original workout completes\n",
    "    complete_idx = tmp_lat.shape[0]-1\n",
    "\n",
    "    # Update distance between last point of original workout route to first point of extended route\n",
    "    tmp_distance[-1] = geodis(tmp_lat[-1], tmp_lon[-1], tmp_alt[-1],\n",
    "                              lat_head_noise[0], lon_head_noise[0], alt_head[0])\n",
    "\n",
    "    # Extend altitude sequence\n",
    "    tmp_alt = np.append(tmp_alt, alt_head)\n",
    "    \n",
    "    # Extend distance sequence\n",
    "    tmp_distance = np.append(tmp_distance, distance_head)\n",
    "\n",
    "    # Total distance\n",
    "    tmp_distance_sum = np.sum(tmp_distance)\n",
    "\n",
    "    return str(list(tmp_alt)), str(list(tmp_distance)), str(list(tmp_heart)), str(list(tmp_speed)), complete_idx, tmp_distance_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "adjust_train_df['altitude_adjusted'], \\\n",
    "    adjust_train_df['distance_adjusted'], \\\n",
    "    adjust_train_df['heart_rate_adjusted'], \\\n",
    "    adjust_train_df['speed_adjusted'], \\\n",
    "    adjust_train_df['complete_idx'], \\\n",
    "    adjust_train_df['distance_adjusted_sum'] = zip(\n",
    "        *adjust_train_df.apply(lambda x: update_sequence(x, 100), axis=1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}