notebooks/LOCI_analytics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup\n",
    "\n",
    "### Full documentation about LOCI is available [here](https://slipo-eu.github.io/loci/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import folium\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# LOCI dependencies:\n",
    "import loci as lc\n",
    "from loci import io\n",
    "from loci import analytics\n",
    "from loci import clustering\n",
    "from loci import plots\n",
    "from loci import topics\n",
    "from loci import index\n",
    "\n",
    "import cluster_diff\n",
    "from cluster_diff import map_cluster_diff\n",
    "\n",
    "# Avoid displaying full WKT in data frames\n",
    "pd.set_option('display.max_colwidth', 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# POI Data Analytics using LOCI\n",
    "\n",
    "#### Perform **spatial analytics** over the integrated POI data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a _LOCI dataframe_ from the integrated POI dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pois = lc.io.read_poi_csv(input_file='./output/corfu_pois.csv',\n",
    "                       col_id='id',\n",
    "                       col_name='name',\n",
    "                       col_lon='lon',\n",
    "                       col_lat='lat',\n",
    "                       col_kwds='generic_category',\n",
    "                       col_sep='|',\n",
    "                       kwds_sep=',',\n",
    "                       source_crs='EPSG:4326',\n",
    "                       target_crs='EPSG:4326',\n",
    "                       keep_other_cols=False)\n",
    "\n",
    "# Utilize the category as keywords for spatial analytics:\n",
    "pois.rename(columns={'generic_category': 'kwds'}, inplace=True)\n",
    "\n",
    "# Turn all names and categories in uppercase characters to facilitate comparison:\n",
    "pois['name'] = pois['name'].apply(lambda x: x.upper())\n",
    "pois['kwds'] = pois['kwds'].apply(lambda x: [element.upper() for element in x])\n",
    "\n",
    "pois.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Draw locations on **map**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = lc.plots.map_points(pois, show_bbox=True)\n",
    "m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a **grid** of fixed-size cells and calculate **density**, i.e., the number of POIs inside each cell:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g, num_columns, num_rows = lc.index.grid(pois, cell_size_ratio=0.05)\n",
    "print('Created ' + str(num_columns) + ' x ' + str(num_columns) + ' grid.')\n",
    "g.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize this grid on map:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings('ignore')\n",
    "lc.plots.map_choropleth(areas=g, id_field='cell_id', value_field='score')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keyword statistics with LOCI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Statistics on the _number of POIs per category_:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = lc.analytics.kwds_freq(pois)\n",
    "kf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **Chart** showing number of POIs per category:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lc.plots.barchart(kf, plot_title='Top Keywords', x_axis_label='Keywords', y_axis_label='Frequency')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **Word cloud** of the various categories in the dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lc.plots.plot_wordcloud(pois)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Map-based analytics with LOCI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **Heatmap** of a particular category (_accommodation_) against the integrated POIs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pois_filtered = lc.analytics.filter_by_kwd(pois, 'ACCOMMODATION')\n",
    "lc.plots.heatmap(pois_filtered, radius=12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load **original** dataset for comparison with the integrated results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pois_original = lc.io.read_poi_csv(input_file='./datasets/osm20_pois_corfu.csv',\n",
    "                       col_id='id',\n",
    "                       col_name='name',\n",
    "                       col_lon='lon',\n",
    "                       col_lat='lat',\n",
    "                       col_kwds='category',\n",
    "                       col_sep='|',\n",
    "                       kwds_sep=',',\n",
    "                       source_crs='EPSG:4326',\n",
    "                       target_crs='EPSG:4326',\n",
    "                       keep_other_cols=False)\n",
    "\n",
    "pois_original.rename(columns={'category': 'kwds'}, inplace=True)\n",
    "\n",
    "lc.plots.plot_wordcloud(pois_original)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **Filter** POIs belonging to a specific category (e.g., _FOOD_):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter on categories of the original data\n",
    "pois_original = lc.analytics.filter_by_kwd(pois_original, kwd_filter='FOOD')\n",
    "\n",
    "# Filter on categories of the final (integrated) data\n",
    "pois_final = pois\n",
    "pois_final = lc.analytics.filter_by_kwd(pois_final, kwd_filter='EAT/DRINK')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create **Areas of Interest** (AOI) using _clustering_ with the DBSCAN algorithm:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DBSCAN parameters\n",
    "alg = 'dbscan'\n",
    "min_pts = 10\n",
    "eps = 0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply _clustering_ separately at each POI dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aois_original, eps_original = lc.clustering.compute_clusters(pois_original, alg=alg, min_pts=min_pts, eps=eps, n_jobs=-1)\n",
    "aois_final, eps_final = lc.clustering.compute_clusters(pois_final, alg=alg, min_pts=min_pts, eps=eps, n_jobs=-1)\n",
    "\n",
    "# Conflate the resulting polygons\n",
    "aois_original = lc.clustering.cluster_shapes(aois_original, 2, eps_original)\n",
    "aois_final = lc.clustering.cluster_shapes(aois_final, 2, eps_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display **AOIs for the original POIs**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots.map_choropleth(aois_original, id_field='cluster_id', value_field='size')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display **AOIs for the integrated POIs**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots.map_choropleth(aois_final, id_field='cluster_id', value_field='size')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compute **geometric difference** between AOIs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf_diff = map_cluster_diff(aois_final, aois_original)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot the geometric differences on **map** with specific symbols and tooltips:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct tooltip\n",
    "fields = list(gdf_diff.columns.values)\n",
    "fields.remove('geometry')\n",
    "if 'style' in fields:\n",
    "    fields.remove('style')\n",
    "tooltip = folium.features.GeoJsonTooltip(fields=fields)\n",
    "\n",
    "# Map styling\n",
    "def style_function(row):\n",
    "    return row['properties']['style']\n",
    "\n",
    "m = lc.plots.map_geometries(gdf_diff, tiles='OpenStreetMap', width='100%', height='100%')\n",
    "\n",
    "folium.GeoJson(\n",
    "    gdf_diff,\n",
    "    style_function=style_function,\n",
    "    tooltip=tooltip\n",
    ").add_to(m)\n",
    "m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Topic Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a _LOCI dataframe_ from the **integrated** POI dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pois = lc.io.read_poi_csv(input_file='./output/corfu_pois.csv',\n",
    "                       col_id='name',\n",
    "                       col_name='id',\n",
    "                       col_lon='lon',\n",
    "                       col_lat='lat',\n",
    "                       col_kwds='category',\n",
    "                       col_sep='|',\n",
    "                       kwds_sep=',',\n",
    "                       source_crs='EPSG:4326',\n",
    "                       target_crs='EPSG:4326',\n",
    "                       keep_other_cols=False)\n",
    "pois['name'] = pois['name'].apply(lambda x: [element.upper() for element in x])\n",
    "pois['category'] = pois['category'].apply(lambda x: [element.upper() for element in x])\n",
    "pois.rename(columns={'category': 'kwds'}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply _DBSCAN_ algorithm to find **spatial clusters** of POIs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pois_in_clusters, eps_per_cluster = lc.clustering.compute_clusters(pois, alg='dbscan', min_pts=10, eps=0.005)\n",
    "cluster_borders = lc.clustering.cluster_shapes(pois_in_clusters, 2, eps_per_cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lc.plots.map_geometries(cluster_borders)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model clusters as documents, extract **3 topics**, and assign topics to clusters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster_topics, topic_keywords, visualized_topics = lc.topics.topic_modeling(pois_in_clusters, num_of_topics=3, kwds_per_topic=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_keywords.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Merge clusters with their topic assigments: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster_borders_topics = pd.merge(cluster_borders, cluster_topics, left_on='cluster_id', right_index=True, how='inner')\n",
    "cluster_borders_topics.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot each cluster with an **RGB color** signifying its adherence to each of the identified topics:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define map styling\n",
    "def style_gen_mixed(row):\n",
    "    r = round(row['properties']['Topic0'] * 255) \n",
    "    g = round(row['properties']['Topic1'] * 255) \n",
    "    b = round(row['properties']['Topic2'] * 255)\n",
    "    color = '#{:02x}{:02x}{:02x}'.format(r, g, b)\n",
    "    return {'fillColor': color, 'weight': 2, 'color': 'black', 'fillOpacity': 0.8}\n",
    "\n",
    "# Construct tooltip\n",
    "fields = list(cluster_borders_topics.columns.values)\n",
    "fields.remove('geometry')\n",
    "if 'style' in fields:\n",
    "    fields.remove('style')\n",
    "tooltip = folium.features.GeoJsonTooltip(fields=fields)\n",
    "\n",
    "m = lc.plots.map_geometries(cluster_borders, tiles='OpenStreetMap', width='100%', height='100%')\n",
    "\n",
    "folium.GeoJson(\n",
    "    cluster_borders_topics,\n",
    "    style_function=style_gen_mixed,\n",
    "    tooltip=tooltip\n",
    ").add_to(m)\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}