notebook/output/turin/data_from_osm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "def initiate_path():\n",
    "    path_temp = 'output'\n",
    "    path_round_about_temp = f'{path_temp}/roundabout'\n",
    "    path_network_temp = f'{path_temp}/networks'\n",
    "    return path_temp, path_round_about_temp, path_network_temp"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import osmnx as ox\n",
    "from geopandas import GeoDataFrame, GeoSeries\n",
    "from osmnx import io\n",
    "\n",
    "path, path_round_about, path_network = initiate_path()\n",
    "project_crs = 'epsg:3857'\n",
    "from sklearn.cluster import DBSCAN\n",
    "from shapely.geometry import Polygon, Point, LineString, MultiPolygon, MultiPoint\n",
    "import math\n",
    "import warnings\n",
    "import pandas as pd\n",
    "\n",
    "warnings.filterwarnings(action='ignore')\n",
    "from momepy import remove_false_nodes,extend_lines"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Run only when download for first time"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# Create graph from OSM server and project it\n",
    "graph = ox.graph_from_place('Torino', network_type='all')\n",
    "graph = ox.bearing.add_edge_bearings(graph, precision=1)\n",
    "graph_pro = ox.projection.project_graph(graph, to_crs=project_crs)\n",
    "io.save_graph_geopackage(graph_pro, filepath='.', encoding='utf-8', directed=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### End"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "# delete segments without names\n",
    "my_gdf = gpd.read_file(f'{path}/edges.shp')\n",
    "not_null = my_gdf.dropna(subset='name')\n",
    "not_null.to_file(f'{path}/not_null.shp')\n",
    "# delete segments present any kind of roundabout or tunnel\n",
    "not_roundaout = not_null[~((not_null['junction'] == 'roundabout') | (not_null['junction'] == 'circular') | (\n",
    "            not_null['tunnel'] == 'building_passage') | (not_null['tunnel'] == 'yes') | (\n",
    "                                       not_null['highway'] == 'cycleway') | (not_null['highway'] == 'path'))]\n",
    "round_about = not_null[not_null['junction'].isin(['roundabout', 'circular'])]\n",
    "not_roundaout.to_file(f'{path}/not_roundabout_tunnel.shp')\n",
    "round_about.to_file(f'{path_round_about}/roundabout.shp')\n",
    "# Project dataframe and calculate angle (0 to 180)\n",
    "df_pro = not_roundaout.to_crs(project_crs)\n",
    "df_pro['angle'] = df_pro['bearing'].apply(lambda x: x if x < 180 else x - 180)\n",
    "df_pro['length'] = df_pro.length\n",
    "df_pro.to_file(f'{path_network}/pro.shp')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "df_pro = gpd.read_file(f'{path_network}/pro.shp')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "def length_of_parallel(my_s_join: GeoDataFrame, the_buffer: GeoSeries, geo_field: str) -> int:\n",
    "    my_s_join['geometry'] = my_s_join[geo_field]\n",
    "    new_data_0 = my_s_join.sjoin(GeoDataFrame(geometry=the_buffer, crs=project_crs), how='inner').reset_index()\n",
    "    if len(new_data_0) == 0:\n",
    "        return 0\n",
    "    return len(new_data_0[new_data_0['index'] != new_data_0['index_right']])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "def check_parallelism(to_translate: GeoDataFrame, is_test_local: bool = False) -> bool:\n",
    "    my_buffer = to_translate['geometry'].buffer(cap_style=2, distance=30, join_style=3)\n",
    "    to_translate['geometry_right'] = to_translate['geometry'].apply(lambda x: x.parallel_offset(35, 'right'))\n",
    "    to_translate['geometry_left'] = to_translate['geometry'].apply(lambda x: x.parallel_offset(35, 'left'))\n",
    "    # Currently is it not working\n",
    "    if is_test_local:\n",
    "        # to_translate.drop(columns= ['geometry', new_geometry[0]]).rename(columns = {new_geometry[1]: 'geometry'}).to_file(f'{path}/test_data/res_translate_{new_geometry[1]}.shp')\n",
    "        # to_translate.drop(columns= ['geometry', new_geometry[1]]).rename(columns = {new_geometry[0]: 'geometry'}).to_file(f'{path}/test_data/res_translate_{new_geometry[0]}.shp')\n",
    "        my_buffer.to_file(f'{path}/test_data/buffers_test.shp')\n",
    "    if length_of_parallel(to_translate, my_buffer, 'geometry_right') > 0 or length_of_parallel(to_translate, my_buffer,                                                                                     'geometry_left') > 0:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "def simplify(my_polygon: Polygon, x: (int, float)) -> Polygon:\n",
    "    simplify_poly = my_polygon.simplify(x, preserve_topology=False)\n",
    "    if simplify_poly.area < 50:\n",
    "        simplify_poly = simplify(my_polygon, x / 2)\n",
    "    return simplify_poly"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "def create_center_line(one_poly):\n",
    "    \"\"\"\n",
    "    This method calculate new line between the farthest points of the simplified polygon\n",
    "    :param one_poly:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "\n",
    "    pnt_list = one_poly.exterior.coords[:-1]\n",
    "    list_shp = [Point(item) for item in pnt_list]\n",
    "    dis, dis_2, dis_3 = 0, 0, 0\n",
    "    third_dis = (-1, -1)\n",
    "    # The new line will be determined by the second-farthest points\n",
    "    for k, point in enumerate(list_shp):\n",
    "        j = k\n",
    "        for point2 in list_shp[k + 1:]:\n",
    "            j += 1\n",
    "            temp_dis = point.distance(point2)\n",
    "            if temp_dis > dis:\n",
    "                dis = point.distance(point2)\n",
    "            elif temp_dis > dis_2:\n",
    "                dis_2 = point.distance(point2)\n",
    "            elif temp_dis > dis_3:\n",
    "                third_dis = (k, j)\n",
    "                dis_3 = point.distance(point2)\n",
    "    if is_test:\n",
    "        dic_sim = {'index': [0], 'geometry': one_poly}\n",
    "        GeoDataFrame(dic_sim, crs='epsg:3857').to_file(f'{path}/test_data/simplify_poly_{unit[0]}_{id_pol + 2}.shp')\n",
    "    max_dist['name'].extend([id_pol + 2, id_pol + 2])\n",
    "    max_dist['geometry'].extend([list_shp[third_dis[0]], list_shp[third_dis[1]]])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "def update_df_with_center_line(new_line):\n",
    "    \"\"\"\n",
    "    update our dictionary with new lines\n",
    "    :param new_line:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    dic_final['name'].append(name)\n",
    "    # dic_final['geometry'].append(LineString(coordinates=(pnt_list[max_dis[0]], pnt_list[max_dis[1]])))\n",
    "    dic_final['geometry'].append(new_line)\n",
    "    dic_final['highway'].append(data.iloc[0]['highway'])\n",
    "    dic_final['bearing'].append(data['angle'].mean())\n",
    "    dic_final['group'].append(unit[0])\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "def update_list(line_local):\n",
    "    \"\"\"\n",
    "    add the first start/end point into the list\n",
    "    :param line_local:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    list_pnts_of_line_group.extend([Point(line_local.coords[0]), Point(line_local.coords[-1])])\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "def add_more_pnts_to_new_lines(pnt_f_loc: Point, pnt_l_loc: Point, line_pnts: list) -> list:\n",
    "    \"\"\"\n",
    "    This method checks if more points should be added to the new lines by checking along the new line if the distance to the old network roads are more than 10 meters\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    # Calculate distance and azimuth between the first and last point\n",
    "    dist = pnt_f_loc.distance(pnt_l_loc)\n",
    "    x_0 = pnt_f_loc.coords[0][0]\n",
    "    y_0 = pnt_f_loc.coords[0][1]\n",
    "    bearing = math.atan2(pnt_l_loc.coords[0][0] - x_0, pnt_l_loc.coords[0][1] - y_0)\n",
    "    bearing = bearing + 2 * math.pi if bearing < 0 else bearing\n",
    "    # Calculate the number of  checks going to carry out\n",
    "    loops = int(dist / 100)\n",
    "\n",
    "    # Calculate  the first point over the line\n",
    "    for dis_on_line in range(1, loops):\n",
    "        x_new = x_0 + 100 * dis_on_line * math.sin(bearing)\n",
    "        y_new = y_0 + 100 * dis_on_line * math.cos(bearing)\n",
    "        # S_joins to all the network lines (same name and group)\n",
    "        # if the distance is less than 10 meters continue, else: find the projection point and add it to the correct location and run the function agein\n",
    "        one_pnt_df = GeoDataFrame(geometry=[Point(x_new, y_new)], crs=project_crs)\n",
    "        s_join_loc = one_pnt_df.sjoin_nearest(data, distance_col='dis').iloc[0]\n",
    "        if s_join_loc['dis'] > 10:\n",
    "            pnt_med = s_join_loc['geometry']\n",
    "            line = data.loc[s_join_loc['index_right']]['geometry']\n",
    "            line_pnts.append(line.interpolate(line.project(pnt_med)))\n",
    "            line_pnts = add_more_pnts_to_new_lines(pnt_med, pnt_l_loc, line_pnts)\n",
    "            return line_pnts\n",
    "    return line_pnts"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\t"
     ]
    }
   ],
   "source": [
    "# Main point to start\n",
    "\n",
    "is_test = True\n",
    "my_groupby = df_pro.groupby('name')\n",
    "dic_final = {'name': [], 'geometry': [], 'highway': [], 'bearing': [], 'group': []}\n",
    "\n",
    "for_time = len(my_groupby)\n",
    "for i, street in enumerate(my_groupby):\n",
    "    # Calculate time to run\n",
    "    print(f'{round(i / for_time * 100, 2)}\\t', end=\"\")\n",
    "    res = street[1]\n",
    "    name = street[0]\n",
    "    if is_test:\n",
    "        name = 'Corso Giovanni Agnelli'\n",
    "        res = my_groupby.get_group(name)\n",
    "    # groupby angle\n",
    "    res = res.dropna(subset=['angle'], axis=0)\n",
    "    if len(res) == 0:\n",
    "        continue\n",
    "    res['group'] = DBSCAN(eps=5, min_samples=2).fit(res['angle'].to_numpy().reshape(-1, 1)).labels_\n",
    "    cur_group = res[res['group'] > -1].groupby('group')\n",
    "    is_parallel = False\n",
    "    for group in cur_group:\n",
    "        data = group[1]\n",
    "        if check_parallelism(data, is_test):\n",
    "            # if among of lines with same angles some are parallel,find the center line for each group\n",
    "            is_parallel = True\n",
    "            for unit in cur_group:\n",
    "                data = unit[1]\n",
    "\n",
    "                # new points DataFrame of start/end line of each group\n",
    "                list_pnts_of_line_group = []\n",
    "                data['geometry'].apply(update_list)\n",
    "                df_pnts = GeoDataFrame(geometry=list_pnts_of_line_group, crs='epsg:3857').drop_duplicates()\n",
    "\n",
    "                # unify lines to one polygon\n",
    "                buffers = data.buffer(cap_style=3, distance=30, join_style=3)\n",
    "                one_buffer = buffers.unary_union\n",
    "\n",
    "                max_dist = {'name': [], 'geometry': []}\n",
    "                # simplify polygon with simplify function. If one_buffer is multipolygon object simplify each one them separately\n",
    "                if isinstance(one_buffer, MultiPolygon):\n",
    "                    for id_pol, polygon in enumerate(one_buffer):\n",
    "                        create_center_line(polygon)\n",
    "                else:\n",
    "                    id_pol = -1\n",
    "                    create_center_line(one_buffer)\n",
    "                max_df = GeoDataFrame(max_dist, crs='epsg:3857')\n",
    "                # find for each points the closet point from the oribinal data. the closet points will create the new line\n",
    "                s_join = max_df.sjoin_nearest(df_pnts).groupby('name')\n",
    "                for geo in s_join:\n",
    "                    same_name = geo[1]\n",
    "                    if same_name.iloc[0]['index_right'] == same_name.iloc[1]['index_right']:\n",
    "                        continue\n",
    "                    in_0 = same_name.iloc[0]['index_right']\n",
    "                    in_1 = same_name.iloc[1]['index_right']\n",
    "                    # These points will be served to be initial reference in order to find more points\n",
    "                    pnt_f = df_pnts.loc[in_0]['geometry']\n",
    "                    pnt_l = df_pnts.loc[in_1]['geometry']\n",
    "                    lines_pnt_geo = add_more_pnts_to_new_lines(pnt_f, pnt_l, [pnt_f])\n",
    "                    lines_pnt_geo.append(pnt_l)\n",
    "                    # Updata dic_final\n",
    "                    update_df_with_center_line(LineString(lines_pnt_geo))\n",
    "                    # dic_final['geometry'].append(LineString(lines_pnt_geo))\n",
    "                    # new_lines['name'].append(geo[0])\n",
    "                if is_test:\n",
    "                    buffers.to_file(f'{path}/test_data/buffers_{unit[0]}.shp')\n",
    "                    # dic_one = {'index':[0],'geometry':one_buffer.geoms}\n",
    "                    # GeoDataFrame(dic_one,crs='epsg:3857').to_file(f'{path}/test_data/one_buffer_{unit[0]}.shp')\n",
    "        # if one group is found as parallel all the groups are calculated as parallel so we don't need to check_parallelism\n",
    "        if is_parallel:\n",
    "            break\n",
    "    # # טסט\n",
    "    if is_test:\n",
    "        final = GeoDataFrame(dic_final, crs=project_crs)\n",
    "        final = final[final.length > 20]\n",
    "        final.to_file(f'{path}/test_data/one_line.shp')\n",
    "        res.to_file(f'{path}/test_data/groups.shp')\n",
    "        break\n",
    "\n",
    "if not is_test:\n",
    "    print('create new files')\n",
    "    # remove short lines\n",
    "    final_cols = ['name', 'geometry', 'highway', 'bearing', 'length']\n",
    "    final = GeoDataFrame(dic_final, crs=project_crs)\n",
    "    final['is_simplify'] = 1\n",
    "    final['length'] = final.length\n",
    "    final = final[final.length > 100]\n",
    "    final.to_file(f'{path}/one_line_third.shp')\n",
    "    # create network\n",
    "    new_network = df_pro[~df_pro['name'].isin(dic_final['name'])][final_cols]\n",
    "    new_network['is_simplify'] = 0\n",
    "    new_network.append(final).to_file(f'{path_network}/new_network_third.shp')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Intersection"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Split in intersection"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "outputs": [],
   "source": [
    "class Intersection:\n",
    "    def __init__(self,network):\n",
    "        self.my_network = network\n",
    "        self.inter_pnt_dic = {'geometry':[],'name':[]}\n",
    "        self.lines_to_delete =[]\n",
    "    def delete_false_intersection(self):\n",
    "        # First clean all the false node\n",
    "        self.my_network = remove_false_nodes(self.my_network)\n",
    "        # the previous function has changed the topology so the length should be updated\n",
    "        self.my_network['length'] =self.my_network.length\n",
    "        self.my_network.to_file(f'{cur_path}/remove_false_nodes.shp')\n",
    "\n",
    "    def intersection_network(self):\n",
    "        # Create buffer around each element\n",
    "        buffer_around_lines= self.my_network['geometry'].buffer(cap_style=3, distance=1, join_style=3)\n",
    "        buffer_around_lines.to_file(f'{cur_path}/buffer.shp')\n",
    "\n",
    "        # s_join between buffer to lines\n",
    "        s_join_0 =gpd.sjoin(left_df=GeoDataFrame(geometry=buffer_around_lines,crs=project_crs),right_df=self.my_network)\n",
    "\n",
    "        # delete lines belong to the buffer\n",
    "        s_join = s_join_0[s_join_0.index!=s_join_0['index_right']]\n",
    "        s_join.to_file(f'{cur_path}/s_join.shp')\n",
    "\n",
    "        # Find new intersections that are not at the beginning or end of the line\n",
    "\n",
    "        s_join.apply(self.__find_intersection_points, axis=1)\n",
    "        inter_pnt_gdf = GeoDataFrame(self.inter_pnt_dic,crs=project_crs)\n",
    "        inter_pnt_gdf.to_file(f'{cur_path}/inter_pnt.shp')\n",
    "\n",
    "        # Split string line by points\n",
    "        segments = {'geometry':[],'org_id':[]}\n",
    "        # Groupby points name (which is the line they should split)\n",
    "        for group_pnts in inter_pnt_gdf.groupby('name'):\n",
    "            points  = group_pnts[1]\n",
    "            points['is_split'] = True\n",
    "\n",
    "            # get the line to split by comparing the name\n",
    "            row = self.my_network.loc[group_pnts[0]]\n",
    "            current = list(row.geometry.coords)\n",
    "            points_line = [Point(x) for x in current]\n",
    "            points_line_gdf = GeoDataFrame(geometry=points_line,crs=project_crs)\n",
    "            points_line_gdf['is_split'] = False\n",
    "\n",
    "            # append all the points together (line points and split points)\n",
    "            line_all_pnts = points_line_gdf.append(points)\n",
    "\n",
    "            # Find the distance of each point form the begining of the line on the line.\n",
    "            line_all_pnts['dis_from_the_start'] = line_all_pnts['geometry'].apply(lambda x:row.geometry.project(x))\n",
    "            line_all_pnts.sort_values('dis_from_the_start',inplace=True)\n",
    "\n",
    "            # split the line\n",
    "            seg =[]\n",
    "            for point in line_all_pnts.iterrows():\n",
    "                prop = point[1]\n",
    "                seg.append(prop['geometry'])\n",
    "                if prop['is_split']:\n",
    "                    segments['geometry'].append(LineString(seg))\n",
    "                    segments['org_id'].append(row.name)\n",
    "                    seg = [prop['geometry']]\n",
    "            segments['geometry'].append(LineString(seg))\n",
    "            segments['org_id'].append(row.name)\n",
    "        network_split = GeoDataFrame(data=segments,crs=project_crs)\n",
    "        cols_no_geometry = self.my_network.columns[:-1]\n",
    "        network_split_final = network_split.set_index('org_id')\n",
    "        network_split_final[cols_no_geometry] =self.my_network[cols_no_geometry]\n",
    "        network_split_final.to_file(f'{cur_path}/only_split.shp')\n",
    "        # remove old and redundant line from our network and update with new one\n",
    "        network_split =self.my_network.drop(index=network_split_final.index.unique()).append(network_split_final).drop(index= self.lines_to_delete)\n",
    "        network_split['length'] = network_split.length\n",
    "        self.my_network = network_split\n",
    "        self.my_network.reset_index().to_file(f'{cur_path}/split.shp')\n",
    "\n",
    "    def __find_intersection_points(self,row):\n",
    "        r\"\"\"\n",
    "        find the intersection points between the two lines\n",
    "        :param row:\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        try:\n",
    "            line_1 = self.my_network.loc[row.name]\n",
    "            line_2 =  self.my_network.loc[row['index_right']]\n",
    "            pnt = line_1.geometry.intersection(line_2.geometry)\n",
    "            # If there are more than one intersection between two lines, one of the lines should be deleted.\n",
    "            if isinstance(pnt,LineString):\n",
    "                print(f\"{row.name},{row['index_right']}:{pnt}\")\n",
    "                return\n",
    "            if isinstance(pnt,MultiPoint):\n",
    "                temp_line= line_1.name if line_1.length< line_2.length else line_2.name\n",
    "                if temp_line not in self.lines_to_delete:\n",
    "                    self.lines_to_delete.append(temp_line)\n",
    "                return\n",
    "            # If it is first or end continue OR if there is no intersection between the two lines\n",
    "            if len(pnt.coords)==0 or pnt.coords[0]==line_1.geometry.coords[0] or pnt.coords[0]==line_1.geometry.coords[-1]:\n",
    "                return\n",
    "            self.inter_pnt_dic['geometry'].append(pnt)\n",
    "            self.inter_pnt_dic['name'].append(row.name)\n",
    "        except:\n",
    "            print(f\"{row.name},{row['index_right']}:{pnt}\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "outputs": [],
   "source": [
    "cur_path = f'{path}/rearrange'\n",
    "new_gpd = gpd.read_file(f'{path_network}/new_network_third.shp')\n",
    "obj_intersection = Intersection(new_gpd)\n",
    "obj_intersection.delete_false_intersection()\n",
    "obj_intersection.intersection_network()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Roundabout"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "class EnvEntity:\n",
    "        def __init__(self,network):\n",
    "            self.pnt_dead_end = None\n",
    "            self.pnt_dic = {}\n",
    "            self.first_last_dic = {'geometry': [], 'line_name': [], 'position': []}\n",
    "            self.network = network\n",
    "\n",
    "\n",
    "        def __populate_pnt_dic(self,point: type, name_of_line: str):\n",
    "            \"\"\"\n",
    "            Make \"pnt_dic\" contain a list of all the lines connected to each point.\n",
    "            :param point:\n",
    "            :param name_of_line:\n",
    "            :return:\n",
    "            \"\"\"\n",
    "            if not point in self.pnt_dic:\n",
    "                self.pnt_dic[point] = []\n",
    "            self.pnt_dic[point].append(name_of_line)\n",
    "\n",
    "        def __send_pnts(self,temp_line: GeoSeries):\n",
    "            \"\"\"\n",
    "            # Send the first and the last points to populate_pnt_dic\n",
    "            :return:\n",
    "            \"\"\"\n",
    "            my_geom = temp_line['geometry']\n",
    "            self.__populate_pnt_dic(my_geom.coords[0], temp_line.name)\n",
    "            self.__populate_pnt_dic(my_geom.coords[-1], temp_line.name)\n",
    "\n",
    "        def get_deadend_gdf(self,delete_short:int =30)-> GeoDataFrame:\n",
    "            self.network.apply(self.__send_pnts, axis=1)\n",
    "\n",
    "            deadend_list = [item[1][0] for item in self.pnt_dic.items() if len(item[1]) == 1]\n",
    "            pnt_dead_end_0 = [item for item in self.pnt_dic.items() if len(item[1]) == 1] # Retain all the line points with deadened\n",
    "            self.pnt_dead_end = [Point(x[0]) for x in pnt_dead_end_0]\n",
    "            # Create shp file of deadened_pnts\n",
    "            geometry,line_name = 'geometry','line_name'\n",
    "            pnt_dead_end_df = GeoDataFrame(data=pnt_dead_end_0)\n",
    "            pnt_dead_end_df[geometry]= pnt_dead_end_df[0].apply(lambda x:Point(x))\n",
    "            pnt_dead_end_df[line_name] = pnt_dead_end_df[1].apply(lambda x:x[0])\n",
    "            pnt_dead_end_df.crs = project_crs\n",
    "            pnt_dead_end_df[[geometry,line_name]].to_file(f'{path_deadend}/deadend_gdf_pnt.shp')\n",
    "\n",
    "            if delete_short>0:\n",
    "                # If it is necessary to eliminate dead-end short segments, it is  important to delete them from the network geodataframe.\n",
    "\n",
    "                deadend_gdf =self.network.loc[deadend_list]\n",
    "                self.network.drop(index=deadend_gdf[deadend_gdf.length<delete_short].index,inplace=True)\n",
    "                return deadend_gdf[deadend_gdf.length>delete_short]\n",
    "            return self.network.loc[deadend_list]\n",
    "\n",
    "        def update_the_current_network(self,temp_network):\n",
    "            r\"\"\"\n",
    "            Update the current network in the new changes\n",
    "            :param temp_network:\n",
    "            :return:\n",
    "            \"\"\"\n",
    "            new_network_temp = self.network.drop(index=temp_network.index)\n",
    "            self.network = new_network_temp.append(temp_network)\n",
    "            self.network['length'] = self.network.length\n",
    "            self.network  = self.network[self.network['length']>1]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "class Roundabout(EnvEntity):\n",
    "    def __init__(self,network: GeoDataFrame):\n",
    "       EnvEntity.__init__(self,network)\n",
    "       self.pnt_dic ={}\n",
    "       self.centroid =self.__from_roundabout_to_centroid()\n",
    "       self.network.rename(columns={'name': 'str_name'}, inplace=True)\n",
    "    def __from_roundabout_to_centroid(self):\n",
    "        # Find the center of each roundabout\n",
    "        # create polygon around each polygon and union\n",
    "        round_about = gpd.read_file(f'{path_round_about}/roundabout.shp')\n",
    "        round_about_buffer = round_about.to_crs(project_crs)['geometry'].buffer(cap_style=1, distance=10,\n",
    "                                                                                join_style=1).unary_union\n",
    "        dic_data = {'name': [], 'geometry': []}\n",
    "        for ii, xx in enumerate(round_about_buffer):\n",
    "            dic_data['name'].append(ii)\n",
    "            dic_data['geometry'].append(xx.centroid)\n",
    "        centroid =GeoDataFrame(dic_data, crs=project_crs)\n",
    "        centroid.to_file(f'{path_round_about}/centroid.shp')\n",
    "        return centroid\n",
    "        # GeoDataFrame(dic_data,crs=project_crs).to_file(f'{path_round_about}/roundabout_union.shp')\n",
    "\n",
    "    def __first_last_pnt_of_line(self,row: GeoSeries):\n",
    "        r\"\"\"\n",
    "        It get geometry of line and fill the first_last_dic with the first and last point and the name of the line\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        geo = list(row['geometry'].coords)\n",
    "        self.first_last_dic['geometry'].extend([Point(geo[0]), Point(geo[-1])])\n",
    "        self.first_last_dic['line_name'].extend([row.name] * 2)\n",
    "        self.first_last_dic['position'].extend([0, -1])\n",
    "    def deadend(self,path_to_save: str, export_files: bool = False):\n",
    "        r\"\"\"\n",
    "        remove not connected line shorter than 100 meters and then return deadend_list lines and their endpoints (as another file)\n",
    "        :param path_to_save:\n",
    "        :param export_files:\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        # Find the first and last points\n",
    "\n",
    "        # Get deadend_gdf\n",
    "        deadend_gdf = self.get_deadend_gdf()\n",
    "\n",
    "        # Create gdf of line points with the reference to the line they belong\n",
    "        deadend_gdf.apply(self.__first_last_pnt_of_line, axis=1)\n",
    "        first_last_gdf = GeoDataFrame(self.first_last_dic, crs=project_crs)\n",
    "\n",
    "        if export_files:\n",
    "            # Optional - Create new file\n",
    "            temp_list = {}\n",
    "            temp_list ['geometry'] = [Point(x) for x in self.pnt_dic.keys()]\n",
    "            temp_list ['lines']  = [str(x) for x in self.pnt_dic.values()]\n",
    "            GeoDataFrame(temp_list,crs=project_crs).to_file(f'{path_deadend}/line_pnts.shp')\n",
    "            first_last_gdf.to_file(f'{path_to_save}/first_last_pnts.shp')\n",
    "            deadend_gdf.to_file(f'{path_to_save}/deadend_gdf_all.shp')\n",
    "\n",
    "        return deadend_gdf, first_last_gdf\n",
    "    def __update_geometry(self,cur,s_join):\n",
    "        r\"\"\"\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        if cur['highway'] == 'footway':\n",
    "            # Don't snap footway to roundabout\n",
    "            return cur['geometry']\n",
    "        # Get only the points that are deadened\n",
    "        points_lines = [item for item in s_join[s_join['line_name'] == cur.name].iterrows()if item[1]['geometry'] in my_roundabout.pnt_dead_end]\n",
    "        if len(points_lines) == 0:\n",
    "            # No roundabout nearby\n",
    "            return cur['geometry']\n",
    "        # get the line geometry to change the first and/ or last point\n",
    "        geo_cur = list(cur['geometry'].coords)\n",
    "\n",
    "        # iterate over the deadened points  near roundabout\n",
    "        for ind in range(len(points_lines)):\n",
    "            points_line = points_lines[ind]\n",
    "            geo_cur[points_line[1]['position']] = my_roundabout.centroid.loc[points_line[1]['index_right']]['geometry'].coords[\n",
    "                0]\n",
    "        return LineString(geo_cur)\n",
    "    def my_spatial_join(self,deadend_lines, deadend_pnts):\n",
    "        # Spatial join between roundabout centroid to nearby dead end lines\n",
    "        # centroid = gpd.read_file(f'{path_round_about}/centroid.shp')\n",
    "        s_join = gpd.sjoin_nearest(left_df=deadend_pnts, right_df=self.centroid, how='left', max_distance=60,\n",
    "                                   distance_col='dist').dropna(subset='dist')\n",
    "        s_join.to_file(f'{path_deadend}/roundabout_near_pnts.shp')\n",
    "        # Update the geometry so the roundabout will be part of the line geometry\n",
    "        change_geo = deadend_lines.copy()\n",
    "\n",
    "        change_geo['geometry'] = change_geo.apply(lambda x:self.__update_geometry(x,s_join), axis=1)\n",
    "        change_geo.reset_index().to_file(f'{path_deadend}/connect_roundabout.shp')\n",
    "        return change_geo\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [],
   "source": [
    "# Network Improvement - to roundabout\n",
    "# Find unconnected lines\n",
    "\n",
    "exist_data= gpd.read_file(f'{cur_path}/split.shp')\n",
    "\n",
    "my_roundabout=Roundabout(exist_data)\n",
    "path_deadend = f'{path}/deadend'\n",
    "deadend_lines, deadend_pnts = my_roundabout.deadend(path_to_save=path_deadend,export_files=True)\n",
    "\n",
    "# update the current network\n",
    "change_geo = my_roundabout.my_spatial_join(deadend_lines, deadend_pnts)\n",
    "my_roundabout.update_the_current_network(change_geo)\n",
    "line_name ='line_name'\n",
    "my_roundabout.network = my_roundabout.network.reset_index().rename(columns={'level_0':line_name})\n",
    "my_roundabout.network.drop_duplicates(subset=line_name,inplace=True)\n",
    "my_roundabout.network.to_file(f'{path_network}/network_ra.shp')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "outputs": [],
   "source": [
    "# First buffer around centroid\n",
    "centr_name= 'centr_name'\n",
    "my_roundabout.network= gpd.read_file(f'{path_network}/network_ra.shp')\n",
    "buffer_around_centroid= my_roundabout.centroid['geometry'].buffer(cap_style=1, distance=30)\n",
    "buffer_around_centroid.to_file(f'{path_round_about}/buffer_around_centroid.shp')\n",
    "\n",
    "# s_join between buffer to lines (reset index to retain the original centroid name which can apper more than one in the results). always stay with data you need and with understandable name\n",
    "roundabout_with_lines =gpd.sjoin(left_df=GeoDataFrame(geometry=buffer_around_centroid,crs=project_crs).reset_index(),right_df=my_roundabout.network[['geometry',line_name]]).drop_duplicates(subset=['index',line_name]).rename(columns={\"index\":centr_name})[['geometry',line_name,centr_name]]\n",
    "roundabout_with_lines.to_file(f'{path_deadend}/roundabout_with_lines.shp')\n",
    "\n",
    "# To facilitate the searching process\n",
    "my_roundabout.network.set_index(line_name,inplace=True)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "outputs": [],
   "source": [
    "# To facilitate easy access to point centroid geometry data, it is advisable to store the information in an object that provides efficient retrieval.\n",
    "pnt_centroid_temp = my_roundabout.centroid['geometry']\n",
    "#  Group the data by centroid\n",
    "for center_line in roundabout_with_lines.groupby(centr_name):\n",
    "    #  Iterate over each group after performing a groupby() operation\n",
    "    for center in center_line[1].itertuples():\n",
    "        # Find the line that connects to the current centroid and obtain its vertices\n",
    "        line_to_test = my_roundabout.network.loc[center[2]]\n",
    "        vertices_line = list(line_to_test['geometry'].coords)\n",
    "        pnt_test = [vertices_line[0],vertices_line[-1]]\n",
    "        # To determine if the current line is already connected to the current centroid,.\n",
    "        is_connected = my_roundabout.centroid[my_roundabout.centroid['geometry'].isin([Point(pnt_test[0]),Point(pnt_test[-1])])]\n",
    "        if len(is_connected)>0 and center[3] in is_connected['name']:\n",
    "            continue\n",
    "\n",
    "        if len(vertices_line)==2:\n",
    "            vertices_line.insert(1, pnt_centroid_temp[center[3]])\n",
    "        else:\n",
    "            my_list = [pnt_centroid_temp[center[3]].distance(Point(temp)) for temp in vertices_line]\n",
    "            # Find the minimum index\n",
    "            min_index = min(range(len(my_list)), key=my_list.__getitem__)\n",
    "            if min_index ==0:\n",
    "                vertices_line.insert(0,pnt_centroid_temp[center[3]])\n",
    "            elif min_index == len(my_list)-1:\n",
    "                vertices_line.append(pnt_centroid_temp[center[3]])\n",
    "            else:\n",
    "                vertices_line[min_index] = pnt_centroid_temp[center[3]]\n",
    "        new_geo = LineString(vertices_line)\n",
    "        my_roundabout.network.at[center[2],'geometry'] = new_geo\n",
    "my_roundabout.network.to_file(f'{path_network}/network_ra_update.shp')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "outputs": [
    {
     "data": {
      "text/plain": "[16.577714574708605,\n 30.518785256092574,\n 46.081763932513134,\n 67.56601779869354,\n 71.10783296217369,\n 141.67607968787638,\n 154.57225224464733]"
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line_to_test = my_roundabout.network.loc[9199]\n",
    "vertices_line = list(line_to_test['geometry'].coords)\n",
    "pnt_test = [vertices_line[0],vertices_line[-1]]\n",
    "# To determine if the current line is already connected to the current centroid,.\n",
    "is_connected = my_roundabout.centroid[my_roundabout.centroid['geometry'].isin([Point(pnt_test[0]),Point(pnt_test[-1])])]\n",
    "my_list = [pnt_centroid_temp[45].distance(Point(temp)) for temp in vertices_line]\n",
    "my_list"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "outputs": [],
   "source": [
    "# Extend\n",
    "new_network2 =gpd.read_file(f'{path_network}/network_ra_update.shp')\n",
    "extend_lines_f= extend_lines(new_network2,100)\n",
    "extend_lines_f['length'] = extend_lines_f.length\n",
    "extend_lines_f.to_file(f'{cur_path}/extend_lines.shp')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "cur_path = f'{path}/finals'\n",
    "obj_intersection2 = Intersection(extend_lines_f)\n",
    "obj_intersection2.delete_false_intersection()\n",
    "obj_intersection2.intersection_network()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "outputs": [],
   "source": [
    "# Clear short segments\n",
    "final = EnvEntity(obj_intersection2.my_network.reset_index())\n",
    "final.update_the_current_network(final.get_deadend_gdf(delete_short=30))\n",
    "final.network.to_file(f'{path_network}/final.shp')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}