From aec51def870db7d6f12fd5c9f31653a4c8057021 Mon Sep 17 00:00:00 2001 From: Nico Alt Date: Wed, 29 Nov 2017 22:12:41 +0100 Subject: [PATCH 01/18] Add config file for Esteli, Nicaragua network --- osm2gtfs/creators/esteli/esteli.json | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 osm2gtfs/creators/esteli/esteli.json diff --git a/osm2gtfs/creators/esteli/esteli.json b/osm2gtfs/creators/esteli/esteli.json new file mode 100644 index 00000000..22b25821 --- /dev/null +++ b/osm2gtfs/creators/esteli/esteli.json @@ -0,0 +1,35 @@ +{ + "query": { + "bbox": { + "n": "13.1308234", + "s": "13.0534171", + "e": "-86.3197351", + "w": "-86.3999775" + }, + "tags": { + "route": "bus", + "network": "NI-Estelí" + } + }, + "stops": { + "name_without": "Parada sin nombre", + "name_auto": "no" + }, + "agency": { + "agency_id": "NI-Estelí", + "agency_name": "Estelí", + "agency_url": "https://wiki.openstreetmap.org/wiki/ES:Wikiproyecto_Nicaragua/Transporte_p%C3%BAblico/Estel%C3%AD", + "agency_timezone": "America/Managua", + "agency_lang": "ES", + "agency_phone": "", + "agency_fare_url": "" + }, + "feed_info": { + "publisher_name": "Nico Alt", + "publisher_url": "https://nico.dorfbrunnen.eu", + "version": "0.1" + }, + "schedule_source": "https://github.com/mapanica/pt-data-esteli/raw/master/timetable.json", + "output_file": "data/ni-esteli.zip", + "selector": "esteli" +} From 2b6de6e09c2a40572dd676a247ea582c00296db3 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Wed, 29 Nov 2017 00:05:50 +0100 Subject: [PATCH 02/18] Add config file for Managua, Nicaragua network --- osm2gtfs/creators/managua/managua.json | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 osm2gtfs/creators/managua/managua.json diff --git a/osm2gtfs/creators/managua/managua.json b/osm2gtfs/creators/managua/managua.json new file mode 100644 index 00000000..0afeb1b3 --- /dev/null +++ b/osm2gtfs/creators/managua/managua.json @@ -0,0 +1,38 @@ +{ + "query": { + "bbox": { + "n": "12.21", + "s": "12.04", + "e": "-86.12", + "w": "-86.38" + }, + "tags": { + "route":"bus", + "network": "IRTRAMMA", + "public_transport:version": "2" + } + }, + "stops": { + "name_without": "Parada sin nombre - Agregaselo en rutas.mapanica.net", + "name_auto": "no" + }, + "agency": { + "agency_id": "NI-IRTRAMMA", + "agency_name": "Instituto Regulador del Transporte del Municipio de Managua", + "agency_url": "http://www.managua.gob.ni/", + "agency_timezone": "America/Managua", + "agency_lang": "es", + "agency_phone": "+505 255-2189", + "agency_fare_url": "" + }, + "feed_info": { + "publisher_name": "MapaNica.net", + "publisher_url": "https://mapanica.net", + "version": "0.1", + "start_date": "20171101", + "end_date": "20181031" + }, + "schedule_source": "https://raw.githubusercontent.com/mapanica/pt-data-managua/master/timetable.json", + "output_file": "data/ni-managua-gtfs.zip", + "selector": "managua" +} From 3c8079a1054503b3fa7a35d279dc00cb9cd4c002 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Sat, 12 Nov 2016 18:59:07 +0100 Subject: [PATCH 03/18] Implement generic data structure Author: Felix Delattre --- osm2gtfs/core/helper.py | 23 ++++ osm2gtfs/core/osm_connector.py | 125 ++++++++++++------ osm2gtfs/core/osm_routes.py | 109 --------------- osm2gtfs/core/osm_stops.py | 76 ----------- osm2gtfs/core/routes.py | 67 ++++++++++ osm2gtfs/core/stops.py | 35 +++++ .../creators/accra/stops_creator_accra.py | 3 +- osm2gtfs/creators/routes_creator.py | 15 ++- osm2gtfs/creators/trips_creator.py | 13 ++ 9 files changed, 242 insertions(+), 224 deletions(-) create mode 100644 osm2gtfs/core/helper.py delete mode 100644 osm2gtfs/core/osm_routes.py delete mode 100644 osm2gtfs/core/osm_stops.py create mode 100644 osm2gtfs/core/routes.py create mode 100644 osm2gtfs/core/stops.py diff --git a/osm2gtfs/core/helper.py b/osm2gtfs/core/helper.py new file mode 100644 index 00000000..9d1df938 --- /dev/null +++ b/osm2gtfs/core/helper.py @@ -0,0 +1,23 @@ +# coding=utf-8 + + +class Helper(object): + """The Helper class contains useful static functions + + """ + + @staticmethod + def print_shape_for_leaflet(shape): + print "var shape = [", + i = 0 + for node in shape: + print "new L.LatLng(" + str(node["lat"]) + ", " + str(node["lon"]) + ")", + if i != len(shape) - 1: + print ",", + i += 1 + print "];" + i = 0 + for node in shape: + print "L.marker([" + str(node["lat"]) + ", " + str(node["lon"]) + "]).addTo(map)" + print " .bindPopup(\"" + str(i) + "\").openPopup();" + i += 1 diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index c2bbb8ec..cdedb6d7 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -3,10 +3,11 @@ import sys import overpy from collections import OrderedDict +from math import cos, sin, atan2, sqrt, radians, degrees from transitfeed import util from osm2gtfs.core.cache import Cache -from osm2gtfs.core.osm_routes import Route, RouteMaster -from osm2gtfs.core.osm_stops import Stop, StopArea +from osm2gtfs.core.routes import Itinerary, Line +from osm2gtfs.core.stops import Stop, StopArea class OsmConnector(object): @@ -125,16 +126,17 @@ def get_routes(self, refresh=False): # Build routes from master relations for rmid, route_master in route_masters.iteritems(): - members = OrderedDict() + itineraries = OrderedDict() # Build route variant members for member in route_master.members: + # Create Itinerary objects from member route variants if member.ref in route_variants: rv = route_variants.pop(member.ref) - members[rv.id] = self._build_route_variant(rv, result) + itineraries[rv.id] = self._build_itinerary(rv, result) - # Route master member was already used before or is not valid + # Route variant was already used or is not valid else: rv = result.get_relations(member.ref) if bool(rv): @@ -142,36 +144,43 @@ def get_routes(self, refresh=False): sys.stderr.write("Route variant was assigned again:\n") sys.stderr.write( "http://osm.org/relation/" + str(rv.id) + "\n") - members[rv.id] = self._build_route_variant(rv, result) + itineraries[rv.id] = self._build_itinerary(rv, result) else: sys.stderr.write( "Member relation is not a valid route variant:\n") sys.stderr.write("http://osm.org/relation/" + str(member.ref) + "\n") - rm = self._build_route_master(route_master, members) + # Create Line object from route master + line = self._build_line(route_master, itineraries) + print line.osm_url - # Make sure ref number is not already taken - if rm.ref in self.routes: + # Make sure route_id (ref) number is not already taken + if line.route_id in self.routes: sys.stderr.write("'Ref' of route_master already taken\n") sys.stderr.write( "http://osm.org/relation/" + str(route_master.id) + "\n") sys.stderr.write("Skipped. Please fix in OpenStreetMap\n") else: - self.routes[rm.ref] = rm + self.routes[line.route_id] = line # Build routes from variants (missing master relation) for rvid, route_variant in route_variants.iteritems(): - sys.stderr.write("Route (variant) without masters\n") - rv = self._build_route_variant(route_variant, result) - # Make sure ref number is not already taken - if rv.ref in self.routes: + sys.stderr.write("Route (variant) without master\n") + itinerary = self._build_itinerary(route_variant, result) + + # Make sure route_id (ref) number is not already taken + if route_variant.tags['ref'] in self.routes: sys.stderr.write("Route (variant) with existing 'Ref'\n") sys.stderr.write( "http://osm.org/relation/" + str(route_variant.id) + "\n") sys.stderr.write("Skipped. Please fix in OpenStreetMap\n") else: - self.routes[rv.ref] = rv + # Create Line from route variant + itineraries = OrderedDict() + itineraries[itinerary.osm_id] = itinerary + line = self._build_line(route_variant, itineraries) + self.routes[line.route_id] = line # Cache data Cache.write_data('routes-' + self.selector, self.routes) @@ -259,48 +268,64 @@ def get_stops(self, refresh=False): return self.stops - def _build_route_master(self, route_master, members): - """Helper function to build a RouteMaster object + def _build_line(self, route_master, itineraries): + """Helper function to build a Line object - Returns a initiated RouteMaster object from raw data + Returns a initiated Line object from raw data """ if 'ref' in route_master.tags: ref = route_master.tags['ref'] else: sys.stderr.write( - "RouteMaster without 'ref'. Please fix in OpenStreetMap\n") + "Relation without 'ref'. Please fix in OpenStreetMap\n") sys.stderr.write( "http://osm.org/relation/" + str(route_master.id) + "\n") - # Check if a ref can be taken from one of the members + # Check if a ref can be taken from one of the itineraries ref = False - for member in list(members.values()): - if not ref and member.ref: - ref = member.ref + for itinerary in list(itineraries.values()): + if not ref and itinerary.ref: + ref = itinerary.ref sys.stderr.write( "Using 'ref' from member variant instead\n") - sys.stderr.write( - "http://osm.org/relation/" + str(member.id) + "\n") + sys.stderr.write(itinerary.osm_url + "\n") # Ignore whole Line if no reference number could be obtained if not ref: sys.stderr.write( - "No 'ref' could be obtained from members. Skipping.\n") + "No 'ref' could be obtained. Skipping whole route.\n") return name = route_master.tags['name'] frequency = None if "frequency" in route_master.tags: frequency = route_master.tags['frequency'] - rm = RouteMaster(route_master.id, ref, name, members, frequency) - print(rm) - return rm - def _build_route_variant(self, route_variant, query_result_set, rm=None): - """Helper function to build a RouteVariant object + if 'route_master' in route_master.tags: + route_type = route_master.tags['route_master'].capitalize() + + # If there was no route_master present we have a route relation here + elif 'route' in route_master.tags: + route_type = route_master.tags['route'].capitalize() + + print route_type + + # Create Line (route master) object + line = Line(osm_id=route_master.id, route_id=ref, + name=name, route_type=route_type, frequency=frequency) + + # Add Itinerary objects (route variants) to Line (route master) + for itinerary in list(itineraries.values()): + line.add_itinerary(itinerary) + + #print(line) + return line + + def _build_itinerary(self, route_variant, query_result_set): + """Helper function to build a Itinerary object - Returns a initiated RouteVariant object from raw data + Returns a initiated Itinerary object from raw data """ if 'ref' in route_variant.tags: @@ -350,9 +375,9 @@ def _build_route_variant(self, route_variant, query_result_set, rm=None): stops.append(otype + "/" + str(stop_candidate.ref)) shape = self._generate_shape(route_variant, query_result_set) - rv = Route(route_variant.id, fr, to, stops, - rm, ref, name, shape, travel_time) - print(rv) + rv = Itinerary(osm_id=route_variant.id, fr=fr, + to=to, stops=stops, shape=shape, ref=ref, + name=name, travel_time=travel_time) return rv def _build_stop(self, stop, stop_type): @@ -368,7 +393,7 @@ def _build_stop(self, stop, stop_type): # Ways don't have coordinates and they have to be calculated if stop_type == "way": - (stop.lat, stop.lon) = Stop.get_center_of_nodes(stop.get_nodes()) + (stop.lat, stop.lon) = OsmConnector.get_center_of_nodes(stop.get_nodes()) s = Stop(stop.id, "node", stop.tags['name'], stop.lat, stop.lon) return s @@ -598,7 +623,7 @@ def _find_best_name_for_unnamed_stop(self, stop): winner_distance = sys.maxint for candidate in candidates: if isinstance(candidate, overpy.Way): - lat, lon = Stop.get_center_of_nodes( + lat, lon = OsmConnector.get_center_of_nodes( candidate.get_nodes(resolve_missing=True)) distance = util.ApproximateDistance( lat, @@ -619,3 +644,29 @@ def _find_best_name_for_unnamed_stop(self, stop): # take name from winner stop.name = winner.tags["name"].encode('utf-8') + + @staticmethod + def get_center_of_nodes(nodes): + """Helper function to get center coordinates of a group of nodes + + """ + x = 0 + y = 0 + z = 0 + + for node in nodes: + lat = radians(float(node.lat)) + lon = radians(float(node.lon)) + + x += cos(lat) * cos(lon) + y += cos(lat) * sin(lon) + z += sin(lat) + + x = float(x / len(nodes)) + y = float(y / len(nodes)) + z = float(z / len(nodes)) + + center_lat = degrees(atan2(z, sqrt(x * x + y * y))) + center_lon = degrees(atan2(y, x)) + + return center_lat, center_lon diff --git a/osm2gtfs/core/osm_routes.py b/osm2gtfs/core/osm_routes.py deleted file mode 100644 index 07621026..00000000 --- a/osm2gtfs/core/osm_routes.py +++ /dev/null @@ -1,109 +0,0 @@ -# coding=utf-8 - - -class BaseRoute(object): - - def __init__(self, osm, ref, name): - self.id = osm - self.ref = ref - if name is not None: - self.name = name.encode('utf-8') - else: - self.name = name - self.last_update = None - - def __repr__(self): - rep = "" - if self.ref is not None: - rep += str(self.ref) + " | " - if self.name is not None: - rep += self.name - return rep - - -class Route(BaseRoute): - - def __init__(self, osm, fr, to, stops, master, ref, name, shape, travel_time=None): - BaseRoute.__init__(self, osm, ref, name) - self.fr = fr - self.to = to - self.stops = stops - self.master = master - self.shape = shape - self.travel_time = travel_time - self.duration = None - - def __repr__(self): - rep = BaseRoute.__repr__(self) - if self.stops is not None: - rep += " | Stops: " + str(len(self.stops)) + " | " - rep += "https://www.openstreetmap.org/relation/" + str(self.id) + " " - rep += "http://www.consorciofenix.com.br/horarios?q=" + str(self.ref) - return rep - - def set_duration(self, duration): - self.duration = duration - - def get_first_stop(self): - if len(self.stops) > 0: - return self.stops[0] - else: - return None - - def get_first_alt_stop(self): - if self.fr is not None: - return self.fr - else: - return "???" - - def has_proper_master(self): - return self.master is not None and len(self.master.routes) > 1 - - # TODO move to debug class? - def print_shape_for_leaflet(self): - print "var shape = [", - i = 0 - for node in self.shape: - print "new L.LatLng(" + str(node["lat"]) + ", " + str(node["lon"]) + ")", - if i != len(self.shape) - 1: - print ",", - i += 1 - print "];" - i = 0 - for node in self.shape: - print "L.marker([" + str(node["lat"]) + ", " + str(node["lon"]) + "]).addTo(map)" - print " .bindPopup(\"" + str(i) + "\").openPopup();" - i += 1 - - -class RouteMaster(BaseRoute): - - def __init__(self, osm, ref, name, routes, frequency=None): - BaseRoute.__init__(self, osm, ref, name) - self.routes = routes - self.frequency = frequency - for route in self.routes.values(): - route.master = self - - def __repr__(self): - rep = BaseRoute.__repr__(self) - rep += " | https://www.openstreetmap.org/relation/" + \ - str(self.id) + "\n" - - i = 1 - for route in self.routes: - rep += " Route %d: " % i - rep += str(self.routes[route]) + "\n" - i += 1 - - return rep - - def set_duration(self, duration): - for route in self.routes.values(): - route.set_duration(duration) - - def get_first_stop(self): - return self.routes.itervalues().next().get_first_stop() - - def get_first_alt_stop(self): - return self.routes.itervalues().next().get_first_alt_stop() diff --git a/osm2gtfs/core/osm_stops.py b/osm2gtfs/core/osm_stops.py deleted file mode 100644 index ce9828b1..00000000 --- a/osm2gtfs/core/osm_stops.py +++ /dev/null @@ -1,76 +0,0 @@ -# coding=utf-8 - -from math import cos, sin, atan2, sqrt, radians, degrees - - -class Stop(object): - - def __init__(self, osm, stop_type, name=None, lat=None, lon=None): - self.id = osm - if name is not None: - self.name = name.encode('utf-8') - else: - self.name = name - self.lat = lat - self.lon = lon - self.type = stop_type - self.added = False - - def __repr__(self): - rep = "" - if self.name is not None: - rep += self.name - if self.lat is not None and self.lon is not None: - rep += " http://www.openstreetmap.org/?mlat=" + \ - str(self.lat) + "&mlon=" + str(self.lon) - rep += " (https://www.openstreetmap.org/" + \ - self.type + "/" + str(self.id) + ")" - return rep - - @staticmethod - def get_center_of_nodes(nodes): - """Helper function to get center coordinates of a group of nodes - - """ - x = 0 - y = 0 - z = 0 - - for node in nodes: - lat = radians(float(node.lat)) - lon = radians(float(node.lon)) - - x += cos(lat) * cos(lon) - y += cos(lat) * sin(lon) - z += sin(lat) - - x = float(x / len(nodes)) - y = float(y / len(nodes)) - z = float(z / len(nodes)) - - center_lat = degrees(atan2(z, sqrt(x * x + y * y))) - center_lon = degrees(atan2(y, x)) - - return center_lat, center_lon - - -class StopArea(object): - - def __init__(self, osm, stop_members, name=None): - self.id = osm - if name is not None: - self.name = name.encode('utf-8') - else: - self.name = name - self.stop_members = stop_members - self.lat, self.lon = Stop.get_center_of_nodes(stop_members.values()) - - def __repr__(self): - rep = "" - if self.name is not None: - rep += self.name - if self.lat is not None and self.lon is not None: - rep += " lat: " + str(self.lat) + " lon: " + str(self.lon) + "\n\t" - for ref, stop in self.stop_members.iteritems(): - rep += " | Stop member: " + ref + " - " + stop.name - return rep diff --git a/osm2gtfs/core/routes.py b/osm2gtfs/core/routes.py new file mode 100644 index 00000000..27ab563c --- /dev/null +++ b/osm2gtfs/core/routes.py @@ -0,0 +1,67 @@ +# coding=utf-8 + +import attr + + +@attr.s +class Line(object): + """A general public transport service Line. + + It's a container of meta information and different Itinerary objects for + variants of the same service line. + + In OpenStreetMap this is usually represented as "route_master" relation. + In GTFS this is usually represented as "route" + + """ + osm_id = attr.ib() + route_id = attr.ib() + name = attr.ib() + route_type = attr.ib() # Required (Tram, Subway, Rail, Bus, ...) + + route_desc = attr.ib(default=None) + route_url = attr.ib(default=None) + route_color = attr.ib(default="FFFFFF") + route_text_color = attr.ib(default="000000") + osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) + frequency = attr.ib() + + # Related route variants + _itineraries = [] + + def add_itinerary(self, itinerary): + self._itineraries.append(itinerary) + + def get_itineraries(self): + return self._itineraries + + +@attr.s +class Itinerary(object): + """A public transport service itinerary. + + It's a representation of a possible variant of a line, grouped together by + a Line object. + + In OpenStreetMap this is usually represented as "route" relation. + In GTFS this is not exlicitly presented but used as based to create "trips" + + """ + osm_id = attr.ib() + ref = attr.ib() + fr = attr.ib() + to = attr.ib() + shape = attr.ib() + stops = attr.ib() + travel_time = attr.ib() + + route_url = attr.ib(default=None) + wheelchair_accessible = attr.ib(default=0) + bikes_allowed = attr.ib(default=0) + osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) + + # Useful information for further calculation + duration = attr.ib(default=None) + + def get_stop_by_position(self, pos): + raise NotImplementedError("Should have implemented this") diff --git a/osm2gtfs/core/stops.py b/osm2gtfs/core/stops.py new file mode 100644 index 00000000..896f9742 --- /dev/null +++ b/osm2gtfs/core/stops.py @@ -0,0 +1,35 @@ +# coding=utf-8 + +import attr + + +@attr.s +class Stop(object): + + osm_id = attr.ib() + osm_type = attr.ib() + name = attr.ib() + lat = attr.ib() + lon = attr.ib() + gtfs_id = attr.ib(default=osm_id) + osm_url = attr.ib(default="http://osm.org/" + + str(osm_type) + "/" + str(osm_id)) + + +class StopArea(object): + + osm_id = attr.ib() + name = attr.ib() + lat = attr.ib() + lon = attr.ib() + + _stop_members = [] + + def __init__(self, osm_id, stop_members, name=None): + self.osm_id = osm_id + if name is not None: + self.name = name.encode('utf-8') + else: + self.name = name + self.stop_members = stop_members + self.lat, self.lon = Stop.get_center_of_nodes(stop_members.values()) diff --git a/osm2gtfs/creators/accra/stops_creator_accra.py b/osm2gtfs/creators/accra/stops_creator_accra.py index 34470aa4..7087d89b 100644 --- a/osm2gtfs/creators/accra/stops_creator_accra.py +++ b/osm2gtfs/creators/accra/stops_creator_accra.py @@ -61,7 +61,8 @@ def add_stops_to_feed(self, feed, data): stops = data.get_stops() stops_by_name = {} - for a_stop in stops.values(): + for a_stop_id, a_stop in stops.items(): + a_stop.osm_id = a_stop_id if a_stop.name not in stops_by_name: stops_by_name[a_stop.name] = [] stops_by_name[a_stop.name].append(a_stop) diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index a3bd14cf..2f88b2d4 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -13,4 +13,17 @@ def __repr__(self): return rep def add_routes_to_feed(self, feed, data): - raise NotImplementedError("Should have implemented this") + return + + """ + route_id # Required: From Line + route_type # Required: From Line + + route_short_name # Required: To be generated from Line or Itinerary + route_long_name # Required: To be generated from Line or Itinerary + + route_desc # From Line + route_url # From Line + route_color: # From Line + route_text_color #From Line + """ diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index cdfa903d..cfaac637 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -15,6 +15,19 @@ def __repr__(self): def add_trips_to_feed(self, feed, data): raise NotImplementedError("Should have implemented this") + """ + route_id # Required: From Line + service_id # Required: To be generated + trip_id # Required: To be generated + + trip_headsign # Itinerary "to" + direction_id # Order of tinieraries in Line object + wheelchair_accessible # Itinerary "wheelchair_accessible" + bikes_allowed: # Itinerary "bikes_allowed" + trip_short_name # To be avoided! + block_id # To be avoided! + """ + @staticmethod def interpolate_stop_times(trip): """ From 54358facafaefdf331c002c51ee4c2afd349c272 Mon Sep 17 00:00:00 2001 From: Nico Alt Date: Thu, 19 Oct 2017 16:52:03 -0600 Subject: [PATCH 04/18] Finish work on data structure --- osm2gtfs/core/osm_connector.py | 120 +++++++++++++----- osm2gtfs/core/routes.py | 18 ++- osm2gtfs/core/stops.py | 8 +- .../creators/accra/stops_creator_accra.py | 6 +- .../creators/accra/trips_creator_accra.py | 16 ++- .../creators/fenix/trips_creator_fenix.py | 8 +- .../incofer/routes_creator_incofer.py | 2 +- .../creators/incofer/trips_creator_incofer.py | 3 +- osm2gtfs/creators/routes_creator.py | 9 +- osm2gtfs/creators/stops_creator.py | 13 +- osm2gtfs/creators/trips_creator.py | 6 +- setup.py | 3 +- 12 files changed, 145 insertions(+), 67 deletions(-) diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index cdedb6d7..86f9c057 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -85,8 +85,8 @@ def get_routes(self, refresh=False): Data about routes is getting obtained from OpenStreetMap through the Overpass API, based on the configuration from the config file. - Then this data gets prepared by building up objects of RouteMaster and - RouteVariant objects that are related to each other. + Then this data gets prepared by building up objects of Line and + Itinerary objects that are related to each other. It uses caching to leverage fast performance and spare the Overpass API. Special commands are used to refresh cached data. @@ -95,8 +95,8 @@ def get_routes(self, refresh=False): :param refresh: A simple boolean indicating a data refresh or use of caching if possible. - :return routes: A dictionary of RouteMaster objects with related - RouteVariant objects constituting the tree of data. + :return routes: A dictionary of Line objects with related + Itinerary objects constituting the tree of data. """ # Preferably return cached data about routes @@ -141,19 +141,17 @@ def get_routes(self, refresh=False): rv = result.get_relations(member.ref) if bool(rv): rv = rv.pop() - sys.stderr.write("Route variant was assigned again:\n") + sys.stderr.write("Itinerary was assigned again:\n") sys.stderr.write( "http://osm.org/relation/" + str(rv.id) + "\n") itineraries[rv.id] = self._build_itinerary(rv, result) else: sys.stderr.write( - "Member relation is not a valid route variant:\n") - sys.stderr.write("http://osm.org/relation/" + - str(member.ref) + "\n") + "Member relation is not a valid itinerary:\n") + sys.stderr.write("http://osm.org/relation/" + str(member.ref) + "\n") # Create Line object from route master line = self._build_line(route_master, itineraries) - print line.osm_url # Make sure route_id (ref) number is not already taken if line.route_id in self.routes: @@ -166,12 +164,15 @@ def get_routes(self, refresh=False): # Build routes from variants (missing master relation) for rvid, route_variant in route_variants.iteritems(): - sys.stderr.write("Route (variant) without master\n") + sys.stderr.write("Itinerary without master\n") + sys.stderr.write( + "http://osm.org/relation/" + str(route_variant.id) + "\n") + sys.stderr.write("Please fix in OpenStreetMap\n") itinerary = self._build_itinerary(route_variant, result) # Make sure route_id (ref) number is not already taken - if route_variant.tags['ref'] in self.routes: - sys.stderr.write("Route (variant) with existing 'Ref'\n") + if itinerary.route_id in self.routes: + sys.stderr.write("Itinerary with existing route id (ref)\n") sys.stderr.write( "http://osm.org/relation/" + str(route_variant.id) + "\n") sys.stderr.write("Skipped. Please fix in OpenStreetMap\n") @@ -218,7 +219,6 @@ def get_stops(self, refresh=False): self.stops = Cache.read_data('stops-' + self.selector) if bool(self.stops): - # Maybe check for unnamed stop names if self.auto_stop_names: self._get_names_for_unnamed_stops() @@ -249,8 +249,11 @@ def get_stops(self, refresh=False): # valid stop_area candidade? if 'public_transport' in relation.tags: if relation.tags["public_transport"] == "stop_area": - self.stops["relation/" + str(relation.id) - ] = self._build_stop_area(relation) + try: + self.stops["relation/" + str(relation.id) + ] = self._build_stop_area(relation) + except RuntimeError: + print('Cannot add stop area', relation.id) # Cache data Cache.write_data('stops-' + self.selector, self.stops) @@ -285,8 +288,8 @@ def _build_line(self, route_master, itineraries): # Check if a ref can be taken from one of the itineraries ref = False for itinerary in list(itineraries.values()): - if not ref and itinerary.ref: - ref = itinerary.ref + if not ref and itinerary.route_id: + ref = itinerary.route_id sys.stderr.write( "Using 'ref' from member variant instead\n") sys.stderr.write(itinerary.osm_url + "\n") @@ -302,6 +305,14 @@ def _build_line(self, route_master, itineraries): if "frequency" in route_master.tags: frequency = route_master.tags['frequency'] + colour = "FFFFFF" + if "colour" in route_master.tags: + colour = OsmConnector.get_hex_code_for_color(route_master.tags['colour']) + + text_colour = OsmConnector.get_complementary_color(colour) + if "text_colour" in route_master.tags: + text_colour = OsmConnector.get_hex_code_for_color(route_master.tags['text_colour']) + if 'route_master' in route_master.tags: route_type = route_master.tags['route_master'].capitalize() @@ -309,17 +320,21 @@ def _build_line(self, route_master, itineraries): elif 'route' in route_master.tags: route_type = route_master.tags['route'].capitalize() - print route_type - # Create Line (route master) object line = Line(osm_id=route_master.id, route_id=ref, - name=name, route_type=route_type, frequency=frequency) + name=name, route_type=route_type, frequency=frequency, + route_color=colour, route_text_color=text_colour) # Add Itinerary objects (route variants) to Line (route master) for itinerary in list(itineraries.values()): - line.add_itinerary(itinerary) + try: + line.add_itinerary(itinerary) + except ValueError: + print('Itinerary ID does not match line ID. Please fix in OSM.') + print(line.osm_url) + itinerary.route_id = line.route_id + line.add_itinerary(itinerary) - #print(line) return line def _build_itinerary(self, route_variant, query_result_set): @@ -335,6 +350,8 @@ def _build_itinerary(self, route_variant, query_result_set): "RouteVariant without 'ref': " + str(route_variant.id) + "\n") sys.stderr.write( "http://osm.org/relation/" + str(route_variant.id) + "\n") + sys.stderr.write( + "Whole Itinerary skipped. Please fix in OpenStreetMap\n") return if 'from' in route_variant.tags: @@ -370,13 +387,13 @@ def _build_itinerary(self, route_variant, query_result_set): otype = "way" else: - raise RuntimeError("Unknown type: " + str(stop_candidate)) + raise RuntimeError("Unknown type of itinerary member: " + str(stop_candidate)) stops.append(otype + "/" + str(stop_candidate.ref)) shape = self._generate_shape(route_variant, query_result_set) rv = Itinerary(osm_id=route_variant.id, fr=fr, - to=to, stops=stops, shape=shape, ref=ref, + to=to, stops=stops, shape=shape, route_id=ref, name=name, travel_time=travel_time) return rv @@ -395,7 +412,7 @@ def _build_stop(self, stop, stop_type): if stop_type == "way": (stop.lat, stop.lon) = OsmConnector.get_center_of_nodes(stop.get_nodes()) - s = Stop(stop.id, "node", stop.tags['name'], stop.lat, stop.lon) + s = Stop(stop.id, "node", unicode(stop.tags['name']), stop.lat, stop.lon) return s def _build_stop_area(self, relation): @@ -407,9 +424,15 @@ def _build_stop_area(self, relation): for member in relation.members: if (isinstance(member, overpy.RelationNode) and member.role == "platform"): - stop = self.stops.pop("node/" + str(member.ref)) - stop_members["node/" + str(member.ref)] = stop - + if "node/" + str(member.ref) in self.stops: + stop = self.stops.pop("node/" + str(member.ref)) + stop_members["node/" + str(member.ref)] = stop + else: + sys.stderr.write("Stop not found in stops: ") + sys.stderr.write("http://osm.org/node/" + + str(member.ref) + "\n") + if len(stop_members) < 1: + raise RuntimeError('Cannot build stop area with no members') if 'name' not in relation.tags: sys.stderr.write("Stop area without name." + " Please fix in OpenStreetMap\n") @@ -568,7 +591,7 @@ def _get_names_for_unnamed_stops(self): # If there is no name, query one intelligently from OSM if stop.name == "[" + self.stop_no_name + "]": self._find_best_name_for_unnamed_stop(stop) - print stop + print(stop) # Cache stops with newly created stop names Cache.write_data('stops-' + self.selector, self.stops) @@ -654,6 +677,9 @@ def get_center_of_nodes(nodes): y = 0 z = 0 + if len(nodes) < 1: + raise ValueError('Cannot find the center of zero nodes') + for node in nodes: lat = radians(float(node.lat)) lon = radians(float(node.lon)) @@ -670,3 +696,39 @@ def get_center_of_nodes(nodes): center_lon = degrees(atan2(y, x)) return center_lat, center_lon + + @staticmethod + def get_hex_code_for_color(color): + color = color.lower() + if color == u'black': + return '000000' + if color == u'blue': + return '0000FF' + if color == u'gray': + return '808080' + if color == u'green': + return '008000' + if color == u'purple': + return '800080' + if color == u'red': + return 'FF0000' + if color == u'silver': + return 'C0C0C0' + if color == u'white': + return 'FFFFFF' + if color == u'yellow': + return 'FFFF00' + print('Color not known: ' + color) + return 'FA8072' + + @staticmethod + def get_complementary_color(color): + """ + Returns complementary RGB color + Source: https://stackoverflow.com/a/38478744 + """ + if color[0] == '#': + color = color[1:] + rgb = (color[0:2], color[2:4], color[4:6]) + comp = ['%02X' % (255 - int(a, 16)) for a in rgb] + return ''.join(comp) diff --git a/osm2gtfs/core/routes.py b/osm2gtfs/core/routes.py index 27ab563c..a828110a 100644 --- a/osm2gtfs/core/routes.py +++ b/osm2gtfs/core/routes.py @@ -24,12 +24,14 @@ class Line(object): route_color = attr.ib(default="FFFFFF") route_text_color = attr.ib(default="000000") osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) - frequency = attr.ib() + frequency = attr.ib(default=None) # Related route variants - _itineraries = [] + _itineraries = attr.ib(default=attr.Factory(list)) def add_itinerary(self, itinerary): + if self.route_id.encode('utf-8') != itinerary.route_id.encode('utf-8'): + raise ValueError('Itinerary route ID (' + itinerary.route_id + ') does not match Line route ID (' + self.route_id + ')') self._itineraries.append(itinerary) def get_itineraries(self): @@ -48,7 +50,8 @@ class Itinerary(object): """ osm_id = attr.ib() - ref = attr.ib() + route_id = attr.ib() + name = attr.ib() fr = attr.ib() to = attr.ib() shape = attr.ib() @@ -63,5 +66,14 @@ class Itinerary(object): # Useful information for further calculation duration = attr.ib(default=None) + # All stop objects of itinerary + _stop_objects = attr.ib(default=attr.Factory(list)) + + def add_stop(self, stop): + self._stop_objects.append(stop) + def get_stop_by_position(self, pos): raise NotImplementedError("Should have implemented this") + + def get_stops(self): + return self._stop_objects diff --git a/osm2gtfs/core/stops.py b/osm2gtfs/core/stops.py index 896f9742..f953c3e9 100644 --- a/osm2gtfs/core/stops.py +++ b/osm2gtfs/core/stops.py @@ -22,8 +22,8 @@ class StopArea(object): name = attr.ib() lat = attr.ib() lon = attr.ib() - - _stop_members = [] + + stop_members = attr.ib(default=attr.Factory(list)) def __init__(self, osm_id, stop_members, name=None): self.osm_id = osm_id @@ -32,4 +32,6 @@ def __init__(self, osm_id, stop_members, name=None): else: self.name = name self.stop_members = stop_members - self.lat, self.lon = Stop.get_center_of_nodes(stop_members.values()) + + from osm2gtfs.core.osm_connector import OsmConnector + self.lat, self.lon = OsmConnector.get_center_of_nodes(self.stop_members.values()) diff --git a/osm2gtfs/creators/accra/stops_creator_accra.py b/osm2gtfs/creators/accra/stops_creator_accra.py index 7087d89b..89b1fbe1 100644 --- a/osm2gtfs/creators/accra/stops_creator_accra.py +++ b/osm2gtfs/creators/accra/stops_creator_accra.py @@ -35,7 +35,7 @@ def create_stop_area(stop_data, feed): lat=float(stop_data.lat), lng=float(stop_data.lon), name=stop_data.name, - stop_id="SA" + str(stop_data.id) + stop_id="SA" + str(stop_data.osm_id) ) gtfs_stop_area.location_type = 1 return gtfs_stop_area @@ -46,13 +46,13 @@ def create_stop_point(stop_data, feed): lat=float(stop_data.lat), lng=float(stop_data.lon), name=stop_data.name, - stop_id=str(stop_data.id) + stop_id=str(stop_data.osm_id) ) return gtfs_stop_point def get_stop_id(stop): - return stop.id + return stop.osm_id class StopsCreatorAccra(StopsCreator): diff --git a/osm2gtfs/creators/accra/trips_creator_accra.py b/osm2gtfs/creators/accra/trips_creator_accra.py index 2498d1b1..056d6ad4 100644 --- a/osm2gtfs/creators/accra/trips_creator_accra.py +++ b/osm2gtfs/creators/accra/trips_creator_accra.py @@ -3,6 +3,7 @@ from datetime import timedelta, datetime from osm2gtfs.creators.trips_creator import TripsCreator +from osm2gtfs.core.routes import Line class TripsCreatorAccra(TripsCreator): @@ -18,11 +19,11 @@ def add_trips_to_feed(self, feed, data): lines = data.routes for route_ref, line in sorted(lines.iteritems()): - if type(line).__name__ != "RouteMaster": + if not isinstance(line, Line): continue line_gtfs = feed.AddRoute( - short_name=line.ref, + short_name=line.route_id, long_name=line.name.decode('utf8'), # we change the route_long_name with the 'from' and 'to' tags # of the last route as the route_master name tag contains @@ -35,7 +36,8 @@ def add_trips_to_feed(self, feed, data): line_gtfs.route_text_color = "ffffff" route_index = 0 - for a_route_ref, a_route in line.routes.iteritems(): + itineraries = line.get_itineraries() + for a_route_ref, a_route in itineraries: trip_gtfs = line_gtfs.AddTrip(feed) trip_gtfs.shape_id = TripsCreator.add_shape( feed, a_route_ref, a_route) @@ -53,10 +55,10 @@ def add_trips_to_feed(self, feed, data): try: ROUTE_FREQUENCY = int(line.frequency) if not ROUTE_FREQUENCY > 0: - print("frequency is invalid for route_master " + str(line.id)) + print("frequency is invalid for route_master " + str(line.osm_id)) ROUTE_FREQUENCY = DEFAULT_ROUTE_FREQUENCY except (ValueError, TypeError) as e: - print("frequency not a number for route_master " + str(line.id)) + print("frequency not a number for route_master " + str(line.osm_id)) ROUTE_FREQUENCY = DEFAULT_ROUTE_FREQUENCY trip_gtfs.AddFrequency( "05:00:00", "22:00:00", ROUTE_FREQUENCY * 60) @@ -64,10 +66,10 @@ def add_trips_to_feed(self, feed, data): try: TRAVEL_TIME = int(a_route.travel_time) if not TRAVEL_TIME > 0: - print("travel_time is invalid for route " + str(a_route.id)) + print("travel_time is invalid for route " + str(a_route.osm_id)) TRAVEL_TIME = DEFAULT_TRAVEL_TIME except (ValueError, TypeError) as e: - print("travel_time not a number for route " + str(a_route.id)) + print("travel_time not a number for route " + str(a_route.osm_id)) TRAVEL_TIME = DEFAULT_TRAVEL_TIME for index_stop, a_stop in enumerate(a_route.stops): diff --git a/osm2gtfs/creators/fenix/trips_creator_fenix.py b/osm2gtfs/creators/fenix/trips_creator_fenix.py index 536e6f10..1e061b4c 100644 --- a/osm2gtfs/creators/fenix/trips_creator_fenix.py +++ b/osm2gtfs/creators/fenix/trips_creator_fenix.py @@ -5,7 +5,7 @@ import transitfeed from datetime import timedelta, datetime from osm2gtfs.creators.trips_creator import TripsCreator -from osm2gtfs.core.osm_routes import Route, RouteMaster +from osm2gtfs.core.routes import Line, Itinerary DEBUG_ROUTE = "" BLACKLIST = [ @@ -82,7 +82,7 @@ def add_trips_to_feed(self, feed, data): def add_route(self, feed, route, horarios, operacoes): line = feed.AddRoute( - short_name=route.ref, + short_name=route.route_id, long_name=route.name.decode('utf8'), route_type="Bus") line.agency_id = feed.GetDefaultAgency().agency_id @@ -139,7 +139,7 @@ def add_trips_by_day(self, feed, line, service, route, horarios, day): if horarios is None or len(horarios) == 0: return - if isinstance(route, RouteMaster): + if isinstance(route, Line): # recurse into "Ida" and "Volta" routes for sub_route in route.routes.values(): self.add_trips_by_day(feed, line, service, sub_route, horarios, day) @@ -270,7 +270,7 @@ def normalize_stop_name(old_name): @staticmethod def add_trip_stops(feed, trip, route, start_time, end_time): - if isinstance(route, Route): + if isinstance(route, Itinerary): i = 1 for stop in route.stops: if i == 1: diff --git a/osm2gtfs/creators/incofer/routes_creator_incofer.py b/osm2gtfs/creators/incofer/routes_creator_incofer.py index 10812773..ded78990 100644 --- a/osm2gtfs/creators/incofer/routes_creator_incofer.py +++ b/osm2gtfs/creators/incofer/routes_creator_incofer.py @@ -16,7 +16,7 @@ def add_routes_to_feed(self, feed, data): # Loop through all lines (master_routes) for line_ref, line in sorted(lines.iteritems()): route = feed.AddRoute( - short_name=line.ref.encode('utf-8'), + short_name=line.route_id.encode('utf-8'), long_name=line.name, # TODO: infer transitfeed "route type" from OSM data route_type="Tram", diff --git a/osm2gtfs/creators/incofer/trips_creator_incofer.py b/osm2gtfs/creators/incofer/trips_creator_incofer.py index 6bec4584..06e7a6a7 100644 --- a/osm2gtfs/creators/incofer/trips_creator_incofer.py +++ b/osm2gtfs/creators/incofer/trips_creator_incofer.py @@ -19,7 +19,8 @@ def add_trips_to_feed(self, feed, data): # print("DEBUG. procesando la línea:", line.name) # itinerary (osm route | non existent gtfs element) - for itinerary_id, itinerary in line.routes.iteritems(): + itineraries = line.get_itineraries() + for itinerary_id, itinerary in itineraries: # debug # print("DEBUG. procesando el itinerario", itinerary.name) diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index 2f88b2d4..f5afb599 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -13,17 +13,16 @@ def __repr__(self): return rep def add_routes_to_feed(self, feed, data): - return - """ route_id # Required: From Line route_type # Required: From Line route_short_name # Required: To be generated from Line or Itinerary - route_long_name # Required: To be generated from Line or Itinerary + route_long_name # Required: To be generated from Line or Itinerary route_desc # From Line route_url # From Line - route_color: # From Line - route_text_color #From Line + route_color # From Line + route_text_color # From Line """ + raise NotImplementedError("Should have implemented this") diff --git a/osm2gtfs/creators/stops_creator.py b/osm2gtfs/creators/stops_creator.py index 5e3ac60b..9c6e4256 100644 --- a/osm2gtfs/creators/stops_creator.py +++ b/osm2gtfs/creators/stops_creator.py @@ -2,8 +2,8 @@ import transitfeed -from osm2gtfs.core.osm_routes import Route, RouteMaster -from osm2gtfs.core.osm_stops import Stop, StopArea +from osm2gtfs.core.routes import Itinerary, Line +from osm2gtfs.core.stops import Stop, StopArea class StopsCreator(object): @@ -75,7 +75,7 @@ def _fill_stops(self, stops, route): """ Fill a route object with stop objects for of linked stop ids """ - if isinstance(route, Route): + if isinstance(route, Itinerary): i = 0 for stop in route.stops: # Replace stop id with Stop objects @@ -83,9 +83,10 @@ def _fill_stops(self, stops, route): route.stops[i] = self._get_stop(stop, stops) i += 1 - elif isinstance(route, RouteMaster): - for route_variant_ref, route_variant in route.routes.iteritems(): - self._fill_stops(stops, route_variant) + elif isinstance(route, Line): + itineraries = route.get_itineraries() + for itinerary_ref, itinerary in itineraries: + self._fill_stops(stops, itinerary) else: raise RuntimeError("Unknown Route: " + str(route)) diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index cfaac637..05b7e6be 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -12,9 +12,8 @@ def __repr__(self): rep += str(self.config) + " | " return rep - def add_trips_to_feed(self, feed, data): - raise NotImplementedError("Should have implemented this") + def add_trips_to_feed(self, feed, data): """ route_id # Required: From Line service_id # Required: To be generated @@ -23,10 +22,11 @@ def add_trips_to_feed(self, feed, data): trip_headsign # Itinerary "to" direction_id # Order of tinieraries in Line object wheelchair_accessible # Itinerary "wheelchair_accessible" - bikes_allowed: # Itinerary "bikes_allowed" + bikes_allowed # Itinerary "bikes_allowed" trip_short_name # To be avoided! block_id # To be avoided! """ + raise NotImplementedError("Should have implemented this") @staticmethod def interpolate_stop_times(trip): diff --git a/setup.py b/setup.py index 2c68069c..6527d3b1 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,10 @@ keywords='openstreetmap gtfs schedule public-transportation python', author='Various collaborators: https://github.com/grote/osm2gtfs/graphs/contributors', - install_requires=['overpy>=0.4', 'transitfeed'], + install_requires=['attrs', 'overpy>=0.4', 'transitfeed'], packages=find_packages(), include_package_data=True, entry_points=''' [console_scripts] osm2gtfs = osm2gtfs.osm2gtfs:main ''' -) From 8b5b4a13e3965d2801b921f58e295431c38f8d8b Mon Sep 17 00:00:00 2001 From: Nico Alt Date: Wed, 29 Nov 2017 00:35:51 +0100 Subject: [PATCH 05/18] Adapt existing creators to new data structure --- .../incofer/routes_creator_incofer.py | 36 +++++-------------- .../creators/incofer/trips_creator_incofer.py | 2 +- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/osm2gtfs/creators/incofer/routes_creator_incofer.py b/osm2gtfs/creators/incofer/routes_creator_incofer.py index ded78990..9c963e5a 100644 --- a/osm2gtfs/creators/incofer/routes_creator_incofer.py +++ b/osm2gtfs/creators/incofer/routes_creator_incofer.py @@ -6,34 +6,16 @@ class RoutesCreatorIncofer(RoutesCreator): def add_routes_to_feed(self, feed, data): + return - # Get routes information - lines = data.get_routes() - # debug - # print("DEBUG: creando itinerarios a partir de", str(len(lines)), - # "lineas") - - # Loop through all lines (master_routes) - for line_ref, line in sorted(lines.iteritems()): - route = feed.AddRoute( - short_name=line.route_id.encode('utf-8'), - long_name=line.name, - # TODO: infer transitfeed "route type" from OSM data - route_type="Tram", - route_id=line_ref) - - # AddRoute method add defaut agency as default - route.agency_id = feed.GetDefaultAgency().agency_id - - route.route_desc = "Esta línea está a prueba" + def _get_route_type(self, line): + return "Tram" - # TODO: get route_url from OSM or other source. - # url = "http://www.incofer.go.cr/tren-urbano-alajuela-rio-segundo" + def _get_route_description(self, line): + return "Esta línea está a prueba" - # line.route_url = url - route.route_color = "ff0000" - route.route_text_color = "ffffff" + def _get_route_color(self, line): + return "ff0000" - # debug - # print("información de la linea:", line.name, "agregada.") - return + def _get_route_text_color(self, line): + return "ffffff" diff --git a/osm2gtfs/creators/incofer/trips_creator_incofer.py b/osm2gtfs/creators/incofer/trips_creator_incofer.py index 06e7a6a7..04b961f9 100644 --- a/osm2gtfs/creators/incofer/trips_creator_incofer.py +++ b/osm2gtfs/creators/incofer/trips_creator_incofer.py @@ -163,8 +163,8 @@ def load_times(route, data, operation): # route_directions = data.schedule["itinerario"][route.ref]["horarios"] times = None - for direction in data.schedule["itinerario"][route.ref]: + for direction in data.schedule["itinerario"][route.route_id]: fr = direction["from"].encode('utf-8') to = direction["to"].encode('utf-8') data_operation = direction["operacion"].encode('utf-8') From 95ddd964b0847594fb1acb73ae5cfbff129f048a Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Wed, 29 Nov 2017 19:55:22 +0100 Subject: [PATCH 06/18] Minor fixes and clean-ups --- osm2gtfs/core/configuration.py | 2 +- osm2gtfs/core/osm_connector.py | 44 ++++++++++--------- osm2gtfs/core/routes.py | 6 ++- osm2gtfs/core/stops.py | 1 + .../creators/accra/stops_creator_accra.py | 6 ++- .../creators/accra/trips_creator_accra.py | 29 +++++++----- osm2gtfs/creators/routes_creator.py | 12 ----- osm2gtfs/creators/stops_creator.py | 8 ++-- osm2gtfs/creators/trips_creator.py | 15 +------ osm2gtfs/tests/tests_accra.py | 8 ++-- setup.py | 10 +++-- 11 files changed, 67 insertions(+), 74 deletions(-) diff --git a/osm2gtfs/core/configuration.py b/osm2gtfs/core/configuration.py index 85327a01..8e0c9a58 100644 --- a/osm2gtfs/core/configuration.py +++ b/osm2gtfs/core/configuration.py @@ -51,7 +51,7 @@ def get_schedule_source(self, refresh=False): else: source_file = self.data['schedule_source'] - cached_file = 'schedule-source-' + self.data['selector'] + cached_file = self.data['selector'] + '-schedule' # Preferably return cached data about schedule if refresh is False: diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index 86f9c057..85a739e2 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -47,7 +47,7 @@ def __init__(self, config): # fallback self.tags = '["public_transport:version" = "2"]' print("No tags found for querying from OpenStreetMap.") - print("Using tag 'public_transport:version=2") + print("Using tag 'public_transport:version=2'") # Define name for stops without one self.stop_no_name = 'No name' @@ -104,7 +104,7 @@ def get_routes(self, refresh=False): # Check if routes data is already built in this object if not self.routes: # If not, try to get routes data from file cache - self.routes = Cache.read_data('routes-' + self.selector) + self.routes = Cache.read_data(self.selector + '-routes') # Return cached data if found if bool(self.routes): return self.routes @@ -184,7 +184,7 @@ def get_routes(self, refresh=False): self.routes[line.route_id] = line # Cache data - Cache.write_data('routes-' + self.selector, self.routes) + Cache.write_data(self.selector + '-routes', self.routes) return self.routes @@ -216,7 +216,7 @@ def get_stops(self, refresh=False): # Check if stops data is already built in this object if not self.stops: # If not, try to get stops data from file cache - self.stops = Cache.read_data('stops-' + self.selector) + self.stops = Cache.read_data(self.selector + '-stops') if bool(self.stops): # Maybe check for unnamed stop names @@ -252,11 +252,11 @@ def get_stops(self, refresh=False): try: self.stops["relation/" + str(relation.id) ] = self._build_stop_area(relation) - except RuntimeError: - print('Cannot add stop area', relation.id) + except (ValueError, TypeError) as e: + sys.stderr.write('Cannot add stop area: ' + str(relation.id)) # Cache data - Cache.write_data('stops-' + self.selector, self.stops) + Cache.write_data(self.selector + '-stops', self.stops) # Maybe check for unnamed stop names if self.auto_stop_names: @@ -305,13 +305,13 @@ def _build_line(self, route_master, itineraries): if "frequency" in route_master.tags: frequency = route_master.tags['frequency'] - colour = "FFFFFF" + color = "FFFFFF" if "colour" in route_master.tags: - colour = OsmConnector.get_hex_code_for_color(route_master.tags['colour']) + color = OsmConnector.get_hex_code_for_color(route_master.tags['colour']) - text_colour = OsmConnector.get_complementary_color(colour) + text_color = OsmConnector.get_complementary_color(color) if "text_colour" in route_master.tags: - text_colour = OsmConnector.get_hex_code_for_color(route_master.tags['text_colour']) + text_color = OsmConnector.get_hex_code_for_color(route_master.tags['text_colour']) if 'route_master' in route_master.tags: route_type = route_master.tags['route_master'].capitalize() @@ -323,15 +323,16 @@ def _build_line(self, route_master, itineraries): # Create Line (route master) object line = Line(osm_id=route_master.id, route_id=ref, name=name, route_type=route_type, frequency=frequency, - route_color=colour, route_text_color=text_colour) + route_color=color, route_text_color=text_color) # Add Itinerary objects (route variants) to Line (route master) for itinerary in list(itineraries.values()): try: line.add_itinerary(itinerary) except ValueError: - print('Itinerary ID does not match line ID. Please fix in OSM.') - print(line.osm_url) + sys.stderr.write( + "Itinerary ID does not match line ID. Please fix in OSM.\n") + sys.stderr.write(line.osm_url) itinerary.route_id = line.route_id line.add_itinerary(itinerary) @@ -387,7 +388,8 @@ def _build_itinerary(self, route_variant, query_result_set): otype = "way" else: - raise RuntimeError("Unknown type of itinerary member: " + str(stop_candidate)) + sys.stderr.write("Unknown type of itinerary member: " + + str(stop_candidate) + "\n") stops.append(otype + "/" + str(stop_candidate.ref)) @@ -410,7 +412,7 @@ def _build_stop(self, stop, stop_type): # Ways don't have coordinates and they have to be calculated if stop_type == "way": - (stop.lat, stop.lon) = OsmConnector.get_center_of_nodes(stop.get_nodes()) + (stop.lat, stop.lon) = self.get_center_of_nodes(stop.get_nodes()) s = Stop(stop.id, "node", unicode(stop.tags['name']), stop.lat, stop.lon) return s @@ -432,7 +434,8 @@ def _build_stop_area(self, relation): sys.stderr.write("http://osm.org/node/" + str(member.ref) + "\n") if len(stop_members) < 1: - raise RuntimeError('Cannot build stop area with no members') + sys.stderr.write("Cannot build stop area with no members\n") + if 'name' not in relation.tags: sys.stderr.write("Stop area without name." + " Please fix in OpenStreetMap\n") @@ -594,7 +597,7 @@ def _get_names_for_unnamed_stops(self): print(stop) # Cache stops with newly created stop names - Cache.write_data('stops-' + self.selector, self.stops) + Cache.write_data(self.selector + '-stops', self.stops) def _find_best_name_for_unnamed_stop(self, stop): """Define name for stop without explicit name based on sourroundings @@ -646,7 +649,7 @@ def _find_best_name_for_unnamed_stop(self, stop): winner_distance = sys.maxint for candidate in candidates: if isinstance(candidate, overpy.Way): - lat, lon = OsmConnector.get_center_of_nodes( + lat, lon = self.get_center_of_nodes( candidate.get_nodes(resolve_missing=True)) distance = util.ApproximateDistance( lat, @@ -678,8 +681,7 @@ def get_center_of_nodes(nodes): z = 0 if len(nodes) < 1: - raise ValueError('Cannot find the center of zero nodes') - + sys.stderr.write("Cannot find the center of zero nodes\n") for node in nodes: lat = radians(float(node.lat)) lon = radians(float(node.lon)) diff --git a/osm2gtfs/core/routes.py b/osm2gtfs/core/routes.py index a828110a..370e8ca3 100644 --- a/osm2gtfs/core/routes.py +++ b/osm2gtfs/core/routes.py @@ -30,8 +30,12 @@ class Line(object): _itineraries = attr.ib(default=attr.Factory(list)) def add_itinerary(self, itinerary): + if self.route_id.encode('utf-8') != itinerary.route_id.encode('utf-8'): - raise ValueError('Itinerary route ID (' + itinerary.route_id + ') does not match Line route ID (' + self.route_id + ')') + raise ValueError('Itinerary route ID (' + + itinerary.route_id + + ') does not match Line route ID (' + + self.route_id + ')') self._itineraries.append(itinerary) def get_itineraries(self): diff --git a/osm2gtfs/core/stops.py b/osm2gtfs/core/stops.py index f953c3e9..7d2e3cac 100644 --- a/osm2gtfs/core/stops.py +++ b/osm2gtfs/core/stops.py @@ -16,6 +16,7 @@ class Stop(object): str(osm_type) + "/" + str(osm_id)) +@attr.s class StopArea(object): osm_id = attr.ib() diff --git a/osm2gtfs/creators/accra/stops_creator_accra.py b/osm2gtfs/creators/accra/stops_creator_accra.py index 89b1fbe1..b2bdf454 100644 --- a/osm2gtfs/creators/accra/stops_creator_accra.py +++ b/osm2gtfs/creators/accra/stops_creator_accra.py @@ -31,22 +31,24 @@ def get_crow_fly_distance(from_tuple, to_tuple): def create_stop_area(stop_data, feed): + stop_id = stop_data.osm_id.split('/')[-1] gtfs_stop_area = feed.AddStop( lat=float(stop_data.lat), lng=float(stop_data.lon), name=stop_data.name, - stop_id="SA" + str(stop_data.osm_id) + stop_id="SA" + str(stop_id) ) gtfs_stop_area.location_type = 1 return gtfs_stop_area def create_stop_point(stop_data, feed): + stop_id = stop_data.osm_id.split('/')[-1] gtfs_stop_point = feed.AddStop( lat=float(stop_data.lat), lng=float(stop_data.lon), name=stop_data.name, - stop_id=str(stop_data.osm_id) + stop_id=str(stop_id) ) return gtfs_stop_point diff --git a/osm2gtfs/creators/accra/trips_creator_accra.py b/osm2gtfs/creators/accra/trips_creator_accra.py index 056d6ad4..9bf65a40 100644 --- a/osm2gtfs/creators/accra/trips_creator_accra.py +++ b/osm2gtfs/creators/accra/trips_creator_accra.py @@ -24,12 +24,12 @@ def add_trips_to_feed(self, feed, data): line_gtfs = feed.AddRoute( short_name=line.route_id, - long_name=line.name.decode('utf8'), + long_name=line.name, # we change the route_long_name with the 'from' and 'to' tags # of the last route as the route_master name tag contains # the line code (route_short_name) route_type="Bus", - route_id=line.id) + route_id=line.osm_id) line_gtfs.agency_id = feed.GetDefaultAgency().agency_id line_gtfs.route_desc = "" line_gtfs.route_color = "1779c2" @@ -37,17 +37,18 @@ def add_trips_to_feed(self, feed, data): route_index = 0 itineraries = line.get_itineraries() - for a_route_ref, a_route in itineraries: + for a_route in itineraries: trip_gtfs = line_gtfs.AddTrip(feed) trip_gtfs.shape_id = TripsCreator.add_shape( - feed, a_route_ref, a_route) + feed, a_route.route_id, a_route) trip_gtfs.direction_id = route_index % 2 route_index += 1 if a_route.fr and a_route.to: trip_gtfs.trip_headsign = a_route.to line_gtfs.route_long_name = a_route.fr.decode( - 'utf8') + " ↔ ".decode('utf8') + a_route.to.decode('utf8') + 'utf8') + " ↔ ".decode( + 'utf8') + a_route.to.decode('utf8') DEFAULT_ROUTE_FREQUENCY = 30 DEFAULT_TRAVEL_TIME = 120 @@ -55,10 +56,12 @@ def add_trips_to_feed(self, feed, data): try: ROUTE_FREQUENCY = int(line.frequency) if not ROUTE_FREQUENCY > 0: - print("frequency is invalid for route_master " + str(line.osm_id)) + print("frequency is invalid for route_master " + str( + line.osm_id)) ROUTE_FREQUENCY = DEFAULT_ROUTE_FREQUENCY except (ValueError, TypeError) as e: - print("frequency not a number for route_master " + str(line.osm_id)) + print("frequency not a number for route_master " + str( + line.osm_id)) ROUTE_FREQUENCY = DEFAULT_ROUTE_FREQUENCY trip_gtfs.AddFrequency( "05:00:00", "22:00:00", ROUTE_FREQUENCY * 60) @@ -66,10 +69,12 @@ def add_trips_to_feed(self, feed, data): try: TRAVEL_TIME = int(a_route.travel_time) if not TRAVEL_TIME > 0: - print("travel_time is invalid for route " + str(a_route.osm_id)) + print("travel_time is invalid for route " + str( + a_route.osm_id)) TRAVEL_TIME = DEFAULT_TRAVEL_TIME except (ValueError, TypeError) as e: - print("travel_time not a number for route " + str(a_route.osm_id)) + print("travel_time not a number for route " + str( + a_route.osm_id)) TRAVEL_TIME = DEFAULT_TRAVEL_TIME for index_stop, a_stop in enumerate(a_route.stops): @@ -78,11 +83,13 @@ def add_trips_to_feed(self, feed, data): if index_stop == 0: trip_gtfs.AddStopTime(feed.GetStop( - str(stop_id)), stop_time=departure_time.strftime("%H:%M:%S")) + str(stop_id)), stop_time=departure_time.strftime( + "%H:%M:%S")) elif index_stop == len(a_route.stops) - 1: departure_time += timedelta(minutes=TRAVEL_TIME) trip_gtfs.AddStopTime(feed.GetStop( - str(stop_id)), stop_time=departure_time.strftime("%H:%M:%S")) + str(stop_id)), stop_time=departure_time.strftime( + "%H:%M:%S")) else: trip_gtfs.AddStopTime(feed.GetStop(str(stop_id))) diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index f5afb599..a3bd14cf 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -13,16 +13,4 @@ def __repr__(self): return rep def add_routes_to_feed(self, feed, data): - """ - route_id # Required: From Line - route_type # Required: From Line - - route_short_name # Required: To be generated from Line or Itinerary - route_long_name # Required: To be generated from Line or Itinerary - - route_desc # From Line - route_url # From Line - route_color # From Line - route_text_color # From Line - """ raise NotImplementedError("Should have implemented this") diff --git a/osm2gtfs/creators/stops_creator.py b/osm2gtfs/creators/stops_creator.py index 9c6e4256..6b8343ad 100644 --- a/osm2gtfs/creators/stops_creator.py +++ b/osm2gtfs/creators/stops_creator.py @@ -1,7 +1,7 @@ # coding=utf-8 +import sys import transitfeed - from osm2gtfs.core.routes import Itinerary, Line from osm2gtfs.core.stops import Stop, StopArea @@ -85,11 +85,11 @@ def _fill_stops(self, stops, route): elif isinstance(route, Line): itineraries = route.get_itineraries() - for itinerary_ref, itinerary in itineraries: + for itinerary in itineraries: self._fill_stops(stops, itinerary) else: - raise RuntimeError("Unknown Route: " + str(route)) + sys.stderr.write("Unknown route: " + str(route) + "\n") def _get_stop(self, stop_id, stops): for ref, elem in stops.iteritems(): @@ -100,4 +100,4 @@ def _get_stop(self, stop_id, stops): if stop_id in elem.stop_members: return elem.stop_members[stop_id] else: - raise RuntimeError("Unknown stop: " + str(stop_id)) + sys.stderr.write("Unknown stop: " + str(stop_id) + "\n") diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index 05b7e6be..c8c6ccac 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -12,26 +12,13 @@ def __repr__(self): rep += str(self.config) + " | " return rep - def add_trips_to_feed(self, feed, data): - """ - route_id # Required: From Line - service_id # Required: To be generated - trip_id # Required: To be generated - - trip_headsign # Itinerary "to" - direction_id # Order of tinieraries in Line object - wheelchair_accessible # Itinerary "wheelchair_accessible" - bikes_allowed # Itinerary "bikes_allowed" - trip_short_name # To be avoided! - block_id # To be avoided! - """ raise NotImplementedError("Should have implemented this") @staticmethod def interpolate_stop_times(trip): """ - interpolate stop_times, because Navitia does not handle this itself by now + Interpolate stop_times, because Navitia does not handle this itself """ for secs, stop_time, is_timepoint in trip.GetTimeInterpolatedStops(): if not is_timepoint: diff --git a/osm2gtfs/tests/tests_accra.py b/osm2gtfs/tests/tests_accra.py index 8c07d97e..fb6d3e45 100644 --- a/osm2gtfs/tests/tests_accra.py +++ b/osm2gtfs/tests/tests_accra.py @@ -85,7 +85,7 @@ def setUp(self): def test_refresh_routes_cache(self): data = OsmConnector(self.config) - cache_file = os.path.join(self.data_dir, "routes-accra.pkl") + cache_file = os.path.join(self.data_dir, "accra-routes.pkl") mocked_overpass_data_file = os.path.join(self.fixture_folder, "overpass-routes.xml") if os.path.isfile(cache_file): os.remove(cache_file) @@ -101,7 +101,7 @@ def test_refresh_routes_cache(self): def test_refresh_stops_cache(self): data = OsmConnector(self.config) - cache_file = os.path.join(self.data_dir, "stops-accra.pkl") + cache_file = os.path.join(self.data_dir, "accra-stops.pkl") mocked_overpass_data_file = os.path.join(self.fixture_folder, "overpass-stops.xml") if os.path.isfile(cache_file): os.remove(cache_file) @@ -117,8 +117,8 @@ def test_refresh_stops_cache(self): def test_gtfs_from_cache(self): # the cache is generated by the previous two functions - routes_cache_file = os.path.join(self.data_dir, "routes-accra.pkl") - stops_file = os.path.join(self.data_dir, "stops-accra.pkl") + routes_cache_file = os.path.join(self.data_dir, "accra-routes.pkl") + stops_file = os.path.join(self.data_dir, "accra-stops.pkl") self.assertTrue(os.path.isfile(stops_file), "The stops cache file doesn't exists") self.assertTrue(os.path.isfile(routes_cache_file), "The routes cache file doesn't exists") diff --git a/setup.py b/setup.py index 6527d3b1..8c56ded2 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,9 @@ install_requires=['attrs', 'overpy>=0.4', 'transitfeed'], packages=find_packages(), include_package_data=True, - entry_points=''' - [console_scripts] - osm2gtfs = osm2gtfs.osm2gtfs:main - ''' + entry_points={ + 'console_scripts': [ + 'osm2gtfs = osm2gtfs.osm2gtfs:main' + ] + }, +) From cdb7b694235d5f41053e1cab8abc98783cf0dd84 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Fri, 8 Dec 2017 13:33:34 +0100 Subject: [PATCH 07/18] Pass all tags from OSM to the creators --- osm2gtfs/core/osm_connector.py | 50 ++------------- osm2gtfs/core/routes.py | 62 ++++++++++++++----- .../creators/accra/trips_creator_accra.py | 24 ++++--- 3 files changed, 66 insertions(+), 70 deletions(-) diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index 85a739e2..f910a402 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -300,30 +300,9 @@ def _build_line(self, route_master, itineraries): "No 'ref' could be obtained. Skipping whole route.\n") return - name = route_master.tags['name'] - frequency = None - if "frequency" in route_master.tags: - frequency = route_master.tags['frequency'] - - color = "FFFFFF" - if "colour" in route_master.tags: - color = OsmConnector.get_hex_code_for_color(route_master.tags['colour']) - - text_color = OsmConnector.get_complementary_color(color) - if "text_colour" in route_master.tags: - text_color = OsmConnector.get_hex_code_for_color(route_master.tags['text_colour']) - - if 'route_master' in route_master.tags: - route_type = route_master.tags['route_master'].capitalize() - - # If there was no route_master present we have a route relation here - elif 'route' in route_master.tags: - route_type = route_master.tags['route'].capitalize() - # Create Line (route master) object line = Line(osm_id=route_master.id, route_id=ref, - name=name, route_type=route_type, frequency=frequency, - route_color=color, route_text_color=text_color) + tags=route_master.tags) # Add Itinerary objects (route variants) to Line (route master) for itinerary in list(itineraries.values()): @@ -331,7 +310,7 @@ def _build_line(self, route_master, itineraries): line.add_itinerary(itinerary) except ValueError: sys.stderr.write( - "Itinerary ID does not match line ID. Please fix in OSM.\n") + "Itinerary ID doesn't match line ID. Please fix in OSM.\n") sys.stderr.write(line.osm_url) itinerary.route_id = line.route_id line.add_itinerary(itinerary) @@ -355,26 +334,6 @@ def _build_itinerary(self, route_variant, query_result_set): "Whole Itinerary skipped. Please fix in OpenStreetMap\n") return - if 'from' in route_variant.tags: - fr = route_variant.tags['from'] - else: - fr = None - - if 'to' in route_variant.tags: - to = route_variant.tags['to'] - else: - to = None - - if 'name' in route_variant.tags: - name = route_variant.tags['name'] - else: - name = None - - if 'travel_time' in route_variant.tags: - travel_time = route_variant.tags['travel_time'] - else: - travel_time = None - stops = [] # Add ids for stops of this route variant @@ -394,9 +353,8 @@ def _build_itinerary(self, route_variant, query_result_set): stops.append(otype + "/" + str(stop_candidate.ref)) shape = self._generate_shape(route_variant, query_result_set) - rv = Itinerary(osm_id=route_variant.id, fr=fr, - to=to, stops=stops, shape=shape, route_id=ref, - name=name, travel_time=travel_time) + rv = Itinerary(osm_id=route_variant.id, route_id=ref, stops=stops, + shape=shape, tags=route_variant.tags) return rv def _build_stop(self, stop, stop_type): diff --git a/osm2gtfs/core/routes.py b/osm2gtfs/core/routes.py index 370e8ca3..409b46fc 100644 --- a/osm2gtfs/core/routes.py +++ b/osm2gtfs/core/routes.py @@ -16,19 +16,42 @@ class Line(object): """ osm_id = attr.ib() route_id = attr.ib() - name = attr.ib() - route_type = attr.ib() # Required (Tram, Subway, Rail, Bus, ...) + tags = attr.ib() + name = attr.ib(default=None) + route_type = attr.ib(default=None) # Required (Tram, Subway, Bus, ...) route_desc = attr.ib(default=None) route_url = attr.ib(default=None) route_color = attr.ib(default="FFFFFF") route_text_color = attr.ib(default="000000") osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) - frequency = attr.ib(default=None) # Related route variants _itineraries = attr.ib(default=attr.Factory(list)) + def __attrs_post_init__(self): + ''' + Populates the object with information obtained from the tags + ''' + self.name = self.tags['name'] + + if "colour" in self.tags: + self.route_color = OsmConnector.get_hex_code_for_color( + self.tags['colour']) + + text_color = OsmConnector.get_complementary_color(self.route_color) + if "text_colour" in self.tags: + self.route_text_color = OsmConnector.get_hex_code_for_color( + self.tags['text_colour']) + + if 'self' in self.tags: + # TODO: Get the type from itineraries/routes or config file + route_type = self.tags['self'].capitalize() + + # If there was no self present we have a route relation here + elif 'route' in self.tags: + route_type = self.tags['route'].capitalize() + def add_itinerary(self, itinerary): if self.route_id.encode('utf-8') != itinerary.route_id.encode('utf-8'): @@ -55,24 +78,35 @@ class Itinerary(object): """ osm_id = attr.ib() route_id = attr.ib() - name = attr.ib() - fr = attr.ib() - to = attr.ib() - shape = attr.ib() stops = attr.ib() - travel_time = attr.ib() - - route_url = attr.ib(default=None) - wheelchair_accessible = attr.ib(default=0) - bikes_allowed = attr.ib(default=0) - osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) + shape = attr.ib() + tags = attr.ib() - # Useful information for further calculation + name = attr.ib(default=None) + fr = attr.ib(default=None) + to = attr.ib(default=None) duration = attr.ib(default=None) + osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) # All stop objects of itinerary _stop_objects = attr.ib(default=attr.Factory(list)) + def __attrs_post_init__(self): + ''' + Populates the object with information obtained from the tags + ''' + if 'from' in self.tags: + self.fr = self.tags['from'] + + if 'to' in self.tags: + self.to = self.tags['to'] + + if 'name' in self.tags: + self.name = self.tags['name'] + + if 'duration' in self.tags: + self.name = self.tags['duration'] + def add_stop(self, stop): self._stop_objects.append(stop) diff --git a/osm2gtfs/creators/accra/trips_creator_accra.py b/osm2gtfs/creators/accra/trips_creator_accra.py index 9bf65a40..edb27003 100644 --- a/osm2gtfs/creators/accra/trips_creator_accra.py +++ b/osm2gtfs/creators/accra/trips_creator_accra.py @@ -53,8 +53,11 @@ def add_trips_to_feed(self, feed, data): DEFAULT_ROUTE_FREQUENCY = 30 DEFAULT_TRAVEL_TIME = 120 + frequency = None + if "frequency" in line.tags: + frequency = line.tags['frequency'] try: - ROUTE_FREQUENCY = int(line.frequency) + ROUTE_FREQUENCY = int(frequency) if not ROUTE_FREQUENCY > 0: print("frequency is invalid for route_master " + str( line.osm_id)) @@ -66,16 +69,17 @@ def add_trips_to_feed(self, feed, data): trip_gtfs.AddFrequency( "05:00:00", "22:00:00", ROUTE_FREQUENCY * 60) - try: - TRAVEL_TIME = int(a_route.travel_time) - if not TRAVEL_TIME > 0: - print("travel_time is invalid for route " + str( - a_route.osm_id)) + if 'travel_time' in a_route.tags: + try: + TRAVEL_TIME = int(a_route.tags['travel_time']) + if not TRAVEL_TIME > 0: + print("travel_time is invalid for route " + str( + a_route.osm_id)) + TRAVEL_TIME = DEFAULT_TRAVEL_TIME + except (ValueError, TypeError) as e: + print("travel_time not a number for route " + str( + a_route.osm_id)) TRAVEL_TIME = DEFAULT_TRAVEL_TIME - except (ValueError, TypeError) as e: - print("travel_time not a number for route " + str( - a_route.osm_id)) - TRAVEL_TIME = DEFAULT_TRAVEL_TIME for index_stop, a_stop in enumerate(a_route.stops): stop_id = a_stop.split('/')[-1] From d42cc66411583225843bb29a339ecb5c19f12336 Mon Sep 17 00:00:00 2001 From: Nico Alt Date: Wed, 29 Nov 2017 00:42:55 +0100 Subject: [PATCH 08/18] Add default trips and routes creators --- osm2gtfs/creators/routes_creator.py | 49 ++++++- osm2gtfs/creators/stops_creator.py | 4 +- osm2gtfs/creators/trips_creator.py | 212 ++++++++++++++++++++++++++-- 3 files changed, 253 insertions(+), 12 deletions(-) diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index a3bd14cf..198522a4 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -13,4 +13,51 @@ def __repr__(self): return rep def add_routes_to_feed(self, feed, data): - raise NotImplementedError("Should have implemented this") + """ + route_id # Required: From Line + route_type # Required: From Line + + route_short_name # Required: To be generated from Line or Itinerary + route_long_name # Required: To be generated from Line or Itinerary + + route_desc # From Line + route_url # From Line + route_color # From Line + route_text_color # From Line + """ + # Get route information + lines = data.get_routes() + + # Loop through all lines + for line_ref, line in sorted(lines.iteritems()): + + # Add route information + route = schedule.AddRoute( + route_id=line_ref, + route_type=self._get_route_type(line), + short_name=line.route_id.encode('utf-8'), + long_name=line.name + ) + + route.agency_id = schedule.GetDefaultAgency().agency_id + + route.route_desc = self._get_route_description(line) + route.route_url = self._get_route_url(line) + route.route_color = self._get_route_color(line) + route.route_text_color = self._get_route_text_color(line) + return + + def _get_route_type(self, line): + return line.route_type + + def _get_route_description(self, line): + return line.route_desc + + def _get_route_url(self, line): + return line.route_url + + def _get_route_color(self, line): + return line.route_color + + def _get_route_text_color(self, line): + return line.route_text_color diff --git a/osm2gtfs/creators/stops_creator.py b/osm2gtfs/creators/stops_creator.py index 6b8343ad..3072667a 100644 --- a/osm2gtfs/creators/stops_creator.py +++ b/osm2gtfs/creators/stops_creator.py @@ -76,12 +76,10 @@ def _fill_stops(self, stops, route): Fill a route object with stop objects for of linked stop ids """ if isinstance(route, Itinerary): - i = 0 for stop in route.stops: # Replace stop id with Stop objects # TODO: Remove here and use references in TripsCreatorFenix - route.stops[i] = self._get_stop(stop, stops) - i += 1 + route.add_stop(self._get_stop(stop, stops)) elif isinstance(route, Line): itineraries = route.get_itineraries() diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index c8c6ccac..540c09d1 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -1,5 +1,10 @@ # coding=utf-8 +import json +import re +from datetime import datetime +from transitfeed import ServicePeriod + class TripsCreator(object): @@ -13,18 +18,209 @@ def __repr__(self): return rep def add_trips_to_feed(self, feed, data): - raise NotImplementedError("Should have implemented this") + """ + route_id # Required: From Line + service_id # Required: To be generated + trip_id # Required: To be generated + + trip_headsign # Itinerary "to" + direction_id # Order of tinieraries in Line object + wheelchair_accessible # Itinerary "wheelchair_accessible" + bikes_allowed # Itinerary "bikes_allowed" + trip_short_name # To be avoided! + block_id # To be avoided! + """ + # Get route information + # print('Getting line information') + lines = data.routes + + # Loop though all lines + for line_id, line in lines.iteritems(): + # print('Loop for line_id', line_id) + if line_id in timetable.excluded_lines: + print('Ignoring line ID: ' + line_id) + continue + # Loop through all itineraries + # print('Getting itinerary information from line', line.route_id) + itineraries = line.get_itineraries() + for itinerary in itineraries: + # print('Loop for itinerary.route_id', itinerary.route_id) + if itinerary.route_id.encode('utf-8') != line_id.encode('utf-8'): + raise RuntimeError('Itinerary route ID (' + itinerary.route_id + ') does not match Line route ID (' + line_id + ')') + + if itinerary.route_id not in timetable.lines: + print('Route ID of itinerary not found in timetable, skipping it', itinerary.route_id) + continue + # Add itinerary shape to schedule + # print('Adding itinerary shape to schedule', itinerary.route_id) + shape_id = TripsCreator.add_shape(schedule, itinerary.route_id, itinerary) + + # Get operations for itinerary + # print('Getting operations for itinerary') + services = self._get_itinerary_services(timetable, itinerary) + + # Loop through all services + for service in services: + # print('Loop for service', service) + # print('Create service period') + service_period = self._create_service_period(schedule, service) + # print('Load timetable') + gtfs_timetable = self._load_timetable(timetable, itinerary, service) + # print('Load stops') + stops = self._load_stops(timetable, itinerary, service) + # print('Get route from line id', line_id) + route = schedule.GetRoute(line_id) + + # print('Add trips for route') + self._add_trips_for_route(schedule, route, itinerary, + service_period, shape_id, stops, + gtfs_timetable) + return + + def _get_itinerary_services(self, timetable, itinerary): + """ + Returns a list with services of given itinerary. + """ + fr = itinerary.fr.encode('utf-8') + to = itinerary.to.encode('utf-8') + + services = [] + + for trip in timetable.lines[itinerary.route_id]: + input_fr = trip["from"].encode('utf-8') + input_to = trip["to"].encode('utf-8') + if input_fr == fr and input_to == to: + trip_services = trip["service"] + for service in trip_services: + services.append(service.encode('utf-8')) + return services + + def _create_service_period(self, schedule, service): + try: + gtfs_service = schedule.GetServicePeriod(service) + if gtfs_service is not None: + return gtfs_service + except KeyError: + pass + + if service == "Mo-Fr": + gtfs_service = ServicePeriod(service) + gtfs_service.SetWeekdayService(True) + gtfs_service.SetWeekendService(False) + elif service == "Mo-Sa": + gtfs_service = ServicePeriod(service) + gtfs_service.SetWeekdayService(True) + gtfs_service.SetWeekendService(False) + gtfs_service.SetDayOfWeekHasService(5, True) + elif service == "Mo-Su": + gtfs_service = ServicePeriod(service) + gtfs_service.SetWeekdayService(True) + gtfs_service.SetWeekendService(True) + elif service == "Sa": + gtfs_service = ServicePeriod(service) + gtfs_service.SetWeekdayService(False) + gtfs_service.SetWeekendService(False) + gtfs_service.SetDayOfWeekHasService(5, True) + elif service == "Su": + gtfs_service = ServicePeriod(service) + gtfs_service.SetWeekdayService(False) + gtfs_service.SetWeekendService(False) + gtfs_service.SetDayOfWeekHasService(6, True) + elif service == "Sa-Su": + gtfs_service = ServicePeriod(service) + gtfs_service.SetWeekdayService(False) + gtfs_service.SetWeekendService(True) + elif re.search(r'^([0-9]{4})-?(1[0-2]|0[1-9])-?(3[01]|0[1-9]|[12][0-9])$', service): + service = service.replace('-', '') + gtfs_service = ServicePeriod(service) + gtfs_service.SetDateHasService(service) + else: + raise KeyError("Unknown service keyword: " + service) + + gtfs_service.SetStartDate(self.config['feed_info']['start_date']) + gtfs_service.SetEndDate(self.config['feed_info']['end_date']) + schedule.AddServicePeriodObject(gtfs_service) + return schedule.GetServicePeriod(service) + + def _load_timetable(self, timetable, itinerary, service): + times = None + for trip in timetable.lines[itinerary.route_id]: + fr = trip["from"].encode('utf-8') + to = trip["to"].encode('utf-8') + trip_services = trip["service"] + if (fr == itinerary.fr.encode('utf-8') and + to == itinerary.to.encode('utf-8') and service in trip_services): + times = trip["times"] + + if times is None: + print("Problems found with Itinerary from " + + itinerary.fr.encode('utf-8') + " to " + + itinerary.to.encode('utf-8') + ) + print("Couldn't load times from timetable.") + return times + + def _load_stops(self, timetable, itinerary, service): + stops = [] + for trip in timetable.lines[itinerary.route_id]: + fr = trip["from"].encode('utf-8') + to = trip["to"].encode('utf-8') + trip_services = trip["service"] + if (fr == itinerary.fr.encode('utf-8') and + to == itinerary.to.encode('utf-8') and service in trip_services): + for stop in trip["stations"]: + stops.append(unicode(stop)) + return stops + + def _add_trips_for_route(self, schedule, gtfs_route, itinerary, service_period, + shape_id, stops, gtfs_timetable): + for trip in gtfs_timetable: + gtfs_trip = gtfs_route.AddTrip(schedule, headsign=itinerary.name, + service_period=service_period) + # print('Count of stops', len(stops)) + # print('Count of itinerary.get_stops()', len(itinerary.get_stops())) + # print('Stops', stops) + for itinerary_stop in itinerary.get_stops(): + gtfs_stop = schedule.GetStop(str(itinerary_stop.osm_id)) + time = "-" + try: + time = trip[stops.index(itinerary_stop.name)] + except ValueError: + pass + if time != "-": + try: + time_at_stop = str(datetime.strptime(time, "%H:%M").time()) + except ValueError: + print('Time seems invalid, skipping time', time) + break + gtfs_trip.AddStopTime(gtfs_stop, stop_time=time_at_stop) + else: + try: + gtfs_trip.AddStopTime(gtfs_stop) + except Exception: + print('Skipping trip because no time were found', itinerary.route_id, stops, itinerary_stop.name) + break + # add empty attributes to make navitia happy + gtfs_trip.block_id = "" + gtfs_trip.wheelchair_accessible = "" + gtfs_trip.bikes_allowed = "" + gtfs_trip.shape_id = shape_id + gtfs_trip.direction_id = "" + TripsCreator.interpolate_stop_times(gtfs_trip) @staticmethod def interpolate_stop_times(trip): """ Interpolate stop_times, because Navitia does not handle this itself """ - for secs, stop_time, is_timepoint in trip.GetTimeInterpolatedStops(): - if not is_timepoint: - stop_time.arrival_secs = secs - stop_time.departure_secs = secs - trip.ReplaceStopTimeObject(stop_time) + try: + for secs, stop_time, is_timepoint in trip.GetTimeInterpolatedStops(): + if not is_timepoint: + stop_time.arrival_secs = secs + stop_time.departure_secs = secs + trip.ReplaceStopTimeObject(stop_time) + except ValueError as e: + print(e) @staticmethod def add_shape(feed, route_id, osm_r): @@ -32,12 +228,12 @@ def add_shape(feed, route_id, osm_r): create GTFS shape and return shape_id to add on GTFS trip """ import transitfeed - shape_id = str(route_id) + shape_id = str(itinerary_id) try: feed.GetShape(shape_id) except KeyError: shape = transitfeed.Shape(shape_id) - for point in osm_r.shape: + for point in itinerary.shape: shape.AddPoint( lat=float(point["lat"]), lon=float(point["lon"])) feed.AddShapeObject(shape) From 1e6995e18b8c318ae7c2cb3b32f6bde4c2ab2ceb Mon Sep 17 00:00:00 2001 From: Ialokim Date: Thu, 23 Nov 2017 19:09:37 -0600 Subject: [PATCH 09/18] Fix same shape for different itineraries and ignore itinerary_stop=None errors --- osm2gtfs/creators/trips_creator.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index 540c09d1..971fd493 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -51,9 +51,9 @@ def add_trips_to_feed(self, feed, data): if itinerary.route_id not in timetable.lines: print('Route ID of itinerary not found in timetable, skipping it', itinerary.route_id) continue - # Add itinerary shape to schedule - # print('Adding itinerary shape to schedule', itinerary.route_id) - shape_id = TripsCreator.add_shape(schedule, itinerary.route_id, itinerary) + # Add itinerary shape to schedule, using osm_id instead of route_id to differ itinerary shapes + # print('Adding itinerary shape to schedule', itinerary.osm_id) + shape_id = TripsCreator.add_shape(schedule, itinerary.osm_id, itinerary) # Get operations for itinerary # print('Getting operations for itinerary') @@ -90,7 +90,7 @@ def _get_itinerary_services(self, timetable, itinerary): input_fr = trip["from"].encode('utf-8') input_to = trip["to"].encode('utf-8') if input_fr == fr and input_to == to: - trip_services = trip["service"] + trip_services = trip["services"] for service in trip_services: services.append(service.encode('utf-8')) return services @@ -147,7 +147,7 @@ def _load_timetable(self, timetable, itinerary, service): for trip in timetable.lines[itinerary.route_id]: fr = trip["from"].encode('utf-8') to = trip["to"].encode('utf-8') - trip_services = trip["service"] + trip_services = trip["services"] if (fr == itinerary.fr.encode('utf-8') and to == itinerary.to.encode('utf-8') and service in trip_services): times = trip["times"] @@ -165,7 +165,7 @@ def _load_stops(self, timetable, itinerary, service): for trip in timetable.lines[itinerary.route_id]: fr = trip["from"].encode('utf-8') to = trip["to"].encode('utf-8') - trip_services = trip["service"] + trip_services = trip["services"] if (fr == itinerary.fr.encode('utf-8') and to == itinerary.to.encode('utf-8') and service in trip_services): for stop in trip["stations"]: @@ -181,6 +181,11 @@ def _add_trips_for_route(self, schedule, gtfs_route, itinerary, service_period, # print('Count of itinerary.get_stops()', len(itinerary.get_stops())) # print('Stops', stops) for itinerary_stop in itinerary.get_stops(): + if itinerary_stop is None: + print('Itinerary stop is None. Seems to be a problem with OSM data. We should really fix that.') + print('itinerary route ID', itinerary.route_id) + print('itinerary stop', itinerary_stop) + continue gtfs_stop = schedule.GetStop(str(itinerary_stop.osm_id)) time = "-" try: @@ -228,7 +233,7 @@ def add_shape(feed, route_id, osm_r): create GTFS shape and return shape_id to add on GTFS trip """ import transitfeed - shape_id = str(itinerary_id) + shape_id = str(osm_id) try: feed.GetShape(shape_id) except KeyError: From 42dca8e22aa4f5088db98c55675776d8adb7d726 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Wed, 6 Dec 2017 21:44:41 +0100 Subject: [PATCH 10/18] Adjust README.md to welcome new cities; Minor fixes. --- README.md | 22 ++++++++++++++++------ osm2gtfs/creators/trips_creator.py | 13 +++++++------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8239fb62..086bf321 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,22 @@ caches on disk for efficient re-use. Then the data is combined with another source of schedule (time) information in order to create a GTFS file using the transitfeed library. -**Attention:** The source code is currently very specific to Florianópolis Buses -and Costa Rica Urban Train, but it can be extended to make it work for your use -case. In the config file any transit network can be specified for download from -OpenStreetMap. And by extending the creator classes in code, different -approaches for time information handling can be easily implemented. You can help -with pull requests to improve this script. +For every new city a new [configuration file](https://github.com/grote/osm2gtfs/wiki/Configuration) +needs to be created and the input of schedule information is preferred +in a certain [format](https://github.com/grote/osm2gtfs/wiki/Schedule). +For any city the script can be easily extended, see the +[developer documentation](https://github.com/grote/osm2gtfs/wiki/Development) +for more information. + +Included cities +----------------- + +* [Florianópolis, Brazil](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/fenix/fenix.json) +* [Suburban trains in Costa Rica](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/incofer/incofer.json) +* [Accra, Ghana](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/accra/accra.json) +* [Managua, Ciudad Sandino](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/managua/managua.json) and [Estelí](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/esteli/esteli.json) in Nicaragua + +*Soon, also in your city* Install ------------ diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index 971fd493..648b9f40 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -1,5 +1,6 @@ # coding=utf-8 +import os import json import re from datetime import datetime @@ -31,15 +32,15 @@ def add_trips_to_feed(self, feed, data): block_id # To be avoided! """ # Get route information - # print('Getting line information') - lines = data.routes + # lines = data.schedule # TODO: For later on + + if os.path.isfile('data/schedule.json'): + with open('data/schedule.json', 'rb') as f: + lines = json.load(f) # Loop though all lines for line_id, line in lines.iteritems(): - # print('Loop for line_id', line_id) - if line_id in timetable.excluded_lines: - print('Ignoring line ID: ' + line_id) - continue + # Loop through all itineraries # print('Getting itinerary information from line', line.route_id) itineraries = line.get_itineraries() From 3f2470b8286db8a0746f2f458c22ce91e9d91dfd Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Fri, 8 Dec 2017 17:08:50 +0100 Subject: [PATCH 11/18] Improve stop and routes standard creators --- osm2gtfs/core/osm_connector.py | 223 ++++++++++++------ osm2gtfs/core/routes.py | 1 + osm2gtfs/core/stops.py | 57 +++-- .../creators/accra/stops_creator_accra.py | 5 +- .../creators/accra/trips_creator_accra.py | 2 +- osm2gtfs/creators/feed_info_creator.py | 5 +- .../creators/fenix/routes_creator_fenix.py | 46 +++- .../creators/fenix/stops_creator_fenix.py | 14 ++ .../incofer/routes_creator_incofer.py | 55 ++++- .../creators/incofer/stops_creator_incofer.py | 14 ++ osm2gtfs/creators/routes_creator.py | 42 +--- osm2gtfs/creators/schedule_creator.py | 2 + osm2gtfs/creators/stops_creator.py | 131 +++++----- osm2gtfs/creators/trips_creator.py | 41 ++-- osm2gtfs/osm2gtfs.py | 2 +- osm2gtfs/tests/tests_accra.py | 4 +- 16 files changed, 398 insertions(+), 246 deletions(-) create mode 100644 osm2gtfs/creators/fenix/stops_creator_fenix.py create mode 100644 osm2gtfs/creators/incofer/stops_creator_incofer.py diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index f910a402..1be3c9eb 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -1,14 +1,13 @@ # coding=utf-8 import sys -import overpy from collections import OrderedDict from math import cos, sin, atan2, sqrt, radians, degrees +import overpy from transitfeed import util from osm2gtfs.core.cache import Cache from osm2gtfs.core.routes import Itinerary, Line -from osm2gtfs.core.stops import Stop, StopArea - +from osm2gtfs.core.stops import Stop, Station class OsmConnector(object): """The OsmConnector class retrieves information about transit networks from @@ -148,7 +147,8 @@ def get_routes(self, refresh=False): else: sys.stderr.write( "Member relation is not a valid itinerary:\n") - sys.stderr.write("http://osm.org/relation/" + str(member.ref) + "\n") + sys.stderr.write("http://osm.org/relation/" + str( + member.ref) + "\n") # Create Line object from route master line = self._build_line(route_master, itineraries) @@ -197,7 +197,7 @@ def get_stops(self, refresh=False): file. Then this data gets prepared by building up objects of the class Stops - and StopArea (when the Stops are members of a stop_area) + and Station (when the Stops are members of a stop_area) It uses caching to leverage fast performance and spare the Overpass API. Special commands are used to refresh cached data. @@ -206,7 +206,7 @@ def get_stops(self, refresh=False): :param refresh: A simple boolean indicating a data refresh or use of caching if possible. - :return stops: A dictionary of Stops and StopAreas constituting the + :return stops: A dictionary of Stops and Stations constituting the obtained data. """ @@ -216,7 +216,8 @@ def get_stops(self, refresh=False): # Check if stops data is already built in this object if not self.stops: # If not, try to get stops data from file cache - self.stops = Cache.read_data(self.selector + '-stops') + self.stops = Cache.read_data( + self.selector + '-stops') if bool(self.stops): # Maybe check for unnamed stop names @@ -231,29 +232,32 @@ def get_stops(self, refresh=False): # Obtain raw data about routes from OpenStreetMap result = self._query_stops() + self.stops['regular'] = {} + self.stops['stations'] = {} # Build stops from ways (polygons) for stop in result.ways: - if self._is_valid_stop_candidate(stop): - self.stops["way/" + str(stop.id) - ] = self._build_stop(stop, "way") + osm_type = "way" + stop_object = self._build_stop(stop, osm_type) + if stop_object: + self.stops['regular'][osm_type + "/" + str( + stop.id)] = stop_object # Build stops from nodes for stop in result.nodes: - if self._is_valid_stop_candidate(stop): - self.stops["node/" + str(stop.id) - ] = self._build_stop(stop, "node") - - # Build stop_areas - for relation in result.relations: - # valid stop_area candidade? - if 'public_transport' in relation.tags: - if relation.tags["public_transport"] == "stop_area": - try: - self.stops["relation/" + str(relation.id) - ] = self._build_stop_area(relation) - except (ValueError, TypeError) as e: - sys.stderr.write('Cannot add stop area: ' + str(relation.id)) + osm_type = "node" + stop_object = self._build_stop(stop, osm_type) + if stop_object: + self.stops['regular'][osm_type + "/" + str( + stop.id)] = stop_object + + # Build stations from stop_area relations + for stop in result.relations: + osm_type = "relation" + stop_object = self._build_station(stop, osm_type) + if stop_object: + self.stops['stations'][osm_type + "/" + str( + stop.id)] = stop_object # Cache data Cache.write_data(self.selector + '-stops', self.stops) @@ -262,13 +266,6 @@ def get_stops(self, refresh=False): if self.auto_stop_names: self._get_names_for_unnamed_stops() - # Warning about stops without stop_area - for ref, elem in self.stops.iteritems(): - if type(elem) is Stop: - sys.stderr.write("Stop is not member of a stop_area." + - " Please fix in OpenStreetMap\n") - sys.stderr.write("http://osm.org/" + ref + "\n") - return self.stops def _build_line(self, route_master, itineraries): @@ -357,54 +354,121 @@ def _build_itinerary(self, route_variant, query_result_set): shape=shape, tags=route_variant.tags) return rv - def _build_stop(self, stop, stop_type): + def _build_stop(self, stop, osm_type): """Helper function to build a Stop object Returns a initiated Stop object from raw data """ - # Make sure name is not empty - if 'name' not in stop.tags: - stop.tags['name'] = "[" + self.stop_no_name + "]" + if self._is_valid_stop_candidate(stop): - # Ways don't have coordinates and they have to be calculated - if stop_type == "way": - (stop.lat, stop.lon) = self.get_center_of_nodes(stop.get_nodes()) + # Make sure name is not empty + if 'name' not in stop.tags: + stop.tags['name'] = "[" + self.stop_no_name + "]" - s = Stop(stop.id, "node", unicode(stop.tags['name']), stop.lat, stop.lon) - return s + # Make sure to allow uft-8 character encoding + stop.tags['name'] = stop.tags['name'].encode('utf-8') - def _build_stop_area(self, relation): - """Helper function to build a StopArea object + # Ways don't have a pair of coordinates and need to be calculated + if osm_type == "way": + (stop.lat, stop.lon) = self.get_center_of_nodes( + stop.get_nodes()) + + # Create and return Stop object + stop = Stop(osm_id=stop.id, osm_type=osm_type, tags=stop.tags, + lat=stop.lat, lon=stop.lon, name=stop.tags['name']) + return stop + + else: + sys.stderr.write( + "Warning: Potential stop was not approved and is ignored") + sys.stderr.write( + " Check tagging: http://osm.org/" + osm_type + "/" + str( + stop.id) + "\n") + return False + + def _build_station(self, stop_area, osm_type): + """Helper function to build Station objects from stop_areas + + The function creates a Station object for the stop_area + flagged as location_type = 1. This means station, that can + group stops. + + The members of this relation add this station their parent. + + Returns a initiated Station object from raw data - Returns a initiated StopArea object from raw data """ - stop_members = {} - for member in relation.members: - if (isinstance(member, overpy.RelationNode) and - member.role == "platform"): - if "node/" + str(member.ref) in self.stops: - stop = self.stops.pop("node/" + str(member.ref)) - stop_members["node/" + str(member.ref)] = stop - else: - sys.stderr.write("Stop not found in stops: ") - sys.stderr.write("http://osm.org/node/" + - str(member.ref) + "\n") - if len(stop_members) < 1: - sys.stderr.write("Cannot build stop area with no members\n") - - if 'name' not in relation.tags: - sys.stderr.write("Stop area without name." + - " Please fix in OpenStreetMap\n") - sys.stderr.write("http://osm.org/relation/" + - str(relation.id) + "\n") - stop_area = StopArea(relation.id, stop_members, self.stop_no_name) + + # Check whether a valid stop_area candidade + if 'public_transport' in stop_area.tags and stop_area.tags[ + 'public_transport'] == 'stop_area': + + # Analzyse member objects (stops) of this stop area + members = {} + for member in stop_area.members: + if (isinstance(member, overpy.RelationNode) and + member.role == "platform"): + + if "node/" + str(member.ref) in self.stops['regular']: + + # Collect the Stop objects that are members + # of this Station + members["node/" + str(member.ref)] = self.stops[ + 'regular']["node/" + str(member.ref)] + else: + sys.stderr.write( + "Error: Station member was not found in data") + sys.stderr.write("http://osm.org/relation/" + + str(stop_area.id) + "\n") + sys.stderr.write("http://osm.org/node/" + + str(member.ref) + "\n") + if len(members) < 1: + # Stop areas with only one stop, are not stations they just + # group different elements of one stop together. + sys.stderr.write( + "Error: Station with no members has been discarted:\n") + sys.stderr.write("http://osm.org/relation/" + + str(stop_area.id) + "\n") + return False + + elif len(members) is 1: + sys.stderr.write( + "Warning: Station has only one platform and is discarted\n") + sys.stderr.write("http://osm.org/relation/" + + str(stop_area.id) + "\n") + return False + + # Check name of stop area + if 'name' not in stop_area.tags: + sys.stderr.write("Warning: Stop area without name." + + " Please fix in OpenStreetMap\n") + sys.stderr.write("http://osm.org/relation/" + + str(stop_area.id) + "\n") + stop_area.name = self.stop_no_name + else: + stop_area.name = stop_area.tags["name"] + + # Calculate coordinates for stop area based on the center of it's + # members + stop_area.lat, stop_area.lon = self.get_center_of_nodes( + members.values()) + + # Create and return Station object + station = Station(osm_id=stop_area.id, tags=stop_area.tags, + lat=stop_area.lat, lon=stop_area.lon, + name=stop_area.name) + station.set_members(members) + return station + else: - stop_area = StopArea(relation.id, stop_members, - relation.tags["name"]) - # print(stop_area) - return stop_area + sys.stderr.write( + "Warning: Potential station was not approved and is ignored") + sys.stderr.write( + " Check tagging: http://osm.org/" + osm_type + "/" + str( + stop_area.id) + "\n") + return False def _query_routes(self): """Helper function to query OpenStreetMap routes @@ -523,21 +587,22 @@ def _generate_shape(self, route_variant, query_result_set): def _is_valid_stop_candidate(self, stop): """Helper function to check if a stop candidate has a valid tagging - Returns True or False + :return bool: Returns True or False """ + valid = False if 'public_transport' in stop.tags: if stop.tags['public_transport'] == 'platform': - return True + valid = True elif stop.tags['public_transport'] == 'station': - return True - elif 'highway' in stop.tags: + valid = True + if 'highway' in stop.tags: if stop.tags['highway'] == 'bus_stop': - return True - elif 'amenity' in stop.tags: + valid = True + if 'amenity' in stop.tags: if stop.tags['amenity'] == 'bus_station': - return True - return False + valid = True + return valid def _get_names_for_unnamed_stops(self): """Intelligently guess stop names for unnamed stops by sourrounding @@ -547,7 +612,7 @@ def _get_names_for_unnamed_stops(self): """ # Loop through all stops - for stop in self.stops.values(): + for stop in self.stops['regular'].values(): # If there is no name, query one intelligently from OSM if stop.name == "[" + self.stop_no_name + "]": @@ -678,6 +743,12 @@ def get_hex_code_for_color(color): return 'FFFFFF' if color == u'yellow': return 'FFFF00' + if color == u'brown': + return 'A52A2A' + if color == u'coral': + return 'FF7F50' + if color == u'turquoise': + return '40E0D0' print('Color not known: ' + color) return 'FA8072' diff --git a/osm2gtfs/core/routes.py b/osm2gtfs/core/routes.py index 409b46fc..20cde17f 100644 --- a/osm2gtfs/core/routes.py +++ b/osm2gtfs/core/routes.py @@ -33,6 +33,7 @@ def __attrs_post_init__(self): ''' Populates the object with information obtained from the tags ''' + from osm2gtfs.core.osm_connector import OsmConnector self.name = self.tags['name'] if "colour" in self.tags: diff --git a/osm2gtfs/core/stops.py b/osm2gtfs/core/stops.py index 7d2e3cac..552b212e 100644 --- a/osm2gtfs/core/stops.py +++ b/osm2gtfs/core/stops.py @@ -1,38 +1,65 @@ # coding=utf-8 +import sys import attr @attr.s -class Stop(object): +class Station(object): osm_id = attr.ib() - osm_type = attr.ib() - name = attr.ib() + tags = attr.ib() lat = attr.ib() lon = attr.ib() - gtfs_id = attr.ib(default=osm_id) + name = attr.ib() + + osm_type = attr.ib(default="relation") + location_type = attr.ib(default=1) osm_url = attr.ib(default="http://osm.org/" + str(osm_type) + "/" + str(osm_id)) + # Stops forming part of this Station + _members = attr.ib(default=attr.Factory(list)) + + def set_members(self, members): + self._members = members + + def get_members(self): + return self._members + @attr.s -class StopArea(object): +class Stop(object): osm_id = attr.ib() - name = attr.ib() + osm_type = attr.ib() + tags = attr.ib() lat = attr.ib() lon = attr.ib() + name = attr.ib() + + location_type = attr.ib(default=0) + osm_url = attr.ib(default="http://osm.org/" + + str(osm_type) + "/" + str(osm_id)) - stop_members = attr.ib(default=attr.Factory(list)) + # The id of the Station this Stop might be part of. + _parent_station = attr.ib(default=None) - def __init__(self, osm_id, stop_members, name=None): - self.osm_id = osm_id - if name is not None: - self.name = name.encode('utf-8') + def set_parent_station(self, identifier, override=False): + """ + Set the parent_station_id on the first time; + Second attempts throw a warning + """ + if self._parent_station is None or override is True: + self._parent_station = identifier else: - self.name = name - self.stop_members = stop_members + sys.stderr.write("Warning: Stop is part of two stop areas:\n") + sys.stderr.write( + "http://osm.org/" + self.osm_type + "/" + str( + self.osm_id) + "\n") + sys.stderr.write("http://osm.org/" + identifier + "\n") + sys.stderr.write("http://osm.org/" + self._parent_station + "\n") + sys.stderr.write("Please fix in OpenStreetMap\n") - from osm2gtfs.core.osm_connector import OsmConnector - self.lat, self.lon = OsmConnector.get_center_of_nodes(self.stop_members.values()) + def get_parent_station(self): + return self._parent_station diff --git a/osm2gtfs/creators/accra/stops_creator_accra.py b/osm2gtfs/creators/accra/stops_creator_accra.py index b2bdf454..a6185bcf 100644 --- a/osm2gtfs/creators/accra/stops_creator_accra.py +++ b/osm2gtfs/creators/accra/stops_creator_accra.py @@ -1,8 +1,7 @@ # coding=utf-8 - -from osm2gtfs.creators.stops_creator import StopsCreator import math +from osm2gtfs.creators.stops_creator import StopsCreator def get_crow_fly_distance(from_tuple, to_tuple): @@ -63,7 +62,7 @@ def add_stops_to_feed(self, feed, data): stops = data.get_stops() stops_by_name = {} - for a_stop_id, a_stop in stops.items(): + for a_stop_id, a_stop in stops['regular'].items(): a_stop.osm_id = a_stop_id if a_stop.name not in stops_by_name: stops_by_name[a_stop.name] = [] diff --git a/osm2gtfs/creators/accra/trips_creator_accra.py b/osm2gtfs/creators/accra/trips_creator_accra.py index edb27003..99a1de39 100644 --- a/osm2gtfs/creators/accra/trips_creator_accra.py +++ b/osm2gtfs/creators/accra/trips_creator_accra.py @@ -23,7 +23,7 @@ def add_trips_to_feed(self, feed, data): continue line_gtfs = feed.AddRoute( - short_name=line.route_id, + short_name=line.osm_id, long_name=line.name, # we change the route_long_name with the 'from' and 'to' tags # of the last route as the route_master name tag contains diff --git a/osm2gtfs/creators/feed_info_creator.py b/osm2gtfs/creators/feed_info_creator.py index 330c2df5..775795a8 100644 --- a/osm2gtfs/creators/feed_info_creator.py +++ b/osm2gtfs/creators/feed_info_creator.py @@ -18,9 +18,8 @@ def add_feed_info_to_feed(self, feed): feed_info = self.prepare_feed_info() feed.AddFeedInfoObject(feed_info) - # Missing feed_info workaround (https://github.com/google/transitfeed/issues/395) - # noinspection PyProtectedMember - # pylint: disable=protected-access + # Missing feed_info workaround + # https://github.com/google/transitfeed/issues/395 feed.AddTableColumns('feed_info', feed_info._ColumnNames()) def prepare_feed_info(self): diff --git a/osm2gtfs/creators/fenix/routes_creator_fenix.py b/osm2gtfs/creators/fenix/routes_creator_fenix.py index 37795535..0985432a 100644 --- a/osm2gtfs/creators/fenix/routes_creator_fenix.py +++ b/osm2gtfs/creators/fenix/routes_creator_fenix.py @@ -1,14 +1,54 @@ # coding=utf-8 +import sys +from osm2gtfs.core.routes import Itinerary, Line +from osm2gtfs.core.stops import Stop, Station from osm2gtfs.creators.routes_creator import RoutesCreator class RoutesCreatorFenix(RoutesCreator): def add_routes_to_feed(self, feed, data): + ''' + Override routes to feed method, to prepare routes with stops + for the handling in the custom trips creators. + ''' + routes = data.routes + stops = data.stops - # Get routes information - data.get_routes() + # Loop through routes + for ref, route in routes.iteritems(): + # Replace stop ids with Stop objects + self._fill_stops(stops, route) - # Fenix logic is doing route aggregation in TripsCreator + data.routes = routes return + + def _fill_stops(self, stops, route): + """ + Fill a route object with stop objects for of linked stop ids + """ + if isinstance(route, Itinerary): + i = 0 + for stop in route.stops: + # Replace stop id with Stop objects + route.stops[i] = self._look_up_stop(stop, stops) + i += 1 + + elif isinstance(route, Line): + itineraries = route.get_itineraries() + for itinerary in itineraries: + self._fill_stops(stops, itinerary) + else: + sys.stderr.write("Unknown route: " + str(route) + "\n") + + def _look_up_stop(self, stop_id, stops): + for ref, elem in stops.iteritems(): + if type(elem) is Stop: + if ref == stop_id: + return elem + elif type(elem) is Station: + if stop_id in elem.stop_members: + return elem.stop_members[stop_id] + else: + sys.stderr.write("Unknown stop: " + str(stop_id) + "\n") diff --git a/osm2gtfs/creators/fenix/stops_creator_fenix.py b/osm2gtfs/creators/fenix/stops_creator_fenix.py new file mode 100644 index 00000000..bad65c59 --- /dev/null +++ b/osm2gtfs/creators/fenix/stops_creator_fenix.py @@ -0,0 +1,14 @@ +# coding=utf-8 + +import transitfeed +from osm2gtfs.creators.stops_creator import StopsCreator + + +class StopsCreatorFenix(StopsCreator): + + # Override construction of stop_id + def get_gtfs_stop_id(self, stop): + + # Simply returns osm_id regardless of the osm_type as only map + # objects of type nodes are assumed. + return stop.osm_id diff --git a/osm2gtfs/creators/incofer/routes_creator_incofer.py b/osm2gtfs/creators/incofer/routes_creator_incofer.py index 9c963e5a..9aa00b47 100644 --- a/osm2gtfs/creators/incofer/routes_creator_incofer.py +++ b/osm2gtfs/creators/incofer/routes_creator_incofer.py @@ -1,21 +1,54 @@ # coding=utf-8 +import sys +from osm2gtfs.core.routes import Itinerary, Line +from osm2gtfs.core.stops import Stop, Station from osm2gtfs.creators.routes_creator import RoutesCreator class RoutesCreatorIncofer(RoutesCreator): def add_routes_to_feed(self, feed, data): + ''' + Override routes to feed method, to prepare routes with stops + for the handling in the custom trips creators. + ''' + routes = data.routes + stops = data.stops + + # Loop through routes + for ref, route in routes.iteritems(): + # Replace stop ids with Stop objects + self._fill_stops(stops, route) + + data.routes = routes return - def _get_route_type(self, line): - return "Tram" - - def _get_route_description(self, line): - return "Esta línea está a prueba" - - def _get_route_color(self, line): - return "ff0000" - - def _get_route_text_color(self, line): - return "ffffff" + def _fill_stops(self, stops, route): + """ + Fill a route object with stop objects for of linked stop ids + """ + if isinstance(route, Itinerary): + i = 0 + for stop in route.stops: + # Replace stop id with Stop objects + route.stops[i] = self._look_up_stop(stop, stops) + i += 1 + + elif isinstance(route, Line): + itineraries = route.get_itineraries() + for itinerary in itineraries: + self._fill_stops(stops, itinerary) + else: + sys.stderr.write("Unknown route: " + str(route) + "\n") + + def _look_up_stop(self, stop_id, stops): + for ref, elem in stops.iteritems(): + if type(elem) is Stop: + if ref == stop_id: + return elem + elif type(elem) is Station: + if stop_id in elem.stop_members: + return elem.stop_members[stop_id] + else: + sys.stderr.write("Unknown stop: " + str(stop_id) + "\n") diff --git a/osm2gtfs/creators/incofer/stops_creator_incofer.py b/osm2gtfs/creators/incofer/stops_creator_incofer.py new file mode 100644 index 00000000..3d7632a3 --- /dev/null +++ b/osm2gtfs/creators/incofer/stops_creator_incofer.py @@ -0,0 +1,14 @@ +# coding=utf-8 + +import transitfeed +from osm2gtfs.creators.stops_creator import StopsCreator + + +class StopsCreatorIncofer(StopsCreator): + + # Override construction of stop_id + def get_gtfs_stop_id(self, stop): + if stop.osm_type == "relation": + return "SA" + str(stop.osm_id) + else: + return stop.osm_id diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index 198522a4..bc4cd9bc 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -14,16 +14,7 @@ def __repr__(self): def add_routes_to_feed(self, feed, data): """ - route_id # Required: From Line - route_type # Required: From Line - - route_short_name # Required: To be generated from Line or Itinerary - route_long_name # Required: To be generated from Line or Itinerary - - route_desc # From Line - route_url # From Line - route_color # From Line - route_text_color # From Line + This function adds the routes from the data to the GTFS feed. """ # Get route information lines = data.get_routes() @@ -32,32 +23,15 @@ def add_routes_to_feed(self, feed, data): for line_ref, line in sorted(lines.iteritems()): # Add route information - route = schedule.AddRoute( + route = feed.AddRoute( route_id=line_ref, - route_type=self._get_route_type(line), + route_type=line.route_type, short_name=line.route_id.encode('utf-8'), long_name=line.name ) - - route.agency_id = schedule.GetDefaultAgency().agency_id - - route.route_desc = self._get_route_description(line) - route.route_url = self._get_route_url(line) - route.route_color = self._get_route_color(line) - route.route_text_color = self._get_route_text_color(line) + route.agency_id = feed.GetDefaultAgency().agency_id + route.route_desc = line.route_desc + route.route_url = line.route_url + route.route_color = line.route_color + route.route_text_color = line.route_text_color return - - def _get_route_type(self, line): - return line.route_type - - def _get_route_description(self, line): - return line.route_desc - - def _get_route_url(self, line): - return line.route_url - - def _get_route_color(self, line): - return line.route_color - - def _get_route_text_color(self, line): - return line.route_text_color diff --git a/osm2gtfs/creators/schedule_creator.py b/osm2gtfs/creators/schedule_creator.py index b482780c..896f9598 100644 --- a/osm2gtfs/creators/schedule_creator.py +++ b/osm2gtfs/creators/schedule_creator.py @@ -32,6 +32,8 @@ def add_schedule_to_data(self, data): def _load_schedule_source(self): """ This function loads and verifies the content of the file. + In the standard schedule creator it assumes a json file. This function + can be overridden to support any type of file format or structure. """ schedule_source = self.config.get_schedule_source() diff --git a/osm2gtfs/creators/stops_creator.py b/osm2gtfs/creators/stops_creator.py index 3072667a..dd511841 100644 --- a/osm2gtfs/creators/stops_creator.py +++ b/osm2gtfs/creators/stops_creator.py @@ -1,9 +1,6 @@ # coding=utf-8 -import sys import transitfeed -from osm2gtfs.core.routes import Itinerary, Line -from osm2gtfs.core.stops import Stop, StopArea class StopsCreator(object): @@ -18,84 +15,70 @@ def __repr__(self): return rep def add_stops_to_feed(self, feed, data): - # Get stops information - stops = data.get_stops() - - # add all stops to GTFS - for elem in stops.values(): - if type(elem) is StopArea: - if len(elem.stop_members) > 1: - parent_station = self.add_stop(feed, elem, None, True) - for stop in elem.stop_members.values(): - self.add_stop(feed, stop, parent_station) - else: - stop = elem.stop_members.values()[0] - self.add_stop(feed, stop) - else: - self.add_stop(feed, elem) - - # Add loose stop objects to route objects - self.add_stops_to_routes(data) - - def add_stop(self, feed, stop, parent_station=None, is_station=False): - stop_dict = {"stop_lat": float(stop.lat), - "stop_lon": float(stop.lon), - "stop_name": stop.name} - - if is_station: - stop_dict["stop_id"] = "SA" + str(stop.id) - stop_dict["location_type"] = "1" - else: - stop_dict["stop_id"] = str(stop.id) - stop_dict["location_type"] = "" + """ + This function adds the Stops from the data to the GTFS feed. + It also unites stops in stations based on stop_areas in OpenStreetMap. + """ + all_stops = data.get_stops() + regular_stops = all_stops['regular'] + parent_stations = all_stops['stations'] - if parent_station is None: - stop_dict["parent_station"] = "" - else: - stop_dict["parent_station"] = parent_station.stop_id + # Loop through all stations and prepare stops + for station in parent_stations.values(): - # Add stop to GTFS object - stop = transitfeed.Stop(field_dict=stop_dict) - feed.AddStopObject(stop) - return stop + # Add station to feed as stop + gtfs_stop_id = self._add_stop_to_feed(station, feed) - def add_stops_to_routes(self, data): - routes = data.routes - stops = data.stops + # Loop through member stops of a station + for member in station.get_members(): - # Loop through routes - for ref, route in routes.iteritems(): - # Replace stop ids with Stop objects - self._fill_stops(stops, route) + # Set parent station of each member Stop + regular_stops[member].set_parent_station(gtfs_stop_id) - data.routes = routes - return + # Loop through regular stops + for stop in regular_stops.values(): + # Add stop to feed + self._add_stop_to_feed(stop, feed) - def _fill_stops(self, stops, route): - """ - Fill a route object with stop objects for of linked stop ids + def get_gtfs_stop_id(self, stop): """ - if isinstance(route, Itinerary): - for stop in route.stops: - # Replace stop id with Stop objects - # TODO: Remove here and use references in TripsCreatorFenix - route.add_stop(self._get_stop(stop, stops)) + This function returns the GTFS stop id to be used for a stop. + It can be overridden by custom cretors to change how stop_ids are made + up. - elif isinstance(route, Line): - itineraries = route.get_itineraries() - for itinerary in itineraries: - self._fill_stops(stops, itinerary) + :return gtfs_stop_id: A string with the stop_id for use in the GTFS + """ + if "gtfs_id" in stop.tags: + # Use a GTFS stop_id coming from OpenStreetMap data + return stop.tags['gtfs_id'] else: - sys.stderr.write("Unknown route: " + str(route) + "\n") - - def _get_stop(self, stop_id, stops): - for ref, elem in stops.iteritems(): - if type(elem) is Stop: - if ref == stop_id: - return elem - elif type(elem) is StopArea: - if stop_id in elem.stop_members: - return elem.stop_members[stop_id] - else: - sys.stderr.write("Unknown stop: " + str(stop_id) + "\n") + # Use a GTFS stop_id mathing to OpenStreetMap objects + return stop.osm_type + "/" + str(stop.osm_id) + + def _add_stop_to_feed(self, stop, feed): + """ + This function adds a Stop or Station object as a stop to GTFS. + It can be overridden by custom cretors to change how stop_ids are made + up. + + :return stop_id: A string with the stop_id in the GTFS + """ + try: + parent_station = stop.get_parent_station() + except AttributeError as e: + parent_station = "" + + field_dict = {'stop_id': self.get_gtfs_stop_id(stop), + 'stop_name': stop.name, + 'stop_lat': float(stop.lat), + 'stop_lon': float(stop.lon), + 'location_type': stop.location_type, + 'parent_station': parent_station + } + + # Add stop to GTFS object + feed.AddStopObject(transitfeed.Stop(field_dict=field_dict)) + + # Return the stop_id of the stop added + return field_dict['stop_id'] diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index 648b9f40..f6a13169 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -1,7 +1,5 @@ # coding=utf-8 -import os -import json import re from datetime import datetime from transitfeed import ServicePeriod @@ -32,11 +30,7 @@ def add_trips_to_feed(self, feed, data): block_id # To be avoided! """ # Get route information - # lines = data.schedule # TODO: For later on - - if os.path.isfile('data/schedule.json'): - with open('data/schedule.json', 'rb') as f: - lines = json.load(f) + lines = data.schedule # Loop though all lines for line_id, line in lines.iteritems(): @@ -52,9 +46,9 @@ def add_trips_to_feed(self, feed, data): if itinerary.route_id not in timetable.lines: print('Route ID of itinerary not found in timetable, skipping it', itinerary.route_id) continue - # Add itinerary shape to schedule, using osm_id instead of route_id to differ itinerary shapes - # print('Adding itinerary shape to schedule', itinerary.osm_id) - shape_id = TripsCreator.add_shape(schedule, itinerary.osm_id, itinerary) + # Add itinerary shape to feed, using osm_id instead of route_id to differ itinerary shapes + # print('Adding itinerary shape to feed', itinerary.osm_id) + shape_id = TripsCreator.add_shape(feed, itinerary.osm_id, itinerary) # Get operations for itinerary # print('Getting operations for itinerary') @@ -64,16 +58,17 @@ def add_trips_to_feed(self, feed, data): for service in services: # print('Loop for service', service) # print('Create service period') - service_period = self._create_service_period(schedule, service) + service_period = self._create_service_period(feed, service) # print('Load timetable') - gtfs_timetable = self._load_timetable(timetable, itinerary, service) + gtfs_timetable = self._load_timetable(timetable, + itinerary, service) # print('Load stops') stops = self._load_stops(timetable, itinerary, service) # print('Get route from line id', line_id) - route = schedule.GetRoute(line_id) + route = feed.GetRoute(line_id) # print('Add trips for route') - self._add_trips_for_route(schedule, route, itinerary, + self._add_trips_for_route(feed, route, itinerary, service_period, shape_id, stops, gtfs_timetable) return @@ -96,9 +91,9 @@ def _get_itinerary_services(self, timetable, itinerary): services.append(service.encode('utf-8')) return services - def _create_service_period(self, schedule, service): + def _create_service_period(self, feed, service): try: - gtfs_service = schedule.GetServicePeriod(service) + gtfs_service = feed.GetServicePeriod(service) if gtfs_service is not None: return gtfs_service except KeyError: @@ -140,8 +135,8 @@ def _create_service_period(self, schedule, service): gtfs_service.SetStartDate(self.config['feed_info']['start_date']) gtfs_service.SetEndDate(self.config['feed_info']['end_date']) - schedule.AddServicePeriodObject(gtfs_service) - return schedule.GetServicePeriod(service) + feed.AddServicePeriodObject(gtfs_service) + return feed.GetServicePeriod(service) def _load_timetable(self, timetable, itinerary, service): times = None @@ -173,10 +168,10 @@ def _load_stops(self, timetable, itinerary, service): stops.append(unicode(stop)) return stops - def _add_trips_for_route(self, schedule, gtfs_route, itinerary, service_period, + def _add_trips_for_route(self, feed, gtfs_route, itinerary, service_period, shape_id, stops, gtfs_timetable): for trip in gtfs_timetable: - gtfs_trip = gtfs_route.AddTrip(schedule, headsign=itinerary.name, + gtfs_trip = gtfs_route.AddTrip(feed, headsign=itinerary.name, service_period=service_period) # print('Count of stops', len(stops)) # print('Count of itinerary.get_stops()', len(itinerary.get_stops())) @@ -187,7 +182,7 @@ def _add_trips_for_route(self, schedule, gtfs_route, itinerary, service_period, print('itinerary route ID', itinerary.route_id) print('itinerary stop', itinerary_stop) continue - gtfs_stop = schedule.GetStop(str(itinerary_stop.osm_id)) + gtfs_stop = feed.GetStop(str(itinerary_stop.osm_id)) time = "-" try: time = trip[stops.index(itinerary_stop.name)] @@ -229,12 +224,12 @@ def interpolate_stop_times(trip): print(e) @staticmethod - def add_shape(feed, route_id, osm_r): + def add_shape(feed, route_id, itinerary): """ create GTFS shape and return shape_id to add on GTFS trip """ import transitfeed - shape_id = str(osm_id) + shape_id = str(route_id) try: feed.GetShape(shape_id) except KeyError: diff --git a/osm2gtfs/osm2gtfs.py b/osm2gtfs/osm2gtfs.py index a60f5c7f..bb4cfa2e 100644 --- a/osm2gtfs/osm2gtfs.py +++ b/osm2gtfs/osm2gtfs.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # coding=utf-8 -import transitfeed import sys import argparse +import transitfeed from core.configuration import Configuration from core.osm_connector import OsmConnector from core.creator_factory import CreatorFactory diff --git a/osm2gtfs/tests/tests_accra.py b/osm2gtfs/tests/tests_accra.py index fb6d3e45..7ccae005 100644 --- a/osm2gtfs/tests/tests_accra.py +++ b/osm2gtfs/tests/tests_accra.py @@ -96,7 +96,7 @@ def test_refresh_routes_cache(self): data.get_routes(refresh=True) self.assertTrue(os.path.isfile(cache_file), 'The routes cache file creation failed') cache = Cache() - routes = cache.read_data('routes-accra') + routes = cache.read_data('accra-routes') self.assertEqual(len(routes), 277, 'Wrong count of routes in the cache file') def test_refresh_stops_cache(self): @@ -112,7 +112,7 @@ def test_refresh_stops_cache(self): data.get_stops(refresh=True) self.assertTrue(os.path.isfile(cache_file), 'The stops cache file creation failed') cache = Cache() - stops = cache.read_data('stops-accra') + stops = cache.read_data('accra-stops') self.assertEqual(len(stops), 2529, 'Wrong count of stops in the cache file') def test_gtfs_from_cache(self): From ec3f729b027daefb66d9e96b00334a12507d678b Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Mon, 11 Dec 2017 00:05:39 +0100 Subject: [PATCH 12/18] Integrate trips creators --- README.md | 8 +- osm2gtfs/core/configuration.py | 2 +- osm2gtfs/core/helper.py | 45 +++ osm2gtfs/core/osm_connector.py | 159 +++----- osm2gtfs/core/routes.py | 79 ++-- osm2gtfs/core/stops.py | 39 +- .../creators/accra/stops_creator_accra.py | 7 +- .../creators/accra/trips_creator_accra.py | 10 +- osm2gtfs/creators/esteli/__init__.py | 2 + osm2gtfs/creators/esteli/esteli.json | 6 +- .../creators/esteli/routes_creator_esteli.py | 32 ++ osm2gtfs/creators/feed_info_creator.py | 1 + .../creators/fenix/routes_creator_fenix.py | 8 +- .../creators/fenix/stops_creator_fenix.py | 5 +- .../creators/fenix/trips_creator_fenix.py | 59 +-- .../incofer/routes_creator_incofer.py | 34 +- .../creators/incofer/stops_creator_incofer.py | 3 +- .../creators/incofer/trips_creator_incofer.py | 53 ++- osm2gtfs/creators/managua/managua.json | 6 +- osm2gtfs/creators/routes_creator.py | 68 +++- osm2gtfs/creators/stops_creator.py | 73 ++-- osm2gtfs/creators/trips_creator.py | 374 +++++++++++------- osm2gtfs/osm2gtfs.py | 2 +- osm2gtfs/tests/tests_accra.py | 3 +- setup.py | 2 +- 25 files changed, 663 insertions(+), 417 deletions(-) create mode 100644 osm2gtfs/creators/esteli/__init__.py create mode 100644 osm2gtfs/creators/esteli/routes_creator_esteli.py diff --git a/README.md b/README.md index 086bf321..4c9a3a39 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,10 @@ for more information. Included cities ----------------- -* [Florianópolis, Brazil](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/fenix/fenix.json) -* [Suburban trains in Costa Rica](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/incofer/incofer.json) -* [Accra, Ghana](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/accra/accra.json) -* [Managua, Ciudad Sandino](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/managua/managua.json) and [Estelí](https://github.com/grote/osm2gtfs/blob/master/osm2gtfs/creators/esteli/esteli.json) in Nicaragua +* [Florianópolis, Brazil](./osm2gtfs/creators/fenix/fenix.json) +* [Suburban trains in Costa Rica](./osm2gtfs/creators/incofer/incofer.json) +* [Accra, Ghana](./osm2gtfs/creators/accra/accra.json) +* [Managua, Ciudad Sandino](./osm2gtfs/creators/managua/managua.json) and [Estelí](./osm2gtfs/creators/esteli/esteli.json) in Nicaragua *Soon, also in your city* diff --git a/osm2gtfs/core/configuration.py b/osm2gtfs/core/configuration.py index 8e0c9a58..fe4c793a 100644 --- a/osm2gtfs/core/configuration.py +++ b/osm2gtfs/core/configuration.py @@ -85,7 +85,7 @@ def get_schedule_source(self, refresh=False): # Cache data Cache.write_file(cached_file, schedule_source) - self._schedule_source = schedule_source + self._schedule_source = schedule_source.read() return self._schedule_source def _load_config(self, args): diff --git a/osm2gtfs/core/helper.py b/osm2gtfs/core/helper.py index 9d1df938..6133139c 100644 --- a/osm2gtfs/core/helper.py +++ b/osm2gtfs/core/helper.py @@ -1,5 +1,8 @@ # coding=utf-8 +import sys +from math import cos, sin, atan2, sqrt, radians, degrees + class Helper(object): """The Helper class contains useful static functions @@ -21,3 +24,45 @@ def print_shape_for_leaflet(shape): print "L.marker([" + str(node["lat"]) + ", " + str(node["lon"]) + "]).addTo(map)" print " .bindPopup(\"" + str(i) + "\").openPopup();" i += 1 + + @staticmethod + def get_center_of_nodes(nodes): + """Helper function to get center coordinates of a group of nodes + + """ + x = 0 + y = 0 + z = 0 + + if len(nodes) < 1: + sys.stderr.write("Cannot find the center of zero nodes\n") + for node in nodes: + lat = radians(float(node.lat)) + lon = radians(float(node.lon)) + + x += cos(lat) * cos(lon) + y += cos(lat) * sin(lon) + z += sin(lat) + + x = float(x / len(nodes)) + y = float(y / len(nodes)) + z = float(z / len(nodes)) + + center_lat = degrees(atan2(z, sqrt(x * x + y * y))) + center_lon = degrees(atan2(y, x)) + + return center_lat, center_lon + + @staticmethod + def interpolate_stop_times(trip): + """ + Interpolate stop_times, because Navitia does not handle this itself + """ + try: + for secs, stop_time, is_timepoint in trip.GetTimeInterpolatedStops(): + if not is_timepoint: + stop_time.arrival_secs = secs + stop_time.departure_secs = secs + trip.ReplaceStopTimeObject(stop_time) + except ValueError as e: + print(e) diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index 1be3c9eb..2f15d0e5 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -2,13 +2,14 @@ import sys from collections import OrderedDict -from math import cos, sin, atan2, sqrt, radians, degrees import overpy from transitfeed import util from osm2gtfs.core.cache import Cache +from osm2gtfs.core.helper import Helper from osm2gtfs.core.routes import Itinerary, Line from osm2gtfs.core.stops import Stop, Station + class OsmConnector(object): """The OsmConnector class retrieves information about transit networks from OpenStreetMap, caches it and serves it when needed to the osm2gtfs script @@ -41,7 +42,7 @@ def __init__(self, config): # tags from config file for querying self.tags = '' for key, value in self.config["query"].get("tags", {}).iteritems(): - self.tags += str('["' + key + '" = "' + value + '"]') + self.tags += unicode('["' + key + '" = "' + value + '"]') if not self.tags: # fallback self.tags = '["public_transport:version" = "2"]' @@ -133,7 +134,8 @@ def get_routes(self, refresh=False): # Create Itinerary objects from member route variants if member.ref in route_variants: rv = route_variants.pop(member.ref) - itineraries[rv.id] = self._build_itinerary(rv, result) + itineraries[rv.id] = self._build_itinerary(rv, result, + route_master) # Route variant was already used or is not valid else: @@ -142,13 +144,19 @@ def get_routes(self, refresh=False): rv = rv.pop() sys.stderr.write("Itinerary was assigned again:\n") sys.stderr.write( - "http://osm.org/relation/" + str(rv.id) + "\n") - itineraries[rv.id] = self._build_itinerary(rv, result) + "https://osm.org/relation/" + str(rv.id) + "\n") + itineraries[rv.id] = self._build_itinerary(rv, result, + route_master) else: sys.stderr.write( - "Member relation is not a valid itinerary:\n") - sys.stderr.write("http://osm.org/relation/" + str( - member.ref) + "\n") + "Warning: This relation route master:\n") + sys.stderr.write(" https://osm.org/relation/" + str( + route_master.id) + "\n") + sys.stderr.write( + " has a member which is not a valid itinerary:\n") + sys.stderr.write(" https://osm.org/" + type( + member).__name__[8:].lower() + "/" + str( + member.ref) + "\n") # Create Line object from route master line = self._build_line(route_master, itineraries) @@ -157,24 +165,24 @@ def get_routes(self, refresh=False): if line.route_id in self.routes: sys.stderr.write("'Ref' of route_master already taken\n") sys.stderr.write( - "http://osm.org/relation/" + str(route_master.id) + "\n") + "https://osm.org/relation/" + str(route_master.id) + "\n") sys.stderr.write("Skipped. Please fix in OpenStreetMap\n") else: self.routes[line.route_id] = line # Build routes from variants (missing master relation) for rvid, route_variant in route_variants.iteritems(): - sys.stderr.write("Itinerary without master\n") + sys.stderr.write("Route (variant) without route_master\n") sys.stderr.write( - "http://osm.org/relation/" + str(route_variant.id) + "\n") + "https://osm.org/relation/" + str(route_variant.id) + "\n") sys.stderr.write("Please fix in OpenStreetMap\n") - itinerary = self._build_itinerary(route_variant, result) + itinerary = self._build_itinerary(route_variant, result, False) # Make sure route_id (ref) number is not already taken if itinerary.route_id in self.routes: - sys.stderr.write("Itinerary with existing route id (ref)\n") + sys.stderr.write("Route with existing route_id (ref)\n") sys.stderr.write( - "http://osm.org/relation/" + str(route_variant.id) + "\n") + "https://osm.org/relation/" + str(route_variant.id) + "\n") sys.stderr.write("Skipped. Please fix in OpenStreetMap\n") else: # Create Line from route variant @@ -188,6 +196,9 @@ def get_routes(self, refresh=False): return self.routes + def set_stops(self, stops): + self.stops = stops + def get_stops(self, refresh=False): """The get_stops function returns the data of stops and stop areas from OpenStreetMap converted into usable objects. @@ -241,7 +252,7 @@ def get_stops(self, refresh=False): stop_object = self._build_stop(stop, osm_type) if stop_object: self.stops['regular'][osm_type + "/" + str( - stop.id)] = stop_object + stop_object.osm_id)] = stop_object # Build stops from nodes for stop in result.nodes: @@ -249,7 +260,7 @@ def get_stops(self, refresh=False): stop_object = self._build_stop(stop, osm_type) if stop_object: self.stops['regular'][osm_type + "/" + str( - stop.id)] = stop_object + stop_object.osm_id)] = stop_object # Build stations from stop_area relations for stop in result.relations: @@ -280,7 +291,7 @@ def _build_line(self, route_master, itineraries): sys.stderr.write( "Relation without 'ref'. Please fix in OpenStreetMap\n") sys.stderr.write( - "http://osm.org/relation/" + str(route_master.id) + "\n") + "https://osm.org/relation/" + str(route_master.id) + "\n") # Check if a ref can be taken from one of the itineraries ref = False @@ -314,7 +325,7 @@ def _build_line(self, route_master, itineraries): return line - def _build_itinerary(self, route_variant, query_result_set): + def _build_itinerary(self, route_variant, query_result_set, route_master): """Helper function to build a Itinerary object Returns a initiated Itinerary object from raw data @@ -326,7 +337,7 @@ def _build_itinerary(self, route_variant, query_result_set): sys.stderr.write( "RouteVariant without 'ref': " + str(route_variant.id) + "\n") sys.stderr.write( - "http://osm.org/relation/" + str(route_variant.id) + "\n") + "https://osm.org/relation/" + str(route_variant.id) + "\n") sys.stderr.write( "Whole Itinerary skipped. Please fix in OpenStreetMap\n") return @@ -349,9 +360,15 @@ def _build_itinerary(self, route_variant, query_result_set): stops.append(otype + "/" + str(stop_candidate.ref)) + if route_master: + route_master_id = "relation/" + str(route_master.id) + else: + route_master_id = None + shape = self._generate_shape(route_variant, query_result_set) rv = Itinerary(osm_id=route_variant.id, route_id=ref, stops=stops, - shape=shape, tags=route_variant.tags) + shape=shape, tags=route_variant.tags, + line=route_master_id) return rv def _build_stop(self, stop, osm_type): @@ -367,12 +384,9 @@ def _build_stop(self, stop, osm_type): if 'name' not in stop.tags: stop.tags['name'] = "[" + self.stop_no_name + "]" - # Make sure to allow uft-8 character encoding - stop.tags['name'] = stop.tags['name'].encode('utf-8') - # Ways don't have a pair of coordinates and need to be calculated if osm_type == "way": - (stop.lat, stop.lon) = self.get_center_of_nodes( + (stop.lat, stop.lon) = Helper.get_center_of_nodes( stop.get_nodes()) # Create and return Stop object @@ -384,7 +398,7 @@ def _build_stop(self, stop, osm_type): sys.stderr.write( "Warning: Potential stop was not approved and is ignored") sys.stderr.write( - " Check tagging: http://osm.org/" + osm_type + "/" + str( + " Check tagging: https://osm.org/" + osm_type + "/" + str( stop.id) + "\n") return False @@ -420,23 +434,23 @@ def _build_station(self, stop_area, osm_type): else: sys.stderr.write( "Error: Station member was not found in data") - sys.stderr.write("http://osm.org/relation/" + + sys.stderr.write("https://osm.org/relation/" + str(stop_area.id) + "\n") - sys.stderr.write("http://osm.org/node/" + + sys.stderr.write("https://osm.org/node/" + str(member.ref) + "\n") if len(members) < 1: # Stop areas with only one stop, are not stations they just # group different elements of one stop together. sys.stderr.write( "Error: Station with no members has been discarted:\n") - sys.stderr.write("http://osm.org/relation/" + + sys.stderr.write("https://osm.org/relation/" + str(stop_area.id) + "\n") return False elif len(members) is 1: sys.stderr.write( "Warning: Station has only one platform and is discarted\n") - sys.stderr.write("http://osm.org/relation/" + + sys.stderr.write("https://osm.org/relation/" + str(stop_area.id) + "\n") return False @@ -444,7 +458,7 @@ def _build_station(self, stop_area, osm_type): if 'name' not in stop_area.tags: sys.stderr.write("Warning: Stop area without name." + " Please fix in OpenStreetMap\n") - sys.stderr.write("http://osm.org/relation/" + + sys.stderr.write("https://osm.org/relation/" + str(stop_area.id) + "\n") stop_area.name = self.stop_no_name else: @@ -452,7 +466,7 @@ def _build_station(self, stop_area, osm_type): # Calculate coordinates for stop area based on the center of it's # members - stop_area.lat, stop_area.lon = self.get_center_of_nodes( + stop_area.lat, stop_area.lon = Helper.get_center_of_nodes( members.values()) # Create and return Station object @@ -466,7 +480,7 @@ def _build_station(self, stop_area, osm_type): sys.stderr.write( "Warning: Potential station was not approved and is ignored") sys.stderr.write( - " Check tagging: http://osm.org/" + osm_type + "/" + str( + " Check tagging: https://osm.org/" + osm_type + "/" + str( stop_area.id) + "\n") return False @@ -573,10 +587,10 @@ def _generate_shape(self, route_variant, query_result_set): shape_sorter.extend(reversed(way_nodes)) else: sys.stderr.write("Route has non-matching ways: " + - "http://osm.org/relation/" + + "https://osm.org/relation/" + str(route_variant.id) + "\n") sys.stderr.write( - " Problem at: http://osm.org/way/" + str(way) + "\n") + " Problem at: https://osm.org/way/" + str(way) + "\n") break for sorted_node in shape_sorter: @@ -617,7 +631,8 @@ def _get_names_for_unnamed_stops(self): # If there is no name, query one intelligently from OSM if stop.name == "[" + self.stop_no_name + "]": self._find_best_name_for_unnamed_stop(stop) - print(stop) + print("* Smartly guessed stop name: " + + stop.name + " - " + stop.osm_url) # Cache stops with newly created stop names Cache.write_data(self.selector + '-stops', self.stops) @@ -672,7 +687,7 @@ def _find_best_name_for_unnamed_stop(self, stop): winner_distance = sys.maxint for candidate in candidates: if isinstance(candidate, overpy.Way): - lat, lon = self.get_center_of_nodes( + lat, lon = Helper.get_center_of_nodes( candidate.get_nodes(resolve_missing=True)) distance = util.ApproximateDistance( lat, @@ -692,74 +707,4 @@ def _find_best_name_for_unnamed_stop(self, stop): winner_distance = distance # take name from winner - stop.name = winner.tags["name"].encode('utf-8') - - @staticmethod - def get_center_of_nodes(nodes): - """Helper function to get center coordinates of a group of nodes - - """ - x = 0 - y = 0 - z = 0 - - if len(nodes) < 1: - sys.stderr.write("Cannot find the center of zero nodes\n") - for node in nodes: - lat = radians(float(node.lat)) - lon = radians(float(node.lon)) - - x += cos(lat) * cos(lon) - y += cos(lat) * sin(lon) - z += sin(lat) - - x = float(x / len(nodes)) - y = float(y / len(nodes)) - z = float(z / len(nodes)) - - center_lat = degrees(atan2(z, sqrt(x * x + y * y))) - center_lon = degrees(atan2(y, x)) - - return center_lat, center_lon - - @staticmethod - def get_hex_code_for_color(color): - color = color.lower() - if color == u'black': - return '000000' - if color == u'blue': - return '0000FF' - if color == u'gray': - return '808080' - if color == u'green': - return '008000' - if color == u'purple': - return '800080' - if color == u'red': - return 'FF0000' - if color == u'silver': - return 'C0C0C0' - if color == u'white': - return 'FFFFFF' - if color == u'yellow': - return 'FFFF00' - if color == u'brown': - return 'A52A2A' - if color == u'coral': - return 'FF7F50' - if color == u'turquoise': - return '40E0D0' - print('Color not known: ' + color) - return 'FA8072' - - @staticmethod - def get_complementary_color(color): - """ - Returns complementary RGB color - Source: https://stackoverflow.com/a/38478744 - """ - if color[0] == '#': - color = color[1:] - rgb = (color[0:2], color[2:4], color[4:6]) - comp = ['%02X' % (255 - int(a, 16)) for a in rgb] - return ''.join(comp) + stop.name = winner.tags["name"] diff --git a/osm2gtfs/core/routes.py b/osm2gtfs/core/routes.py index 20cde17f..a19c0bda 100644 --- a/osm2gtfs/core/routes.py +++ b/osm2gtfs/core/routes.py @@ -11,7 +11,7 @@ class Line(object): variants of the same service line. In OpenStreetMap this is usually represented as "route_master" relation. - In GTFS this is usually represented as "route" + In GTFS this is usually represented as "route". """ osm_id = attr.ib() @@ -19,47 +19,53 @@ class Line(object): tags = attr.ib() name = attr.ib(default=None) - route_type = attr.ib(default=None) # Required (Tram, Subway, Bus, ...) + route_type = attr.ib(default=None) route_desc = attr.ib(default=None) - route_url = attr.ib(default=None) - route_color = attr.ib(default="FFFFFF") - route_text_color = attr.ib(default="000000") - osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) + route_color = attr.ib(default="#FFFFFF") + route_text_color = attr.ib(default="#000000") + osm_url = attr.ib(default=None) # Related route variants _itineraries = attr.ib(default=attr.Factory(list)) def __attrs_post_init__(self): + '''Populates the object with information obtained from the tags + ''' - Populates the object with information obtained from the tags - ''' - from osm2gtfs.core.osm_connector import OsmConnector - self.name = self.tags['name'] + from osm2gtfs.core.helper import Helper + + # Disabling some pylint errors as pylint currently doesn't support any + # custom decorators or descriptors + # https://github.com/PyCQA/pylint/issues/1694 - if "colour" in self.tags: - self.route_color = OsmConnector.get_hex_code_for_color( - self.tags['colour']) + # pylint: disable=unsupported-membership-test,unsubscriptable-object + self.name = self.tags['name'] + self.osm_url = "https://osm.org/relation/" + str(self.osm_id) - text_color = OsmConnector.get_complementary_color(self.route_color) - if "text_colour" in self.tags: - self.route_text_color = OsmConnector.get_hex_code_for_color( - self.tags['text_colour']) + # pylint: disable=unsupported-membership-test,unsubscriptable-object + if 'colour' in self.tags: + self.route_color = self.tags['colour'] - if 'self' in self.tags: - # TODO: Get the type from itineraries/routes or config file - route_type = self.tags['self'].capitalize() + # pylint: disable=unsupported-membership-test,unsubscriptable-object + if 'ref:colour_tx' in self.tags: + self.route_text_color = self.tags['ref:colour_tx'] - # If there was no self present we have a route relation here + # pylint: disable=unsupported-membership-test,unsubscriptable-object + if 'route_master' in self.tags: + self.route_type = self.tags['route_master'].capitalize() elif 'route' in self.tags: - route_type = self.tags['route'].capitalize() + self.route_type = self.tags['route'].capitalize() + else: + self.route_type = "Bus" def add_itinerary(self, itinerary): - if self.route_id.encode('utf-8') != itinerary.route_id.encode('utf-8'): + if self.route_id != itinerary.route_id: raise ValueError('Itinerary route ID (' + itinerary.route_id + ') does not match Line route ID (' + self.route_id + ')') + # pylint: disable=no-member self._itineraries.append(itinerary) def get_itineraries(self): @@ -79,40 +85,49 @@ class Itinerary(object): """ osm_id = attr.ib() route_id = attr.ib() - stops = attr.ib() shape = attr.ib() tags = attr.ib() name = attr.ib(default=None) + line = attr.ib(default=None) + osm_url = attr.ib(default=None) fr = attr.ib(default=None) to = attr.ib(default=None) duration = attr.ib(default=None) - osm_url = attr.ib(default="http://osm.org/relation/" + str(osm_id)) # All stop objects of itinerary - _stop_objects = attr.ib(default=attr.Factory(list)) + stops = attr.ib(default=attr.Factory(list)) def __attrs_post_init__(self): ''' Populates the object with information obtained from the tags ''' + + self.osm_url = "https://osm.org/relation/" + str(self.osm_id) + + # Disabling some pylint errors as pylint currently doesn't support any + # custom decorators or descriptors + # https://github.com/PyCQA/pylint/issues/1694 + + # pylint: disable=unsupported-membership-test,unsubscriptable-object if 'from' in self.tags: self.fr = self.tags['from'] + # pylint: disable=unsupported-membership-test,unsubscriptable-object if 'to' in self.tags: self.to = self.tags['to'] + # pylint: disable=unsupported-membership-test,unsubscriptable-object if 'name' in self.tags: self.name = self.tags['name'] + # pylint: disable=unsupported-membership-test,unsubscriptable-object if 'duration' in self.tags: - self.name = self.tags['duration'] + self.duration = self.tags['duration'] def add_stop(self, stop): - self._stop_objects.append(stop) - - def get_stop_by_position(self, pos): - raise NotImplementedError("Should have implemented this") + # pylint: disable=no-member + self.stops.append(stop) def get_stops(self): - return self._stop_objects + return self.stops diff --git a/osm2gtfs/core/stops.py b/osm2gtfs/core/stops.py index 552b212e..10f38261 100644 --- a/osm2gtfs/core/stops.py +++ b/osm2gtfs/core/stops.py @@ -13,20 +13,33 @@ class Station(object): lon = attr.ib() name = attr.ib() + stop_id = attr.ib(default="") osm_type = attr.ib(default="relation") location_type = attr.ib(default=1) - osm_url = attr.ib(default="http://osm.org/" + - str(osm_type) + "/" + str(osm_id)) + osm_url = attr.ib(default=None) # Stops forming part of this Station _members = attr.ib(default=attr.Factory(list)) + def __attrs_post_init__(self): + '''Populates the object with information obtained from the tags + + ''' + self.osm_url = "https://osm.org/" + str( + self.osm_type) + "/" + str(self.osm_id) + def set_members(self, members): self._members = members def get_members(self): return self._members + def get_stop_id(self): + return self.stop_id + + def set_stop_id(self, stop_id): + self.stop_id = stop_id + @attr.s class Stop(object): @@ -38,13 +51,20 @@ class Stop(object): lon = attr.ib() name = attr.ib() + stop_id = attr.ib("") location_type = attr.ib(default=0) - osm_url = attr.ib(default="http://osm.org/" + - str(osm_type) + "/" + str(osm_id)) + osm_url = attr.ib(default=None) # The id of the Station this Stop might be part of. _parent_station = attr.ib(default=None) + def __attrs_post_init__(self): + '''Populates the object with information obtained from the tags + + ''' + self.osm_url = "https://osm.org/" + str( + self.osm_type) + "/" + str(self.osm_id) + def set_parent_station(self, identifier, override=False): """ Set the parent_station_id on the first time; @@ -55,11 +75,14 @@ def set_parent_station(self, identifier, override=False): else: sys.stderr.write("Warning: Stop is part of two stop areas:\n") sys.stderr.write( - "http://osm.org/" + self.osm_type + "/" + str( + "https://osm.org/" + self.osm_type + "/" + str( self.osm_id) + "\n") - sys.stderr.write("http://osm.org/" + identifier + "\n") - sys.stderr.write("http://osm.org/" + self._parent_station + "\n") - sys.stderr.write("Please fix in OpenStreetMap\n") def get_parent_station(self): return self._parent_station + + def get_stop_id(self): + return self.stop_id + + def set_stop_id(self, stop_id): + self.stop_id = stop_id diff --git a/osm2gtfs/creators/accra/stops_creator_accra.py b/osm2gtfs/creators/accra/stops_creator_accra.py index a6185bcf..62bb50b3 100644 --- a/osm2gtfs/creators/accra/stops_creator_accra.py +++ b/osm2gtfs/creators/accra/stops_creator_accra.py @@ -30,7 +30,7 @@ def get_crow_fly_distance(from_tuple, to_tuple): def create_stop_area(stop_data, feed): - stop_id = stop_data.osm_id.split('/')[-1] + stop_id = stop_data.osm_id gtfs_stop_area = feed.AddStop( lat=float(stop_data.lat), lng=float(stop_data.lon), @@ -42,7 +42,7 @@ def create_stop_area(stop_data, feed): def create_stop_point(stop_data, feed): - stop_id = stop_data.osm_id.split('/')[-1] + stop_id = stop_data.osm_id gtfs_stop_point = feed.AddStop( lat=float(stop_data.lat), lng=float(stop_data.lon), @@ -62,8 +62,7 @@ def add_stops_to_feed(self, feed, data): stops = data.get_stops() stops_by_name = {} - for a_stop_id, a_stop in stops['regular'].items(): - a_stop.osm_id = a_stop_id + for internal_stop_id, a_stop in stops['regular'].items(): if a_stop.name not in stops_by_name: stops_by_name[a_stop.name] = [] stops_by_name[a_stop.name].append(a_stop) diff --git a/osm2gtfs/creators/accra/trips_creator_accra.py b/osm2gtfs/creators/accra/trips_creator_accra.py index 99a1de39..365fb13d 100644 --- a/osm2gtfs/creators/accra/trips_creator_accra.py +++ b/osm2gtfs/creators/accra/trips_creator_accra.py @@ -3,6 +3,7 @@ from datetime import timedelta, datetime from osm2gtfs.creators.trips_creator import TripsCreator +from osm2gtfs.core.helper import Helper from osm2gtfs.core.routes import Line @@ -21,9 +22,10 @@ def add_trips_to_feed(self, feed, data): for route_ref, line in sorted(lines.iteritems()): if not isinstance(line, Line): continue + print("Generating schedule for line: " + route_ref) line_gtfs = feed.AddRoute( - short_name=line.osm_id, + short_name=str(line.route_id), long_name=line.name, # we change the route_long_name with the 'from' and 'to' tags # of the last route as the route_master name tag contains @@ -39,8 +41,8 @@ def add_trips_to_feed(self, feed, data): itineraries = line.get_itineraries() for a_route in itineraries: trip_gtfs = line_gtfs.AddTrip(feed) - trip_gtfs.shape_id = TripsCreator.add_shape( - feed, a_route.route_id, a_route) + trip_gtfs.shape_id = self._add_shape_to_feed( + feed, a_route.osm_id, a_route) trip_gtfs.direction_id = route_index % 2 route_index += 1 @@ -103,4 +105,4 @@ def add_trips_to_feed(self, feed, data): stop_time.departure_secs = secs trip_gtfs.ReplaceStopTimeObject(stop_time) - TripsCreator.interpolate_stop_times(trip_gtfs) + Helper.interpolate_stop_times(trip_gtfs) diff --git a/osm2gtfs/creators/esteli/__init__.py b/osm2gtfs/creators/esteli/__init__.py new file mode 100644 index 00000000..5c9136ad --- /dev/null +++ b/osm2gtfs/creators/esteli/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# coding=utf-8 diff --git a/osm2gtfs/creators/esteli/esteli.json b/osm2gtfs/creators/esteli/esteli.json index 22b25821..85a1904b 100644 --- a/osm2gtfs/creators/esteli/esteli.json +++ b/osm2gtfs/creators/esteli/esteli.json @@ -20,13 +20,13 @@ "agency_name": "Estelí", "agency_url": "https://wiki.openstreetmap.org/wiki/ES:Wikiproyecto_Nicaragua/Transporte_p%C3%BAblico/Estel%C3%AD", "agency_timezone": "America/Managua", - "agency_lang": "ES", + "agency_lang": "es", "agency_phone": "", "agency_fare_url": "" }, "feed_info": { - "publisher_name": "Nico Alt", - "publisher_url": "https://nico.dorfbrunnen.eu", + "publisher_name": "MapaNica.net", + "publisher_url": "https://mapanica.net", "version": "0.1" }, "schedule_source": "https://github.com/mapanica/pt-data-esteli/raw/master/timetable.json", diff --git a/osm2gtfs/creators/esteli/routes_creator_esteli.py b/osm2gtfs/creators/esteli/routes_creator_esteli.py new file mode 100644 index 00000000..1c69544e --- /dev/null +++ b/osm2gtfs/creators/esteli/routes_creator_esteli.py @@ -0,0 +1,32 @@ +# coding=utf-8 + +import webcolors +from osm2gtfs.creators.routes_creator import RoutesCreator + + +class RoutesCreatorEsteli(RoutesCreator): + + def _define_route_color(self, route): + """ + Overriden to support color names + """ + if not route.route_color == '#FFFFFF': + route.route_color = webcolors.name_to_hex(route.route_color) + return route.route_color[1:] + + def _define_route_text_color(self, route): + """ + Overriden to support automatic guessing + """ + return self._get_complementary_color(route.route_color) + + def _get_complementary_color(self, color): + """ + Returns complementary RGB color + Source: https://stackoverflow.com/a/38478744 + """ + if color[0] == '#': + color = color[1:] + rgb = (color[0:2], color[2:4], color[4:6]) + comp = ['%02X' % (255 - int(a, 16)) for a in rgb] + return ''.join(comp) diff --git a/osm2gtfs/creators/feed_info_creator.py b/osm2gtfs/creators/feed_info_creator.py index 775795a8..1ed1edcb 100644 --- a/osm2gtfs/creators/feed_info_creator.py +++ b/osm2gtfs/creators/feed_info_creator.py @@ -20,6 +20,7 @@ def add_feed_info_to_feed(self, feed): # Missing feed_info workaround # https://github.com/google/transitfeed/issues/395 + # pylint: disable=protected-access feed.AddTableColumns('feed_info', feed_info._ColumnNames()) def prepare_feed_info(self): diff --git a/osm2gtfs/creators/fenix/routes_creator_fenix.py b/osm2gtfs/creators/fenix/routes_creator_fenix.py index 0985432a..277c61f6 100644 --- a/osm2gtfs/creators/fenix/routes_creator_fenix.py +++ b/osm2gtfs/creators/fenix/routes_creator_fenix.py @@ -13,15 +13,13 @@ def add_routes_to_feed(self, feed, data): Override routes to feed method, to prepare routes with stops for the handling in the custom trips creators. ''' - routes = data.routes - stops = data.stops + routes = data.get_routes() + stops = data.get_stops() # Loop through routes for ref, route in routes.iteritems(): # Replace stop ids with Stop objects - self._fill_stops(stops, route) - - data.routes = routes + self._fill_stops(stops['regular'], route) return def _fill_stops(self, stops, route): diff --git a/osm2gtfs/creators/fenix/stops_creator_fenix.py b/osm2gtfs/creators/fenix/stops_creator_fenix.py index bad65c59..71f12461 100644 --- a/osm2gtfs/creators/fenix/stops_creator_fenix.py +++ b/osm2gtfs/creators/fenix/stops_creator_fenix.py @@ -1,14 +1,13 @@ # coding=utf-8 -import transitfeed from osm2gtfs.creators.stops_creator import StopsCreator class StopsCreatorFenix(StopsCreator): # Override construction of stop_id - def get_gtfs_stop_id(self, stop): + def _define_stop_id(self, stop): # Simply returns osm_id regardless of the osm_type as only map # objects of type nodes are assumed. - return stop.osm_id + return str(stop.osm_id) diff --git a/osm2gtfs/creators/fenix/trips_creator_fenix.py b/osm2gtfs/creators/fenix/trips_creator_fenix.py index 1e061b4c..7a19926b 100644 --- a/osm2gtfs/creators/fenix/trips_creator_fenix.py +++ b/osm2gtfs/creators/fenix/trips_creator_fenix.py @@ -2,9 +2,10 @@ import sys import re -import transitfeed from datetime import timedelta, datetime +import transitfeed from osm2gtfs.creators.trips_creator import TripsCreator +from osm2gtfs.core.helper import Helper from osm2gtfs.core.routes import Line, Itinerary DEBUG_ROUTE = "" @@ -54,7 +55,7 @@ def __init__(self, config): self.exceptions = None def add_trips_to_feed(self, feed, data): - routes = data.routes + routes = data.get_routes() feed.AddServicePeriodObject(self.service_weekday) feed.AddServicePeriodObject(self.service_saturday) feed.AddServicePeriodObject(self.service_sunday) @@ -64,6 +65,9 @@ def add_trips_to_feed(self, feed, data): # Try to find OSM routes in Fenix data for route_ref, route in sorted(routes.iteritems()): + + print("Generating schedule for line: " + route_ref) + if route_ref not in BLACKLIST and route_ref in linhas: linha = linhas[route_ref] route.name = linha['nome'].encode('utf-8') @@ -75,10 +79,12 @@ def add_trips_to_feed(self, feed, data): continue duration_str = linha['tempo_de_percurso'].replace('aproximado', '') (hours, tmp, minutes) = duration_str.partition(':') - route.set_duration(timedelta(hours=int(hours), minutes=int(minutes))) + route.duration = timedelta(hours=int(hours), minutes=int(minutes)) self.add_route(feed, route, linha['horarios'], linha['operacoes']) elif route_ref not in BLACKLIST: - sys.stderr.write("Route not found in Fenix data: " + str(route) + "\n") + sys.stderr.write( + "Route not found in Fenix data: [" + route.route_id + "] " + str( + route.osm_url) + "\n") def add_route(self, feed, route, horarios, operacoes): line = feed.AddRoute( @@ -87,7 +93,7 @@ def add_route(self, feed, route, horarios, operacoes): route_type="Bus") line.agency_id = feed.GetDefaultAgency().agency_id line.route_desc = "TEST DESCRIPTION" - line.route_url = "http://www.consorciofenix.com.br/horarios?q=" + str(route.ref) + line.route_url = "http://www.consorciofenix.com.br/horarios?q=" + str(route.route_id) line.route_color = "1779c2" line.route_text_color = "ffffff" @@ -135,13 +141,15 @@ def add_route(self, feed, route, horarios, operacoes): self.add_trips_by_day(feed, line, self.service_sunday, route, sunday, SUNDAY) def add_trips_by_day(self, feed, line, service, route, horarios, day): + # check if we even have service if horarios is None or len(horarios) == 0: return if isinstance(route, Line): # recurse into "Ida" and "Volta" routes - for sub_route in route.routes.values(): + for sub_route in route.get_itineraries(): + sub_route.duration = route.duration self.add_trips_by_day(feed, line, service, sub_route, horarios, day) return @@ -157,12 +165,12 @@ def add_trips_by_day(self, feed, line, service, route, horarios, day): # Do not print debug output here, because already done in route.match_first_stops() return - if route.ref == DEBUG_ROUTE: + if route.route_id == DEBUG_ROUTE: print "\n\n\n" + str(route) print day + " - " + key # get shape id - shape_id = str(route.id) + shape_id = str(route.route_id) try: feed.GetShape(shape_id) except KeyError: @@ -171,8 +179,10 @@ def add_trips_by_day(self, feed, line, service, route, horarios, day): shape.AddPoint(lat=float(point["lat"]), lon=float(point["lon"])) feed.AddShapeObject(shape) - if len(horarios) > 1 and not route.has_proper_master(): - sys.stderr.write("Route should have a master: " + str(route) + "\n") + if len(horarios) > 1 and route.line is None: + sys.stderr.write( + "Route should have a master: [" + route.route_id + "] " + str( + route.osm_url) + "\n") for time_group in horarios[key]: for time_point in time_group: @@ -183,7 +193,7 @@ def add_trips_by_day(self, feed, line, service, route, horarios, day): # calculate last arrival time for GTFS start_sec = transitfeed.TimeToSecondsSinceMidnight(start_time) factor = 1 - if len(horarios) > 1 and not route.has_proper_master(): + if len(horarios) > 1 and route.line is None: # since this route has only one instead of two trips, double the duration factor = 2 end_sec = start_sec + route.duration.seconds * factor @@ -199,12 +209,12 @@ def add_trips_by_day(self, feed, line, service, route, horarios, day): trip.bikes_allowed = "" trip.shape_id = shape_id trip.direction_id = "" - if route.ref == DEBUG_ROUTE: + if route.route_id == DEBUG_ROUTE: print "ADD TRIP " + str(trip.trip_id) + ":" self.add_trip_stops(feed, trip, route, start_time, end_time) # interpolate times, because Navitia can not handle this itself - TripsCreator.interpolate_stop_times(trip) + Helper.interpolate_stop_times(trip) def get_exception_service_period(self, feed, date, day): date_string = date.strftime("%Y%m%d") @@ -230,14 +240,17 @@ def get_exception_service_period(self, feed, date, day): @staticmethod def match_first_stops(route, sim_stops): # get the first stop of the route - stop = route.get_first_stop() + stop = route.stops[0] # normalize its name stop.name = TripsCreatorFenix.normalize_stop_name(stop.name) # get first stop from relation 'from' tag - alt_stop_name = route.get_first_alt_stop() - alt_stop_name = TripsCreatorFenix.normalize_stop_name(alt_stop_name.encode('utf-8')) + if 'from' in route.tags: + alt_stop_name = route.tags['from'] + else: + alt_stop_name = "" + alt_stop_name = TripsCreatorFenix.normalize_stop_name(alt_stop_name) # trying to match first stop from OSM with SIM for o_sim_stop in sim_stops: @@ -262,7 +275,9 @@ def match_first_stops(route, sim_stops): @staticmethod def normalize_stop_name(old_name): name = STOP_REGEX.sub(r'\1', old_name) - name = name.replace('Terminal de Integração da Lagoa da Conceição', 'TILAG') + if type(name).__name__ == 'str': + name = name.decode('utf-8') + name = name.replace('Terminal de Integração da Lagoa da Conceição'.decode('utf-8'), 'TILAG') name = name.replace('Terminal Centro', 'TICEN') name = name.replace('Terminal Rio Tavares', 'TIRIO') name = name.replace('Itacurubi', 'Itacorubi') @@ -275,16 +290,16 @@ def add_trip_stops(feed, trip, route, start_time, end_time): for stop in route.stops: if i == 1: # timepoint="1" (Times are considered exact) - trip.AddStopTime(feed.GetStop(str(stop.id)), stop_time=start_time) - if route.ref == DEBUG_ROUTE: + trip.AddStopTime(feed.GetStop(str(stop.stop_id)), stop_time=start_time) + if route.route_id == DEBUG_ROUTE: print "START: " + start_time + " at " + str(stop) elif i == len(route.stops): # timepoint="0" (Times are considered approximate) - trip.AddStopTime(feed.GetStop(str(stop.id)), stop_time=end_time) - if route.ref == DEBUG_ROUTE: + trip.AddStopTime(feed.GetStop(str(stop.stop_id)), stop_time=end_time) + if route.route_id == DEBUG_ROUTE: print "END: " + end_time + " at " + str(stop) else: # timepoint="0" (Times are considered approximate) - trip.AddStopTime(feed.GetStop(str(stop.id))) + trip.AddStopTime(feed.GetStop(str(stop.stop_id))) # print "INTER: " + str(stop) i += 1 diff --git a/osm2gtfs/creators/incofer/routes_creator_incofer.py b/osm2gtfs/creators/incofer/routes_creator_incofer.py index 9aa00b47..cf92189b 100644 --- a/osm2gtfs/creators/incofer/routes_creator_incofer.py +++ b/osm2gtfs/creators/incofer/routes_creator_incofer.py @@ -13,15 +13,41 @@ def add_routes_to_feed(self, feed, data): Override routes to feed method, to prepare routes with stops for the handling in the custom trips creators. ''' - routes = data.routes - stops = data.stops + routes = data.get_routes() + stops = data.get_stops() # Loop through routes for ref, route in routes.iteritems(): # Replace stop ids with Stop objects - self._fill_stops(stops, route) + self._fill_stops(stops['regular'], route) - data.routes = routes + # debug + # print("DEBUG: creando itinerarios a partir de", str(len(lines)), + # "lineas") + + # Loop through all lines (master_routes) + for line_ref, line in sorted(routes.iteritems()): + route = feed.AddRoute( + short_name=line.route_id.encode('utf-8'), + long_name=line.name, + # TODO: infer transitfeed "route type" from OSM data + route_type="Tram", + route_id=line_ref) + + # AddRoute method add defaut agency as default + route.agency_id = feed.GetDefaultAgency().agency_id + + route.route_desc = "Test line" + + # TODO: get route_url from OSM or other source. + # url = "http://www.incofer.go.cr/tren-urbano-alajuela-rio-segundo" + + # line.route_url = url + route.route_color = "ff0000" + route.route_text_color = "ffffff" + + # debug + # print("información de la linea:", line.name, "agregada.") return def _fill_stops(self, stops, route): diff --git a/osm2gtfs/creators/incofer/stops_creator_incofer.py b/osm2gtfs/creators/incofer/stops_creator_incofer.py index 3d7632a3..d0127e52 100644 --- a/osm2gtfs/creators/incofer/stops_creator_incofer.py +++ b/osm2gtfs/creators/incofer/stops_creator_incofer.py @@ -1,13 +1,12 @@ # coding=utf-8 -import transitfeed from osm2gtfs.creators.stops_creator import StopsCreator class StopsCreatorIncofer(StopsCreator): # Override construction of stop_id - def get_gtfs_stop_id(self, stop): + def _define_stop_id(self, stop): if stop.osm_type == "relation": return "SA" + str(stop.osm_id) else: diff --git a/osm2gtfs/creators/incofer/trips_creator_incofer.py b/osm2gtfs/creators/incofer/trips_creator_incofer.py index 04b961f9..b141dbce 100644 --- a/osm2gtfs/creators/incofer/trips_creator_incofer.py +++ b/osm2gtfs/creators/incofer/trips_creator_incofer.py @@ -11,7 +11,7 @@ class TripsCreatorIncofer(TripsCreator): def add_trips_to_feed(self, feed, data): - lines = data.routes + lines = data.get_routes() # line (osm rounte master | gtfs route) for line_id, line in lines.iteritems(): @@ -20,12 +20,13 @@ def add_trips_to_feed(self, feed, data): # itinerary (osm route | non existent gtfs element) itineraries = line.get_itineraries() - for itinerary_id, itinerary in itineraries: + for itinerary in itineraries: # debug # print("DEBUG. procesando el itinerario", itinerary.name) # shape for itinerary - shape_id = TripsCreator.add_shape(feed, itinerary_id, itinerary) + shape_id = self._add_shape_to_feed( + feed, itinerary.route_id, itinerary) # service periods | días de opearación (c/u con sus horarios) operations = self._get_itinerary_operation(itinerary, data) @@ -52,25 +53,23 @@ def _get_itinerary_operation(self, itinerary, data): incluyen en el archivo de entrada. """ - fr = itinerary.fr.encode('utf-8') - to = itinerary.to.encode('utf-8') start_date = self.config['feed_info']['start_date'] enda_date = self.config['feed_info']['end_date'] operations = [] - for operation in data.schedule["itinerario"][itinerary.ref]: - input_fr = operation["from"].encode('utf-8') - input_to = operation["to"].encode('utf-8') - if input_fr == fr and input_to == to: + for operation in data.schedule["itinerario"][itinerary.route_id]: + input_fr = operation["from"] + input_to = operation["to"] + if input_fr == itinerary.fr and input_to == itinerary.to: - if operation["operacion"].encode('utf-8') == "weekday": + if operation["operacion"] == "weekday": operations.append("weekday") - if operation["operacion"].encode('utf-8') == "saturday": + if operation["operacion"] == "saturday": operations.append("saturday") - if operation["operacion"].encode('utf-8') == "sunday": + if operation["operacion"] == "sunday": operations.append("sunday") return operations @@ -80,8 +79,8 @@ def _create_service_period(self, feed, operation): if service is not None: return service except KeyError: - print("INFO. No existe el service_period para la operación:", - operation, " por lo que será creado") + print("INFO. There is no service_period for this service:", + operation, " therefore it will be created.") if operation == "weekday": service = transitfeed.ServicePeriod("weekday") @@ -124,7 +123,7 @@ def add_trips_for_route(feed, gtfs_route, itinerary, service_period, for stop in itinerary.stops: if stop.name == estacion: - parada = feed.GetStop(str(stop.id)) + parada = feed.GetStop(str(stop.stop_id)) trip.AddStopTime(parada, stop_time=str(tiempo_parada)) continue @@ -142,14 +141,12 @@ def add_trips_for_route(feed, gtfs_route, itinerary, service_period, def load_stations(route, data, operation): stations = [] - for direction in data.schedule["itinerario"][route.ref]: - fr = direction["from"].encode('utf-8') - to = direction["to"].encode('utf-8') - data_operation = direction["operacion"].encode('utf-8') - if (fr == route.fr.encode('utf-8') and - to == route.to.encode('utf-8') and data_operation == operation): + for direction in data.schedule["itinerario"][route.route_id]: + data_operation = direction["operacion"] + if (direction["from"] == route.fr and + direction["to"] == route.to and data_operation == operation): for station in direction["estaciones"]: - stations = stations + [station.encode('utf-8')] + stations = stations + [station] # debug # print("(json) estaciones encontradas: " + str(len(stations))) @@ -165,16 +162,14 @@ def load_times(route, data, operation): times = None for direction in data.schedule["itinerario"][route.route_id]: - fr = direction["from"].encode('utf-8') - to = direction["to"].encode('utf-8') - data_operation = direction["operacion"].encode('utf-8') - if (fr == route.fr.encode('utf-8') and - to == route.to.encode('utf-8') and data_operation == operation): + data_operation = direction["operacion"] + if (direction["from"] == route.fr and + direction["to"] == route.to and data_operation == operation): times = direction["horarios"] if times is None: - print("debug: ruta va de", route.fr.encode('utf-8'), - "hacia", route.to.encode('utf-8')) + print("debug: ruta va de", route.fr, + "hacia", route.to) print("error consiguiendo los tiempos de la ruta") return times diff --git a/osm2gtfs/creators/managua/managua.json b/osm2gtfs/creators/managua/managua.json index 0afeb1b3..36b90910 100644 --- a/osm2gtfs/creators/managua/managua.json +++ b/osm2gtfs/creators/managua/managua.json @@ -13,7 +13,7 @@ } }, "stops": { - "name_without": "Parada sin nombre - Agregaselo en rutas.mapanica.net", + "name_without": "Parada sin nombre", "name_auto": "no" }, "agency": { @@ -29,8 +29,8 @@ "publisher_name": "MapaNica.net", "publisher_url": "https://mapanica.net", "version": "0.1", - "start_date": "20171101", - "end_date": "20181031" + "start_date": "20171201", + "end_date": "20180331" }, "schedule_source": "https://raw.githubusercontent.com/mapanica/pt-data-managua/master/timetable.json", "output_file": "data/ni-managua-gtfs.zip", diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index bc4cd9bc..ab18c1ae 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -17,21 +17,63 @@ def add_routes_to_feed(self, feed, data): This function adds the routes from the data to the GTFS feed. """ # Get route information - lines = data.get_routes() + routes = data.get_routes() - # Loop through all lines - for line_ref, line in sorted(lines.iteritems()): + # Loop through all routes + for route_ref, route in sorted(routes.iteritems()): # Add route information - route = feed.AddRoute( - route_id=line_ref, - route_type=line.route_type, - short_name=line.route_id.encode('utf-8'), - long_name=line.name + gtfs_route = feed.AddRoute( + route_id=self._define_route_id(route), + route_type=self._define_route_type(route), + short_name=route.route_id.encode('utf-8'), + long_name=route.name ) - route.agency_id = feed.GetDefaultAgency().agency_id - route.route_desc = line.route_desc - route.route_url = line.route_url - route.route_color = line.route_color - route.route_text_color = line.route_text_color + gtfs_route.agency_id = feed.GetDefaultAgency().agency_id + gtfs_route.route_desc = self._define_route_description(route) + gtfs_route.route_url = self._define_route_url(route) + gtfs_route.route_color = self._define_route_color(route) + gtfs_route.route_text_color = self._define_route_text_color(route) return + + def _define_route_id(self, route): + """ + Returns the route_id for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.route_id + + def _define_route_type(self, route): + """ + Returns the route_id for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.route_type + + def _define_route_description(self, route): + """ + Returns the route_desc for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.route_desc + + def _define_route_url(self, route): + """ + Returns the route_url for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.osm_url + + def _define_route_color(self, route): + """ + Returns the route_route_color for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.route_color[1:] + + def _define_route_text_color(self, route): + """ + Returns the route_text_color for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.route_text_color[1:] diff --git a/osm2gtfs/creators/stops_creator.py b/osm2gtfs/creators/stops_creator.py index dd511841..f6e1e280 100644 --- a/osm2gtfs/creators/stops_creator.py +++ b/osm2gtfs/creators/stops_creator.py @@ -37,40 +37,35 @@ def add_stops_to_feed(self, feed, data): # Loop through regular stops for stop in regular_stops.values(): - # Add stop to feed - self._add_stop_to_feed(stop, feed) - - def get_gtfs_stop_id(self, stop): - """ - This function returns the GTFS stop id to be used for a stop. - It can be overridden by custom cretors to change how stop_ids are made - up. - - :return gtfs_stop_id: A string with the stop_id for use in the GTFS - """ - if "gtfs_id" in stop.tags: - # Use a GTFS stop_id coming from OpenStreetMap data - return stop.tags['gtfs_id'] - else: - # Use a GTFS stop_id mathing to OpenStreetMap objects - return stop.osm_type + "/" + str(stop.osm_id) + # Add stop to feed + gtfs_stop_id = self._add_stop_to_feed(stop, feed) def _add_stop_to_feed(self, stop, feed): """ - This function adds a Stop or Station object as a stop to GTFS. - It can be overridden by custom cretors to change how stop_ids are made + This function adds a single Stop or Station object as a stop to GTFS. + It can be overridden by custom creators to change how stop_ids are made up. :return stop_id: A string with the stop_id in the GTFS """ try: parent_station = stop.get_parent_station() - except AttributeError as e: + except AttributeError: parent_station = "" - field_dict = {'stop_id': self.get_gtfs_stop_id(stop), - 'stop_name': stop.name, + # Send stop_id creation through overridable function + gtfs_stop_id = self._define_stop_id(stop) + + # Save defined stop_id to the object for furhter use in other creators + stop.set_stop_id(gtfs_stop_id) + + # Set stop name + stop_name = self._define_stop_name(stop) + + # Collect all data together for the stop creation + field_dict = {'stop_id': self._define_stop_id(stop), + 'stop_name': stop_name, 'stop_lat': float(stop.lat), 'stop_lon': float(stop.lon), 'location_type': stop.location_type, @@ -80,5 +75,39 @@ def _add_stop_to_feed(self, stop, feed): # Add stop to GTFS object feed.AddStopObject(transitfeed.Stop(field_dict=field_dict)) + if type(stop_name).__name__ == "unicode": + stop_name = stop_name.encode('utf-8') + + print("Added stop: " + stop_name + + " - " + stop.osm_url) + # Return the stop_id of the stop added return field_dict['stop_id'] + + def _define_stop_id(self, stop): + """ + This function returns the GTFS stop id to be used for a stop. + It can be overridden by custom creators to change how stop_ids are made + up. + + :return stop_id: A string with the stop_id for use in the GTFS + """ + + # Use a GTFS stop_id coming from OpenStreetMap data + if "ref:gtfs" in stop.tags: + stop_id = stop.tags['ref:gtfs'] + elif "ref" in stop.tags: + stop_id = stop.tags['ref'] + + # Use a GTFS stop_id matching to OpenStreetMap objects + else: + stop_id = stop.osm_type + "/" + str(stop.osm_id) + + return stop_id + + def _define_stop_name(self, stop): + """ + Returns the stop name for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return stop.name diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index f6a13169..31a05b5c 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -2,7 +2,9 @@ import re from datetime import datetime +import transitfeed from transitfeed import ServicePeriod +from osm2gtfs.core.helper import Helper class TripsCreator(object): @@ -18,80 +20,222 @@ def __repr__(self): def add_trips_to_feed(self, feed, data): """ - route_id # Required: From Line - service_id # Required: To be generated - trip_id # Required: To be generated + This function generates and adds trips to the GTFS feed. - trip_headsign # Itinerary "to" - direction_id # Order of tinieraries in Line object - wheelchair_accessible # Itinerary "wheelchair_accessible" - bikes_allowed # Itinerary "bikes_allowed" - trip_short_name # To be avoided! - block_id # To be avoided! + It is the place where geographic information and schedule is + getting joined to produce a routable GTFS. """ - # Get route information - lines = data.schedule + all_trips_count = 0 - # Loop though all lines - for line_id, line in lines.iteritems(): + # Go though all lines + for line_id, line in data.routes.iteritems(): - # Loop through all itineraries - # print('Getting itinerary information from line', line.route_id) + print("\nGenerating schedule for line: [" + str( + line_id) + "] - " + line.name) + + # Loop through it's itineraries itineraries = line.get_itineraries() for itinerary in itineraries: - # print('Loop for itinerary.route_id', itinerary.route_id) - if itinerary.route_id.encode('utf-8') != line_id.encode('utf-8'): - raise RuntimeError('Itinerary route ID (' + itinerary.route_id + ') does not match Line route ID (' + line_id + ')') + trips_count = 0 - if itinerary.route_id not in timetable.lines: - print('Route ID of itinerary not found in timetable, skipping it', itinerary.route_id) - continue - # Add itinerary shape to feed, using osm_id instead of route_id to differ itinerary shapes - # print('Adding itinerary shape to feed', itinerary.osm_id) - shape_id = TripsCreator.add_shape(feed, itinerary.osm_id, itinerary) - - # Get operations for itinerary - # print('Getting operations for itinerary') - services = self._get_itinerary_services(timetable, itinerary) - - # Loop through all services - for service in services: - # print('Loop for service', service) - # print('Create service period') - service_period = self._create_service_period(feed, service) - # print('Load timetable') - gtfs_timetable = self._load_timetable(timetable, - itinerary, service) - # print('Load stops') - stops = self._load_stops(timetable, itinerary, service) - # print('Get route from line id', line_id) - route = feed.GetRoute(line_id) - - # print('Add trips for route') - self._add_trips_for_route(feed, route, itinerary, - service_period, shape_id, stops, - gtfs_timetable) + # Verify data before proceeding + if self._verify_data(data.schedule, line, itinerary): + + # Prepare itinerary's trips and schedule + prepared_trips = self._prepare_trips(feed, data.schedule, + itinerary) + + # Add itinerary shape to feed. + shape_id = self._add_shape_to_feed( + feed, "relation/" + str(itinerary.osm_id), itinerary) + + # Add trips of each itinerary to the GTFS feed + for trip_builder in prepared_trips: + + trip_builder['all_stops'] = data.get_stops() + trips_count += self._add_itinerary_trips( + feed, itinerary, line, trip_builder, shape_id) + + # Print out status messge about added trips + print(" Itinerary: [" + str(itinerary.route_id) + "] " + + itinerary.to.encode("utf-8") + " (added " + str( + trips_count) + " trips, serving " + str( + len(itinerary.get_stops())) + " stops) - " + + itinerary.osm_url) + all_trips_count += trips_count + + print("\nTotal of added trips to this GTFS: " + + str(all_trips_count) + "\n\n") return - def _get_itinerary_services(self, timetable, itinerary): + def _prepare_trips(self, feed, schedule, itinerary): """ - Returns a list with services of given itinerary. + Prepare information necessary to generate trips + + :return trips: List of different objects """ - fr = itinerary.fr.encode('utf-8') - to = itinerary.to.encode('utf-8') + # Define a list with service days of given itinerary. services = [] - - for trip in timetable.lines[itinerary.route_id]: - input_fr = trip["from"].encode('utf-8') - input_to = trip["to"].encode('utf-8') - if input_fr == fr and input_to == to: + for trip in schedule['lines'][itinerary.route_id]: + input_fr = trip["from"] + input_to = trip["to"] + if input_fr == itinerary.fr and input_to == itinerary.to: trip_services = trip["services"] for service in trip_services: - services.append(service.encode('utf-8')) - return services + services.append(service) + + if not services: + print(" Warning: From and to values didn't match with schedule.") + + # Loop through all service days + trips = [] + for service in services: + + # Define GTFS feed service period + service_period = self._create_gtfs_service_period(feed, service) + + # Get schedule for this itinierary's trips + trips_schedule = self._load_itinerary_schedule(schedule, + itinerary, service) + + # Get the stops, which are listed in the schedule + scheduled_stops = self._load_scheduled_stops( + schedule, itinerary, service) + + # Prepare a trips builder container with useful data for later + trips.append({'service_period': service_period, + 'stops': scheduled_stops, 'schedule': trips_schedule}) + return trips + + def _verify_data(self, schedule, line, itinerary): + """ + Verifies line, itinerary and it's schedule data for trip creation + """ + + # Check if itinerary and line are having the same reference + if itinerary.route_id != line.route_id: + print("Warning: The route id of the itinerary (" + + str(itinerary.route_id) + ") doesn't match route id of line (" + + str(line.route_id) + ")") + print(" " + itinerary.osm_url) + print(" " + line.osm_url) + return True + + # Check if time information in schedule can be found for + # the itinerary + if itinerary.route_id not in schedule['lines']: + print(" Warning: Route not found in schedule.") + return False + + return True + + def _add_shape_to_feed(self, feed, shape_id, itinerary): + """ + Create GTFS shape and return shape_id to add on GTFS trip + """ + shape_id = str(shape_id) + + # Only add a shape if there isn't one with this shape_id + try: + feed.GetShape(shape_id) + except KeyError: + shape = transitfeed.Shape(shape_id) + for point in itinerary.shape: + shape.AddPoint( + lat=float(point["lat"]), lon=float(point["lon"])) + feed.AddShapeObject(shape) + return shape_id + + def _add_itinerary_trips(self, feed, itinerary, line, trip_builder, + shape_id): + """ + Add all trips of an itinerary to the GTFS feed. + """ + # Obtain GTFS route to add trips to it. + route = feed.GetRoute(line.route_id) + trips_count = 0 + + # Loop through each timeslot for a trip + for trip in trip_builder['schedule']: + gtfs_trip = route.AddTrip(feed, headsign=itinerary.to, + service_period=trip_builder['service_period']) + trips_count += 1 + + # Go through all stops of an itinerary + for itinerary_stop_id in itinerary.get_stops(): + + # Load full stop object + try: + itinerary_stop = trip_builder[ + 'all_stops']['regular'][itinerary_stop_id] + except ValueError: + print("Itinerary (" + itinerary.route_url + + ") misses a stop:") + print(" Please review:" + itinerary_stop.osm_url) + continue + + try: + # Load respective GTFS stop object + gtfs_stop = feed.GetStop(str(itinerary_stop.stop_id)) + except ValueError: + print("Warning: Stop in itinerary was not found in GTFS.") + print(" " + itinerary_stop.osm_url) + + # Make sure we compare same unicode encoding + if type(itinerary_stop.name) is str: + itinerary_stop.name = itinerary_stop.name.decode('utf-8') + + time = "-" + # Check if we have specific time information for this stop. + try: + time = trip[trip_builder['stops'].index(itinerary_stop.name)] + except ValueError: + pass + + # Validate time information + if time != "-": + try: + time_at_stop = str( + datetime.strptime(time, "%H:%M").time()) + except ValueError: + print("Warning: Time for a stop was not valid.") + print(" " + itinerary_stop.name + + " - " + itinerary_stop.osm_id) + break + gtfs_trip.AddStopTime(gtfs_stop, stop_time=time_at_stop) - def _create_service_period(self, feed, service): + # Add stop without time information, too (we interpolate later) + else: + try: + gtfs_trip.AddStopTime(gtfs_stop) + except ValueError: + print("Warning: Could not add first a stop to trip.") + print(" " + itinerary_stop.name + + " - " + itinerary_stop.osm_id) + break + + # Add reference to shape + gtfs_trip.shape_id = shape_id + + # Add empty attributes to make navitia happy + gtfs_trip.block_id = "" + gtfs_trip.wheelchair_accessible = "" + gtfs_trip.bikes_allowed = "" + gtfs_trip.direction_id = "" + + # Calculate all times of stops, which were added with no time + Helper.interpolate_stop_times(gtfs_trip) + return trips_count + + def _create_gtfs_service_period(self, feed, service): + """ + Generate a transitfeed ServicePeriod object + from a time string according to the standard schedule: + https://github.com/grote/osm2gtfs/wiki/Schedule + + :return: ServicePeriod object + """ try: gtfs_service = feed.GetServicePeriod(service) if gtfs_service is not None: @@ -138,104 +282,38 @@ def _create_service_period(self, feed, service): feed.AddServicePeriodObject(gtfs_service) return feed.GetServicePeriod(service) - def _load_timetable(self, timetable, itinerary, service): + def _load_itinerary_schedule(self, schedule, itinerary, service): + """ + Load the part of the provided schedule that fits to a particular + itinerary. + + :return times: List of strings + """ times = None - for trip in timetable.lines[itinerary.route_id]: - fr = trip["from"].encode('utf-8') - to = trip["to"].encode('utf-8') + for trip in schedule['lines'][itinerary.route_id]: trip_services = trip["services"] - if (fr == itinerary.fr.encode('utf-8') and - to == itinerary.to.encode('utf-8') and service in trip_services): + if (trip[ + "from"] == itinerary.fr and trip[ + "to"] == itinerary.to and service in trip_services): times = trip["times"] if times is None: - print("Problems found with Itinerary from " + - itinerary.fr.encode('utf-8') + " to " + - itinerary.to.encode('utf-8') - ) - print("Couldn't load times from timetable.") + print("Warning: Couldn't load times from schedule for route") return times - def _load_stops(self, timetable, itinerary, service): + def _load_scheduled_stops(self, schedule, itinerary, service): + """ + Load the name of stops that have time information in the provided + schedule. + + :return stops: List of strings + """ stops = [] - for trip in timetable.lines[itinerary.route_id]: - fr = trip["from"].encode('utf-8') - to = trip["to"].encode('utf-8') + for trip in schedule['lines'][itinerary.route_id]: trip_services = trip["services"] - if (fr == itinerary.fr.encode('utf-8') and - to == itinerary.to.encode('utf-8') and service in trip_services): + if (trip[ + "from"] == itinerary.fr and trip[ + "to"] == itinerary.to and service in trip_services): for stop in trip["stations"]: - stops.append(unicode(stop)) + stops.append(stop) return stops - - def _add_trips_for_route(self, feed, gtfs_route, itinerary, service_period, - shape_id, stops, gtfs_timetable): - for trip in gtfs_timetable: - gtfs_trip = gtfs_route.AddTrip(feed, headsign=itinerary.name, - service_period=service_period) - # print('Count of stops', len(stops)) - # print('Count of itinerary.get_stops()', len(itinerary.get_stops())) - # print('Stops', stops) - for itinerary_stop in itinerary.get_stops(): - if itinerary_stop is None: - print('Itinerary stop is None. Seems to be a problem with OSM data. We should really fix that.') - print('itinerary route ID', itinerary.route_id) - print('itinerary stop', itinerary_stop) - continue - gtfs_stop = feed.GetStop(str(itinerary_stop.osm_id)) - time = "-" - try: - time = trip[stops.index(itinerary_stop.name)] - except ValueError: - pass - if time != "-": - try: - time_at_stop = str(datetime.strptime(time, "%H:%M").time()) - except ValueError: - print('Time seems invalid, skipping time', time) - break - gtfs_trip.AddStopTime(gtfs_stop, stop_time=time_at_stop) - else: - try: - gtfs_trip.AddStopTime(gtfs_stop) - except Exception: - print('Skipping trip because no time were found', itinerary.route_id, stops, itinerary_stop.name) - break - # add empty attributes to make navitia happy - gtfs_trip.block_id = "" - gtfs_trip.wheelchair_accessible = "" - gtfs_trip.bikes_allowed = "" - gtfs_trip.shape_id = shape_id - gtfs_trip.direction_id = "" - TripsCreator.interpolate_stop_times(gtfs_trip) - - @staticmethod - def interpolate_stop_times(trip): - """ - Interpolate stop_times, because Navitia does not handle this itself - """ - try: - for secs, stop_time, is_timepoint in trip.GetTimeInterpolatedStops(): - if not is_timepoint: - stop_time.arrival_secs = secs - stop_time.departure_secs = secs - trip.ReplaceStopTimeObject(stop_time) - except ValueError as e: - print(e) - - @staticmethod - def add_shape(feed, route_id, itinerary): - """ - create GTFS shape and return shape_id to add on GTFS trip - """ - import transitfeed - shape_id = str(route_id) - try: - feed.GetShape(shape_id) - except KeyError: - shape = transitfeed.Shape(shape_id) - for point in itinerary.shape: - shape.AddPoint( - lat=float(point["lat"]), lon=float(point["lon"])) - feed.AddShapeObject(shape) - return shape_id diff --git a/osm2gtfs/osm2gtfs.py b/osm2gtfs/osm2gtfs.py index bb4cfa2e..e1ea6fb3 100644 --- a/osm2gtfs/osm2gtfs.py +++ b/osm2gtfs/osm2gtfs.py @@ -65,8 +65,8 @@ def main(): # Add data to feed agency_creator.add_agency_to_feed(feed) feed_info_creator.add_feed_info_to_feed(feed) - routes_creator.add_routes_to_feed(feed, data) stops_creator.add_stops_to_feed(feed, data) + routes_creator.add_routes_to_feed(feed, data) schedule_creator.add_schedule_to_data(data) trips_creator.add_trips_to_feed(feed, data) diff --git a/osm2gtfs/tests/tests_accra.py b/osm2gtfs/tests/tests_accra.py index 7ccae005..f161a184 100644 --- a/osm2gtfs/tests/tests_accra.py +++ b/osm2gtfs/tests/tests_accra.py @@ -113,7 +113,8 @@ def test_refresh_stops_cache(self): self.assertTrue(os.path.isfile(cache_file), 'The stops cache file creation failed') cache = Cache() stops = cache.read_data('accra-stops') - self.assertEqual(len(stops), 2529, 'Wrong count of stops in the cache file') + amount_of_stops = len(stops['regular']) + len(stops['stations']) + self.assertEqual(amount_of_stops, 2529, 'Wrong count of stops in the cache file') def test_gtfs_from_cache(self): # the cache is generated by the previous two functions diff --git a/setup.py b/setup.py index 8c56ded2..a74bac73 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ keywords='openstreetmap gtfs schedule public-transportation python', author='Various collaborators: https://github.com/grote/osm2gtfs/graphs/contributors', - install_requires=['attrs', 'overpy>=0.4', 'transitfeed'], + install_requires=['attrs', 'overpy>=0.4', 'transitfeed', 'mock', 'webcolors'], packages=find_packages(), include_package_data=True, entry_points={ From fbb8dd5bbd273452d9536e321c8c995c93297c0f Mon Sep 17 00:00:00 2001 From: Ialokim Date: Fri, 15 Dec 2017 10:20:24 -0600 Subject: [PATCH 13/18] fixed two bugs --- osm2gtfs/core/cache.py | 2 +- osm2gtfs/core/configuration.py | 5 +++-- setup.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/osm2gtfs/core/cache.py b/osm2gtfs/core/cache.py index a424f498..d885731c 100644 --- a/osm2gtfs/core/cache.py +++ b/osm2gtfs/core/cache.py @@ -50,7 +50,7 @@ def write_file(name, content): if not os.path.isdir('data'): os.mkdir('data') with open(os.path.join('data', name), 'wb') as f: - f.write(content.read()) + f.write(content) @staticmethod def read_file(name): diff --git a/osm2gtfs/core/configuration.py b/osm2gtfs/core/configuration.py index fe4c793a..52332b85 100644 --- a/osm2gtfs/core/configuration.py +++ b/osm2gtfs/core/configuration.py @@ -83,9 +83,10 @@ def get_schedule_source(self, refresh=False): sys.exit(0) schedule_source = schedule_source_file - # Cache data - Cache.write_file(cached_file, schedule_source) self._schedule_source = schedule_source.read() + + # Cache data + Cache.write_file(cached_file, self._schedule_source) return self._schedule_source def _load_config(self, args): diff --git a/setup.py b/setup.py index a74bac73..a7d18794 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ keywords='openstreetmap gtfs schedule public-transportation python', author='Various collaborators: https://github.com/grote/osm2gtfs/graphs/contributors', - install_requires=['attrs', 'overpy>=0.4', 'transitfeed', 'mock', 'webcolors'], + install_requires=['attrs>=17.1.0', 'overpy>=0.4', 'transitfeed', 'mock', 'webcolors'], packages=find_packages(), include_package_data=True, entry_points={ From a306be087e7f924dd361ef1eb1f2063a76078b30 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Sat, 16 Dec 2017 18:58:28 +0100 Subject: [PATCH 14/18] Introduce inheritance in data structure elements --- osm2gtfs/core/elements.py | 186 ++++++++++++++++++ osm2gtfs/core/osm_connector.py | 87 ++++++-- osm2gtfs/core/routes.py | 133 ------------- osm2gtfs/core/stops.py | 88 --------- .../creators/accra/trips_creator_accra.py | 2 +- .../creators/fenix/routes_creator_fenix.py | 3 +- .../creators/fenix/trips_creator_fenix.py | 2 +- .../incofer/routes_creator_incofer.py | 3 +- .../creators/incofer/stops_creator_incofer.py | 2 +- osm2gtfs/creators/routes_creator.py | 4 +- osm2gtfs/creators/stops_creator.py | 2 +- osm2gtfs/creators/trips_creator.py | 17 +- 12 files changed, 272 insertions(+), 257 deletions(-) create mode 100644 osm2gtfs/core/elements.py delete mode 100644 osm2gtfs/core/routes.py delete mode 100644 osm2gtfs/core/stops.py diff --git a/osm2gtfs/core/elements.py b/osm2gtfs/core/elements.py new file mode 100644 index 00000000..2b6e561c --- /dev/null +++ b/osm2gtfs/core/elements.py @@ -0,0 +1,186 @@ +# coding=utf-8 + +import sys +import attr + + +@attr.s +class Element(object): + """The basic data element. + Contains the common attributes all other data classes share. + + """ + osm_id = attr.ib() + osm_type = attr.ib() + osm_url = attr.ib() + + tags = attr.ib() + name = attr.ib() + + +@attr.s +class Line(Element): + """A general public transport service Line. + + It's a container of meta information and different Itinerary objects for + variants of the same service line. + + In OpenStreetMap this is usually represented as "route_master" relation. + In GTFS this is usually represented as "route". + + """ + route_id = attr.ib() + + route_type = attr.ib(default=None) + route_desc = attr.ib(default=None) + route_color = attr.ib(default="#FFFFFF") + route_text_color = attr.ib(default="#000000") + + # Related route variants + _itineraries = attr.ib(default=attr.Factory(list)) + + def __attrs_post_init__(self): + ''' + Populates the object with information obtained from the tags + ''' + # pylint: disable=unsupported-membership-test,unsubscriptable-object + if 'colour' in self.tags: + self.route_color = self.tags['colour'] + + # pylint: disable=unsupported-membership-test,unsubscriptable-object + if 'route_master' in self.tags: + self.route_type = self.tags['route_master'].capitalize() + else: + sys.stderr.write( + "Warning: Route master relation without a route_master tag:\n") + sys.stderr.write(" " + self.osm_url) + + # Try to guess the type differently + if 'route' in self.tags: + self.route_type = self.tags['route'].capitalize() + else: + self.route_type = "Bus" + + def add_itinerary(self, itinerary): + + if self.route_id != itinerary.route_id: + raise ValueError('Itinerary route ID (' + + itinerary.route_id + + ') does not match Line route ID (' + + self.route_id + ')') + # pylint: disable=no-member + self._itineraries.append(itinerary) + + def get_itineraries(self): + return self._itineraries + + +@attr.s +class Itinerary(Element): + """A public transport service itinerary. + + It's a representation of a possible variant of a line, grouped together by + a Line object. + + In OpenStreetMap this is usually represented as "route" relation. + In GTFS this is not exlicitly presented but used as base to create "trips" + + """ + route_id = attr.ib() + shape = attr.ib() + + line = attr.ib(default=None) + fr = attr.ib(default=None) + to = attr.ib(default=None) + duration = attr.ib(default=None) + + # All stop objects of itinerary + stops = attr.ib(default=attr.Factory(list)) + + def __attrs_post_init__(self): + ''' + Populates the object with information obtained from the tags + ''' + # pylint: disable=unsupported-membership-test,unsubscriptable-object + if 'from' in self.tags: + self.fr = self.tags['from'] + + # pylint: disable=unsupported-membership-test,unsubscriptable-object + if 'to' in self.tags: + self.to = self.tags['to'] + + def get_stops(self): + return self.stops + + +@attr.s +class Station(Element): + """A public transport stop of the type station. + + It's a representation of a possible group of stops. + + In OpenStreetMap this is usually represented as "stop_area" relation. + In GTFS it is handled as a stop with a location_type=1. Regular Stops with + location_type=0 might specify a station as parent_station. + + """ + lat = attr.ib() + lon = attr.ib() + + stop_id = attr.ib(default="") + location_type = attr.ib(default=1) + + # Stops forming part of this Station + _members = attr.ib(default=attr.Factory(list)) + + def set_members(self, members): + self._members = members + + def get_members(self): + return self._members + + def get_stop_id(self): + return self.stop_id + + def set_stop_id(self, stop_id): + self.stop_id = stop_id + + +@attr.s +class Stop(Element): + """A public transport stop. + + In OpenStreetMap this is usually represented as an object of the role + "plattform" in the route. + + """ + lat = attr.ib() + lon = attr.ib() + + stop_id = attr.ib("") + location_type = attr.ib(default=0) + + # The id of the Station this Stop might be part of. + _parent_station = attr.ib(default=None) + + def set_parent_station(self, identifier, override=False): + """ + Set the parent_station_id on the first time; + Second attempts throw a warning + """ + if self._parent_station is None or override is True: + self._parent_station = identifier + else: + sys.stderr.write("Warning: Stop is part of two stop areas:\n") + sys.stderr.write( + "https://osm.org/" + self.osm_type + "/" + str( + self.osm_id) + "\n") + + def get_parent_station(self): + return self._parent_station + + def get_stop_id(self): + return self.stop_id + + def set_stop_id(self, stop_id): + self.stop_id = stop_id diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index 2f15d0e5..fbc49ee6 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -6,8 +6,7 @@ from transitfeed import util from osm2gtfs.core.cache import Cache from osm2gtfs.core.helper import Helper -from osm2gtfs.core.routes import Itinerary, Line -from osm2gtfs.core.stops import Stop, Station +from osm2gtfs.core.elements import Line, Itinerary, Station, Stop class OsmConnector(object): @@ -285,6 +284,7 @@ def _build_line(self, route_master, itineraries): Returns a initiated Line object from raw data """ + osm_type = "relation" if 'ref' in route_master.tags: ref = route_master.tags['ref'] else: @@ -308,9 +308,20 @@ def _build_line(self, route_master, itineraries): "No 'ref' could be obtained. Skipping whole route.\n") return + # Move to Elements class, once attributes with defaults play well + # with inheritance https://github.com/python-attrs/attrs/issues/38 + osm_url = "https://osm.org/" + str( + osm_type) + "/" + str(route_master.id) + if 'name' in route_master.tags: + name = route_master.tags['name'] + elif 'ref' in route_master.tags: + name = route_master.tags['ref'] + else: + name = None + # Create Line (route master) object - line = Line(osm_id=route_master.id, route_id=ref, - tags=route_master.tags) + line = Line(osm_id=route_master.id, osm_type=osm_type, osm_url=osm_url, + tags=route_master.tags, name=name, route_id=ref) # Add Itinerary objects (route variants) to Line (route master) for itinerary in list(itineraries.values()): @@ -331,6 +342,7 @@ def _build_itinerary(self, route_variant, query_result_set, route_master): Returns a initiated Itinerary object from raw data """ + osm_type = "relation" if 'ref' in route_variant.tags: ref = route_variant.tags['ref'] else: @@ -361,14 +373,27 @@ def _build_itinerary(self, route_variant, query_result_set, route_master): stops.append(otype + "/" + str(stop_candidate.ref)) if route_master: - route_master_id = "relation/" + str(route_master.id) + parent_identifier = osm_type + "/" + str(route_master.id) + else: + parent_identifier = None + + # Move to Elements class, once attributes with defaults play well with + # inheritance https://github.com/python-attrs/attrs/issues/38 + osm_url = "https://osm.org/" + str( + osm_type) + "/" + str(route_variant.id) + if 'name' in route_variant.tags: + name = route_variant.tags['name'] + elif 'ref' in route_variant.tags: + name = route_variant.tags['ref'] else: - route_master_id = None + name = None shape = self._generate_shape(route_variant, query_result_set) - rv = Itinerary(osm_id=route_variant.id, route_id=ref, stops=stops, - shape=shape, tags=route_variant.tags, - line=route_master_id) + + rv = Itinerary(osm_id=route_variant.id, osm_type=osm_type, + osm_url=osm_url, name=name, tags=route_variant.tags, + route_id=ref, shape=shape, line=parent_identifier, + stops=stops) return rv def _build_stop(self, stop, osm_type): @@ -389,9 +414,15 @@ def _build_stop(self, stop, osm_type): (stop.lat, stop.lon) = Helper.get_center_of_nodes( stop.get_nodes()) + # Move to Elements class, once attributes with defaults play well + # with inheritance https://github.com/python-attrs/attrs/issues/38 + osm_url = "https://osm.org/" + str( + osm_type) + "/" + str(stop.id) + # Create and return Stop object - stop = Stop(osm_id=stop.id, osm_type=osm_type, tags=stop.tags, - lat=stop.lat, lon=stop.lon, name=stop.tags['name']) + stop = Stop(osm_id=stop.id, osm_type=osm_type, osm_url=osm_url, + tags=stop.tags, name=stop.tags['name'], lat=stop.lat, + lon=stop.lon) return stop else: @@ -415,6 +446,12 @@ def _build_station(self, stop_area, osm_type): """ + # Check tagging whether this is a stop area. + if 'public_transport' not in stop_area.tags: + return False + elif not stop_area.tags['public_transport'] == 'stop_area': + return False + # Check whether a valid stop_area candidade if 'public_transport' in stop_area.tags and stop_area.tags[ 'public_transport'] == 'stop_area': @@ -422,15 +459,21 @@ def _build_station(self, stop_area, osm_type): # Analzyse member objects (stops) of this stop area members = {} for member in stop_area.members: - if (isinstance(member, overpy.RelationNode) and - member.role == "platform"): - if "node/" + str(member.ref) in self.stops['regular']: + if member.role == "platform": + + if isinstance(member, overpy.RelationNode): + member_osm_type = "node" + elif isinstance(member, overpy.RelationWay): + member_osm_type = "way" + + identifier = member_osm_type + "/" + str(member.ref) + + if identifier in self.stops['regular']: # Collect the Stop objects that are members # of this Station - members["node/" + str(member.ref)] = self.stops[ - 'regular']["node/" + str(member.ref)] + members[identifier] = self.stops['regular'][identifier] else: sys.stderr.write( "Error: Station member was not found in data") @@ -469,10 +512,16 @@ def _build_station(self, stop_area, osm_type): stop_area.lat, stop_area.lon = Helper.get_center_of_nodes( members.values()) + # Move to Elements class, once attributes with defaults play well + # with inheritance https://github.com/python-attrs/attrs/issues/38 + osm_url = "https://osm.org/" + str( + osm_type) + "/" + str(stop_area.id) + # Create and return Station object - station = Station(osm_id=stop_area.id, tags=stop_area.tags, - lat=stop_area.lat, lon=stop_area.lon, - name=stop_area.name) + station = Station(osm_id=stop_area.id, osm_type=osm_type, + osm_url=osm_url, tags=stop_area.tags, + name=stop_area.name, lat=stop_area.lat, + lon=stop_area.lon) station.set_members(members) return station diff --git a/osm2gtfs/core/routes.py b/osm2gtfs/core/routes.py deleted file mode 100644 index a19c0bda..00000000 --- a/osm2gtfs/core/routes.py +++ /dev/null @@ -1,133 +0,0 @@ -# coding=utf-8 - -import attr - - -@attr.s -class Line(object): - """A general public transport service Line. - - It's a container of meta information and different Itinerary objects for - variants of the same service line. - - In OpenStreetMap this is usually represented as "route_master" relation. - In GTFS this is usually represented as "route". - - """ - osm_id = attr.ib() - route_id = attr.ib() - tags = attr.ib() - - name = attr.ib(default=None) - route_type = attr.ib(default=None) - route_desc = attr.ib(default=None) - route_color = attr.ib(default="#FFFFFF") - route_text_color = attr.ib(default="#000000") - osm_url = attr.ib(default=None) - - # Related route variants - _itineraries = attr.ib(default=attr.Factory(list)) - - def __attrs_post_init__(self): - '''Populates the object with information obtained from the tags - - ''' - from osm2gtfs.core.helper import Helper - - # Disabling some pylint errors as pylint currently doesn't support any - # custom decorators or descriptors - # https://github.com/PyCQA/pylint/issues/1694 - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - self.name = self.tags['name'] - self.osm_url = "https://osm.org/relation/" + str(self.osm_id) - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - if 'colour' in self.tags: - self.route_color = self.tags['colour'] - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - if 'ref:colour_tx' in self.tags: - self.route_text_color = self.tags['ref:colour_tx'] - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - if 'route_master' in self.tags: - self.route_type = self.tags['route_master'].capitalize() - elif 'route' in self.tags: - self.route_type = self.tags['route'].capitalize() - else: - self.route_type = "Bus" - - def add_itinerary(self, itinerary): - - if self.route_id != itinerary.route_id: - raise ValueError('Itinerary route ID (' + - itinerary.route_id + - ') does not match Line route ID (' + - self.route_id + ')') - # pylint: disable=no-member - self._itineraries.append(itinerary) - - def get_itineraries(self): - return self._itineraries - - -@attr.s -class Itinerary(object): - """A public transport service itinerary. - - It's a representation of a possible variant of a line, grouped together by - a Line object. - - In OpenStreetMap this is usually represented as "route" relation. - In GTFS this is not exlicitly presented but used as based to create "trips" - - """ - osm_id = attr.ib() - route_id = attr.ib() - shape = attr.ib() - tags = attr.ib() - - name = attr.ib(default=None) - line = attr.ib(default=None) - osm_url = attr.ib(default=None) - fr = attr.ib(default=None) - to = attr.ib(default=None) - duration = attr.ib(default=None) - - # All stop objects of itinerary - stops = attr.ib(default=attr.Factory(list)) - - def __attrs_post_init__(self): - ''' - Populates the object with information obtained from the tags - ''' - - self.osm_url = "https://osm.org/relation/" + str(self.osm_id) - - # Disabling some pylint errors as pylint currently doesn't support any - # custom decorators or descriptors - # https://github.com/PyCQA/pylint/issues/1694 - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - if 'from' in self.tags: - self.fr = self.tags['from'] - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - if 'to' in self.tags: - self.to = self.tags['to'] - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - if 'name' in self.tags: - self.name = self.tags['name'] - - # pylint: disable=unsupported-membership-test,unsubscriptable-object - if 'duration' in self.tags: - self.duration = self.tags['duration'] - - def add_stop(self, stop): - # pylint: disable=no-member - self.stops.append(stop) - - def get_stops(self): - return self.stops diff --git a/osm2gtfs/core/stops.py b/osm2gtfs/core/stops.py deleted file mode 100644 index 10f38261..00000000 --- a/osm2gtfs/core/stops.py +++ /dev/null @@ -1,88 +0,0 @@ -# coding=utf-8 - -import sys -import attr - - -@attr.s -class Station(object): - - osm_id = attr.ib() - tags = attr.ib() - lat = attr.ib() - lon = attr.ib() - name = attr.ib() - - stop_id = attr.ib(default="") - osm_type = attr.ib(default="relation") - location_type = attr.ib(default=1) - osm_url = attr.ib(default=None) - - # Stops forming part of this Station - _members = attr.ib(default=attr.Factory(list)) - - def __attrs_post_init__(self): - '''Populates the object with information obtained from the tags - - ''' - self.osm_url = "https://osm.org/" + str( - self.osm_type) + "/" + str(self.osm_id) - - def set_members(self, members): - self._members = members - - def get_members(self): - return self._members - - def get_stop_id(self): - return self.stop_id - - def set_stop_id(self, stop_id): - self.stop_id = stop_id - - -@attr.s -class Stop(object): - - osm_id = attr.ib() - osm_type = attr.ib() - tags = attr.ib() - lat = attr.ib() - lon = attr.ib() - name = attr.ib() - - stop_id = attr.ib("") - location_type = attr.ib(default=0) - osm_url = attr.ib(default=None) - - # The id of the Station this Stop might be part of. - _parent_station = attr.ib(default=None) - - def __attrs_post_init__(self): - '''Populates the object with information obtained from the tags - - ''' - self.osm_url = "https://osm.org/" + str( - self.osm_type) + "/" + str(self.osm_id) - - def set_parent_station(self, identifier, override=False): - """ - Set the parent_station_id on the first time; - Second attempts throw a warning - """ - if self._parent_station is None or override is True: - self._parent_station = identifier - else: - sys.stderr.write("Warning: Stop is part of two stop areas:\n") - sys.stderr.write( - "https://osm.org/" + self.osm_type + "/" + str( - self.osm_id) + "\n") - - def get_parent_station(self): - return self._parent_station - - def get_stop_id(self): - return self.stop_id - - def set_stop_id(self, stop_id): - self.stop_id = stop_id diff --git a/osm2gtfs/creators/accra/trips_creator_accra.py b/osm2gtfs/creators/accra/trips_creator_accra.py index 365fb13d..1aa8605f 100644 --- a/osm2gtfs/creators/accra/trips_creator_accra.py +++ b/osm2gtfs/creators/accra/trips_creator_accra.py @@ -4,7 +4,7 @@ from osm2gtfs.creators.trips_creator import TripsCreator from osm2gtfs.core.helper import Helper -from osm2gtfs.core.routes import Line +from osm2gtfs.core.elements import Line class TripsCreatorAccra(TripsCreator): diff --git a/osm2gtfs/creators/fenix/routes_creator_fenix.py b/osm2gtfs/creators/fenix/routes_creator_fenix.py index 277c61f6..0e1566ef 100644 --- a/osm2gtfs/creators/fenix/routes_creator_fenix.py +++ b/osm2gtfs/creators/fenix/routes_creator_fenix.py @@ -1,8 +1,7 @@ # coding=utf-8 import sys -from osm2gtfs.core.routes import Itinerary, Line -from osm2gtfs.core.stops import Stop, Station +from osm2gtfs.core.elements import Line, Itinerary, Station, Stop from osm2gtfs.creators.routes_creator import RoutesCreator diff --git a/osm2gtfs/creators/fenix/trips_creator_fenix.py b/osm2gtfs/creators/fenix/trips_creator_fenix.py index 7a19926b..817fe2d8 100644 --- a/osm2gtfs/creators/fenix/trips_creator_fenix.py +++ b/osm2gtfs/creators/fenix/trips_creator_fenix.py @@ -6,7 +6,7 @@ import transitfeed from osm2gtfs.creators.trips_creator import TripsCreator from osm2gtfs.core.helper import Helper -from osm2gtfs.core.routes import Line, Itinerary +from osm2gtfs.core.elements import Line, Itinerary DEBUG_ROUTE = "" BLACKLIST = [ diff --git a/osm2gtfs/creators/incofer/routes_creator_incofer.py b/osm2gtfs/creators/incofer/routes_creator_incofer.py index cf92189b..69bff29b 100644 --- a/osm2gtfs/creators/incofer/routes_creator_incofer.py +++ b/osm2gtfs/creators/incofer/routes_creator_incofer.py @@ -1,8 +1,7 @@ # coding=utf-8 import sys -from osm2gtfs.core.routes import Itinerary, Line -from osm2gtfs.core.stops import Stop, Station +from osm2gtfs.core.elements import Line, Itinerary, Station, Stop from osm2gtfs.creators.routes_creator import RoutesCreator diff --git a/osm2gtfs/creators/incofer/stops_creator_incofer.py b/osm2gtfs/creators/incofer/stops_creator_incofer.py index d0127e52..19a68ff0 100644 --- a/osm2gtfs/creators/incofer/stops_creator_incofer.py +++ b/osm2gtfs/creators/incofer/stops_creator_incofer.py @@ -10,4 +10,4 @@ def _define_stop_id(self, stop): if stop.osm_type == "relation": return "SA" + str(stop.osm_id) else: - return stop.osm_id + return str(stop.osm_id) diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index ab18c1ae..f5ec7e6e 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -66,7 +66,7 @@ def _define_route_url(self, route): def _define_route_color(self, route): """ - Returns the route_route_color for the use in the GTFS feed. + Returns the route_color for the use in the GTFS feed. Can be easily overridden in any creator. """ return route.route_color[1:] @@ -76,4 +76,6 @@ def _define_route_text_color(self, route): Returns the route_text_color for the use in the GTFS feed. Can be easily overridden in any creator. """ + if 'ref:colour_tx' in route.tags: + route.route_text_color = route.tags['ref:colour_tx'] return route.route_text_color[1:] diff --git a/osm2gtfs/creators/stops_creator.py b/osm2gtfs/creators/stops_creator.py index f6e1e280..51afce73 100644 --- a/osm2gtfs/creators/stops_creator.py +++ b/osm2gtfs/creators/stops_creator.py @@ -57,7 +57,7 @@ def _add_stop_to_feed(self, stop, feed): # Send stop_id creation through overridable function gtfs_stop_id = self._define_stop_id(stop) - # Save defined stop_id to the object for furhter use in other creators + # Save defined stop_id to the object for further use in other creators stop.set_stop_id(gtfs_stop_id) # Set stop name diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index 31a05b5c..f34c9f40 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -47,7 +47,8 @@ def add_trips_to_feed(self, feed, data): # Add itinerary shape to feed. shape_id = self._add_shape_to_feed( - feed, "relation/" + str(itinerary.osm_id), itinerary) + feed, itinerary.osm_type + "/" + str( + itinerary.osm_id), itinerary) # Add trips of each itinerary to the GTFS feed for trip_builder in prepared_trips: @@ -172,7 +173,7 @@ def _add_itinerary_trips(self, feed, itinerary, line, trip_builder, except ValueError: print("Itinerary (" + itinerary.route_url + ") misses a stop:") - print(" Please review:" + itinerary_stop.osm_url) + print(" Please review stop:" + itinerary_stop_id) continue try: @@ -199,9 +200,10 @@ def _add_itinerary_trips(self, feed, itinerary, line, trip_builder, time_at_stop = str( datetime.strptime(time, "%H:%M").time()) except ValueError: - print("Warning: Time for a stop was not valid.") + print('Warning: Time "' + + time + '" for the stop was not valid:') print(" " + itinerary_stop.name + - " - " + itinerary_stop.osm_id) + " - " + itinerary_stop.osm_url) break gtfs_trip.AddStopTime(gtfs_stop, stop_time=time_at_stop) @@ -292,11 +294,10 @@ def _load_itinerary_schedule(self, schedule, itinerary, service): times = None for trip in schedule['lines'][itinerary.route_id]: trip_services = trip["services"] - if (trip[ - "from"] == itinerary.fr and trip[ - "to"] == itinerary.to and service in trip_services): + if (trip["from"] == itinerary.fr and + trip["to"] == itinerary.to and + service in trip_services): times = trip["times"] - if times is None: print("Warning: Couldn't load times from schedule for route") return times From 1399c21ad631bd5f07b542791efc1eed8f42b710 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Sat, 23 Dec 2017 11:36:01 +0100 Subject: [PATCH 15/18] Improve automatic color guessing for route_text_color --- .../creators/esteli/routes_creator_esteli.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/osm2gtfs/creators/esteli/routes_creator_esteli.py b/osm2gtfs/creators/esteli/routes_creator_esteli.py index 1c69544e..1be6f234 100644 --- a/osm2gtfs/creators/esteli/routes_creator_esteli.py +++ b/osm2gtfs/creators/esteli/routes_creator_esteli.py @@ -1,5 +1,6 @@ # coding=utf-8 +import math import webcolors from osm2gtfs.creators.routes_creator import RoutesCreator @@ -18,15 +19,20 @@ def _define_route_text_color(self, route): """ Overriden to support automatic guessing """ - return self._get_complementary_color(route.route_color) - def _get_complementary_color(self, color): - """ - Returns complementary RGB color - Source: https://stackoverflow.com/a/38478744 - """ + # Prepare the color information + color = route.route_color if color[0] == '#': color = color[1:] - rgb = (color[0:2], color[2:4], color[4:6]) - comp = ['%02X' % (255 - int(a, 16)) for a in rgb] - return ''.join(comp) + + # Slice RGB and convert to decimal numbers + red, green, blue = (int(color[0:2], 16), int(color[2:4], 16), + int(color[4:6], 16)) + + # Calculate the route_text_color; based on + # http://www.nbdtech.com/Blog/archive/2008/04/27/Calculating-the-Perceived-Brightness-of-a-Color.aspx + brightness = math.sqrt( + red * red * .241 + green * green * .691 + blue * blue * .068) + route_text_color = "ffffff" if brightness <= 130 else "000000" + + return route_text_color From 459a22e036065168bb65e3e3074cf20dfda95cf1 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Sat, 16 Dec 2017 18:58:28 +0100 Subject: [PATCH 16/18] Introduce inheritance in data structure elements --- osm2gtfs/creators/routes_creator.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/osm2gtfs/creators/routes_creator.py b/osm2gtfs/creators/routes_creator.py index f5ec7e6e..0d06e406 100644 --- a/osm2gtfs/creators/routes_creator.py +++ b/osm2gtfs/creators/routes_creator.py @@ -26,8 +26,8 @@ def add_routes_to_feed(self, feed, data): gtfs_route = feed.AddRoute( route_id=self._define_route_id(route), route_type=self._define_route_type(route), - short_name=route.route_id.encode('utf-8'), - long_name=route.name + short_name=self._define_short_name(route), + long_name=self._define_long_name(route) ) gtfs_route.agency_id = feed.GetDefaultAgency().agency_id gtfs_route.route_desc = self._define_route_description(route) @@ -43,6 +43,20 @@ def _define_route_id(self, route): """ return route.route_id + def _define_short_name(self, route): + """ + Returns the short name for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.route_id.encode('utf-8') + + def _define_long_name(self, route): + """ + Returns the long name for the use in the GTFS feed. + Can be easily overridden in any creator. + """ + return route.name + def _define_route_type(self, route): """ Returns the route_id for the use in the GTFS feed. From 756fcbde60770ec46ad8069f3d9a68f75e694a96 Mon Sep 17 00:00:00 2001 From: jamescr Date: Thu, 18 Jan 2018 00:35:49 -0600 Subject: [PATCH 17/18] Fix shapes in Incofer custom creators --- osm2gtfs/creators/incofer/trips_creator_incofer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osm2gtfs/creators/incofer/trips_creator_incofer.py b/osm2gtfs/creators/incofer/trips_creator_incofer.py index b141dbce..de53491d 100644 --- a/osm2gtfs/creators/incofer/trips_creator_incofer.py +++ b/osm2gtfs/creators/incofer/trips_creator_incofer.py @@ -25,8 +25,7 @@ def add_trips_to_feed(self, feed, data): # print("DEBUG. procesando el itinerario", itinerary.name) # shape for itinerary - shape_id = self._add_shape_to_feed( - feed, itinerary.route_id, itinerary) + shape_id = self._add_shape_to_feed(feed, itinerary.osm_id, itinerary) # service periods | días de opearación (c/u con sus horarios) operations = self._get_itinerary_operation(itinerary, data) From 057ca2f2a51106d775d4a1132bd2f9765467a17e Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 24 Jan 2018 15:36:55 +0100 Subject: [PATCH 18/18] Improve code based on review comments --- README.md | 4 +- osm2gtfs/core/configuration.py | 4 +- osm2gtfs/core/elements.py | 2 +- osm2gtfs/core/osm_connector.py | 196 +++++++++--------- osm2gtfs/creators/feed_info_creator.py | 1 + .../creators/fenix/trips_creator_fenix.py | 11 +- osm2gtfs/creators/stops_creator.py | 2 +- osm2gtfs/creators/trips_creator.py | 12 +- 8 files changed, 117 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index 4c9a3a39..3e11ca81 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,7 @@ source of schedule (time) information in order to create a GTFS file using the transitfeed library. For every new city a new [configuration file](https://github.com/grote/osm2gtfs/wiki/Configuration) -needs to be created and the input of schedule information is preferred -in a certain [format](https://github.com/grote/osm2gtfs/wiki/Schedule). -For any city the script can be easily extended, see the +needs to be created. Additionally, schedule information should be provided. By-default the schedule information is expected to be provided in a [certain format](https://github.com/grote/osm2gtfs/wiki/Schedule). However other formats are supported through extending the code. For any city and schedule format the script can be easily extended, see the [developer documentation](https://github.com/grote/osm2gtfs/wiki/Development) for more information. diff --git a/osm2gtfs/core/configuration.py b/osm2gtfs/core/configuration.py index 52332b85..8ac44c2d 100644 --- a/osm2gtfs/core/configuration.py +++ b/osm2gtfs/core/configuration.py @@ -81,9 +81,9 @@ def get_schedule_source(self, refresh=False): sys.stderr.write( "Error: Couldn't find schedule_source file.\n") sys.exit(0) - schedule_source = schedule_source_file + schedule_source = schedule_source_file.read() - self._schedule_source = schedule_source.read() + self._schedule_source = schedule_source # Cache data Cache.write_file(cached_file, self._schedule_source) diff --git a/osm2gtfs/core/elements.py b/osm2gtfs/core/elements.py index 2b6e561c..9310bbdb 100644 --- a/osm2gtfs/core/elements.py +++ b/osm2gtfs/core/elements.py @@ -53,7 +53,7 @@ def __attrs_post_init__(self): else: sys.stderr.write( "Warning: Route master relation without a route_master tag:\n") - sys.stderr.write(" " + self.osm_url) + sys.stderr.write(" " + self.osm_url + "\n") # Try to guess the type differently if 'route' in self.tags: diff --git a/osm2gtfs/core/osm_connector.py b/osm2gtfs/core/osm_connector.py index fbc49ee6..e79acadc 100644 --- a/osm2gtfs/core/osm_connector.py +++ b/osm2gtfs/core/osm_connector.py @@ -153,8 +153,7 @@ def get_routes(self, refresh=False): route_master.id) + "\n") sys.stderr.write( " has a member which is not a valid itinerary:\n") - sys.stderr.write(" https://osm.org/" + type( - member).__name__[8:].lower() + "/" + str( + sys.stderr.write(" https://osm.org/relation/" + str( member.ref) + "\n") # Create Line object from route master @@ -178,17 +177,18 @@ def get_routes(self, refresh=False): itinerary = self._build_itinerary(route_variant, result, False) # Make sure route_id (ref) number is not already taken - if itinerary.route_id in self.routes: - sys.stderr.write("Route with existing route_id (ref)\n") - sys.stderr.write( - "https://osm.org/relation/" + str(route_variant.id) + "\n") - sys.stderr.write("Skipped. Please fix in OpenStreetMap\n") - else: - # Create Line from route variant - itineraries = OrderedDict() - itineraries[itinerary.osm_id] = itinerary - line = self._build_line(route_variant, itineraries) - self.routes[line.route_id] = line + if itinerary is not None: + if itinerary.route_id in self.routes: + sys.stderr.write("Route with existing route_id (ref)\n") + sys.stderr.write( + "https://osm.org/relation/" + str(route_variant.id) + "\n") + sys.stderr.write("Skipped. Please fix in OpenStreetMap\n") + else: + # Create Line from route variant + itineraries = OrderedDict() + itineraries[itinerary.osm_id] = itinerary + line = self._build_line(route_variant, itineraries) + self.routes[line.route_id] = line # Cache data Cache.write_data(self.selector + '-routes', self.routes) @@ -306,7 +306,7 @@ def _build_line(self, route_master, itineraries): if not ref: sys.stderr.write( "No 'ref' could be obtained. Skipping whole route.\n") - return + return None # Move to Elements class, once attributes with defaults play well # with inheritance https://github.com/python-attrs/attrs/issues/38 @@ -330,7 +330,7 @@ def _build_line(self, route_master, itineraries): except ValueError: sys.stderr.write( "Itinerary ID doesn't match line ID. Please fix in OSM.\n") - sys.stderr.write(line.osm_url) + sys.stderr.write(line.osm_url + "\n") itinerary.route_id = line.route_id line.add_itinerary(itinerary) @@ -352,7 +352,7 @@ def _build_itinerary(self, route_variant, query_result_set, route_master): "https://osm.org/relation/" + str(route_variant.id) + "\n") sys.stderr.write( "Whole Itinerary skipped. Please fix in OpenStreetMap\n") - return + return None stops = [] @@ -369,6 +369,7 @@ def _build_itinerary(self, route_variant, query_result_set, route_master): else: sys.stderr.write("Unknown type of itinerary member: " + str(stop_candidate) + "\n") + continue stops.append(otype + "/" + str(stop_candidate.ref)) @@ -427,11 +428,11 @@ def _build_stop(self, stop, osm_type): else: sys.stderr.write( - "Warning: Potential stop was not approved and is ignored") + "Warning: Potential stop in invalid and has been ignored.\n") sys.stderr.write( " Check tagging: https://osm.org/" + osm_type + "/" + str( stop.id) + "\n") - return False + return None def _build_station(self, stop_area, osm_type): """Helper function to build Station objects from stop_areas @@ -448,90 +449,88 @@ def _build_station(self, stop_area, osm_type): # Check tagging whether this is a stop area. if 'public_transport' not in stop_area.tags: - return False - elif not stop_area.tags['public_transport'] == 'stop_area': - return False - - # Check whether a valid stop_area candidade - if 'public_transport' in stop_area.tags and stop_area.tags[ - 'public_transport'] == 'stop_area': - - # Analzyse member objects (stops) of this stop area - members = {} - for member in stop_area.members: + sys.stderr.write( + "Warning: Potential station has no public_transport tag.\n") + sys.stderr.write( + " Please fix on OSM: https://osm.org/" + osm_type + "/" + str( + stop_area.id) + "\n") + return None + elif stop_area.tags['public_transport'] != 'stop_area': + sys.stderr.write( + "Warning: Potential station is not tagged as stop_area.\n") + sys.stderr.write( + " Please fix on OSM: https://osm.org/" + osm_type + "/" + str( + stop_area.id) + "\n") + return None - if member.role == "platform": + # Analzyse member objects (stops) of this stop area + members = {} + for member in stop_area.members: - if isinstance(member, overpy.RelationNode): - member_osm_type = "node" - elif isinstance(member, overpy.RelationWay): - member_osm_type = "way" + if member.role == "platform": - identifier = member_osm_type + "/" + str(member.ref) + if isinstance(member, overpy.RelationNode): + member_osm_type = "node" + elif isinstance(member, overpy.RelationWay): + member_osm_type = "way" - if identifier in self.stops['regular']: + identifier = member_osm_type + "/" + str(member.ref) - # Collect the Stop objects that are members - # of this Station - members[identifier] = self.stops['regular'][identifier] - else: - sys.stderr.write( - "Error: Station member was not found in data") - sys.stderr.write("https://osm.org/relation/" + - str(stop_area.id) + "\n") - sys.stderr.write("https://osm.org/node/" + - str(member.ref) + "\n") - if len(members) < 1: - # Stop areas with only one stop, are not stations they just - # group different elements of one stop together. - sys.stderr.write( - "Error: Station with no members has been discarted:\n") - sys.stderr.write("https://osm.org/relation/" + - str(stop_area.id) + "\n") - return False + if identifier in self.stops['regular']: - elif len(members) is 1: - sys.stderr.write( - "Warning: Station has only one platform and is discarted\n") - sys.stderr.write("https://osm.org/relation/" + - str(stop_area.id) + "\n") - return False - - # Check name of stop area - if 'name' not in stop_area.tags: - sys.stderr.write("Warning: Stop area without name." + - " Please fix in OpenStreetMap\n") - sys.stderr.write("https://osm.org/relation/" + - str(stop_area.id) + "\n") - stop_area.name = self.stop_no_name - else: - stop_area.name = stop_area.tags["name"] + # Collect the Stop objects that are members + # of this Station + members[identifier] = self.stops['regular'][identifier] + else: + sys.stderr.write( + "Error: Station member was not found in data") + sys.stderr.write("https://osm.org/relation/" + + str(stop_area.id) + "\n") + sys.stderr.write("https://osm.org/node/" + + str(member.ref) + "\n") + if len(members) < 1: + # Stop areas with only one stop, are not stations they just + # group different elements of one stop together. + sys.stderr.write( + "Error: Station with no members has been discarted:\n") + sys.stderr.write("https://osm.org/relation/" + + str(stop_area.id) + "\n") + return None - # Calculate coordinates for stop area based on the center of it's - # members - stop_area.lat, stop_area.lon = Helper.get_center_of_nodes( - members.values()) + elif len(members) == 1: + sys.stderr.write( + "Warning: Station has only one platform and is discarted\n") + sys.stderr.write("https://osm.org/relation/" + + str(stop_area.id) + "\n") + return None + + # Check name of stop area + if 'name' not in stop_area.tags: + sys.stderr.write("Warning: Stop area without name." + + " Please fix in OpenStreetMap\n") + sys.stderr.write("https://osm.org/relation/" + + str(stop_area.id) + "\n") + stop_area.name = self.stop_no_name + else: + stop_area.name = stop_area.tags["name"] - # Move to Elements class, once attributes with defaults play well - # with inheritance https://github.com/python-attrs/attrs/issues/38 - osm_url = "https://osm.org/" + str( - osm_type) + "/" + str(stop_area.id) + # Calculate coordinates for stop area based on the center of it's + # members + stop_area.lat, stop_area.lon = Helper.get_center_of_nodes( + members.values()) - # Create and return Station object - station = Station(osm_id=stop_area.id, osm_type=osm_type, - osm_url=osm_url, tags=stop_area.tags, - name=stop_area.name, lat=stop_area.lat, - lon=stop_area.lon) - station.set_members(members) - return station + # Move to Elements class, once attributes with defaults play well + # with inheritance https://github.com/python-attrs/attrs/issues/38 + osm_url = "https://osm.org/" + str( + osm_type) + "/" + str(stop_area.id) - else: - sys.stderr.write( - "Warning: Potential station was not approved and is ignored") - sys.stderr.write( - " Check tagging: https://osm.org/" + osm_type + "/" + str( - stop_area.id) + "\n") - return False + # Create and return Station object + station = Station(osm_id=stop_area.id, osm_type=osm_type, + osm_url=osm_url, tags=stop_area.tags, + name=stop_area.name, lat=stop_area.lat, + lon=stop_area.lon) + station.set_members(members) + return station def _query_routes(self): """Helper function to query OpenStreetMap routes @@ -653,19 +652,18 @@ def _is_valid_stop_candidate(self, stop): :return bool: Returns True or False """ - valid = False if 'public_transport' in stop.tags: if stop.tags['public_transport'] == 'platform': - valid = True + return True elif stop.tags['public_transport'] == 'station': - valid = True + return True if 'highway' in stop.tags: if stop.tags['highway'] == 'bus_stop': - valid = True + return True if 'amenity' in stop.tags: if stop.tags['amenity'] == 'bus_station': - valid = True - return valid + return True + return False def _get_names_for_unnamed_stops(self): """Intelligently guess stop names for unnamed stops by sourrounding @@ -680,7 +678,7 @@ def _get_names_for_unnamed_stops(self): # If there is no name, query one intelligently from OSM if stop.name == "[" + self.stop_no_name + "]": self._find_best_name_for_unnamed_stop(stop) - print("* Smartly guessed stop name: " + + print("* Found alternative stop name: " + stop.name + " - " + stop.osm_url) # Cache stops with newly created stop names diff --git a/osm2gtfs/creators/feed_info_creator.py b/osm2gtfs/creators/feed_info_creator.py index 1ed1edcb..c5a5323a 100644 --- a/osm2gtfs/creators/feed_info_creator.py +++ b/osm2gtfs/creators/feed_info_creator.py @@ -20,6 +20,7 @@ def add_feed_info_to_feed(self, feed): # Missing feed_info workaround # https://github.com/google/transitfeed/issues/395 + # noinspection PyProtectedMember # pylint: disable=protected-access feed.AddTableColumns('feed_info', feed_info._ColumnNames()) diff --git a/osm2gtfs/creators/fenix/trips_creator_fenix.py b/osm2gtfs/creators/fenix/trips_creator_fenix.py index 817fe2d8..59659a52 100644 --- a/osm2gtfs/creators/fenix/trips_creator_fenix.py +++ b/osm2gtfs/creators/fenix/trips_creator_fenix.py @@ -237,24 +237,23 @@ def get_exception_service_period(self, feed, date, day): feed.AddServicePeriodObject(service) return service - @staticmethod - def match_first_stops(route, sim_stops): + def match_first_stops(self, route, sim_stops, ): # get the first stop of the route stop = route.stops[0] # normalize its name - stop.name = TripsCreatorFenix.normalize_stop_name(stop.name) + stop.name = self.normalize_stop_name(stop.name) # get first stop from relation 'from' tag if 'from' in route.tags: alt_stop_name = route.tags['from'] else: alt_stop_name = "" - alt_stop_name = TripsCreatorFenix.normalize_stop_name(alt_stop_name) + alt_stop_name = self.normalize_stop_name(alt_stop_name) # trying to match first stop from OSM with SIM for o_sim_stop in sim_stops: - sim_stop = TripsCreatorFenix.normalize_stop_name(o_sim_stop) + sim_stop = self.normalize_stop_name(o_sim_stop) if sim_stop == stop.name: return o_sim_stop elif sim_stop == alt_stop_name: @@ -267,7 +266,7 @@ def match_first_stops(route, sim_stops): sys.stderr.write("OSM Stop: '" + stop.name + "'\n") sys.stderr.write("OSM ALT Stop: '" + alt_stop_name + "'\n") for sim_stop in sim_stops: - sim_stop = TripsCreatorFenix.normalize_stop_name(sim_stop) + sim_stop = self.normalize_stop_name(sim_stop) sys.stderr.write("SIM Stop: '" + sim_stop + "'\n") print return None diff --git a/osm2gtfs/creators/stops_creator.py b/osm2gtfs/creators/stops_creator.py index 51afce73..d36d3cf8 100644 --- a/osm2gtfs/creators/stops_creator.py +++ b/osm2gtfs/creators/stops_creator.py @@ -52,7 +52,7 @@ def _add_stop_to_feed(self, stop, feed): try: parent_station = stop.get_parent_station() except AttributeError: - parent_station = "" + parent_station = None # Send stop_id creation through overridable function gtfs_stop_id = self._define_stop_id(stop) diff --git a/osm2gtfs/creators/trips_creator.py b/osm2gtfs/creators/trips_creator.py index f34c9f40..22b51d45 100644 --- a/osm2gtfs/creators/trips_creator.py +++ b/osm2gtfs/creators/trips_creator.py @@ -1,6 +1,7 @@ # coding=utf-8 import re +import sys from datetime import datetime import transitfeed from transitfeed import ServicePeriod @@ -24,6 +25,9 @@ def add_trips_to_feed(self, feed, data): It is the place where geographic information and schedule is getting joined to produce a routable GTFS. + + The default format of the schedule information: + https://github.com/grote/osm2gtfs/wiki/Schedule """ all_trips_count = 0 @@ -171,9 +175,11 @@ def _add_itinerary_trips(self, feed, itinerary, line, trip_builder, itinerary_stop = trip_builder[ 'all_stops']['regular'][itinerary_stop_id] except ValueError: - print("Itinerary (" + itinerary.route_url + - ") misses a stop:") - print(" Please review stop:" + itinerary_stop_id) + + sys.stderr.write( + "Itinerary (" + itinerary.route_url + ") misses a stop: \n") + sys.stderr.write( + "Please review: " + itinerary_stop_id + "\n") continue try: