diff --git a/bremen-tests.pbf b/bremen-tests.osm.pbf similarity index 100% rename from bremen-tests.pbf rename to bremen-tests.osm.pbf diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d9cfad3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +version: '2' +services: + openpoiservice-flask: + build: . + #volumes: + #- ./:/var/www/crypto + ports: + - "5000:5000" \ No newline at end of file diff --git a/gunicorn_config.py b/gunicorn_config.py new file mode 100644 index 0000000..09951a6 --- /dev/null +++ b/gunicorn_config.py @@ -0,0 +1,2 @@ +bind = "0.0.0.0:5000" +workers = 2 diff --git a/manage.py b/manage.py index fd20a46..45ae3ec 100755 --- a/manage.py +++ b/manage.py @@ -1,11 +1,16 @@ # manage.py + import unittest from flask.cli import FlaskGroup from openpoiservice.server import create_app, db from openpoiservice.server.db_import import parser from openpoiservice.server import ops_settings import os +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) app = create_app() @@ -46,6 +51,9 @@ def import_data(): db.create_all() + logger.info("Starting to import data...") + + # add option to add multiple osm pbf files parser.run_import(os.path.join(os.getcwd(), ops_settings['osm_file'])) diff --git a/openpoiservice/__init__.py b/openpoiservice/__init__.py index e11030e..73ecb9a 100755 --- a/openpoiservice/__init__.py +++ b/openpoiservice/__init__.py @@ -1 +1 @@ -# openpoiservice/__init__.py \ No newline at end of file +# openpoiservice/__init__.py diff --git a/openpoiservice/server/__init__.py b/openpoiservice/server/__init__.py index 111196d..c23b6df 100755 --- a/openpoiservice/server/__init__.py +++ b/openpoiservice/server/__init__.py @@ -8,6 +8,9 @@ import yaml import os import time +import logging + +logger = logging.getLogger(__name__) # instantiate the extensions @@ -52,15 +55,16 @@ def create_app(script_info=None): Swagger(app) - @app.before_request - def before_request(): - g.start = time.time() + if "DEVELOPMENT" in os.environ: + @app.before_request + def before_request(): + g.start = time.time() - @app.teardown_request - def teardown_request(exception=None): - if 'start' in g: + @app.teardown_request + def teardown_request(exception=None): + # if 'start' in g: diff = time.time() - g.start - print "Request took: {} seconds".format(diff) + logger.info("Request took: {} seconds".format(diff)) # error handlers @app.errorhandler(401) diff --git a/openpoiservice/server/api/query_builder.py b/openpoiservice/server/api/query_builder.py index 9390df2..b8c2a7a 100644 --- a/openpoiservice/server/api/query_builder.py +++ b/openpoiservice/server/api/query_builder.py @@ -12,6 +12,9 @@ from sqlalchemy import dialects import geojson as geojson from itertools import tee, islice, chain, izip +import logging + +logger = logging.getLogger(__name__) class QueryBuilder(object): @@ -58,7 +61,7 @@ def request_pois(self): geom_filters, geom = self.generate_geom_filters(params['geometry'], Pois) - print geom_filters, geom + logger.debug('geometry filters: {}, geometry: {}'.format(geom_filters, geom)) if 'filters' in params and 'category_ids' in params['filters']: geom_filters.append(Pois.category.in_(params['filters']['category_ids'])) @@ -80,7 +83,7 @@ def request_pois(self): .group_by(bbox_query.c.category) \ .all() - print len(stats_query) + logger.debug('number of poi stats: {}'.format(len(stats_query))) places_json = self.generate_category_stats(stats_query) return places_json @@ -113,7 +116,7 @@ def request_pois(self): # for dude in pois_query: # print wkb.loads(str(dude[6]), hex=True) - print len(pois_query) + logger.debug("number of pois: {}".format(len(pois_query))) # response as geojson feature collection features = self.generate_geojson_features(pois_query) @@ -143,7 +146,6 @@ def generate_sortby(params, geom, query): @staticmethod def generate_geom_filters(geometry, Pois): - print geometry, geometry['radius'] filters, geom = [], None if 'bbox' in geometry and 'geom' not in geometry: @@ -171,7 +173,6 @@ def generate_geom_filters(geometry, Pois): elif 'bbox' not in geometry and 'geom' in geometry: - print geometry['radius'] geom = geometry['geom'].wkt filters.append( # buffer around geom @@ -310,7 +311,7 @@ def generate_geojson_features(cls, query): feature_collection = geojson.FeatureCollection(features) - print len(features) + logger.debug("Amount of features {}".format(len(features))) return feature_collection @staticmethod diff --git a/openpoiservice/server/api/views.py b/openpoiservice/server/api/views.py index fa9f618..57ee984 100755 --- a/openpoiservice/server/api/views.py +++ b/openpoiservice/server/api/views.py @@ -7,7 +7,7 @@ from shapely.geometry import Point, Polygon, LineString, MultiPoint from openpoiservice.server import api_exceptions, ops_settings from openpoiservice.server.api.query_builder import QueryBuilder -from openpoiservice.server.utils.geometries import parse_geometry, validate_limit, validate_area, transform_geom +from openpoiservice.server.utils.geometries import parse_geometry, validate_limit, transform_geom from flasgger.utils import swag_from @@ -83,7 +83,6 @@ def custom_schema(): Optional('id'): Required(Coerce(str), msg='must be a string') }) -print schema main_blueprint = Blueprint('main', __name__, ) diff --git a/openpoiservice/server/db_import/models.py b/openpoiservice/server/db_import/models.py index 6ff9a0b..0bdf199 100755 --- a/openpoiservice/server/db_import/models.py +++ b/openpoiservice/server/db_import/models.py @@ -2,11 +2,14 @@ from openpoiservice.server import db, ops_settings from geoalchemy2 import Geography +import logging + +logger = logging.getLogger(__name__) class Pois(db.Model): __tablename__ = ops_settings['provider_parameters']['table_name'] - print __tablename__ + logger.info('table name for pois: {}'.format(__tablename__)) osm_id = db.Column(db.BigInteger, primary_key=True) osm_type = db.Column(db.Integer, nullable=False) category = db.Column(db.Integer, index=True, nullable=False) @@ -22,7 +25,7 @@ def __repr__(self): class Tags(db.Model): __tablename__ = ops_settings['provider_parameters']['table_name'] + "_tags" - print __tablename__ + logger.info('Table name for tags: {}'.format(__tablename__)) id = db.Column(db.Integer, primary_key=True) osm_id = db.Column(db.BigInteger, diff --git a/openpoiservice/server/db_import/parse_pbf.py b/openpoiservice/server/db_import/parse_pbf.py index d935b55..78d234e 100644 --- a/openpoiservice/server/db_import/parse_pbf.py +++ b/openpoiservice/server/db_import/parse_pbf.py @@ -6,7 +6,9 @@ from openpoiservice.server.db_import.objects import PoiObject, TagsObject import shapely as shapely from shapely.geometry import Point, Polygon, LineString, MultiPoint +import logging +logger = logging.getLogger(__name__) class LatLng(object): @@ -73,7 +75,7 @@ def __init__(self): def parse_relations(self, relations): """ - Callback function called by imposm after relations are parsed. The idea is to extract polygons which may + Callback function called by imposm while relations are parsed. The idea is to extract polygons which may contain poi tags of interest. For this we are currently using osm_type=multipolygon. The osm ids of the found objects are then used in parse_ways. @@ -107,13 +109,13 @@ def parse_relations(self, relations): self.relations_cnt += 1 if self.relations_cnt % 10000 == 0: - print 'Relations found: {} '.format(self.relations_cnt) + logger.info('Relations found: {} '.format(self.relations_cnt)) self.entity_cnt += 1 def parse_ways(self, ways): """ - Callback function called by imposm after ways are parsed. If a category can't be found it may likely + Callback function called by imposm while ways are parsed. If a category can't be found it may likely be that the osmid of this way can be found in self.relation_ways which will contain additional tags and therefore eventually a category. A way object is added to a list process_ways which at this point is lacking coordinates -> next step. @@ -151,7 +153,7 @@ def parse_ways(self, ways): self.ways_cnt += 1 if self.ways_cnt % 50000 == 0: - print 'Ways found: {} '.format(self.ways_cnt) + logger.info('Ways found: {} '.format(self.ways_cnt)) ways_obj = WayObject(osmid, tags, refs, category_id) self.process_ways.append(ways_obj) @@ -173,7 +175,7 @@ def store_poi(self, poi_object): )) if self.pois_cnt % 1000 == 0: - print 'Pois: {}, tags: {}'.format(self.pois_cnt, self.tags_cnt) + logger.info('Pois: {}, tags: {}'.format(self.pois_cnt, self.tags_cnt)) db.session.add_all(self.poi_objects) db.session.add_all(self.tags_objects) db.session.commit() @@ -236,7 +238,7 @@ def create_poi(self, tags, osmid, lat_lng, osm_type, category=0): def parse_coords(self, coords): """ - Callback function called by imposm after coordinates are parsed. Saves coordinates to nodes dictionary for + Callback function called by imposm while coordinates are parsed. Saves coordinates to nodes dictionary for way nodes that so far don't comprise coordinates. :param coords: osm coordinate objects @@ -251,7 +253,7 @@ def parse_coords(self, coords): def parse_nodes(self, osm_nodes): """ - Callback function called by imposm after nodes are parsed. + Callback function called by imposm while nodes are parsed. :param osm_nodes: osm node objects :type osm_nodes: list of osm nodes @@ -270,9 +272,6 @@ def parse_nodes_of_ways(self): It tries to find coordinates of these nodes and creates a simple average position of these which is the poi. Saved to DB afterwards. """ - - print self.nodes - osm_type = 2 for way in self.process_ways: diff --git a/openpoiservice/server/db_import/parser.py b/openpoiservice/server/db_import/parser.py index 8cdb478..b0f6972 100644 --- a/openpoiservice/server/db_import/parser.py +++ b/openpoiservice/server/db_import/parser.py @@ -3,6 +3,9 @@ from openpoiservice.server.db_import.parse_pbf import PbfImporter from imposm.parser import OSMParser from timeit import Timer +import logging + +logger = logging.getLogger(__name__) def run_import(path): @@ -12,22 +15,24 @@ def run_import(path): relations = OSMParser(concurrency=4, relations_callback=pbf_importer.parse_relations) t = Timer(lambda: relations.parse(osm_file_path)) - print 'Time passed: {}s'.format(t.timeit(number=1)) - print 'Found {} ways in relations'.format(pbf_importer.relations_cnt) + logger.info('Start reading database') + + logger.info('Time passed: {}s'.format(t.timeit(number=1))) + logger.info('Found {} ways in relations'.format(pbf_importer.relations_cnt)) ways = OSMParser(concurrency=4, ways_callback=pbf_importer.parse_ways) t = Timer(lambda: ways.parse(osm_file_path)) - print 'Time passed: {}s'.format(t.timeit(number=1)) - print 'Found {} ways'.format(pbf_importer.ways_cnt) + logger.info('Time passed: {}s'.format(t.timeit(number=1))) + logger.info('Found {} ways'.format(pbf_importer.ways_cnt)) nodes = OSMParser(concurrency=4, nodes_callback=pbf_importer.parse_nodes) t = Timer(lambda: nodes.parse(osm_file_path)) - print 'Time passed to parse nodes: {}s'.format(t.timeit(number=1)) + logger.info('Time passed to parse nodes: {}s'.format(t.timeit(number=1))) coords = OSMParser(concurrency=4, coords_callback=pbf_importer.parse_coords) t = Timer(lambda: coords.parse(osm_file_path)) - print 'Time passed to parse coords for nodes from ways: {}s'.format(t.timeit(number=1)) + logger.info('Time passed to parse coords for nodes from ways: {}s'.format(t.timeit(number=1))) t = Timer(lambda: pbf_importer.parse_nodes_of_ways()) - print 'Time passed total: {}s'.format(t.timeit(number=1)) - print 'Found {} pois'.format(pbf_importer.pois_cnt) + logger.info('Time passed total: {}s'.format(t.timeit(number=1))) + logger.info('Found {} pois'.format(pbf_importer.pois_cnt)) diff --git a/openpoiservice/server/ops_settings.yml b/openpoiservice/server/ops_settings.yml index b01b695..fe4a64a 100644 --- a/openpoiservice/server/ops_settings.yml +++ b/openpoiservice/server/ops_settings.yml @@ -11,8 +11,9 @@ maximum_area: 25000 maximum_linestring_length: 500000 # limit of pois response_limit: 2000 -osm_file: bremen-tests.pbf -osm_file_tests: bremen-tests.pbf +osm_file: bremen-latest.osm.pbf +osm_file_tests: bremen-tests.osm.pbf + provider_parameters: table_name: ops_planet_pois db_name: gis diff --git a/openpoiservice/server/utils/geometries.py b/openpoiservice/server/utils/geometries.py index db9335b..4c22f7e 100644 --- a/openpoiservice/server/utils/geometries.py +++ b/openpoiservice/server/utils/geometries.py @@ -34,10 +34,6 @@ def transform_geom(g1, src_proj, dest_proj): return g2 -def validate_area(geometry, limit): - pass - - def validate_limit(radius, limit): """ Returns True if radius is in custom specific limits.