- Added docker setup with gunicorn, closes #1

- Added logging

docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2'
+services:
+ openpoiservice-flask:
+  build: .
+  #volumes:
+  #- ./:/var/www/crypto
+  ports:
+   - "5000:5000"

gunicorn_config.py

@@ -0,0 +1,2 @@
+bind = "0.0.0.0:5000"
+workers = 2

manage.py

@@ -1,11 +1,16 @@
 # manage.py
 
+
 import unittest
 from flask.cli import FlaskGroup
 from openpoiservice.server import create_app, db
 from openpoiservice.server.db_import import parser
 from openpoiservice.server import ops_settings
 import os
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
 
 app = create_app()
@@ -46,6 +51,9 @@ def import_data():
 
     db.create_all()
 
+    logger.info("Starting to import data...")
+
+    # add option to add multiple osm pbf files
     parser.run_import(os.path.join(os.getcwd(), ops_settings['osm_file']))
 
 

openpoiservice/__init__.py

@@ -1 +1 @@
-# openpoiservice/__init__.py
+# openpoiservice/__init__.py

openpoiservice/server/__init__.py

@@ -8,6 +8,9 @@
 import yaml
 import os
 import time
+import logging
+
+logger = logging.getLogger(__name__)
 
 # instantiate the extensions
 
@@ -52,15 +55,16 @@ def create_app(script_info=None):
 
     Swagger(app)
 
-    @app.before_request
-    def before_request():
-        g.start = time.time()
+    if "DEVELOPMENT" in os.environ:
+        @app.before_request
+        def before_request():
+            g.start = time.time()
 
-    @app.teardown_request
-    def teardown_request(exception=None):
-        if 'start' in g:
+        @app.teardown_request
+        def teardown_request(exception=None):
+            # if 'start' in g:
             diff = time.time() - g.start
-            print "Request took: {} seconds".format(diff)
+            logger.info("Request took: {} seconds".format(diff))
 
     # error handlers
     @app.errorhandler(401)

openpoiservice/server/api/query_builder.py

@@ -12,6 +12,9 @@
 from sqlalchemy import dialects
 import geojson as geojson
 from itertools import tee, islice, chain, izip
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class QueryBuilder(object):
@@ -58,7 +61,7 @@ def request_pois(self):
 
         geom_filters, geom = self.generate_geom_filters(params['geometry'], Pois)
 
-        print geom_filters, geom
+        logger.debug('geometry filters: {}, geometry: {}'.format(geom_filters, geom))
         if 'filters' in params and 'category_ids' in params['filters']:
             geom_filters.append(Pois.category.in_(params['filters']['category_ids']))
 
@@ -80,7 +83,7 @@ def request_pois(self):
                 .group_by(bbox_query.c.category) \
                 .all()
 
-            print len(stats_query)
+            logger.debug('number of poi stats: {}'.format(len(stats_query)))
             places_json = self.generate_category_stats(stats_query)
 
             return places_json
@@ -113,7 +116,7 @@ def request_pois(self):
 
             # for dude in pois_query:
             #    print wkb.loads(str(dude[6]), hex=True)
-            print len(pois_query)
+            logger.debug("number of pois: {}".format(len(pois_query)))
 
             # response as geojson feature collection
             features = self.generate_geojson_features(pois_query)
@@ -143,7 +146,6 @@ def generate_sortby(params, geom, query):
 
     @staticmethod
     def generate_geom_filters(geometry, Pois):
-        print geometry, geometry['radius']
         filters, geom = [], None
 
         if 'bbox' in geometry and 'geom' not in geometry:
@@ -171,7 +173,6 @@ def generate_geom_filters(geometry, Pois):
 
         elif 'bbox' not in geometry and 'geom' in geometry:
 
-            print geometry['radius']
             geom = geometry['geom'].wkt
 
             filters.append(  # buffer around geom
@@ -310,7 +311,7 @@ def generate_geojson_features(cls, query):
 
         feature_collection = geojson.FeatureCollection(features)
 
-        print len(features)
+        logger.debug("Amount of features {}".format(len(features)))
         return feature_collection
 
     @staticmethod

openpoiservice/server/api/views.py

@@ -7,7 +7,7 @@
 from shapely.geometry import Point, Polygon, LineString, MultiPoint
 from openpoiservice.server import api_exceptions, ops_settings
 from openpoiservice.server.api.query_builder import QueryBuilder
-from openpoiservice.server.utils.geometries import parse_geometry, validate_limit, validate_area, transform_geom
+from openpoiservice.server.utils.geometries import parse_geometry, validate_limit, transform_geom
 from flasgger.utils import swag_from
 
 
@@ -83,7 +83,6 @@ def custom_schema():
     Optional('id'): Required(Coerce(str), msg='must be a string')
 })
 
-print schema
 main_blueprint = Blueprint('main', __name__, )
 
 

openpoiservice/server/db_import/models.py

@@ -2,11 +2,14 @@
 
 from openpoiservice.server import db, ops_settings
 from geoalchemy2 import Geography
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class Pois(db.Model):
     __tablename__ = ops_settings['provider_parameters']['table_name']
-    print __tablename__
+    logger.info('table name for pois: {}'.format(__tablename__))
     osm_id = db.Column(db.BigInteger, primary_key=True)
     osm_type = db.Column(db.Integer, nullable=False)
     category = db.Column(db.Integer, index=True, nullable=False)
@@ -22,7 +25,7 @@ def __repr__(self):
 
 class Tags(db.Model):
     __tablename__ = ops_settings['provider_parameters']['table_name'] + "_tags"
-    print __tablename__
+    logger.info('Table name for tags: {}'.format(__tablename__))
 
     id = db.Column(db.Integer, primary_key=True)
     osm_id = db.Column(db.BigInteger,

openpoiservice/server/db_import/parse_pbf.py

@@ -6,7 +6,9 @@
 from openpoiservice.server.db_import.objects import PoiObject, TagsObject
 import shapely as shapely
 from shapely.geometry import Point, Polygon, LineString, MultiPoint
+import logging
 
+logger = logging.getLogger(__name__)
 
 
 class LatLng(object):
@@ -73,7 +75,7 @@ def __init__(self):
 
     def parse_relations(self, relations):
         """
-        Callback function called by imposm after relations are parsed. The idea is to extract polygons which may
+        Callback function called by imposm while relations are parsed. The idea is to extract polygons which may
         contain poi tags of interest. For this we are currently using osm_type=multipolygon.
         The osm ids of the found objects are then used in parse_ways.
 
@@ -107,13 +109,13 @@ def parse_relations(self, relations):
                             self.relations_cnt += 1
 
                             if self.relations_cnt % 10000 == 0:
-                                print 'Relations found: {} '.format(self.relations_cnt)
+                                logger.info('Relations found: {} '.format(self.relations_cnt))
 
             self.entity_cnt += 1
 
     def parse_ways(self, ways):
         """
-        Callback function called by imposm after ways are parsed. If a category can't be found it may likely
+        Callback function called by imposm while ways are parsed. If a category can't be found it may likely
         be that the osmid of this way can be found in self.relation_ways which will contain additional tags
         and therefore eventually a category. A way object is added to a list process_ways which at this point
         is lacking coordinates -> next step.
@@ -151,7 +153,7 @@ def parse_ways(self, ways):
                     self.ways_cnt += 1
 
                     if self.ways_cnt % 50000 == 0:
-                        print 'Ways found: {} '.format(self.ways_cnt)
+                        logger.info('Ways found: {} '.format(self.ways_cnt))
 
                     ways_obj = WayObject(osmid, tags, refs, category_id)
                     self.process_ways.append(ways_obj)
@@ -173,7 +175,7 @@ def store_poi(self, poi_object):
                                      ))
 
         if self.pois_cnt % 1000 == 0:
-            print 'Pois: {}, tags: {}'.format(self.pois_cnt, self.tags_cnt)
+            logger.info('Pois: {}, tags: {}'.format(self.pois_cnt, self.tags_cnt))
             db.session.add_all(self.poi_objects)
             db.session.add_all(self.tags_objects)
             db.session.commit()
@@ -236,7 +238,7 @@ def create_poi(self, tags, osmid, lat_lng, osm_type, category=0):
 
     def parse_coords(self, coords):
         """
-        Callback function called by imposm after coordinates are parsed. Saves coordinates to nodes dictionary for
+        Callback function called by imposm while coordinates are parsed. Saves coordinates to nodes dictionary for
         way nodes that so far don't comprise coordinates.
 
         :param coords: osm coordinate objects
@@ -251,7 +253,7 @@ def parse_coords(self, coords):
 
     def parse_nodes(self, osm_nodes):
         """
-        Callback function called by imposm after nodes are parsed.
+        Callback function called by imposm while nodes are parsed.
 
         :param osm_nodes: osm node objects
         :type osm_nodes: list of osm nodes
@@ -270,9 +272,6 @@ def parse_nodes_of_ways(self):
         It tries to find coordinates of these nodes and creates a simple average position of these which is the
         poi. Saved to DB afterwards.
         """
-
-        print self.nodes
-
         osm_type = 2
 
         for way in self.process_ways:

openpoiservice/server/db_import/parser.py

@@ -3,6 +3,9 @@
 from openpoiservice.server.db_import.parse_pbf import PbfImporter
 from imposm.parser import OSMParser
 from timeit import Timer
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 def run_import(path):
@@ -12,22 +15,24 @@ def run_import(path):
 
     relations = OSMParser(concurrency=4, relations_callback=pbf_importer.parse_relations)
     t = Timer(lambda: relations.parse(osm_file_path))
-    print 'Time passed: {}s'.format(t.timeit(number=1))
-    print 'Found {} ways in relations'.format(pbf_importer.relations_cnt)
+    logger.info('Start reading database')
+
+    logger.info('Time passed: {}s'.format(t.timeit(number=1)))
+    logger.info('Found {} ways in relations'.format(pbf_importer.relations_cnt))
 
     ways = OSMParser(concurrency=4, ways_callback=pbf_importer.parse_ways)
     t = Timer(lambda: ways.parse(osm_file_path))
-    print 'Time passed: {}s'.format(t.timeit(number=1))
-    print 'Found {} ways'.format(pbf_importer.ways_cnt)
+    logger.info('Time passed: {}s'.format(t.timeit(number=1)))
+    logger.info('Found {} ways'.format(pbf_importer.ways_cnt))
 
     nodes = OSMParser(concurrency=4, nodes_callback=pbf_importer.parse_nodes)
     t = Timer(lambda: nodes.parse(osm_file_path))
-    print 'Time passed to parse nodes: {}s'.format(t.timeit(number=1))
+    logger.info('Time passed to parse nodes: {}s'.format(t.timeit(number=1)))
 
     coords = OSMParser(concurrency=4, coords_callback=pbf_importer.parse_coords)
     t = Timer(lambda: coords.parse(osm_file_path))
-    print 'Time passed to parse coords for nodes from ways: {}s'.format(t.timeit(number=1))
+    logger.info('Time passed to parse coords for nodes from ways: {}s'.format(t.timeit(number=1)))
 
     t = Timer(lambda: pbf_importer.parse_nodes_of_ways())
-    print 'Time passed total: {}s'.format(t.timeit(number=1))
-    print 'Found {} pois'.format(pbf_importer.pois_cnt)
+    logger.info('Time passed total: {}s'.format(t.timeit(number=1)))
+    logger.info('Found {} pois'.format(pbf_importer.pois_cnt))

openpoiservice/server/ops_settings.yml

@@ -11,8 +11,9 @@ maximum_area: 25000
 maximum_linestring_length: 500000
 # limit of pois
 response_limit: 2000
-osm_file: bremen-tests.pbf
-osm_file_tests: bremen-tests.pbf
+osm_file: bremen-latest.osm.pbf
+osm_file_tests: bremen-tests.osm.pbf
+
 provider_parameters:
   table_name: ops_planet_pois
   db_name: gis

openpoiservice/server/utils/geometries.py

@@ -34,10 +34,6 @@ def transform_geom(g1, src_proj, dest_proj):
     return g2
 
 
-def validate_area(geometry, limit):
-    pass
-
-
 def validate_limit(radius, limit):
     """
     Returns True if radius is in custom specific limits.