test

pyextra/overpy/__init__.py

@@ -5,6 +5,8 @@
 import json
 import re
 import sys
+import time
+import requests
 
 from overpy import exception
 from overpy.__about__ import (
@@ -18,12 +20,15 @@
 XML_PARSER_DOM = 1
 XML_PARSER_SAX = 2
 
-if PY2:
-    from urllib2 import urlopen
-    from urllib2 import HTTPError
-elif PY3:
-    from urllib.request import urlopen
-    from urllib.error import HTTPError
+# Try to convert some common attributes
+# http://wiki.openstreetmap.org/wiki/Elements#Common_attributes
+GLOBAL_ATTRIBUTE_MODIFIERS = {
+    "changeset": int,
+    "timestamp": lambda ts: datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ"),
+    "uid": int,
+    "version": int,
+    "visible": lambda v: v.lower() == "true"
+}
 
 
 def is_valid_type(element, cls):
@@ -41,18 +46,31 @@ def is_valid_type(element, cls):
 class Overpass(object):
     """
     Class to access the Overpass API
+
+    :cvar default_max_retry_count: Global max number of retries (Default: 0)
+    :cvar default_retry_timeout: Global time to wait between tries (Default: 1.0s)
     """
+    default_max_retry_count = 0
     default_read_chunk_size = 4096
+    default_retry_timeout = 1.0
     default_url = "http://overpass-api.de/api/interpreter"
 
-    def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX):
+    def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX, max_retry_count=None, retry_timeout=None, timeout=5.0, headers=None):
         """
         :param read_chunk_size: Max size of each chunk read from the server response
         :type read_chunk_size: Integer
         :param url: Optional URL of the Overpass server. Defaults to http://overpass-api.de/api/interpreter
         :type url: str
         :param xml_parser: The xml parser to use
         :type xml_parser: Integer
+        :param max_retry_count: Max number of retries (Default: default_max_retry_count)
+        :type max_retry_count: Integer
+        :param retry_timeout: Time to wait between tries (Default: default_retry_timeout)
+        :type retry_timeout: float
+        :param timeout: HTTP request timeout
+        :type timeout: float
+        :param headers: HTTP request headers
+        :type headers: dict
         """
         self.url = self.default_url
         if url is not None:
@@ -63,7 +81,34 @@ def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX):
         if read_chunk_size is None:
             read_chunk_size = self.default_read_chunk_size
         self.read_chunk_size = read_chunk_size
+
+        if max_retry_count is None:
+            max_retry_count = self.default_max_retry_count
+        self.max_retry_count = max_retry_count
+
+        if retry_timeout is None:
+            retry_timeout = self.default_retry_timeout
+        self.retry_timeout = retry_timeout
+
         self.xml_parser = xml_parser
+        self.timeout = timeout
+        self.headers = headers
+
+    def _handle_remark_msg(self, msg):
+        """
+        Try to parse the message provided with the remark tag or element.
+
+        :param str msg: The message
+        :raises overpy.exception.OverpassRuntimeError: If message starts with 'runtime error:'
+        :raises overpy.exception.OverpassRuntimeRemark: If message starts with 'runtime remark:'
+        :raises overpy.exception.OverpassUnknownError: If we are unable to identify the error
+        """
+        msg = msg.strip()
+        if msg.startswith("runtime error:"):
+            raise exception.OverpassRuntimeError(msg=msg)
+        elif msg.startswith("runtime remark:"):
+            raise exception.OverpassRuntimeRemark(msg=msg)
+        raise exception.OverpassUnknownError(msg=msg)
 
     def query(self, query):
         """
@@ -76,56 +121,79 @@ def query(self, query):
         if not isinstance(query, bytes):
             query = query.encode("utf-8")
 
-        try:
-            f = urlopen(self.url, query)
-        except HTTPError as e:
-            f = e
-
-        response = f.read(self.read_chunk_size)
-        while True:
-            data = f.read(self.read_chunk_size)
-            if len(data) == 0:
-                break
-            response = response + data
-        f.close()
-
-        if f.code == 200:
-            if PY2:
-                http_info = f.info()
-                content_type = http_info.getheader("content-type")
-            else:
-                content_type = f.getheader("Content-Type")
-
-            if content_type == "application/json":
-                return self.parse_json(response)
+        retry_num = 0
+        retry_exceptions = []
+        do_retry = True if self.max_retry_count > 0 else False
+        while retry_num <= self.max_retry_count:
+            if retry_num > 0:
+                time.sleep(self.retry_timeout)
+            retry_num += 1
 
-            if content_type == "application/osm3s+xml":
-                return self.parse_xml(response)
+            try:
+                if self.headers is not None:
+                    r = requests.post(self.url, query, timeout=self.timeout, headers=self.headers)
+                else:
+                    r = requests.post(self.url, query, timeout=self.timeout)
+                response = r.content
+            except (requests.exceptions.BaseHTTPError, requests.exceptions.RequestException) as e:
+                if not do_retry:
+                    raise e
+                retry_exceptions.append(e)
+                continue
 
-            raise exception.OverpassUnknownContentType(content_type)
+            if r.status_code == 200:
+                content_type = r.headers["Content-Type"]
 
-        if f.code == 400:
-            msgs = []
-            for msg in self._regex_extract_error_msg.finditer(response):
-                tmp = self._regex_remove_tag.sub(b"", msg.group("msg"))
-                try:
-                    tmp = tmp.decode("utf-8")
-                except UnicodeDecodeError:
-                    tmp = repr(tmp)
-                msgs.append(tmp)
+                if content_type == "application/json":
+                    return self.parse_json(response)
 
-            raise exception.OverpassBadRequest(
-                query,
-                msgs=msgs
-            )
+                if content_type == "application/osm3s+xml":
+                    return self.parse_xml(response)
 
-        if f.code == 429:
-            raise exception.OverpassTooManyRequests
+                e = exception.OverpassUnknownContentType(content_type)
+                if not do_retry:
+                    raise e
+                retry_exceptions.append(e)
+                continue
+            elif r.status_code == 400:
+                msgs = []
+                for msg in self._regex_extract_error_msg.finditer(response):
+                    tmp = self._regex_remove_tag.sub(b"", msg.group("msg"))
+                    try:
+                        tmp = tmp.decode("utf-8")
+                    except UnicodeDecodeError:
+                        tmp = repr(tmp)
+                    msgs.append(tmp)
+
+                e = exception.OverpassBadRequest(
+                    query,
+                    msgs=msgs
+                )
+                if not do_retry:
+                    raise e
+                retry_exceptions.append(e)
+                continue
+            elif r.status_code == 429:
+                e = exception.OverpassTooManyRequests
+                if not do_retry:
+                    raise e
+                retry_exceptions.append(e)
+                continue
+            elif r.status_code == 504:
+                e = exception.OverpassGatewayTimeout
+                if not do_retry:
+                    raise e
+                retry_exceptions.append(e)
+                continue
 
-        if f.code == 504:
-            raise exception.OverpassGatewayTimeout
+            # No valid response code
+            e = exception.OverpassUnknownHTTPStatusCode(r.status_code)
+            if not do_retry:
+                raise e
+            retry_exceptions.append(e)
+            continue
 
-        raise exception.OverpassUnknownHTTPStatusCode(f.code)
+        raise exception.MaxRetriesReached(retry_count=retry_num, exceptions=retry_exceptions)
 
     def parse_json(self, data, encoding="utf-8"):
         """
@@ -139,8 +207,11 @@ def parse_json(self, data, encoding="utf-8"):
         :rtype: overpy.Result
         """
         if isinstance(data, bytes):
-            data = data.decode(encoding)
+          data = data.decode(encoding)
+
         data = json.loads(data, parse_float=Decimal)
+        if "remark" in data:
+            self._handle_remark_msg(msg=data.get("remark"))
         return Result.from_json(data, api=self)
 
     def parse_xml(self, data, encoding="utf-8", parser=None):
@@ -155,13 +226,16 @@ def parse_xml(self, data, encoding="utf-8", parser=None):
         """
         if parser is None:
             parser = self.xml_parser
-
         if isinstance(data, bytes):
             data = data.decode(encoding)
         if PY2 and not isinstance(data, str):
             # Python 2.x: Convert unicode strings
             data = data.encode(encoding)
 
+        m = re.compile("<remark>(?P<msg>[^<>]*)</remark>").search(data)
+        if m:
+            self._handle_remark_msg(m.group("msg"))
+
         return Result.from_xml(data, api=self, parser=parser)
 
 
@@ -279,23 +353,39 @@ def from_json(cls, data, api=None):
         return result
 
     @classmethod
-    def from_xml(cls, data, api=None, parser=XML_PARSER_SAX):
+    def from_xml(cls, data, api=None, parser=None):
         """
-        Create a new instance and load data from xml object.
+        Create a new instance and load data from xml data or object.
+
+        .. note::
+            If parser is set to None, the functions tries to find the best parse.
+            By default the SAX parser is chosen if a string is provided as data.
+            The parser is set to DOM if an xml.etree.ElementTree.Element is provided as data value.
 
         :param data: Root element
-        :type data: xml.etree.ElementTree.Element
-        :param api:
+        :type data: str | xml.etree.ElementTree.Element
+        :param api: The instance to query additional information if required.
         :type api: Overpass
-        :param parser: Specify the parser to use(DOM or SAX)
-        :type parser: Integer
+        :param parser: Specify the parser to use(DOM or SAX)(Default: None = autodetect, defaults to SAX)
+        :type parser: Integer | None
         :return: New instance of Result object
         :rtype: Result
         """
+        if parser is None:
+            if isinstance(data, str):
+                parser = XML_PARSER_SAX
+            else:
+                parser = XML_PARSER_DOM
+
         result = cls(api=api)
         if parser == XML_PARSER_DOM:
             import xml.etree.ElementTree as ET
-            root = ET.fromstring(data)
+            if isinstance(data, str):
+                root = ET.fromstring(data)
+            elif isinstance(data, ET.Element):
+                root = data
+            else:
+                raise exception.OverPyException("Unable to detect data type.")
 
             for elem_cls in [Node, Way, Relation, Area]:
                 for child in root:
@@ -522,17 +612,10 @@ def __init__(self, attributes=None, result=None, tags=None):
         """
 
         self._result = result
-        # Try to convert some common attributes
-        # http://wiki.openstreetmap.org/wiki/Elements#Common_attributes
-        self._attribute_modifiers = {
-            "changeset": int,
-            "timestamp": lambda ts: datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ"),
-            "uid": int,
-            "version": int,
-            "visible": lambda v: v.lower() == "true"
-        }
         self.attributes = attributes
-        for n, m in self._attribute_modifiers.items():
+        # ToDo: Add option to modify attribute modifiers
+        attribute_modifiers = dict(GLOBAL_ATTRIBUTE_MODIFIERS.items())
+        for n, m in attribute_modifiers.items():
             if n in self.attributes:
                 self.attributes[n] = m(self.attributes[n])
         self.id = None

pyextra/overpy/exception.py

@@ -37,6 +37,18 @@ def __str__(self):
         )
 
 
+class MaxRetriesReached(OverPyException):
+    """
+    Raised if max retries reached and the Overpass server didn't respond with a result.
+    """
+    def __init__(self, retry_count, exceptions):
+        self.exceptions = exceptions
+        self.retry_count = retry_count
+
+    def __str__(self):
+        return "Unable get any result from the Overpass API server after %d retries." % self.retry_count
+
+
 class OverpassBadRequest(OverPyException):
     """
     Raised if the Overpass API service returns a syntax error.
@@ -62,6 +74,29 @@ def __str__(self):
         return "\n".join(tmp_msgs)
 
 
+class OverpassError(OverPyException):
+    """
+    Base exception to report errors if the response returns a remark tag or element.
+    
+    .. note::
+        If you are not sure which of the subexceptions you should use, use this one and try to parse the message.
+
+        For more information have a look at https://github.com/DinoTools/python-overpy/issues/62
+    
+    :param str msg: The message from the remark tag or element
+    """
+    def __init__(self, msg=None):
+        #: The message from the remark tag or element
+        self.msg = msg
+
+    def __str__(self):
+        if self.msg is None:
+            return "No error message provided"
+        if not isinstance(self.msg, str):
+            return str(self.msg)
+        return self.msg
+
+
 class OverpassGatewayTimeout(OverPyException):
     """
     Raised if load of the Overpass API service is too high and it can't handle the request.
@@ -70,6 +105,22 @@ def __init__(self):
         OverPyException.__init__(self, "Server load too high")
 
 
+class OverpassRuntimeError(OverpassError):
+    """
+    Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with
+    'runtime error:'.
+    """
+    pass
+
+
+class OverpassRuntimeRemark(OverpassError):
+    """
+    Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with
+    'runtime remark:'.
+    """
+    pass
+
+
 class OverpassTooManyRequests(OverPyException):
     """
     Raised if the Overpass API service returns a 429 status code.
@@ -94,6 +145,13 @@ def __str__(self):
         return "Unknown content type: %s" % self.content_type
 
 
+class OverpassUnknownError(OverpassError):
+    """
+    Raised if the server returns a remark-tag(xml) or remark element(json) and we are unable to find any reason.
+    """
+    pass
+
+
 class OverpassUnknownHTTPStatusCode(OverPyException):
     """
     Raised if the returned HTTP status code isn't handled by OverPy.