Use requests instead of urllib behind six

pierky/arouteserver/arin_db_dump.py

@@ -17,11 +17,9 @@
 import json
 import logging
 import os
+import requests
 from packaging import version
 
-from six.moves.urllib.request import urlopen
-from six.moves.urllib.error import HTTPError
-
 from .ipaddresses import IPNetwork
 from .cached_objects import CachedObject
 from .errors import ARINWhoisDBDumpError
@@ -129,8 +127,8 @@ def _get_data(self):
 
             url = self.source
             try:
-                response = urlopen(url).read()
-            except HTTPError as e:
+                response = requests.get(url).content
+            except requests.exceptions.HTTPError as e:
                 raise ARINWhoisDBDumpError(
                     "HTTP error while retrieving ARIN Whois DB dump "
                     "from {}: "

pierky/arouteserver/euro_ix.py

@@ -16,8 +16,8 @@
 import logging
 import json
 import re
+import requests
 import six
-from six.moves.urllib.request import urlopen
 
 from .peering_db import PeeringDBNet, PeeringDBNoInfoError
 from .errors import EuroIXError, EuroIXSchemaError
@@ -47,10 +47,11 @@ def __init__(self, input_object, cache_dir, cache_expiry):
         if isinstance(input_object, dict):
             self.raw_data = input_object
         elif isinstance(input_object, six.string_types):
+            response = requests.get(input_object)
+            raw = response.content.decode("utf-8")
             try:
-                response = urlopen(input_object)
-                raw = response.read().decode("utf-8")
-            except Exception as e:
+                response.raise_for_status()
+            except requests.exceptions.HTTPError as e:
                 raise EuroIXError(
                     "Error while retrieving Euro-IX "
                     "JSON file from {}: {}".format(

pierky/arouteserver/peering_db.py

@@ -16,8 +16,7 @@
 import logging
 import json
 import re
-from six.moves.urllib.request import urlopen
-from six.moves.urllib.error import HTTPError
+import requests
 
 from .cached_objects import CachedObject
 from .config.validators import ValidatorASSet
@@ -36,9 +35,10 @@ def _get_peeringdb_url(self):
 
     @staticmethod
     def _read_from_url(url):
+        response = requests.get(url)
         try:
-            response = urlopen(url)
-        except HTTPError as e:
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
             if e.code == 404:
                 return "{}"
             else:
@@ -55,7 +55,7 @@ def _read_from_url(url):
                 )
             )
 
-        return response.read().decode("utf-8")
+        return response.content.decode("utf-8")
 
     def _get_data_from_peeringdb(self):
         plain_text = self._read_from_url(self._get_peeringdb_url())

pierky/arouteserver/ripe_rpki_cache.py

@@ -16,8 +16,7 @@
 import json
 import logging
 
-from six.moves.urllib.request import urlopen
-from six.moves.urllib.error import HTTPError
+import requests
 
 from .cached_objects import CachedObject
 from .errors import RPKIValidatorCacheError
@@ -50,9 +49,12 @@ def _get_object_filename(self):
 
     def _get_data(self):
         if self.url.lower().startswith(("http://", "https://")):
+            response = requests.get(self.url,
+                                    headers={'Accept': 'text/json'})
             try:
-                raw = urlopen(self.url)
-            except HTTPError as e:
+                response.raise_for_status()
+                raw = response.content
+            except requests.exceptions.HTTPError as e:
                 raise RPKIValidatorCacheError(
                     "HTTP error while retrieving ROAs from "
                     "RIPE RPKI Validator cache ({}): "
@@ -69,7 +71,7 @@ def _get_data(self):
                 )
         else:
             try:
-                raw = open(self.url, "rb")
+                raw = open(self.url, "rb").read()
             except Exception as e:
                 raise RPKIValidatorCacheError(
                     "Error while reading ROAs from file "
@@ -79,7 +81,7 @@ def _get_data(self):
                 )
 
         try:
-            roas = json.loads(raw.read().decode("utf-8"))
+            roas = json.loads(raw.decode("utf-8"))
         except Exception as e:
             raise RPKIValidatorCacheError(
                 "Error while parsing ROAs from "

requirements.txt

@@ -4,5 +4,6 @@ Jinja2>=2.9.4
 mock>=2.0.0;python_version<"3.3"
 nose>=1.3.7
 PyYAML>=3.12
+requests>=2.18.4
 packaging
 six