Updated libraries, supporting python 3.8 (#3)

Kept marshmallow 2, will upgrade to 3 in a future version

README.md

@@ -9,7 +9,7 @@ databases.
 General python set-up:
 
 ```
-virtualenv -ppython3.5 .
+virtualenv -ppython3.8 .
 source bin/activate
 pip install -r requirements.txt
 pip install -r requirements-dev.txt

requirements-dev.txt

@@ -1 +1,2 @@
 nose==1.3.7
+freezegun==1.1.0

requirements.txt

@@ -1,6 +1,6 @@
-aiohttp>3.1,<3.3
+aiohttp>=3.7.3,<4.0
 easyinject==0.3
-marshmallow==2.15.6
-packaging==16.7
-lxml>=3.6.4,<5
-yarl>1.4,<1.5
+marshmallow>=2.21.0,<3
+packaging==20.9
+lxml==4.6.2
+yarl==1.6.3

setup.py

@@ -1,27 +1,26 @@
 #!/usr/bin/env python
-
-try:  # for pip >= 10
-    from pip._internal.req import parse_requirements
-except ImportError:  # for pip <= 9.0.3
-    from pip.req import parse_requirements
 from setuptools import setup
 
-
 version_file = "openwebvulndb/__version__.py"
 version_data = {}
 with open(version_file) as f:
     code = compile(f.read(), version_file, 'exec')
     exec(code, globals(), version_data)
 
-reqs = [str(x.req) for x in parse_requirements('./requirements.txt', session=False)]
-
-
 setup(name='openwebvulndb-tools',
       version=version_data['__version__'],
       description='A collection of tools to maintain vulnerability databases.',
       author='Delve Labs inc.',
+      python_requires='>=3.8.0,<3.9.0',
       author_email='info@delvelabs.ca',
       url='https://github.com/delvelabs/openwebvulndb-tools',
       packages=['openwebvulndb.common', 'openwebvulndb.wordpress'],
       license="GPLv2",
-      install_requires=reqs)
+      install_requires=[
+          "aiohttp>=3.7.3,<4.0",
+          "easyinject==0.3",
+          "marshmallow>=2.21.0,<3",
+          "packaging==20.9",
+          "lxml==4.6.2",
+          "yarl==1.6.3"
+      ])

tests/common_test/vcs_test.py

@@ -113,7 +113,7 @@ async def test_execute_checkout(self, loop):
             proc = MagicMock()
             proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop)
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
             svn.has_recursive_externals = MagicMock(return_value=fake_future(False, loop))
 
@@ -134,7 +134,7 @@ async def test_execute_handle_error(self, loop):
             proc = MagicMock()
             proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop)
             proc.returncode = 1
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
             svn.has_recursive_externals = MagicMock(return_value=fake_future(False, loop))
 
@@ -156,7 +156,7 @@ async def test_execute_switch(self, loop):
             proc = MagicMock()
             proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop)
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
             svn.has_recursive_externals = MagicMock(return_value=fake_future(False, loop))
 
@@ -177,7 +177,7 @@ async def test_checkout_ignore_externals_if_any_recursive_external(self, loop):
             proc = MagicMock()
             proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop)
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
             svn.has_recursive_externals = MagicMock(return_value=fake_future(True, loop))
 
@@ -194,7 +194,7 @@ async def test_switch_ignore_externals_if_any_recursive_external(self, loop):
             proc = MagicMock()
             proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop)
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
             svn.has_recursive_externals = MagicMock(return_value=fake_future(True, loop))
 
@@ -230,7 +230,7 @@ async def test_list_externals(self, loop):
                   b"https://plugins.svn.wordpress.org/plugin/tags/1.0/languages - external https://www.some-external.example\n\n"
             proc.communicate.return_value = fake_future((out, b""), loop=loop)
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
 
             svn = Subversion(loop=loop)
             svn.info = MagicMock(return_value=fake_future({"url": "https://plugins.svn.wordpress.org/plugin/tags/1.0",
@@ -259,7 +259,7 @@ async def test_list_externals_with_relative_path(self, loop):
                   b"https://svn.example.com/plugins/plugin/tags/1.0/subdir4 - external ^/../../../repo/external\n\n"
             proc.communicate.return_value = fake_future((out, b""), loop=loop)
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
 
             svn = Subversion(loop=loop)
             svn.info = MagicMock(return_value=fake_future({"url": "https://svn.example.com/plugins/plugin/tags/1.0",
@@ -290,7 +290,7 @@ async def test_list_externals_no_external(self, loop):
             proc.communicate.return_value = fake_future(
                 (b"", b""), loop=loop)
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
 
             externals = await svn.list_externals("https://plugins.svn.wordpress.org/plugin/tags/1.0",
@@ -310,7 +310,7 @@ async def test_svn_info(self, loop):
                                                         loop=loop),
                                             fake_future((b"https://plugins.svn.wordpress.org\n", b""), loop=loop)]
             proc.returncode = 0
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
 
             info = await svn.info("https://plugins.svn.wordpress.org/plugin/tags/1.0", workdir="/tmp/plugin")
@@ -319,6 +319,7 @@ async def test_svn_info(self, loop):
                 [call(*("svn", "info", "--show-item", "url", "https://plugins.svn.wordpress.org/plugin/tags/1.0"),
                       cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
                       stdin=asyncio.subprocess.PIPE),
+                 call().communicate(),
                  call(*("svn", "info", "--show-item", "repos-root-url", "https://plugins.svn.wordpress.org/plugin/tags/1.0"),
                       cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
                       stdin=asyncio.subprocess.PIPE)])
@@ -338,7 +339,7 @@ async def test_svn_get_last_release_date_of_components_return_last_modification_
                 fake_future(b"svn: E200009: Could not list all targets because some targets don't exist", loop=loop)]
             proc.stdout.at_eof.side_effect = [False, False, False, False, False, True]
             proc.wait.return_value = fake_future(0, loop=loop)
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
 
             plugins = await svn._get_last_release_date_of_components("plugins", "http://plugins.svn.wordpress.org/")
@@ -357,7 +358,7 @@ async def test_svn_get_last_release_date_of_components_replace_hours_with_curren
                 fake_future(b"1749964 user1               Oct 20 11:15 plugin-1/\n", loop=loop)
             proc.stdout.at_eof.side_effect = [False, True]
             proc.wait.return_value = fake_future(0, loop=loop)
-            cse.return_value = fake_future(proc, loop=loop)
+            cse.return_value = proc
             svn = Subversion(loop=loop)
 
             plugins = await svn._get_last_release_date_of_components("plugins", "http://plugins.svn.wordpress.org/")

tests/fixtures.py

@@ -37,14 +37,13 @@ def setup(f):
 
 
 def ClientResponse(method, url, *,
-                   writer=None, continue100=None, timer=None, request_info=None, auto_decompress=True,
+                   writer=None, continue100=None, timer=None, request_info=None,
                    traces=None, loop=None, session=None):
     return BaseClientResponse(method, url,
                               writer=writer or Mock(),
                               continue100=continue100,
                               timer=timer or TimerNoop(),
                               request_info=request_info or Mock(),
-                              auto_decompress=auto_decompress,
                               traces=traces or [],
                               loop=loop or asyncio.get_event_loop(),
                               session=session or None)
@@ -95,7 +94,7 @@ def __setattr__(self, name, value):
         super().__setattr__(name, value)
 
 
-class AsyncContextManagerMock(MagicMock):
+class AsyncContextManagerMock:
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

tests/wordpress_test/enumerate_plugins_test.py

@@ -151,7 +151,7 @@ async def test_when_fetch_fails_bad_repo(self, loop):
     async def test_fetch_plugin_data(self, loop):
         my_response = ClientResponse('GET', URL('https://api.wordpress.org/plugins/info/1.0/better-wp-security.json'))
         my_response.status = 200
-        my_response.headers = {'Content-Type': 'application/json'}
+        my_response._headers = {'Content-Type': 'application/json'}
         my_response._body = read_file(__file__, 'better-wp-security.json').encode('utf8')
 
         aiohttp_session = ClientSessionMock(get_response=my_response)
@@ -181,7 +181,7 @@ async def test_flag_as_popular_from_api(self, loop):
         my_response = ClientResponse('GET', URL('http://api.wordpress.org/plugins/info/1.1/?action=query_plugins'
                                                 '&request[browse]=popular&request[per_page]=200'))
         my_response.status = 200
-        my_response.headers = {'Content-Type': 'application/json'}
+        my_response._headers = {'Content-Type': 'application/json'}
         my_response._body = read_file(__file__, 'popular-plugins.json').encode('utf8')
 
         aiohttp_session = ClientSessionMock(get_response=my_response)

tests/wordpress_test/enumerate_themes_test.py

@@ -153,7 +153,7 @@ async def test_when_fetch_fails_bad_repo(self, loop):
     async def test_fetch_theme_data(self, loop):
         my_response = ClientResponse('GET', URL('https://api.wordpress.org/themes/info/1.1/?action=theme_information&request[slug]=twentyeleven'))  # noqa
         my_response.status = 200
-        my_response.headers = {'Content-Type': 'application/json'}
+        my_response._headers = {'Content-Type': 'application/json'}
         my_response._body = read_file(__file__, 'twentyeleven.json').encode('utf8')
 
         aiohttp_session = ClientSessionMock(get_response=my_response)
@@ -184,7 +184,7 @@ async def test_flag_as_popular_from_api(self, loop):
         my_response = ClientResponse('GET', URL('http://api.wordpress.org/themes/info/1.1/?action=query_plugins'
                                                 '&request[browse]=popular&request[per_page]=100'))
         my_response.status = 100
-        my_response.headers = {'Content-Type': 'application/json'}
+        my_response._headers = {'Content-Type': 'application/json'}
         my_response._body = read_file(__file__, 'popular-themes.json').encode('utf8')
 
         aiohttp_session = ClientSessionMock(get_response=my_response)