From 0c94a9a6947c05e1efe5639d5534346c60c204f1 Mon Sep 17 00:00:00 2001 From: Pierre Delagrave Date: Tue, 16 Mar 2021 13:53:04 -0400 Subject: [PATCH] Updated libraries, supporting python 3.8 (#3) Kept marshmallow 2, will upgrade to 3 in a future version --- README.md | 2 +- requirements-dev.txt | 1 + requirements.txt | 10 ++++---- setup.py | 19 ++++++++------- tests/common_test/vcs_test.py | 23 ++++++++++--------- tests/fixtures.py | 5 ++-- .../wordpress_test/enumerate_plugins_test.py | 4 ++-- tests/wordpress_test/enumerate_themes_test.py | 4 ++-- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index eb0cba8..afca9cf 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ databases. General python set-up: ``` -virtualenv -ppython3.5 . +virtualenv -ppython3.8 . source bin/activate pip install -r requirements.txt pip install -r requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt index 9eccdff..ee51f06 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ nose==1.3.7 +freezegun==1.1.0 diff --git a/requirements.txt b/requirements.txt index 5b982b6..0c43340 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -aiohttp>3.1,<3.3 +aiohttp>=3.7.3,<4.0 easyinject==0.3 -marshmallow==2.15.6 -packaging==16.7 -lxml>=3.6.4,<5 -yarl>1.4,<1.5 +marshmallow>=2.21.0,<3 +packaging==20.9 +lxml==4.6.2 +yarl==1.6.3 diff --git a/setup.py b/setup.py index 5f9d3dc..b82598f 100755 --- a/setup.py +++ b/setup.py @@ -1,27 +1,26 @@ #!/usr/bin/env python - -try: # for pip >= 10 - from pip._internal.req import parse_requirements -except ImportError: # for pip <= 9.0.3 - from pip.req import parse_requirements from setuptools import setup - version_file = "openwebvulndb/__version__.py" version_data = {} with open(version_file) as f: code = compile(f.read(), version_file, 'exec') exec(code, globals(), version_data) -reqs = [str(x.req) for x in parse_requirements('./requirements.txt', session=False)] - - setup(name='openwebvulndb-tools', version=version_data['__version__'], description='A collection of tools to maintain vulnerability databases.', author='Delve Labs inc.', + python_requires='>=3.8.0,<3.9.0', author_email='info@delvelabs.ca', url='https://github.com/delvelabs/openwebvulndb-tools', packages=['openwebvulndb.common', 'openwebvulndb.wordpress'], license="GPLv2", - install_requires=reqs) + install_requires=[ + "aiohttp>=3.7.3,<4.0", + "easyinject==0.3", + "marshmallow>=2.21.0,<3", + "packaging==20.9", + "lxml==4.6.2", + "yarl==1.6.3" + ]) diff --git a/tests/common_test/vcs_test.py b/tests/common_test/vcs_test.py index 9a94efa..1e70a01 100644 --- a/tests/common_test/vcs_test.py +++ b/tests/common_test/vcs_test.py @@ -113,7 +113,7 @@ async def test_execute_checkout(self, loop): proc = MagicMock() proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop) proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) svn.has_recursive_externals = MagicMock(return_value=fake_future(False, loop)) @@ -134,7 +134,7 @@ async def test_execute_handle_error(self, loop): proc = MagicMock() proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop) proc.returncode = 1 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) svn.has_recursive_externals = MagicMock(return_value=fake_future(False, loop)) @@ -156,7 +156,7 @@ async def test_execute_switch(self, loop): proc = MagicMock() proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop) proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) svn.has_recursive_externals = MagicMock(return_value=fake_future(False, loop)) @@ -177,7 +177,7 @@ async def test_checkout_ignore_externals_if_any_recursive_external(self, loop): proc = MagicMock() proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop) proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) svn.has_recursive_externals = MagicMock(return_value=fake_future(True, loop)) @@ -194,7 +194,7 @@ async def test_switch_ignore_externals_if_any_recursive_external(self, loop): proc = MagicMock() proc.communicate.return_value = fake_future((b"out", b"err"), loop=loop) proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) svn.has_recursive_externals = MagicMock(return_value=fake_future(True, loop)) @@ -230,7 +230,7 @@ async def test_list_externals(self, loop): b"https://plugins.svn.wordpress.org/plugin/tags/1.0/languages - external https://www.some-external.example\n\n" proc.communicate.return_value = fake_future((out, b""), loop=loop) proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) svn.info = MagicMock(return_value=fake_future({"url": "https://plugins.svn.wordpress.org/plugin/tags/1.0", @@ -259,7 +259,7 @@ async def test_list_externals_with_relative_path(self, loop): b"https://svn.example.com/plugins/plugin/tags/1.0/subdir4 - external ^/../../../repo/external\n\n" proc.communicate.return_value = fake_future((out, b""), loop=loop) proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) svn.info = MagicMock(return_value=fake_future({"url": "https://svn.example.com/plugins/plugin/tags/1.0", @@ -290,7 +290,7 @@ async def test_list_externals_no_external(self, loop): proc.communicate.return_value = fake_future( (b"", b""), loop=loop) proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) externals = await svn.list_externals("https://plugins.svn.wordpress.org/plugin/tags/1.0", @@ -310,7 +310,7 @@ async def test_svn_info(self, loop): loop=loop), fake_future((b"https://plugins.svn.wordpress.org\n", b""), loop=loop)] proc.returncode = 0 - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) info = await svn.info("https://plugins.svn.wordpress.org/plugin/tags/1.0", workdir="/tmp/plugin") @@ -319,6 +319,7 @@ async def test_svn_info(self, loop): [call(*("svn", "info", "--show-item", "url", "https://plugins.svn.wordpress.org/plugin/tags/1.0"), cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE), + call().communicate(), call(*("svn", "info", "--show-item", "repos-root-url", "https://plugins.svn.wordpress.org/plugin/tags/1.0"), cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE)]) @@ -338,7 +339,7 @@ async def test_svn_get_last_release_date_of_components_return_last_modification_ fake_future(b"svn: E200009: Could not list all targets because some targets don't exist", loop=loop)] proc.stdout.at_eof.side_effect = [False, False, False, False, False, True] proc.wait.return_value = fake_future(0, loop=loop) - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) plugins = await svn._get_last_release_date_of_components("plugins", "http://plugins.svn.wordpress.org/") @@ -357,7 +358,7 @@ async def test_svn_get_last_release_date_of_components_replace_hours_with_curren fake_future(b"1749964 user1 Oct 20 11:15 plugin-1/\n", loop=loop) proc.stdout.at_eof.side_effect = [False, True] proc.wait.return_value = fake_future(0, loop=loop) - cse.return_value = fake_future(proc, loop=loop) + cse.return_value = proc svn = Subversion(loop=loop) plugins = await svn._get_last_release_date_of_components("plugins", "http://plugins.svn.wordpress.org/") diff --git a/tests/fixtures.py b/tests/fixtures.py index c9bd0d0..af27f87 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -37,14 +37,13 @@ def setup(f): def ClientResponse(method, url, *, - writer=None, continue100=None, timer=None, request_info=None, auto_decompress=True, + writer=None, continue100=None, timer=None, request_info=None, traces=None, loop=None, session=None): return BaseClientResponse(method, url, writer=writer or Mock(), continue100=continue100, timer=timer or TimerNoop(), request_info=request_info or Mock(), - auto_decompress=auto_decompress, traces=traces or [], loop=loop or asyncio.get_event_loop(), session=session or None) @@ -95,7 +94,7 @@ def __setattr__(self, name, value): super().__setattr__(name, value) -class AsyncContextManagerMock(MagicMock): +class AsyncContextManagerMock: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/wordpress_test/enumerate_plugins_test.py b/tests/wordpress_test/enumerate_plugins_test.py index 76934df..0cc1db0 100644 --- a/tests/wordpress_test/enumerate_plugins_test.py +++ b/tests/wordpress_test/enumerate_plugins_test.py @@ -151,7 +151,7 @@ async def test_when_fetch_fails_bad_repo(self, loop): async def test_fetch_plugin_data(self, loop): my_response = ClientResponse('GET', URL('https://api.wordpress.org/plugins/info/1.0/better-wp-security.json')) my_response.status = 200 - my_response.headers = {'Content-Type': 'application/json'} + my_response._headers = {'Content-Type': 'application/json'} my_response._body = read_file(__file__, 'better-wp-security.json').encode('utf8') aiohttp_session = ClientSessionMock(get_response=my_response) @@ -181,7 +181,7 @@ async def test_flag_as_popular_from_api(self, loop): my_response = ClientResponse('GET', URL('http://api.wordpress.org/plugins/info/1.1/?action=query_plugins' '&request[browse]=popular&request[per_page]=200')) my_response.status = 200 - my_response.headers = {'Content-Type': 'application/json'} + my_response._headers = {'Content-Type': 'application/json'} my_response._body = read_file(__file__, 'popular-plugins.json').encode('utf8') aiohttp_session = ClientSessionMock(get_response=my_response) diff --git a/tests/wordpress_test/enumerate_themes_test.py b/tests/wordpress_test/enumerate_themes_test.py index af8dafe..2d4f45a 100644 --- a/tests/wordpress_test/enumerate_themes_test.py +++ b/tests/wordpress_test/enumerate_themes_test.py @@ -153,7 +153,7 @@ async def test_when_fetch_fails_bad_repo(self, loop): async def test_fetch_theme_data(self, loop): my_response = ClientResponse('GET', URL('https://api.wordpress.org/themes/info/1.1/?action=theme_information&request[slug]=twentyeleven')) # noqa my_response.status = 200 - my_response.headers = {'Content-Type': 'application/json'} + my_response._headers = {'Content-Type': 'application/json'} my_response._body = read_file(__file__, 'twentyeleven.json').encode('utf8') aiohttp_session = ClientSessionMock(get_response=my_response) @@ -184,7 +184,7 @@ async def test_flag_as_popular_from_api(self, loop): my_response = ClientResponse('GET', URL('http://api.wordpress.org/themes/info/1.1/?action=query_plugins' '&request[browse]=popular&request[per_page]=100')) my_response.status = 100 - my_response.headers = {'Content-Type': 'application/json'} + my_response._headers = {'Content-Type': 'application/json'} my_response._body = read_file(__file__, 'popular-themes.json').encode('utf8') aiohttp_session = ClientSessionMock(get_response=my_response)