diff --git a/.travis.yml b/.travis.yml index 4d80a75..a9abbc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ python: - '3.6' - '3.7' - '3.8' + - '3.9' + - '3.10' install: pip install --ignore-installed -r requirements.txt && pip install --ignore-installed -r requirements-dev.txt script: - nosetests diff --git a/README.md b/README.md index afca9cf..33a708a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ databases. General python set-up: ``` -virtualenv -ppython3.8 . +virtualenv -ppython3.10 . source bin/activate pip install -r requirements.txt pip install -r requirements-dev.txt diff --git a/openwebvulndb/common/parallel.py b/openwebvulndb/common/parallel.py index a70fe88..bc449ac 100644 --- a/openwebvulndb/common/parallel.py +++ b/openwebvulndb/common/parallel.py @@ -28,7 +28,7 @@ class ParallelWorker: def __init__(self, worker_count, *, loop, name="Worker", timeout_per_job=None): self.loop = loop self.name = name - self.queue = asyncio.Queue(loop=loop) + self.queue = asyncio.Queue() self.workers = [loop.create_task(self.consume(i)) for i in range(worker_count)] self.timeout_per_job = timeout_per_job diff --git a/openwebvulndb/common/vcs.py b/openwebvulndb/common/vcs.py index b76bef0..427d8cc 100644 --- a/openwebvulndb/common/vcs.py +++ b/openwebvulndb/common/vcs.py @@ -85,14 +85,13 @@ def build_ls(url): async def ls(self, url): try: command = self.build_ls(url) - return await asyncio.wait_for(self.read_lines(command), 30.0, loop=self.loop) + return await asyncio.wait_for(self.read_lines(command), 30.0) except asyncio.TimeoutError: raise ExecutionFailure('Timeout reached') async def read_lines(self, command, *, ignore_errors=False): process = await create_subprocess_exec( *command, - loop=self.loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, stdin=asyncio.subprocess.DEVNULL @@ -231,7 +230,7 @@ def parse_line(line): match = line_pattern.match(line) if match: component_key, day, month, year = match.group("component", "day", "month", "year") - if component_key is not ".": + if component_key != ".": component_key = "%s/%s" % (key, component_key) year = datetime.today().year if year is None else year date = datetime.strptime("%s %s %s" % (day, month, year), "%d %b %Y") @@ -240,7 +239,7 @@ def parse_line(line): try: command = ["svn", "ls", "-v", "^/tags", repository_url] - out = await asyncio.wait_for(self.read_lines(command, ignore_errors=True), timeout=60, loop=self.loop) + out = await asyncio.wait_for(self.read_lines(command, ignore_errors=True), timeout=60) update_dates = {} for line in out: component_key, date = parse_line(line) @@ -254,7 +253,6 @@ async def _process(self, command, workdir): process = await create_subprocess_exec( *command, cwd=workdir, - loop=self.loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE diff --git a/requirements-dev.txt b/requirements-dev.txt index ee51f06..a39c618 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ nose==1.3.7 -freezegun==1.1.0 +freezegun==1.2.2 diff --git a/requirements.txt b/requirements.txt index 0c43340..2cb2109 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiohttp>=3.7.3,<4.0 easyinject==0.3 marshmallow>=2.21.0,<3 -packaging==20.9 -lxml==4.6.2 -yarl==1.6.3 +packaging==21.3 +lxml==4.9.1 +yarl==1.8.1 diff --git a/setup.py b/setup.py index 6df0901..01c0fd1 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ version=version_data['__version__'], description='A collection of tools to maintain vulnerability databases.', author='Delve Labs inc.', - python_requires='>=3.6.0,<3.9.0', + python_requires='>=3.6.0,<3.11.0', author_email='info@delvelabs.ca', url='https://github.com/delvelabs/openwebvulndb-tools', packages=['openwebvulndb.common', 'openwebvulndb.wordpress'], @@ -20,7 +20,7 @@ "aiohttp>=3.7.3,<4.0", "easyinject==0.3", "marshmallow>=2.21.0,<3", - "packaging==20.9", - "lxml==4.6.2", - "yarl==1.6.3" + "packaging==21.3", + "lxml==4.9.1", + "yarl==1.8.1" ]) diff --git a/tests/common_test/parallel_test.py b/tests/common_test/parallel_test.py index 3673d40..7dd3fdd 100644 --- a/tests/common_test/parallel_test.py +++ b/tests/common_test/parallel_test.py @@ -55,7 +55,7 @@ async def coro_with_exception(): worker = ParallelWorker(1, loop=loop) await worker.request(coro_with_exception) await worker.request(coro) - with async_timeout.timeout(timeout=0.01, loop=loop): + with async_timeout.timeout(0.01): await worker.wait() coro.assert_called_once_with() diff --git a/tests/common_test/vcs_test.py b/tests/common_test/vcs_test.py index 1e70a01..bc86446 100644 --- a/tests/common_test/vcs_test.py +++ b/tests/common_test/vcs_test.py @@ -121,7 +121,6 @@ async def test_execute_checkout(self, loop): cse.assert_called_with("svn", "checkout", "https://svn.example.com/tags/1.2.3", ".", cwd="/tmp/foobar", - loop=loop, stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) @@ -143,7 +142,6 @@ async def test_execute_handle_error(self, loop): cse.assert_called_with("svn", "checkout", "https://svn.example.com/tags/1.2.3", ".", cwd="/tmp/foobar", - loop=loop, stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) @@ -164,7 +162,6 @@ async def test_execute_switch(self, loop): cse.assert_called_with("svn", "switch", "--ignore-ancestry", "https://svn.example.com/tags/1.2.3", cwd="/tmp/foobar", - loop=loop, stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) @@ -184,7 +181,7 @@ async def test_checkout_ignore_externals_if_any_recursive_external(self, loop): await svn.checkout("https://svn.example.com/tags/1.2.3", workdir="/tmp/foobar") cse.assert_called_with("svn", "checkout", "--ignore-externals", "https://svn.example.com/tags/1.2.3", ".", - cwd="/tmp/foobar", loop=loop, stdout=asyncio.subprocess.PIPE, + cwd="/tmp/foobar", stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) proc.communicate.assert_called_with() @@ -201,7 +198,7 @@ async def test_switch_ignore_externals_if_any_recursive_external(self, loop): await svn.switch("https://svn.example.com/tags/1.2.3", workdir="/tmp/foobar") cse.assert_called_with("svn", "switch", "--ignore-ancestry", "--ignore-externals", - "https://svn.example.com/tags/1.2.3", cwd="/tmp/foobar", loop=loop, + "https://svn.example.com/tags/1.2.3", cwd="/tmp/foobar", stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) proc.communicate.assert_called_with() @@ -241,7 +238,7 @@ async def test_list_externals(self, loop): cse.assert_called_once_with(*("svn", "propget", "-R", "svn:externals", "https://plugins.svn.wordpress.org/plugin/tags/1.0"), - cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, + cwd="/tmp/plugin", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE) self.assertEqual(externals, [{"name": "external", "url": "https://www.some-external.example"}]*4) @@ -270,7 +267,7 @@ async def test_list_externals_with_relative_path(self, loop): cse.assert_called_once_with(*("svn", "propget", "-R", "svn:externals", "https://svn.example.com/plugins/plugin/tags/1.0"), - cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, + cwd="/tmp/plugin", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE) self.assertEqual(externals, [{"name": "external", "url": "https://svn.example.com/external"}, {"name": "external", "url": "https://svn.example.com/external"}, @@ -298,7 +295,7 @@ async def test_list_externals_no_external(self, loop): cse.assert_called_once_with(*("svn", "propget", "-R", "svn:externals", "https://plugins.svn.wordpress.org/plugin/tags/1.0"), - cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, + cwd="/tmp/plugin", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE) self.assertEqual(externals, []) @@ -317,11 +314,11 @@ async def test_svn_info(self, loop): cse.assert_has_calls( [call(*("svn", "info", "--show-item", "url", "https://plugins.svn.wordpress.org/plugin/tags/1.0"), - cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + cwd="/tmp/plugin", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE), call().communicate(), call(*("svn", "info", "--show-item", "repos-root-url", "https://plugins.svn.wordpress.org/plugin/tags/1.0"), - cwd="/tmp/plugin", loop=loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + cwd="/tmp/plugin", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE)]) self.assertEqual(info, {"url": "https://plugins.svn.wordpress.org/plugin/tags/1.0", @@ -344,7 +341,7 @@ async def test_svn_get_last_release_date_of_components_return_last_modification_ plugins = await svn._get_last_release_date_of_components("plugins", "http://plugins.svn.wordpress.org/") - cse.assert_has_calls([call(*("svn", "ls", "-v", "^/tags", "http://plugins.svn.wordpress.org/"), loop=loop, + cse.assert_has_calls([call(*("svn", "ls", "-v", "^/tags", "http://plugins.svn.wordpress.org/"), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, stdin=asyncio.subprocess.DEVNULL)]) self.assertEqual(plugins, {"plugins/plugin-1": date(year=2015, month=1, day=28),