diff --git a/openwebvulndb/common/parallel.py b/openwebvulndb/common/parallel.py index 898d412..a70fe88 100644 --- a/openwebvulndb/common/parallel.py +++ b/openwebvulndb/common/parallel.py @@ -51,6 +51,8 @@ async def consume(self, n): self._handle_task_timeout(task) else: await coroutine(*args, **kwargs) + except Exception as e: + logger.exception(e) finally: self.queue.task_done() diff --git a/openwebvulndb/common/vcs.py b/openwebvulndb/common/vcs.py index 8bc9ef0..8edb4f6 100644 --- a/openwebvulndb/common/vcs.py +++ b/openwebvulndb/common/vcs.py @@ -25,6 +25,7 @@ from .errors import ExecutionFailure, DirectoryExpected from .logs import logger from urllib.parse import urljoin, urlparse, urlunparse +import re class Workspace: @@ -264,7 +265,7 @@ async def to_version(self, version): async def list_versions(self): versions = await self.subversion.ls(self.repository) - return [v.strip("/") for v in versions] + return [v.strip("/") for v in versions if re.search("\d", v)] def destroy(self): for path, dirs, files in walk(self.workdir, topdown=False): diff --git a/openwebvulndb/wordpress/repository.py b/openwebvulndb/wordpress/repository.py index 94fc49b..67d37de 100644 --- a/openwebvulndb/wordpress/repository.py +++ b/openwebvulndb/wordpress/repository.py @@ -60,7 +60,6 @@ async def fetch(self, url, parser): try: async with self.session.get(url) as response: data = await response.text() - response.close() return parser.parse(data) except SoftwareNotFound: raise diff --git a/tests/common_test/parallel_test.py b/tests/common_test/parallel_test.py index fe67899..3673d40 100644 --- a/tests/common_test/parallel_test.py +++ b/tests/common_test/parallel_test.py @@ -17,8 +17,10 @@ from unittest import TestCase from fixtures import async_test +import async_timeout +from aiohttp.test_utils import make_mocked_coro -from openwebvulndb.common.parallel import BackgroundRunner +from openwebvulndb.common.parallel import BackgroundRunner, ParallelWorker class ParallelTest(TestCase): @@ -43,3 +45,17 @@ async def test_configured_runner(self, loop): def test_no_loop_uses_default(self): runner = BackgroundRunner(None) self.assertIs(runner.run, BackgroundRunner.default) + + @async_test() + async def test_consume_do_not_block_on_exception(self, loop): + async def coro_with_exception(): + raise Exception() + + coro = make_mocked_coro() + worker = ParallelWorker(1, loop=loop) + await worker.request(coro_with_exception) + await worker.request(coro) + with async_timeout.timeout(timeout=0.01, loop=loop): + await worker.wait() + + coro.assert_called_once_with() diff --git a/tests/common_test/vcs_test.py b/tests/common_test/vcs_test.py index 133f8d6..93f39e3 100644 --- a/tests/common_test/vcs_test.py +++ b/tests/common_test/vcs_test.py @@ -422,3 +422,14 @@ async def test_checkout_and_switch(self, fake_future): await workspace.to_version("1.1") svn.switch.assert_called_with("https://svn.example.com/1.1", workdir="/tmp/foo") + + @async_test() + async def test_list_versions_skip_versions_without_digit(self, fake_future): + svn = MagicMock() + svn.ls.return_value = fake_future(["1.0/", "1.1/", "trunk/"]) + + workspace = SubversionWorkspace(workdir="/tmp/foo", subversion=svn, repository="https://svn.test.com/") + + versions = await workspace.list_versions() + + self.assertEqual(versions, ["1.0", "1.1"])