From 088c6a6af599b33aa2a34865e17cb00134810bef Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 10 Aug 2017 16:05:01 -0400 Subject: [PATCH 1/4] WRN-1841: Directory in /tags without digit in the name are ignored. --- openwebvulndb/common/vcs.py | 3 ++- tests/common_test/vcs_test.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/openwebvulndb/common/vcs.py b/openwebvulndb/common/vcs.py index 8bc9ef0..8edb4f6 100644 --- a/openwebvulndb/common/vcs.py +++ b/openwebvulndb/common/vcs.py @@ -25,6 +25,7 @@ from .errors import ExecutionFailure, DirectoryExpected from .logs import logger from urllib.parse import urljoin, urlparse, urlunparse +import re class Workspace: @@ -264,7 +265,7 @@ async def to_version(self, version): async def list_versions(self): versions = await self.subversion.ls(self.repository) - return [v.strip("/") for v in versions] + return [v.strip("/") for v in versions if re.search("\d", v)] def destroy(self): for path, dirs, files in walk(self.workdir, topdown=False): diff --git a/tests/common_test/vcs_test.py b/tests/common_test/vcs_test.py index 133f8d6..93f39e3 100644 --- a/tests/common_test/vcs_test.py +++ b/tests/common_test/vcs_test.py @@ -422,3 +422,14 @@ async def test_checkout_and_switch(self, fake_future): await workspace.to_version("1.1") svn.switch.assert_called_with("https://svn.example.com/1.1", workdir="/tmp/foo") + + @async_test() + async def test_list_versions_skip_versions_without_digit(self, fake_future): + svn = MagicMock() + svn.ls.return_value = fake_future(["1.0/", "1.1/", "trunk/"]) + + workspace = SubversionWorkspace(workdir="/tmp/foo", subversion=svn, repository="https://svn.test.com/") + + versions = await workspace.list_versions() + + self.assertEqual(versions, ["1.0", "1.1"]) From 53c4abff798239c35f121b158f354607eb54085a Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 15 Aug 2017 10:41:13 -0400 Subject: [PATCH 2/4] Fix parallel workers blocking when tasks raise exception. --- openwebvulndb/common/parallel.py | 2 ++ openwebvulndb/wordpress/repository.py | 1 - tests/common_test/parallel_test.py | 18 +++++++++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/openwebvulndb/common/parallel.py b/openwebvulndb/common/parallel.py index 898d412..039e4c1 100644 --- a/openwebvulndb/common/parallel.py +++ b/openwebvulndb/common/parallel.py @@ -51,6 +51,8 @@ async def consume(self, n): self._handle_task_timeout(task) else: await coroutine(*args, **kwargs) + except Exception as e: + logger.warn("Unexpected exception in {} {}: {}".format(self.name, n, repr(e))) finally: self.queue.task_done() diff --git a/openwebvulndb/wordpress/repository.py b/openwebvulndb/wordpress/repository.py index 94fc49b..67d37de 100644 --- a/openwebvulndb/wordpress/repository.py +++ b/openwebvulndb/wordpress/repository.py @@ -60,7 +60,6 @@ async def fetch(self, url, parser): try: async with self.session.get(url) as response: data = await response.text() - response.close() return parser.parse(data) except SoftwareNotFound: raise diff --git a/tests/common_test/parallel_test.py b/tests/common_test/parallel_test.py index fe67899..5ddecb9 100644 --- a/tests/common_test/parallel_test.py +++ b/tests/common_test/parallel_test.py @@ -17,8 +17,10 @@ from unittest import TestCase from fixtures import async_test +import async_timeout +from aiohttp.test_utils import make_mocked_coro -from openwebvulndb.common.parallel import BackgroundRunner +from openwebvulndb.common.parallel import BackgroundRunner, ParallelWorker class ParallelTest(TestCase): @@ -43,3 +45,17 @@ async def test_configured_runner(self, loop): def test_no_loop_uses_default(self): runner = BackgroundRunner(None) self.assertIs(runner.run, BackgroundRunner.default) + + @async_test() + async def test_consume_do_not_block_on_exception(self, loop): + async def coro0(): + raise Exception() + + coro = make_mocked_coro() + worker = ParallelWorker(1, loop=loop) + await worker.request(coro0) + await worker.request(coro) + with async_timeout.timeout(timeout=0.01, loop=loop): + await worker.wait() + + coro.assert_called_once_with() From 09cc457ba823410c8daf3c1b038a29f9c601ca61 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 15 Aug 2017 10:44:53 -0400 Subject: [PATCH 3/4] minor test refactoring --- tests/common_test/parallel_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common_test/parallel_test.py b/tests/common_test/parallel_test.py index 5ddecb9..3673d40 100644 --- a/tests/common_test/parallel_test.py +++ b/tests/common_test/parallel_test.py @@ -48,12 +48,12 @@ def test_no_loop_uses_default(self): @async_test() async def test_consume_do_not_block_on_exception(self, loop): - async def coro0(): + async def coro_with_exception(): raise Exception() coro = make_mocked_coro() worker = ParallelWorker(1, loop=loop) - await worker.request(coro0) + await worker.request(coro_with_exception) await worker.request(coro) with async_timeout.timeout(timeout=0.01, loop=loop): await worker.wait() From d2eb4ee272850d0529b6835966a47db1c25bb230 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 15 Aug 2017 15:15:29 -0400 Subject: [PATCH 4/4] WRN-1841: log exception with logger.exception. --- openwebvulndb/common/parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwebvulndb/common/parallel.py b/openwebvulndb/common/parallel.py index 039e4c1..a70fe88 100644 --- a/openwebvulndb/common/parallel.py +++ b/openwebvulndb/common/parallel.py @@ -52,7 +52,7 @@ async def consume(self, n): else: await coroutine(*args, **kwargs) except Exception as e: - logger.warn("Unexpected exception in {} {}: {}".format(self.name, n, repr(e))) + logger.exception(e) finally: self.queue.task_done()