Merged in WRN-1841 (pull request #11)

WRN-1841

Approved-by: Louis-Philippe Huberdeau <lp@huberdeau.info>

openwebvulndb/common/parallel.py

@@ -51,6 +51,8 @@ async def consume(self, n):
                         self._handle_task_timeout(task)
                 else:
                     await coroutine(*args, **kwargs)
+            except Exception as e:
+                logger.exception(e)
             finally:
                 self.queue.task_done()
 

openwebvulndb/common/vcs.py

@@ -25,6 +25,7 @@
 from .errors import ExecutionFailure, DirectoryExpected
 from .logs import logger
 from urllib.parse import urljoin, urlparse, urlunparse
+import re
 
 
 class Workspace:
@@ -264,7 +265,7 @@ async def to_version(self, version):
 
     async def list_versions(self):
         versions = await self.subversion.ls(self.repository)
-        return [v.strip("/") for v in versions]
+        return [v.strip("/") for v in versions if re.search("\d", v)]
 
     def destroy(self):
         for path, dirs, files in walk(self.workdir, topdown=False):

openwebvulndb/wordpress/repository.py

@@ -60,7 +60,6 @@ async def fetch(self, url, parser):
         try:
             async with self.session.get(url) as response:
                 data = await response.text()
-                response.close()
                 return parser.parse(data)
         except SoftwareNotFound:
             raise

tests/common_test/parallel_test.py

@@ -17,8 +17,10 @@
 
 from unittest import TestCase
 from fixtures import async_test
+import async_timeout
+from aiohttp.test_utils import make_mocked_coro
 
-from openwebvulndb.common.parallel import BackgroundRunner
+from openwebvulndb.common.parallel import BackgroundRunner, ParallelWorker
 
 
 class ParallelTest(TestCase):
@@ -43,3 +45,17 @@ async def test_configured_runner(self, loop):
     def test_no_loop_uses_default(self):
         runner = BackgroundRunner(None)
         self.assertIs(runner.run, BackgroundRunner.default)
+
+    @async_test()
+    async def test_consume_do_not_block_on_exception(self, loop):
+        async def coro_with_exception():
+            raise Exception()
+
+        coro = make_mocked_coro()
+        worker = ParallelWorker(1, loop=loop)
+        await worker.request(coro_with_exception)
+        await worker.request(coro)
+        with async_timeout.timeout(timeout=0.01, loop=loop):
+            await worker.wait()
+
+        coro.assert_called_once_with()

tests/common_test/vcs_test.py

@@ -422,3 +422,14 @@ async def test_checkout_and_switch(self, fake_future):
         await workspace.to_version("1.1")
 
         svn.switch.assert_called_with("https://svn.example.com/1.1", workdir="/tmp/foo")
+
+    @async_test()
+    async def test_list_versions_skip_versions_without_digit(self, fake_future):
+        svn = MagicMock()
+        svn.ls.return_value = fake_future(["1.0/", "1.1/", "trunk/"])
+
+        workspace = SubversionWorkspace(workdir="/tmp/foo", subversion=svn, repository="https://svn.test.com/")
+
+        versions = await workspace.list_versions()
+
+        self.assertEqual(versions, ["1.0", "1.1"])