Add timeout for listing of plugins/themes update date.

openwebvulndb/common/vcs.py

@@ -29,6 +29,10 @@
 from .logs import logger
 
 
+line_pattern = re.compile("(?P<revision>\d+)\s+(?P<username>[\w\s\.-]+)\s+(?P<month>[A-Z][a-z]{2})\s+"
+                          "(?P<day>\d{2})\s+(?:(?P<year>\d{4})|(?P<time>\d\d:\d\d))\s+(?P<component>\S+)/$")
+
+
 class Workspace:
 
     async def to_version(self, version):
@@ -209,29 +213,42 @@ async def get_themes_with_new_release(self, date):
         return await self.get_components_with_new_release("themes", "http://themes.svn.wordpress.org/", date)
 
     async def get_components_with_new_release(self, key, repository_url, date):
-        components_update_date = await self._get_last_release_date_of_components(key, repository_url)
-        components = set()
-        for key, update_date in components_update_date.items():
-            if update_date >= date:
-                components.add(key)
-        return components
+        try:
+            components_update_date = await self._get_last_release_date_of_components(key, repository_url)
+            components = set()
+            for key, update_date in components_update_date.items():
+                if update_date >= date:
+                    components.add(key)
+            return components
+        except ExecutionFailure as e:
+            logger.warn("A command failed to execute: %s", e)
+            return set()
 
     async def _get_last_release_date_of_components(self, key, repository_url):
-        out = await self.read_lines(["svn", "ls", "-v", "^/tags", repository_url], ignore_errors=True)
-        line_pattern = re.compile("(?P<revision>\d+)\s+(?P<username>[\w\s\.-]+)\s+(?P<month>[A-Z][a-z]{2})\s+"
-                                  "(?P<day>\d{2})\s+(?:(?P<year>\d{4})|(?P<time>\d\d:\d\d))\s+(?P<component>\S+)/$")
-        update_dates = {}
-        for line in out:
-            line = line.lstrip()  # Themes are listed with whitespace at beginning because revision has less digits.
+
+        def parse_line(line):
+            line = line.lstrip()  # Whitespace at beginning of line when revision has less digits.
             match = line_pattern.match(line)
             if match:
                 component_key, day, month, year = match.group("component", "day", "month", "year")
                 if component_key is not ".":
                     component_key = "%s/%s" % (key, component_key)
                     year = datetime.today().year if year is None else year
                     date = datetime.strptime("%s %s %s" % (day, month, year), "%d %b %Y")
+                    return component_key, date
+            return None, None
+
+        try:
+            command = ["svn", "ls", "-v", "^/tags", repository_url]
+            out = await asyncio.wait_for(self.read_lines(command, ignore_errors=True), timeout=60, loop=self.loop)
+            update_dates = {}
+            for line in out:
+                component_key, date = parse_line(line)
+                if component_key is not None:
                     update_dates[component_key] = date.date()
-        return update_dates
+            return update_dates
+        except asyncio.TimeoutError:
+            raise ExecutionFailure('Timeout reached')
 
     async def _process(self, command, workdir):
         process = await create_subprocess_exec(

openwebvulndb/wordpress/__main__.py

@@ -18,7 +18,7 @@
 import os
 from argparse import ArgumentParser
 from os.path import join
-from random import shuffle
+from datetime import date, timedelta
 
 from openwebvulndb import app
 from openwebvulndb.common.release import GitHubRelease
@@ -97,7 +97,7 @@ def vane2_export(storage, aiohttp_session, loop, create_release=False, target_co
     aiohttp_session.close()
 
 
-def populate_versions(loop, repository_hasher, storage):
+def populate_versions(loop, repository_hasher, storage, subversion, interval):
     async def load_input():
         worker = ParallelWorker(8, loop=loop, timeout_per_job=1800)  # Half an hour at most
         meta = storage.read_meta("wordpress")
@@ -106,13 +106,16 @@ async def load_input():
         meta = storage.read_meta("mu")
         await worker.request(repository_hasher.collect_from_meta, meta)
 
-        # When restarting the job, shuffle so that we don't spend so much time doing those already done
-        task_list = list(storage.list_meta("plugins")) + list(storage.list_meta("themes"))
-        shuffle(task_list)
+        plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval))
+        themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval))
+        task_list = plugins | themes
+        metas = list(storage.list_meta("themes")) + list(storage.list_meta("themes"))
+        existing_keys = {meta.key for meta in metas}
+        task_list &= existing_keys
 
-        for meta in task_list:
+        for key in task_list:
+            meta = storage.read_meta(key)
             await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}")
-
         await worker.wait()
 
     loop.run_until_complete(load_input())
@@ -164,6 +167,8 @@ def change_version_format(storage):
 parser.add_argument('--target-commitish', dest='target_commitish', help='Branch name or SHA number of the commit used '
                                                                         'for the new release')
 parser.add_argument('--release-version', dest='release_version', help='print version of the new release')
+parser.add_argument('--interval', dest='interval', help='The interval in days since the last update of plugins and '
+                                                        'themes versions. 7 days by default', default=7, type=int)
 
 args = parser.parse_args()
 
@@ -177,7 +182,8 @@ def change_version_format(storage):
                     dest_folder=args.dest_folder,
                     create_release=args.create_release,
                     target_commitish=args.target_commitish,
-                    release_version=args.release_version)
+                    release_version=args.release_version,
+                    interval=args.interval)
     local.call(operations[args.action])
 except KeyboardInterrupt:
     pass

tests/common_test/vcs_test.py

@@ -374,12 +374,22 @@ async def test_svn_get_components_with_new_release(self, loop):
         svn = Subversion(loop=None)
         svn._get_last_release_date_of_components = MagicMock(return_value=fake_future(themes, loop=loop))
 
-        recently_updated = await svn.get_components_with_new_release("themes",
-                                                                              "http://themes.svn.wordpress.org/",
-                                                                              date(year=2017, day=6, month=10))
+        recently_updated = await svn.get_components_with_new_release("themes", "http://themes.svn.wordpress.org/",
+                                                                     date(year=2017, day=6, month=10))
 
         self.assertEqual(recently_updated, {"themes/theme-0", "themes/theme-3"})
 
+    @async_test()
+    async def test_svn_get_components_with_new_release_return_empty_set_if_command_timeout(self, loop):
+        svn = Subversion(loop=None)
+        fut = asyncio.Future(loop=loop)
+        fut.set_exception(ExecutionFailure("Timeout reached"))
+        svn._get_last_release_date_of_components = MagicMock(return_value=fut)
+
+        result = await svn.get_components_with_new_release("plugins", "http://plugins.svn.example.com/", date.today())
+
+        self.assertEqual(result, set())
+
 
 class SubversionWorkspaceTest(TestCase):