Merged in WRN-1934 (pull request #12)

WRN-1934

Approved-by: Louis-Philippe Huberdeau <lp@huberdeau.info>

openwebvulndb/common/vcs.py

@@ -21,11 +21,16 @@
 from os.path import join
 from os import mkdir, walk, rmdir, remove
 from contextlib import contextmanager
+from urllib.parse import urljoin, urlparse, urlunparse
+import re
+from datetime import datetime
 
 from .errors import ExecutionFailure, DirectoryExpected
 from .logs import logger
-from urllib.parse import urljoin, urlparse, urlunparse
-import re
+
+
+line_pattern = re.compile("(?P<revision>\d+)\s+(?P<author>[\w\s\.-]+)\s+(?P<month>[A-Z][a-z]{2})\s+"
+                          "(?P<day>\d{2})\s+(?:(?P<year>\d{4})|(?P<time>\d\d:\d\d))\s+(?P<component>\S+)/$")
 
 
 class Workspace:
@@ -84,7 +89,7 @@ async def ls(self, url):
         except asyncio.TimeoutError:
             raise ExecutionFailure('Timeout reached')
 
-    async def read_lines(self, command):
+    async def read_lines(self, command, *, ignore_errors=False):
         process = await create_subprocess_exec(
             *command,
             loop=self.loop,
@@ -101,7 +106,7 @@ async def read_lines(self, command):
 
         # No need to wait for a long time, we're at EOF
         code = await process.wait()
-        if code == 0:
+        if code == 0 or ignore_errors:
             return out
 
         raise ExecutionFailure("Listing failure")
@@ -201,6 +206,50 @@ async def info(self, path, *, workdir):
         root = out.rstrip("\n")
         return {"url": url, "root": root}
 
+    async def get_plugins_with_new_release(self, date):
+        return await self.get_components_with_new_release("plugins", "http://plugins.svn.wordpress.org/", date)
+
+    async def get_themes_with_new_release(self, date):
+        return await self.get_components_with_new_release("themes", "http://themes.svn.wordpress.org/", date)
+
+    async def get_components_with_new_release(self, key, repository_url, date):
+        try:
+            components_update_date = await self._get_last_release_date_of_components(key, repository_url)
+            components = set()
+            for key, update_date in components_update_date.items():
+                if update_date >= date:
+                    components.add(key)
+            return components
+        except ExecutionFailure as e:
+            logger.warn("A command failed to execute: %s", e)
+            return set()
+
+    async def _get_last_release_date_of_components(self, key, repository_url):
+
+        def parse_line(line):
+            line = line.lstrip()  # Remove whitespace added at beginning of line when revision has less digits.
+            match = line_pattern.match(line)
+            if match:
+                component_key, day, month, year = match.group("component", "day", "month", "year")
+                if component_key is not ".":
+                    component_key = "%s/%s" % (key, component_key)
+                    year = datetime.today().year if year is None else year
+                    date = datetime.strptime("%s %s %s" % (day, month, year), "%d %b %Y")
+                    return component_key, date
+            return None, None
+
+        try:
+            command = ["svn", "ls", "-v", "^/tags", repository_url]
+            out = await asyncio.wait_for(self.read_lines(command, ignore_errors=True), timeout=60, loop=self.loop)
+            update_dates = {}
+            for line in out:
+                component_key, date = parse_line(line)
+                if component_key is not None:
+                    update_dates[component_key] = date.date()
+            return update_dates
+        except asyncio.TimeoutError:
+            raise ExecutionFailure('Timeout reached')
+
     async def _process(self, command, workdir):
         process = await create_subprocess_exec(
             *command,

openwebvulndb/wordpress/__main__.py

@@ -18,7 +18,7 @@
 import os
 from argparse import ArgumentParser
 from os.path import join
-from random import shuffle
+from datetime import date, timedelta
 
 from openwebvulndb import app
 from openwebvulndb.common.release import GitHubRelease
@@ -97,7 +97,7 @@ def vane2_export(storage, aiohttp_session, loop, create_release=False, target_co
     aiohttp_session.close()
 
 
-def populate_versions(loop, repository_hasher, storage):
+def populate_versions(loop, repository_hasher, storage, subversion, interval):
     async def load_input():
         worker = ParallelWorker(8, loop=loop, timeout_per_job=1800)  # Half an hour at most
         meta = storage.read_meta("wordpress")
@@ -106,13 +106,16 @@ async def load_input():
         meta = storage.read_meta("mu")
         await worker.request(repository_hasher.collect_from_meta, meta)
 
-        # When restarting the job, shuffle so that we don't spend so much time doing those already done
-        task_list = list(storage.list_meta("plugins")) + list(storage.list_meta("themes"))
-        shuffle(task_list)
+        plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval))
+        themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval))
+        task_list = plugins | themes
+        metas = list(storage.list_meta("themes")) + list(storage.list_meta("themes"))
+        existing_keys = {meta.key for meta in metas}
+        task_list &= existing_keys
 
-        for meta in task_list:
+        for key in task_list:
+            meta = storage.read_meta(key)
             await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}")
-
         await worker.wait()
 
     loop.run_until_complete(load_input())
@@ -164,6 +167,8 @@ def change_version_format(storage):
 parser.add_argument('--target-commitish', dest='target_commitish', help='Branch name or SHA number of the commit used '
                                                                         'for the new release')
 parser.add_argument('--release-version', dest='release_version', help='print version of the new release')
+parser.add_argument('--interval', dest='interval', help='The interval in days since the last update of plugins and '
+                                                        'themes versions. 30 days by default', default=30, type=int)
 
 args = parser.parse_args()
 
@@ -177,7 +182,8 @@ def change_version_format(storage):
                     dest_folder=args.dest_folder,
                     create_release=args.create_release,
                     target_commitish=args.target_commitish,
-                    release_version=args.release_version)
+                    release_version=args.release_version,
+                    interval=args.interval)
     local.call(operations[args.action])
 except KeyboardInterrupt:
     pass

tests/common_test/vcs_test.py

@@ -18,7 +18,8 @@
 import asyncio
 from unittest import TestCase
 from unittest.mock import MagicMock, call, patch
-from fixtures import async_test, fake_future, file_path
+from fixtures import async_test, fake_future, file_path, freeze_time
+from datetime import date
 
 from openwebvulndb.common import RepositoryChecker, Repository
 from openwebvulndb.common.vcs import Subversion, SubversionWorkspace
@@ -325,6 +326,70 @@ async def test_svn_info(self, loop):
             self.assertEqual(info, {"url": "https://plugins.svn.wordpress.org/plugin/tags/1.0",
                                     "root": "https://plugins.svn.wordpress.org"})
 
+    @async_test()
+    async def test_svn_get_last_release_date_of_components_return_last_modification_date_of_tags_folder(self, loop):
+        with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
+            proc = MagicMock()
+            proc.stdout.readline.side_effect = [
+                fake_future(b"svn: warning: W160013: URL 'http://themes.svn.wordpress.org/tags' non-existent in revision 83065", loop=loop),
+                fake_future(b"1749964 user1               Oct 20 11:15 ./\n", loop=loop),
+                fake_future(b"1077807 user 2              Jan 28  2015 plugin-1/\n", loop=loop),
+                fake_future(b"1385952 user.3              Apr 04  2016 plugin-2/", loop=loop),
+                fake_future(b"svn: E200009: Could not list all targets because some targets don't exist", loop=loop)]
+            proc.stdout.at_eof.side_effect = [False, False, False, False, False, True]
+            proc.wait.return_value = fake_future(0, loop=loop)
+            cse.return_value = fake_future(proc, loop=loop)
+            svn = Subversion(loop=loop)
+
+            plugins = await svn._get_last_release_date_of_components("plugins", "http://plugins.svn.wordpress.org/")
+
+            cse.assert_has_calls([call(*("svn", "ls", "-v", "^/tags", "http://plugins.svn.wordpress.org/"), loop=loop,
+                                       stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
+                                       stdin=asyncio.subprocess.DEVNULL)])
+            self.assertEqual(plugins, {"plugins/plugin-1": date(year=2015, month=1, day=28),
+                                       "plugins/plugin-2": date(year=2016, month=4, day=4)})
+
+    @async_test()
+    async def test_svn_get_last_release_date_of_components_replace_hours_with_current_year(self, loop):
+        with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
+            proc = MagicMock()
+            proc.stdout.readline.return_value = \
+                fake_future(b"1749964 user1               Oct 20 11:15 plugin-1/\n", loop=loop)
+            proc.stdout.at_eof.side_effect = [False, True]
+            proc.wait.return_value = fake_future(0, loop=loop)
+            cse.return_value = fake_future(proc, loop=loop)
+            svn = Subversion(loop=loop)
+
+            plugins = await svn._get_last_release_date_of_components("plugins", "http://plugins.svn.wordpress.org/")
+
+            self.assertEqual(plugins, {"plugins/plugin-1": date(year=date.today().year, month=10, day=20)})
+
+    @freeze_time(date(year=2017, day=22, month=10))
+    @async_test()
+    async def test_svn_get_components_with_new_release(self, loop):
+        themes = {"themes/theme-0": date(year=2017, month=10, day=20),
+                   "themes/theme-1": date(year=2016, month=4, day=4),
+                   "themes/theme-2": date(year=2015, month=10, day=21),
+                   "themes/theme-3": date(year=2017, month=10, day=6)}
+        svn = Subversion(loop=None)
+        svn._get_last_release_date_of_components = MagicMock(return_value=fake_future(themes, loop=loop))
+
+        recently_updated = await svn.get_components_with_new_release("themes", "http://themes.svn.wordpress.org/",
+                                                                     date(year=2017, day=6, month=10))
+
+        self.assertEqual(recently_updated, {"themes/theme-0", "themes/theme-3"})
+
+    @async_test()
+    async def test_svn_get_components_with_new_release_return_empty_set_if_command_timeout(self, loop):
+        svn = Subversion(loop=None)
+        fut = asyncio.Future(loop=loop)
+        fut.set_exception(ExecutionFailure("Timeout reached"))
+        svn._get_last_release_date_of_components = MagicMock(return_value=fut)
+
+        result = await svn.get_components_with_new_release("plugins", "http://plugins.svn.example.com/", date.today())
+
+        self.assertEqual(result, set())
+
 
 class SubversionWorkspaceTest(TestCase):