Merged in WRN-2239 (pull request #13)

WRN-2239

Approved-by: Louis-Philippe Huberdeau <lp@huberdeau.info>

README.md

@@ -46,6 +46,18 @@ python -m openwebvulndb.common find_unclosed_vulnerabilities --filter popular
 # Regenerate the Vane WordPress Scanner vulnerability data
 python -m openwebvulndb.wordpress vane_export -i ~/vane/data/
 
+# Export the Vane 2.0 WordPress Scanner vulnerability data.
+# Add Vane 2 data as an asset of a release on the GitHub repository configured in the virtual environment.
+# The environment variables required are:
+#   - VANE2_REPO_NAME=name-of-the-repository
+#   - VANE2_REPO_OWNER=github-username
+#   - VANE2_REPO_PASSWORD=password-or-personal-access-token
+# With no argument, the data will be added to the latest release. To create a new release for the data,
+# use the --create-release option. The current date will be used for the release number. A custom version number can
+# be specified with --release-version
+# --target-commitish can be ignored for now, as the default is master.
+python -m openwebvulndb.wordpress vane2_export [--create-release] [--target-commitish branch|commit] [--release-version]
+
 # Re-load CVE data
 python -m openwebvulndb.wordpress load_cve
 
@@ -54,8 +66,12 @@ python -m openwebvulndb.wordpress list_plugins
 python -m openwebvulndb.wordpress list_themes
 
 # Populate versions (takes a really long time, but you can stop at any point)
-#  - Searches through repositories for new versions and populate file hashes
-python -m openwebvulndb.wordpress populate_versions
+# Searches through repositories updated in the last 30 days and populate versions file hashes.
+# --interval is used to change the default value of 30 days. -w or --wp-only only update WordPress core versions.
+python -m openwebvulndb.wordpress populate_versions [--interval days] [-w | --wp-only]
+
+# Fetch the latest vulnerabilities about WordPress on Security Focus and update the vulnerability database.
+python -m openwebvulndb.wordpress update_securityfocus_database
 ```
 
 # License

openwebvulndb/wordpress/__main__.py

@@ -89,33 +89,33 @@ def vane2_export(storage, aiohttp_session, loop, create_release=False, target_co
     github_release.set_repository_settings(os.environ["VANE2_REPO_OWNER"], os.environ["VANE2_REPO_PASSWORD"],
                                            os.environ["VANE2_REPO_NAME"])
     try:
-        loop.run_until_complete(github_release.release_data(export_path, "vane2_data_", create_release, target_commitish,
-                                                            release_version))
+        loop.run_until_complete(github_release.release_data(export_path, "vane2_data_", create_release,
+                                                            target_commitish, release_version or str(date.today())))
         logger.info("Vane data successfully released.")
     except (Exception, RuntimeError, ValueError) as e:
         logger.exception(e)
     aiohttp_session.close()
 
 
-def populate_versions(loop, repository_hasher, storage, subversion, interval):
+def populate_versions(loop, repository_hasher, storage, subversion, interval, wp_only):
     async def load_input():
         worker = ParallelWorker(8, loop=loop, timeout_per_job=1800)  # Half an hour at most
         meta = storage.read_meta("wordpress")
         await worker.request(repository_hasher.collect_from_meta, meta)
-
-        meta = storage.read_meta("mu")
-        await worker.request(repository_hasher.collect_from_meta, meta)
-
-        plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval))
-        themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval))
-        task_list = plugins | themes
-        metas = list(storage.list_meta("plugins")) + list(storage.list_meta("themes"))
-        existing_keys = {meta.key for meta in metas}
-        task_list &= existing_keys
-
-        for key in task_list:
-            meta = storage.read_meta(key)
-            await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}")
+        if not wp_only:
+            meta = storage.read_meta("mu")
+            await worker.request(repository_hasher.collect_from_meta, meta)
+
+            plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval))
+            themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval))
+            task_list = plugins | themes
+            metas = list(storage.list_meta("plugins")) + list(storage.list_meta("themes"))
+            existing_keys = {meta.key for meta in metas}
+            task_list &= existing_keys
+
+            for key in task_list:
+                meta = storage.read_meta(key)
+                await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}")
         await worker.wait()
 
     loop.run_until_complete(load_input())
@@ -159,16 +159,19 @@ def change_version_format(storage):
 parser.add_argument("--pages-to-fetch", dest="vulnerabilities_pages_to_fetch",
                     help="Amount of pages of latest vulnerabilities on security focus website to fetch to update "
                          "the database (1 by default, -1 for all pages).", default=1, type=int)
-parser.add_argument('-i', '--input-path', dest='input_path',
-                    help='Data source path (vane import)')
-parser.add_argument('-f', '--input-file', dest='input_file',
-                    help='Cached input file')
-parser.add_argument('--create-release', dest='create_release', action='store_true', help='Create a new GitHub release')
-parser.add_argument('--target-commitish', dest='target_commitish', help='Branch name or SHA number of the commit used '
-                                                                        'for the new release')
-parser.add_argument('--release-version', dest='release_version', help='print version of the new release')
-parser.add_argument('--interval', dest='interval', help='The interval in days since the last update of plugins and '
-                                                        'themes versions. 30 days by default', default=30, type=int)
+parser.add_argument("-i", "--input-path", dest="input_path",
+                    help="Data source path (vane import)")
+parser.add_argument("-f", "--input-file", dest="input_file",
+                    help="Cached input file")
+parser.add_argument("--create-release", dest="create_release", action="store_true", help="Create a new GitHub release")
+parser.add_argument("--target-commitish", dest="target_commitish", help="Branch name or SHA number of the commit used "
+                                                                        "for the new release", default="master")
+parser.add_argument("--release-version", dest="release_version", help="Version number for the new release. The "
+                                                                      "current is used by default.")
+parser.add_argument("--interval", dest="interval", help="The interval in days since the last update of plugins and "
+                                                        "themes versions. 30 days by default", default=30, type=int)
+parser.add_argument("-w", "--wp-only", dest="wp_only", help="Only populate versions for WordPress core, skip plugins "
+                                                            "and themes", action="store_true")
 
 args = parser.parse_args()
 
@@ -183,7 +186,9 @@ def change_version_format(storage):
                     create_release=args.create_release,
                     target_commitish=args.target_commitish,
                     release_version=args.release_version,
-                    interval=args.interval)
+                    interval=args.interval,
+                    wp_only=args.wp_only,
+                    )
     local.call(operations[args.action])
 except KeyboardInterrupt:
     pass