diff --git a/README.md b/README.md index a33d643..eb0cba8 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,18 @@ python -m openwebvulndb.common find_unclosed_vulnerabilities --filter popular # Regenerate the Vane WordPress Scanner vulnerability data python -m openwebvulndb.wordpress vane_export -i ~/vane/data/ +# Export the Vane 2.0 WordPress Scanner vulnerability data. +# Add Vane 2 data as an asset of a release on the GitHub repository configured in the virtual environment. +# The environment variables required are: +# - VANE2_REPO_NAME=name-of-the-repository +# - VANE2_REPO_OWNER=github-username +# - VANE2_REPO_PASSWORD=password-or-personal-access-token +# With no argument, the data will be added to the latest release. To create a new release for the data, +# use the --create-release option. The current date will be used for the release number. A custom version number can +# be specified with --release-version +# --target-commitish can be ignored for now, as the default is master. +python -m openwebvulndb.wordpress vane2_export [--create-release] [--target-commitish branch|commit] [--release-version] + # Re-load CVE data python -m openwebvulndb.wordpress load_cve @@ -54,8 +66,12 @@ python -m openwebvulndb.wordpress list_plugins python -m openwebvulndb.wordpress list_themes # Populate versions (takes a really long time, but you can stop at any point) -# - Searches through repositories for new versions and populate file hashes -python -m openwebvulndb.wordpress populate_versions +# Searches through repositories updated in the last 30 days and populate versions file hashes. +# --interval is used to change the default value of 30 days. -w or --wp-only only update WordPress core versions. +python -m openwebvulndb.wordpress populate_versions [--interval days] [-w | --wp-only] + +# Fetch the latest vulnerabilities about WordPress on Security Focus and update the vulnerability database. +python -m openwebvulndb.wordpress update_securityfocus_database ``` # License diff --git a/openwebvulndb/wordpress/__main__.py b/openwebvulndb/wordpress/__main__.py index 52a8d10..7deed82 100644 --- a/openwebvulndb/wordpress/__main__.py +++ b/openwebvulndb/wordpress/__main__.py @@ -89,33 +89,33 @@ def vane2_export(storage, aiohttp_session, loop, create_release=False, target_co github_release.set_repository_settings(os.environ["VANE2_REPO_OWNER"], os.environ["VANE2_REPO_PASSWORD"], os.environ["VANE2_REPO_NAME"]) try: - loop.run_until_complete(github_release.release_data(export_path, "vane2_data_", create_release, target_commitish, - release_version)) + loop.run_until_complete(github_release.release_data(export_path, "vane2_data_", create_release, + target_commitish, release_version or str(date.today()))) logger.info("Vane data successfully released.") except (Exception, RuntimeError, ValueError) as e: logger.exception(e) aiohttp_session.close() -def populate_versions(loop, repository_hasher, storage, subversion, interval): +def populate_versions(loop, repository_hasher, storage, subversion, interval, wp_only): async def load_input(): worker = ParallelWorker(8, loop=loop, timeout_per_job=1800) # Half an hour at most meta = storage.read_meta("wordpress") await worker.request(repository_hasher.collect_from_meta, meta) - - meta = storage.read_meta("mu") - await worker.request(repository_hasher.collect_from_meta, meta) - - plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval)) - themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval)) - task_list = plugins | themes - metas = list(storage.list_meta("plugins")) + list(storage.list_meta("themes")) - existing_keys = {meta.key for meta in metas} - task_list &= existing_keys - - for key in task_list: - meta = storage.read_meta(key) - await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}") + if not wp_only: + meta = storage.read_meta("mu") + await worker.request(repository_hasher.collect_from_meta, meta) + + plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval)) + themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval)) + task_list = plugins | themes + metas = list(storage.list_meta("plugins")) + list(storage.list_meta("themes")) + existing_keys = {meta.key for meta in metas} + task_list &= existing_keys + + for key in task_list: + meta = storage.read_meta(key) + await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}") await worker.wait() loop.run_until_complete(load_input()) @@ -159,16 +159,19 @@ def change_version_format(storage): parser.add_argument("--pages-to-fetch", dest="vulnerabilities_pages_to_fetch", help="Amount of pages of latest vulnerabilities on security focus website to fetch to update " "the database (1 by default, -1 for all pages).", default=1, type=int) -parser.add_argument('-i', '--input-path', dest='input_path', - help='Data source path (vane import)') -parser.add_argument('-f', '--input-file', dest='input_file', - help='Cached input file') -parser.add_argument('--create-release', dest='create_release', action='store_true', help='Create a new GitHub release') -parser.add_argument('--target-commitish', dest='target_commitish', help='Branch name or SHA number of the commit used ' - 'for the new release') -parser.add_argument('--release-version', dest='release_version', help='print version of the new release') -parser.add_argument('--interval', dest='interval', help='The interval in days since the last update of plugins and ' - 'themes versions. 30 days by default', default=30, type=int) +parser.add_argument("-i", "--input-path", dest="input_path", + help="Data source path (vane import)") +parser.add_argument("-f", "--input-file", dest="input_file", + help="Cached input file") +parser.add_argument("--create-release", dest="create_release", action="store_true", help="Create a new GitHub release") +parser.add_argument("--target-commitish", dest="target_commitish", help="Branch name or SHA number of the commit used " + "for the new release", default="master") +parser.add_argument("--release-version", dest="release_version", help="Version number for the new release. The " + "current is used by default.") +parser.add_argument("--interval", dest="interval", help="The interval in days since the last update of plugins and " + "themes versions. 30 days by default", default=30, type=int) +parser.add_argument("-w", "--wp-only", dest="wp_only", help="Only populate versions for WordPress core, skip plugins " + "and themes", action="store_true") args = parser.parse_args() @@ -183,7 +186,9 @@ def change_version_format(storage): create_release=args.create_release, target_commitish=args.target_commitish, release_version=args.release_version, - interval=args.interval) + interval=args.interval, + wp_only=args.wp_only, + ) local.call(operations[args.action]) except KeyboardInterrupt: pass