From 9f6739f9afba47b3cb551d0a60b5e19d7ee33a59 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 11 Apr 2018 15:48:58 -0400 Subject: [PATCH 1/4] WRN-2239: Add option to only populate WordPress versions --- openwebvulndb/wordpress/__main__.py | 55 +++++++++++++++-------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/openwebvulndb/wordpress/__main__.py b/openwebvulndb/wordpress/__main__.py index 52a8d10..a207948 100644 --- a/openwebvulndb/wordpress/__main__.py +++ b/openwebvulndb/wordpress/__main__.py @@ -97,25 +97,25 @@ def vane2_export(storage, aiohttp_session, loop, create_release=False, target_co aiohttp_session.close() -def populate_versions(loop, repository_hasher, storage, subversion, interval): +def populate_versions(loop, repository_hasher, storage, subversion, interval, wp_only): async def load_input(): worker = ParallelWorker(8, loop=loop, timeout_per_job=1800) # Half an hour at most meta = storage.read_meta("wordpress") await worker.request(repository_hasher.collect_from_meta, meta) - - meta = storage.read_meta("mu") - await worker.request(repository_hasher.collect_from_meta, meta) - - plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval)) - themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval)) - task_list = plugins | themes - metas = list(storage.list_meta("plugins")) + list(storage.list_meta("themes")) - existing_keys = {meta.key for meta in metas} - task_list &= existing_keys - - for key in task_list: - meta = storage.read_meta(key) - await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}") + if not wp_only: + meta = storage.read_meta("mu") + await worker.request(repository_hasher.collect_from_meta, meta) + + plugins = await subversion.get_plugins_with_new_release(date.today() - timedelta(days=interval)) + themes = await subversion.get_themes_with_new_release(date.today() - timedelta(days=interval)) + task_list = plugins | themes + metas = list(storage.list_meta("plugins")) + list(storage.list_meta("themes")) + existing_keys = {meta.key for meta in metas} + task_list &= existing_keys + + for key in task_list: + meta = storage.read_meta(key) + await worker.request(repository_hasher.collect_from_meta, meta, prefix_pattern="wp-content/{meta.key}") await worker.wait() loop.run_until_complete(load_input()) @@ -159,16 +159,18 @@ def change_version_format(storage): parser.add_argument("--pages-to-fetch", dest="vulnerabilities_pages_to_fetch", help="Amount of pages of latest vulnerabilities on security focus website to fetch to update " "the database (1 by default, -1 for all pages).", default=1, type=int) -parser.add_argument('-i', '--input-path', dest='input_path', - help='Data source path (vane import)') -parser.add_argument('-f', '--input-file', dest='input_file', - help='Cached input file') -parser.add_argument('--create-release', dest='create_release', action='store_true', help='Create a new GitHub release') -parser.add_argument('--target-commitish', dest='target_commitish', help='Branch name or SHA number of the commit used ' - 'for the new release') -parser.add_argument('--release-version', dest='release_version', help='print version of the new release') -parser.add_argument('--interval', dest='interval', help='The interval in days since the last update of plugins and ' - 'themes versions. 30 days by default', default=30, type=int) +parser.add_argument("-i", "--input-path", dest="input_path", + help="Data source path (vane import)") +parser.add_argument("-f", "--input-file", dest="input_file", + help="Cached input file") +parser.add_argument("--create-release", dest="create_release", action="store_true", help="Create a new GitHub release") +parser.add_argument("--target-commitish", dest="target_commitish", help="Branch name or SHA number of the commit used " + "for the new release") +parser.add_argument("--release-version", dest="release_version", help="print version of the new release") +parser.add_argument("--interval", dest="interval", help="The interval in days since the last update of plugins and " + "themes versions. 30 days by default", default=30, type=int) +parser.add_argument("-w", "--wp-only", dest="wp_only", help="Only populate versions for WordPress core, skip plugins " + "and themes", action="store_true") args = parser.parse_args() @@ -183,7 +185,8 @@ def change_version_format(storage): create_release=args.create_release, target_commitish=args.target_commitish, release_version=args.release_version, - interval=args.interval) + interval=args.interval, + wp_only=args.wp_only) local.call(operations[args.action]) except KeyboardInterrupt: pass From 1db9ea0bcdc3604a2ed86442ea7bc65fbddd6d69 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 12 Apr 2018 10:38:00 -0400 Subject: [PATCH 2/4] WRN-2239: WRN-2239: update README --- README.md | 16 ++++++++++++++-- openwebvulndb/wordpress/__main__.py | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a33d643..e015d0d 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,14 @@ python -m openwebvulndb.common find_unclosed_vulnerabilities --filter popular # Regenerate the Vane WordPress Scanner vulnerability data python -m openwebvulndb.wordpress vane_export -i ~/vane/data/ +# Export the Vane 2.0 WordPress Scanner vulnerability data. +# Add Vane 2 data as an asset of a release on the GitHub repository configured in the virtual environment. +# With no argument, the data will be added to the latest release. To create a new release for the data, +# use the --create-release option, specify the version with --release-version. +# --target-commitish can be ignored for now, as the default is master. +python -m openwebvulndb.wordpress vane2_export [--create-release] [--target-commitish branch|tag|commit] +[--release-version version] + # Re-load CVE data python -m openwebvulndb.wordpress load_cve @@ -54,8 +62,12 @@ python -m openwebvulndb.wordpress list_plugins python -m openwebvulndb.wordpress list_themes # Populate versions (takes a really long time, but you can stop at any point) -# - Searches through repositories for new versions and populate file hashes -python -m openwebvulndb.wordpress populate_versions +# Searches through repositories updated in the last 30 days and populate versions file hashes. +# --interval is used to change the default value of 30 days. -w or --wp-only only update WordPress core versions. +python -m openwebvulndb.wordpress populate_versions [--interval days] [-w | --wp-only] + +# Fetch the latest vulnerabilities about WordPress on Security Focus and update the vulnerability database. +python -m openwebvulndb.wordpress update_securityfocus_database ``` # License diff --git a/openwebvulndb/wordpress/__main__.py b/openwebvulndb/wordpress/__main__.py index a207948..70b14df 100644 --- a/openwebvulndb/wordpress/__main__.py +++ b/openwebvulndb/wordpress/__main__.py @@ -165,7 +165,7 @@ def change_version_format(storage): help="Cached input file") parser.add_argument("--create-release", dest="create_release", action="store_true", help="Create a new GitHub release") parser.add_argument("--target-commitish", dest="target_commitish", help="Branch name or SHA number of the commit used " - "for the new release") + "for the new release", default="master") parser.add_argument("--release-version", dest="release_version", help="print version of the new release") parser.add_argument("--interval", dest="interval", help="The interval in days since the last update of plugins and " "themes versions. 30 days by default", default=30, type=int) From f3be716e23914bb83e12d8dac90ed504ca5f2ad7 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 12 Apr 2018 14:35:57 -0400 Subject: [PATCH 3/4] WRN-2239: Use date to identify release versions --- README.md | 5 ++--- openwebvulndb/wordpress/__main__.py | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e015d0d..8bca848 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,9 @@ python -m openwebvulndb.wordpress vane_export -i ~/vane/data/ # Export the Vane 2.0 WordPress Scanner vulnerability data. # Add Vane 2 data as an asset of a release on the GitHub repository configured in the virtual environment. # With no argument, the data will be added to the latest release. To create a new release for the data, -# use the --create-release option, specify the version with --release-version. +# use the --create-release option. The current date will be used for the release number. # --target-commitish can be ignored for now, as the default is master. -python -m openwebvulndb.wordpress vane2_export [--create-release] [--target-commitish branch|tag|commit] -[--release-version version] +python -m openwebvulndb.wordpress vane2_export [--create-release] [--target-commitish branch|commit] # Re-load CVE data python -m openwebvulndb.wordpress load_cve diff --git a/openwebvulndb/wordpress/__main__.py b/openwebvulndb/wordpress/__main__.py index 70b14df..34b602e 100644 --- a/openwebvulndb/wordpress/__main__.py +++ b/openwebvulndb/wordpress/__main__.py @@ -59,7 +59,7 @@ def vane_export(vane_importer, storage, input_path): rebuild.write() -def vane2_export(storage, aiohttp_session, loop, create_release=False, target_commitish=None, release_version=None): +def vane2_export(storage, aiohttp_session, loop, create_release=False, target_commitish=None): export_path = EXPORT_PATH os.makedirs(export_path, exist_ok=True) exporter = Exporter(storage) @@ -89,8 +89,8 @@ def vane2_export(storage, aiohttp_session, loop, create_release=False, target_co github_release.set_repository_settings(os.environ["VANE2_REPO_OWNER"], os.environ["VANE2_REPO_PASSWORD"], os.environ["VANE2_REPO_NAME"]) try: - loop.run_until_complete(github_release.release_data(export_path, "vane2_data_", create_release, target_commitish, - release_version)) + loop.run_until_complete(github_release.release_data(export_path, "vane2_data_", create_release, + target_commitish, str(date.today()))) logger.info("Vane data successfully released.") except (Exception, RuntimeError, ValueError) as e: logger.exception(e) @@ -166,7 +166,6 @@ def change_version_format(storage): parser.add_argument("--create-release", dest="create_release", action="store_true", help="Create a new GitHub release") parser.add_argument("--target-commitish", dest="target_commitish", help="Branch name or SHA number of the commit used " "for the new release", default="master") -parser.add_argument("--release-version", dest="release_version", help="print version of the new release") parser.add_argument("--interval", dest="interval", help="The interval in days since the last update of plugins and " "themes versions. 30 days by default", default=30, type=int) parser.add_argument("-w", "--wp-only", dest="wp_only", help="Only populate versions for WordPress core, skip plugins " @@ -184,7 +183,6 @@ def change_version_format(storage): dest_folder=args.dest_folder, create_release=args.create_release, target_commitish=args.target_commitish, - release_version=args.release_version, interval=args.interval, wp_only=args.wp_only) local.call(operations[args.action]) From 47a4ea6ede473f0f52122097c8a73b034fec9821 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 16 Apr 2018 13:16:57 -0400 Subject: [PATCH 4/4] WRN-2239: release-version option is back. --- README.md | 9 +++++++-- openwebvulndb/wordpress/__main__.py | 10 +++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8bca848..eb0cba8 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,15 @@ python -m openwebvulndb.wordpress vane_export -i ~/vane/data/ # Export the Vane 2.0 WordPress Scanner vulnerability data. # Add Vane 2 data as an asset of a release on the GitHub repository configured in the virtual environment. +# The environment variables required are: +# - VANE2_REPO_NAME=name-of-the-repository +# - VANE2_REPO_OWNER=github-username +# - VANE2_REPO_PASSWORD=password-or-personal-access-token # With no argument, the data will be added to the latest release. To create a new release for the data, -# use the --create-release option. The current date will be used for the release number. +# use the --create-release option. The current date will be used for the release number. A custom version number can +# be specified with --release-version # --target-commitish can be ignored for now, as the default is master. -python -m openwebvulndb.wordpress vane2_export [--create-release] [--target-commitish branch|commit] +python -m openwebvulndb.wordpress vane2_export [--create-release] [--target-commitish branch|commit] [--release-version] # Re-load CVE data python -m openwebvulndb.wordpress load_cve diff --git a/openwebvulndb/wordpress/__main__.py b/openwebvulndb/wordpress/__main__.py index 34b602e..7deed82 100644 --- a/openwebvulndb/wordpress/__main__.py +++ b/openwebvulndb/wordpress/__main__.py @@ -59,7 +59,7 @@ def vane_export(vane_importer, storage, input_path): rebuild.write() -def vane2_export(storage, aiohttp_session, loop, create_release=False, target_commitish=None): +def vane2_export(storage, aiohttp_session, loop, create_release=False, target_commitish=None, release_version=None): export_path = EXPORT_PATH os.makedirs(export_path, exist_ok=True) exporter = Exporter(storage) @@ -90,7 +90,7 @@ def vane2_export(storage, aiohttp_session, loop, create_release=False, target_co os.environ["VANE2_REPO_NAME"]) try: loop.run_until_complete(github_release.release_data(export_path, "vane2_data_", create_release, - target_commitish, str(date.today()))) + target_commitish, release_version or str(date.today()))) logger.info("Vane data successfully released.") except (Exception, RuntimeError, ValueError) as e: logger.exception(e) @@ -166,6 +166,8 @@ def change_version_format(storage): parser.add_argument("--create-release", dest="create_release", action="store_true", help="Create a new GitHub release") parser.add_argument("--target-commitish", dest="target_commitish", help="Branch name or SHA number of the commit used " "for the new release", default="master") +parser.add_argument("--release-version", dest="release_version", help="Version number for the new release. The " + "current is used by default.") parser.add_argument("--interval", dest="interval", help="The interval in days since the last update of plugins and " "themes versions. 30 days by default", default=30, type=int) parser.add_argument("-w", "--wp-only", dest="wp_only", help="Only populate versions for WordPress core, skip plugins " @@ -183,8 +185,10 @@ def change_version_format(storage): dest_folder=args.dest_folder, create_release=args.create_release, target_commitish=args.target_commitish, + release_version=args.release_version, interval=args.interval, - wp_only=args.wp_only) + wp_only=args.wp_only, + ) local.call(operations[args.action]) except KeyboardInterrupt: pass