diff --git a/requirements.txt b/requirements.txt index 9f63d1969..90701d507 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,11 @@ asgiref==3.2.7 attrs==19.3.0 +backcall==0.1.0 beautifulsoup4==4.7.1 cached-property==1.5.1 cffi==1.14.0 +contextlib2==0.5.5 +decorator==4.4.2 dephell-specifier==0.2.1 dj-database-url==0.4.2 Django==3.0.3 @@ -10,29 +13,41 @@ django-filter==2.2.0 djangorestframework==3.11.0 gunicorn==19.7.1 importlib-metadata==1.3.0 +ipython==7.13.0 +ipython-genutils==0.2.0 +jedi==0.17.0 lxml==4.3.3 more-itertools==8.0.2 -https://github.com/package-url/packageurl-python/archive/master.zip +packageurl-python==0.9.0 packaging==19.2 +parso==0.7.0 +pexpect==4.8.0 +pickleshare==0.7.5 pluggy==0.13.1 +prompt-toolkit==3.0.5 psycopg2==2.8.4 +ptyprocess==0.6.0 py==1.8.0 pycodestyle==2.5.0 pycparser==2.20 pygit2==1.2.0 +Pygments==2.6.1 pyparsing==2.4.5 pytest==5.3.2 pytest-dependency==0.4.0 pytest-django==3.7.0 pytest-mock==1.13.0 +python-dateutil==2.8.1 pytoml==0.1.21 pytz==2019.3 PyYAML==5.3.1 +saneyaml==0.4 schema==0.7.1 six==1.13.0 soupsieve==1.9.5 sqlparse==0.3.0 tqdm==4.41.1 +traitlets==4.3.3 wcwidth==0.1.7 whitenoise==5.0.1 zipp==0.6.0 diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index ddee1ae75..a1a604558 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -29,6 +29,7 @@ from typing import Tuple import packageurl +from django.db import DataError from vulnerabilities import models from vulnerabilities.data_source import Advisory, DataSource @@ -83,12 +84,18 @@ def run(self, cutoff_date: datetime.datetime = None) -> None: def _process_added_advisories(data_source: DataSource) -> None: for batch in data_source.added_advisories(): - impacted, resolved = _collect_package_urls(batch) - impacted, resolved = _bulk_insert_packages(impacted, resolved) + try: + impacted, resolved = _collect_package_urls(batch) + impacted, resolved = _bulk_insert_packages(impacted, resolved) - vulnerabilities = _insert_vulnerabilities_and_references(batch) + vulnerabilities = _insert_vulnerabilities_and_references(batch) - _bulk_insert_impacted_and_resolved_packages(batch, vulnerabilities, impacted, resolved) + _bulk_insert_impacted_and_resolved_packages(batch, vulnerabilities, impacted, resolved) + except (DataError, RuntimeError) as e: + # FIXME This exception might happen when the max. length of a VARCHAR column is exceeded. + # Skipping an entire batch because one version number might be too long is obviously a terrible way to + # handle this case. + logger.exception(e) def _process_updated_advisories(data_source: DataSource) -> None: @@ -148,10 +155,6 @@ def _get_or_create_vulnerability(advisory: Advisory) -> Tuple[models.Vulnerabili def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]: version = packageurl.normalize_version(p.version, encode=True) - # FIXME terrible hack, remove after https://github.com/package-url/packageurl-python/pull/30 was merged - if len(version) > 50: - version = version[:50] - query_kwargs = { 'name': packageurl.normalize_name(p.name, p.type, encode=True), 'version': version, diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 3f0c1209d..9734e1f2e 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -20,8 +20,9 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from vulnerabilities.importers.archlinux import ArchlinuxDataSource from vulnerabilities.importers.alpine_linux import AlpineDataSource +from vulnerabilities.importers.archlinux import ArchlinuxDataSource from vulnerabilities.importers.debian import DebianDataSource +from vulnerabilities.importers.npm import NpmDataSource from vulnerabilities.importers.rust import RustDataSource from vulnerabilities.importers.safety_db import SafetyDbDataSource diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 133132aec..1a6304c20 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -22,106 +22,219 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import json -from dephell_specifier import RangeSpecifier -from urllib.request import urlopen +from typing import Any +from typing import List +from typing import Mapping +from typing import Set +from typing import Tuple from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import urlopen +from dateutil.parser import parse +from dephell_specifier import RangeSpecifier +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import DataSource NPM_URL = 'https://registry.npmjs.org{}' -PAGE = '/-/npm/v1/security/advisories?page=0' +PAGE = '/-/npm/v1/security/advisories?perPage=100&page=0' -def get_all_versions(package_name): - """ - Returns all versions available for a module - """ - package_name = package_name.strip() - package_url = NPM_URL.format(f'/{package_name}') - try: - with urlopen(package_url) as response: - data = json.load(response) - except HTTPError as e: - if e.code == 404: - return [] - else: - raise - # NPM registry has no data regarding this package, we skip these - return [v for v in data.get('versions', {})] +class NpmDataSource(DataSource): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._api_response = None + self._versions = VersionAPI() + self._added_records, self._updated_records = [], [] + self._added_advisories, self._updated_advisories = [], [] + + def __enter__(self): + self._api_response = self._fetch() + self._categorize_records() + + @property + def versions(self): # quick hack to make it patchable + return self._versions + + def _fetch(self) -> Mapping[str, Any]: + data = None + nextpage = PAGE + while nextpage: + try: + with urlopen(NPM_URL.format(nextpage)) as response: + response = json.load(response) + + if data is None: + data = response + else: + data['objects'].extend(response.get('objects', [])) + + nextpage = response.get('urls', {}).get('next') + + except HTTPError as error: + if error.code == 404: + return data + else: + raise + + return data + + def _categorize_records(self) -> None: + for advisory in self._api_response['objects']: + created = parse(advisory['created']).timestamp() + updated = parse(advisory['updated']).timestamp() + + if created > self.cutoff_timestamp: + self._added_records.append(advisory) + elif updated > self.cutoff_timestamp: + self._updated_records.append(advisory) + + def _parse(self, records: List[Mapping[str, Any]]) -> List[Advisory]: + advisories = [] + + for record in records: + package_name = record['module_name'] + all_versions = self.versions.get(package_name) + aff_range, fixed_range = record.get('vulnerable_versions', ''), record.get('patched_versions', '') + + impacted_versions, resolved_versions = categorize_versions(all_versions, aff_range, fixed_range) + impacted_purls = _versions_to_purls(package_name, impacted_versions) + resolved_purls = _versions_to_purls(package_name, resolved_versions) + + for cve_id in record.get('cves') or ['']: + advisories.append(Advisory( + summary=record.get('overview', ''), + cve_id=cve_id, + impacted_package_urls=impacted_purls, + resolved_package_urls=resolved_purls, + reference_urls=[NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}')], + )) + return advisories -def extract_versions(package_name, aff_version_range, fixed_version_range): + def added_advisories(self) -> Set[Advisory]: + return self.batch_advisories(self._parse(self._added_records)) + + def updated_advisories(self) -> Set[Advisory]: + return self.batch_advisories(self._parse(self._updated_records)) + + +def _versions_to_purls(package_name, versions): + purls = {f'pkg:npm/{quote(package_name)}@{v}' for v in versions} + return {PackageURL.from_string(s) for s in purls} + + +def categorize_versions( + all_versions: Set[str], + aff_version_range: str, + fixed_version_range: str, +) -> Tuple[Set[str], Set[str]]: """ Seperate list of affected versions and unaffected versions from all versions using the ranges specified. + + :return: impacted, resolved versions """ + if not all_versions: + # NPM registry has no data regarding this package, we skip these + return set(), set() + aff_spec = RangeSpecifier(aff_version_range) fix_spec = RangeSpecifier(fixed_version_range) - all_ver = get_all_versions(package_name) - if not all_ver: - # NPM registry has no data regarding this package, we skip these - return ([], []) - aff_ver = [] - fix_ver = [] - # Unaffected version is that version which is in the fixed_version_range + aff_ver, fix_ver = set(), set() + + # Unaffected version is that version which is in the fixed_version_range # or which is absent in the aff_version_range - for ver in all_ver: + for ver in all_versions: if ver in fix_spec or ver not in aff_spec: - fix_ver.append(ver) + fix_ver.add(ver) else: - aff_ver.append(ver) - return (aff_ver, fix_ver) - - -def extract_data(JSON): - """ - Extract package name, summary, CVE IDs, severity and - fixed & affected versions - """ - package_vulnerabilities = [] - for obj in JSON.get('objects', []): - if 'module_name' not in obj: - continue - - package_name = obj['module_name'] - - affected_versions, fixed_versions = extract_versions( - package_name, - obj.get('vulnerable_versions', ''), - obj.get('patched_versions', '') - ) - if not affected_versions and not fixed_versions: - continue - # NPM registry has no data regarding this package finally we skip these - - package_vulnerabilities.append({ - 'package_name': package_name, - 'summary': obj.get('overview', ''), - 'cve_ids': obj.get('cves', []), - 'fixed_versions': fixed_versions, - 'affected_versions': affected_versions, - 'severity': obj.get('severity', ''), - 'advisory': obj.get('url', ''), - }) - return package_vulnerabilities - - -def scrape_vulnerabilities(): - """ - Extract JSON From NPM registry - """ - package_vulnerabilities = [] - nextpage = PAGE - while nextpage: - try: - cururl = NPM_URL.format(nextpage) - response = json.load(urlopen(cururl)) - package_vulnerabilities.extend(extract_data(response)) - nextpage = response.get('urls', {}).get('next') - - except HTTPError as error: - if error.code == 404: - break - else: - raise - - return package_vulnerabilities + aff_ver.add(ver) + + return aff_ver, fix_ver + + +class VersionAPI: + def __init__(self, cache: Mapping[str, Set[str]] = None): + self.cache = cache or {} + + def get(self, package_name: str) -> Set[str]: + """ + Returns all versions available for a module + """ + package_name = package_name.strip() + + if package_name not in self.cache: + releases = set() + try: + with urlopen(f'https://registry.npmjs.org/{package_name}') as response: + data = json.load(response) + releases = {v for v in data.get('versions', {})} + except HTTPError as e: + if e.code == 404: + # NPM registry has no data regarding this package, we skip these + pass + else: + raise + + self.cache[package_name] = releases + + return self.cache[package_name] + + +# def extract_data(JSON): +# """ +# Extract package name, summary, CVE IDs, severity and +# fixed & affected versions +# """ +# package_vulnerabilities = [] +# for obj in JSON.get('objects', []): +# if 'module_name' not in obj: +# continue +# +# package_name = obj['module_name'] +# +# affected_versions, fixed_versions = extract_versions( +# package_name, +# obj.get('vulnerable_versions', ''), +# obj.get('patched_versions', '') +# ) +# if not affected_versions and not fixed_versions: +# continue +# # NPM registry has no data regarding this package finally we skip these +# +# package_vulnerabilities.append({ +# 'package_name': package_name, +# 'summary': obj.get('overview', ''), +# 'cve_ids': obj.get('cves', []), +# 'fixed_versions': fixed_versions, +# 'affected_versions': affected_versions, +# 'severity': obj.get('severity', ''), +# 'advisory': obj.get('url', ''), +# }) +# return package_vulnerabilities +# +# +# def scrape_vulnerabilities(): +# """ +# Extract JSON From NPM registry +# """ +# package_vulnerabilities = [] +# nextpage = PAGE +# while nextpage: +# try: +# cururl = NPM_URL.format(nextpage) +# response = json.load(urlopen(cururl)) +# package_vulnerabilities.extend(extract_data(response)) +# nextpage = response.get('urls', {}).get('next') +# +# except HTTPError as error: +# if error.code == 404: +# break +# else: +# raise +# +# return package_vulnerabilities diff --git a/vulnerabilities/migrations/0006_npm_importer.py b/vulnerabilities/migrations/0006_npm_importer.py new file mode 100644 index 000000000..d499d6e05 --- /dev/null +++ b/vulnerabilities/migrations/0006_npm_importer.py @@ -0,0 +1,53 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from django.db import migrations + + +def add_npm_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + + Importer.objects.create( + name='npm', + license='', + last_run=None, + data_source='NpmDataSource', + data_source_cfg={}, + ) + + +def remove_npm_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + qs = Importer.objects.filter(name='npm') + if qs: + qs[0].delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0005_debian_importer'), + ] + + operations = [ + migrations.RunPython(add_npm_importer, remove_npm_importer), + ] diff --git a/vulnerabilities/tests/test_data/npm_test.json b/vulnerabilities/tests/test_data/npm_test.json index 89262a0e2..7c616f284 100644 --- a/vulnerabilities/tests/test_data/npm_test.json +++ b/vulnerabilities/tests/test_data/npm_test.json @@ -1,40 +1,104 @@ { - "objects": [ - { - "id": 12, - "created": "2015-10-17T19:41:46.382Z", - "updated": "2019-06-24T14:13:54.355Z", - "deleted": null, - "title": "Rosetta-Flash JSONP Vulnerability", - "found_by": { - "name": "Michele Spagnuolo" - }, - "reported_by": { - "name": "Michele Spagnuolo" - }, - "module_name": "hapi", - "cves": [ - "CVE-2014-4671" - ], - "vulnerable_versions": "< 6.1.0", - "patched_versions": ">= 6.1.0", - "overview": "This description taken from the pull request provided by Patrick Kettner.\n\n\n\nVersions 6.1.0 and earlier of hapi are vulnerable to a rosetta-flash attack, which can be used by attackers to send data across domains and break the browser same-origin-policy.\n\n\n", - "recommendation": "- Update hapi to version 6.1.1 or later.\n\nAlternatively, a solution previously implemented by Google, Facebook, and Github is to prepend callbacks with an empty inline comment. This will cause the flash parser to break on invalid inputs and prevent the issue, and how the issue has been resolved internally in hapi.", - "references": "- [PR #1766 - prepend jsonp callbacks with a comment to prevent the rosetta-flash vulnerability](https://github.com/spumko/hapi/pull/1766)\n\n- [Background from Michele Spagnuolo](http://miki.it/blog/2014/7/8/abusing-jsonp-with-rosetta-flash/)\n\nThanks to [Patrick Kettner](https://github.com/patrickkettner) for submitting a pull request to address this in hapi.", - "access": "public", - "severity": "moderate", - "cwe": "CWE-538", - "metadata": { - "module_type": "Network.Library", - "exploitability": 3, - "affected_components": "" - }, - "url": "https://npmjs.com/advisories/12" - } - ], - "total": 1179, - "urls": { - "next": "/-/npm/v1/security/advisories?page=2", - "prev": "/-/npm/v1/security/advisories?page=0" + "objects": [ + { + "id": 1518, + "created": "2020-04-30T18:19:09.542Z", + "updated": "2020-04-30T18:19:09.542Z", + "deleted": null, + "title": "Cross-Site Scripting", + "found_by": { + "link": "", + "name": "Masato Kinugawa", + "email": "" + }, + "reported_by": { + "link": "", + "name": "Masato Kinugawa", + "email": "" + }, + "module_name": "jquery", + "cves": [ + "CVE-2020-11022" + ], + "vulnerable_versions": "<3.5.0", + "patched_versions": ">=3.5.0", + "overview": "Versions of `jquery` prior to 3.5.0 are vulnerable to Cross-Site Scripting. Passing HTML from untrusted sources - even after sanitizing it - to one of jQuery's DOM manipulation methods (i.e. .html(), .append(), and others) may execute arbitrary JavaScript in a victim's browser.", + "recommendation": "Upgrade to version 3.5.0 or later.", + "references": "- [GitHub Advisory](https://github.com/advisories/GHSA-gxr4-xjj5-5px2)", + "access": "public", + "severity": "moderate", + "cwe": "CWE-79", + "metadata": { + "module_type": "", + "exploitability": 3, + "affected_components": "" + } + }, + { + "id": 1514, + "created": "2020-04-14T21:44:49.820Z", + "updated": "2020-04-14T21:44:49.820Z", + "deleted": null, + "title": "DLL Injection", + "found_by": { + "link": "", + "name": "Dan Shallom, OP Innovate Ltd", + "email": "" + }, + "reported_by": { + "link": "", + "name": "Dan Shallom, OP Innovate Ltd", + "email": "" + }, + "module_name": "kerberos", + "cves": [], + "vulnerable_versions": "<1.0.0", + "patched_versions": ">=1.0.0", + "overview": "Version of `kerberos` prior to 1.0.0 are vulnerable to DLL Injection. The package loads DLLs without specifying a full path. This may allow attackers to create a file with the same name in a folder that precedes the intended file in the DLL path search. Doing so would allow attackers to execute arbitrary code in the machine.", + "recommendation": "Upgrade to version 1.0.0 or later.", + "references": "", + "access": "public", + "severity": "high", + "cwe": "CWE-114", + "metadata": { + "module_type": "", + "exploitability": 4, + "affected_components": "" + } + }, + { + "id": 1476, + "created": "2020-02-17T13:39:01.39", + "updated": "2020-02-18T18:00:29.843", + "deleted": null, + "title": "Denial of Service", + "found_by": { + "link": "", + "name": "Eran Hammer", + "email": "" + }, + "reported_by": { + "link": "", + "name": "Eran Hammer", + "email": "" + }, + "module_name": "@hapi/subtext", + "cves": [], + "vulnerable_versions": ">=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3", + "patched_versions": ">=6.1.3 <7.0.0 || >=7.0.3", + "overview": "Versions of `@hapi/subtext` prior to 6.1.3 or 7.0.3 are vulnerable to Denial of Service. The Content-Encoding HTTP header parser has a vulnerability which will cause the function to throw a system error if the header contains some invalid values. Because hapi rethrows system errors (as opposed to catching expected application errors), the error is thrown all the way up the stack. If no unhandled exception handler is available, the application will exist, allowing an attacker to shut down services.", + "recommendation": "Upgrade to version 6.1.3 or 7.0.3", + "references": "", + "access": "public", + "severity": "high", + "cwe": "CWE-400", + "metadata": { + "module_type": "", + "exploitability": 6, + "affected_components": "" + } } -} \ No newline at end of file + ], + "total": 3, + "urls": {} +} diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/test_npm.py index 45820a65f..96568a7f2 100644 --- a/vulnerabilities/tests/test_npm.py +++ b/vulnerabilities/tests/test_npm.py @@ -20,90 +20,103 @@ # for any legal advice. # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import os import json -import pytest +import os +from unittest.mock import patch + +from django.test import TestCase -from vulnerabilities.importers.npm import extract_data -from vulnerabilities.importers.npm import get_all_versions +from vulnerabilities import models +from vulnerabilities.import_runner import ImportRunner +from vulnerabilities.importers.npm import VersionAPI +from vulnerabilities.importers.npm import categorize_versions BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/') -@pytest.mark.webtest -def test_get_all_versions(): - x = get_all_versions('electron') - expected = ['0.1.2', '2.0.0', '3.0.0', - '4.0.0', '5.0.0', '6.0.0', '7.0.0'] - assert set(expected) <= set(x) - - -@pytest.mark.webtest -def test_extract_data(): - with open(os.path.join(TEST_DATA, 'npm_test.json')) as f: - test_data = json.load(f) - - expected = { - 'package_name': 'hapi', - 'cve_ids': ['CVE-2014-4671'], - 'fixed_versions': [ - '6.1.0', '6.2.0', '6.2.1', '6.2.2', '6.3.0', '6.4.0', - '6.5.0', '6.5.1', '6.6.0', '6.7.0', '6.7.1', '6.8.0', - '6.8.1', '6.9.0', '6.10.0', '6.11.0', '6.11.1', '7.0.0', - '7.0.1', '7.1.0', '7.1.1', '7.2.0', '7.3.0', '7.4.0', - '7.5.0', '7.5.1', '7.5.2', '8.0.0', '7.5.3', '8.1.0', - '8.2.0', '8.3.0', '8.3.1', '8.4.0', '8.5.0', '8.5.1', - '8.5.2', '8.5.3', '8.6.0', '8.6.1', '8.8.0', '8.8.1', - '9.0.0', '9.0.1', '9.0.2', '9.0.3', '9.0.4', '9.1.0', - '9.2.0', '9.3.0', '9.3.1', '10.0.0', '10.0.1', '10.1.0', - '10.2.1', '10.4.0', '10.4.1', '10.5.0', '11.0.0', '11.0.1', - '11.0.2', '11.0.3', '11.0.4', '11.0.5', '11.1.0', '11.1.1', - '11.1.2', '11.1.3', '11.1.4', '12.0.0', '12.0.1', '12.1.0', - '9.5.1', '13.0.0', '13.1.0', '13.2.0', '13.2.1', '13.2.2', - '13.3.0', '13.4.0', '13.4.1', '13.4.2', '13.5.0', '14.0.0', - '13.5.3', '14.1.0', '14.2.0', '15.0.1', '15.0.2', '15.0.3', - '15.1.0', '15.1.1', '15.2.0', '16.0.0', '16.0.1', '16.0.2', - '16.0.3', '16.1.0', '16.1.1', '16.2.0', '16.3.0', '16.3.1', - '16.4.0', '16.4.1', '16.4.2', '16.4.3', '16.5.0', '16.5.1', - '16.5.2', '16.6.0', '16.6.1', '16.6.2', '17.0.0', '17.0.1', - '17.0.2', '17.1.0', '17.1.1', '17.2.0', '17.2.1', '16.6.3', - '17.2.2', '17.2.3', '17.3.0', '17.3.1', '17.4.0', '17.5.0', - '17.5.1', '17.5.2', '17.5.3', '17.5.4', '17.5.5', '17.6.0', - '17.6.1', '17.6.2', '17.6.3', '16.6.4', '17.6.4', '16.6.5', - '17.7.0', '16.7.0', '17.8.0', '17.8.1', '18.0.0', '17.8.2', - '17.8.3', '18.0.1', '17.8.4', '18.1.0', '17.8.5'], - 'affected_versions': [ - '0.0.1', '0.0.2', '0.0.3', '0.0.4', '0.0.5', '0.0.6', '0.1.0', - '0.1.1', '0.1.2', '0.1.3', '0.2.0', '0.2.1', '0.3.0', '0.4.0', - '0.4.1', '0.4.2', '0.4.3', '0.4.4', '0.5.0', '0.5.1', '0.6.0', - '0.6.1', '0.5.2', '0.7.0', '0.7.1', '0.8.0', '0.8.1', '0.8.2', - '0.8.3', '0.8.4', '0.9.0', '0.9.1', '0.9.2', '0.10.0', '0.10.1', - '0.11.0', '0.11.1', '0.11.2', '0.11.3', '0.12.0', '0.13.0', - '0.13.1', '0.13.2', '0.11.4', '0.13.3', '0.14.0', '0.14.1', - '0.14.2', '0.15.0', '0.15.1', '0.15.2', '0.15.3', '0.15.4', - '0.15.5', '0.15.6', '0.15.7', '0.15.8', '0.15.9', '0.16.0', - '1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.2.0', '1.3.0', - '1.4.0', '1.5.0', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.7.1', - '1.7.2', '1.7.3', '1.8.0', '1.8.1', '1.8.2', '1.8.3', '1.9.0', - '1.9.1', '1.9.2', '1.9.3', '1.9.4', '1.9.5', '1.9.6', '1.9.7', - '1.10.0', '1.11.0', '1.11.1', '1.12.0', '1.13.0', '1.14.0', - '1.15.0', '1.16.0', '1.16.1', '1.17.0', '1.18.0', '1.19.0', - '1.19.1', '1.19.2', '1.19.3', '1.19.4', '1.19.5', '1.20.0', - '2.0.0', '2.1.0', '2.1.1', '2.1.2', '2.2.0', '2.3.0', '2.4.0', - '2.5.0', '2.6.0', '3.0.0', '3.0.1', '3.0.2', '3.1.0', '4.0.0', - '4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3', - '4.1.4', '5.0.0', '5.1.0', '6.0.0', '6.0.1', '6.0.2'], - 'severity': 'moderate' - } - got = extract_data(test_data)[0] - # Check if expected affected version and fixed version is subset of what we get from online - assert set(expected['affected_versions']) <= set( - got['affected_versions']) - assert set(expected['fixed_versions']) <= set( - got['fixed_versions']) - - assert expected['package_name'] == got['package_name'] - assert expected['severity'] == got['severity'] - assert expected['cve_ids'] == got['cve_ids'] +MOCK_VERSION_API = VersionAPI(cache={ + 'jquery': {'3.4', '3.8'}, + 'kerberos': {'0.5.8', '1.2'}, + '@hapi/subtext': {'3.7', '4.1.1', '6.1.3', '7.0.0', '7.0.5'}, +}) + + +@patch('vulnerabilities.importers.NpmDataSource.versions', new=MOCK_VERSION_API) +class NpmImportTest(TestCase): + @classmethod + def setUpClass(cls) -> None: + fixture_path = os.path.join(TEST_DATA, 'npm_test.json') + with open(fixture_path) as f: + cls.mock_response = json.load(f) + + cls.importer = models.Importer.objects.create( + name='npm_unittests', + license='', + last_run=None, + data_source='NpmDataSource', + data_source_cfg={}, + ) + + @classmethod + def tearDownClass(cls) -> None: + # Make sure no requests for unexpected package names have been made during the tests. + assert len(MOCK_VERSION_API.cache) == 3, MOCK_VERSION_API.cache + + def test_import(self): + runner = ImportRunner(self.importer, 5) + + with patch('vulnerabilities.importers.NpmDataSource._fetch', return_value=self.mock_response): + runner.run() + + assert models.Vulnerability.objects.count() == 3 + assert models.VulnerabilityReference.objects.count() == 3 + assert models.ResolvedPackage.objects.count() == 5 + assert models.ImpactedPackage.objects.count() == 4 + + expected_package_count = sum([len(v) for v in MOCK_VERSION_API.cache.values()]) + assert models.Package.objects.count() == expected_package_count + + self.assert_for_package('jquery', {'3.4'}, {'3.8'}, '1518', cve_id='CVE-2020-11022') + self.assert_for_package('kerberos', {'0.5.8'}, {'1.2'}, '1514') + self.assert_for_package('subtext', {'4.1.1', '7.0.0'}, {'3.7', '6.1.3', '7.0.5'}, '1476') + + def assert_for_package(self, package_name, impacted_versions, resolved_versions, vuln_id, cve_id=None): + vuln = None + + for version in impacted_versions: + pkg = models.Package.objects.get(name=package_name, version=version) + + assert pkg.vulnerabilities.count() == 1 + vuln = pkg.vulnerabilities.first() + if cve_id: + assert vuln.cve_id == cve_id + + ref_url = f'https://registry.npmjs.org/-/npm/v1/advisories/{vuln_id}' + assert models.VulnerabilityReference.objects.get(url=ref_url, vulnerability=vuln) + + for version in resolved_versions: + pkg = models.Package.objects.get(name=package_name, version=version) + assert models.ResolvedPackage.objects.filter(package=pkg, vulnerability=vuln) + + +def test_categorize_versions_simple_ranges(): + all_versions = {'3.4', '3.8'} + impacted_ranges = '<3.5.0' + resolved_ranges = '>=3.5.0' + + impacted_versions, resolved_versions = categorize_versions(all_versions, impacted_ranges, resolved_ranges) + + assert impacted_versions == {'3.4'} + assert resolved_versions == {'3.8'} + + +def test_categorize_versions_complex_ranges(): + all_versions = {'3.7', '4.1.1', '6.1.3', '7.0.0', '7.0.5'} + impacted_ranges = '>=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3' + resolved_ranges = '>=6.1.3 <7.0.0 || >=7.0.3' + + impacted_versions, resolved_versions = categorize_versions(all_versions, impacted_ranges, resolved_ranges) + + assert impacted_versions == {'4.1.1', '7.0.0'} + assert resolved_versions == {'3.7', '6.1.3', '7.0.5'}