Implement NPM importer based on DataSource

Signed-off-by: Haiko Schol <hs@haikoschol.com>

requirements.txt

@@ -1,38 +1,53 @@
 asgiref==3.2.7
 attrs==19.3.0
+backcall==0.1.0
 beautifulsoup4==4.7.1
 cached-property==1.5.1
 cffi==1.14.0
+contextlib2==0.5.5
+decorator==4.4.2
 dephell-specifier==0.2.1
 dj-database-url==0.4.2
 Django==3.0.3
 django-filter==2.2.0
 djangorestframework==3.11.0
 gunicorn==19.7.1
 importlib-metadata==1.3.0
+ipython==7.13.0
+ipython-genutils==0.2.0
+jedi==0.17.0
 lxml==4.3.3
 more-itertools==8.0.2
-https://github.com/package-url/packageurl-python/archive/master.zip
+packageurl-python==0.9.0
 packaging==19.2
+parso==0.7.0
+pexpect==4.8.0
+pickleshare==0.7.5
 pluggy==0.13.1
+prompt-toolkit==3.0.5
 psycopg2==2.8.4
+ptyprocess==0.6.0
 py==1.8.0
 pycodestyle==2.5.0
 pycparser==2.20
 pygit2==1.2.0
+Pygments==2.6.1
 pyparsing==2.4.5
 pytest==5.3.2
 pytest-dependency==0.4.0
 pytest-django==3.7.0
 pytest-mock==1.13.0
+python-dateutil==2.8.1
 pytoml==0.1.21
 pytz==2019.3
 PyYAML==5.3.1
+saneyaml==0.4
 schema==0.7.1
 six==1.13.0
 soupsieve==1.9.5
 sqlparse==0.3.0
 tqdm==4.41.1
+traitlets==4.3.3
 wcwidth==0.1.7
 whitenoise==5.0.1
 zipp==0.6.0

vulnerabilities/import_runner.py

@@ -29,6 +29,7 @@
 from typing import Tuple
 
 import packageurl
+from django.db import DataError
 
 from vulnerabilities import models
 from vulnerabilities.data_source import Advisory, DataSource
@@ -83,12 +84,18 @@ def run(self, cutoff_date: datetime.datetime = None) -> None:
 
 def _process_added_advisories(data_source: DataSource) -> None:
     for batch in data_source.added_advisories():
-        impacted, resolved = _collect_package_urls(batch)
-        impacted, resolved = _bulk_insert_packages(impacted, resolved)
+        try:
+            impacted, resolved = _collect_package_urls(batch)
+            impacted, resolved = _bulk_insert_packages(impacted, resolved)
 
-        vulnerabilities = _insert_vulnerabilities_and_references(batch)
+            vulnerabilities = _insert_vulnerabilities_and_references(batch)
 
-        _bulk_insert_impacted_and_resolved_packages(batch, vulnerabilities, impacted, resolved)
+            _bulk_insert_impacted_and_resolved_packages(batch, vulnerabilities, impacted, resolved)
+        except (DataError, RuntimeError) as e:
+            # FIXME This exception might happen when the max. length of a VARCHAR column is exceeded.
+            # Skipping an entire batch because one version number might be too long is obviously a terrible way to
+            # handle this case.
+            logger.exception(e)
 
 
 def _process_updated_advisories(data_source: DataSource) -> None:
@@ -148,10 +155,6 @@ def _get_or_create_vulnerability(advisory: Advisory) -> Tuple[models.Vulnerabili
 def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:
     version = packageurl.normalize_version(p.version, encode=True)
 
-    # FIXME terrible hack, remove after https://github.com/package-url/packageurl-python/pull/30 was merged
-    if len(version) > 50:
-        version = version[:50]
-
     query_kwargs = {
         'name': packageurl.normalize_name(p.name, p.type, encode=True),
         'version': version,

vulnerabilities/importers/__init__.py

@@ -20,8 +20,9 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-from vulnerabilities.importers.archlinux import ArchlinuxDataSource
 from vulnerabilities.importers.alpine_linux import AlpineDataSource
+from vulnerabilities.importers.archlinux import ArchlinuxDataSource
 from vulnerabilities.importers.debian import DebianDataSource
+from vulnerabilities.importers.npm import NpmDataSource
 from vulnerabilities.importers.rust import RustDataSource
 from vulnerabilities.importers.safety_db import SafetyDbDataSource