Manage vulnerability sources in database

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 3.6
+python: 3.8
 
 services:
   - postgresql
@@ -11,7 +11,7 @@ env:
   - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7"
 
 before_script:
-  - pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py,test_suse.py --max-line-length=100 .
+  - pycodestyle --exclude=migrations,settings.py,venv,lib_oval.py,test_ubuntu.py,test_suse.py,test_data_source.py --max-line-length=100 .
   - psql -c "CREATE DATABASE vulnerablecode;" -U postgres
   - ./manage.py migrate
 

README.md

@@ -12,7 +12,7 @@ git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode
 
 ### System requirements
 
-- Python 3.6+
+- Python 3.8+
 
 - PostgreSQL 9+ or [Docker](https://hub.docker.com/search/?type=edition&offering=community)
 

requirements.txt

@@ -1,32 +1,53 @@
+asgiref==3.2.7
 attrs==19.3.0
+backcall==0.1.0
 beautifulsoup4==4.7.1
+cached-property==1.5.1
+cffi==1.14.0
+contextlib2==0.5.5
+decorator==4.4.2
 dephell-specifier==0.2.1
 dj-database-url==0.4.2
 Django==3.0.3
 django-filter==2.2.0
 djangorestframework==3.11.0
 gunicorn==19.7.1
 importlib-metadata==1.3.0
+ipython==7.13.0
+ipython-genutils==0.2.0
+jedi==0.17.0
 lxml==4.3.3
 more-itertools==8.0.2
-packageurl-python==0.8.7
+packageurl-python==0.9.0
 packaging==19.2
+parso==0.7.0
+pexpect==4.8.0
+pickleshare==0.7.5
 pluggy==0.13.1
+prompt-toolkit==3.0.5
 psycopg2==2.8.4
+ptyprocess==0.6.0
 py==1.8.0
 pycodestyle==2.5.0
+pycparser==2.20
+pygit2==1.2.0
+Pygments==2.6.1
 pyparsing==2.4.5
 pytest==5.3.2
 pytest-dependency==0.4.0
 pytest-django==3.7.0
 pytest-mock==1.13.0
+python-dateutil==2.8.1
+pytoml==0.1.21
 pytz==2019.3
-PyYAML==5.3
+PyYAML==5.3.1
 saneyaml==0.4
+schema==0.7.1
 six==1.13.0
 soupsieve==1.9.5
 sqlparse==0.3.0
+tqdm==4.41.1
+traitlets==4.3.3
+wcwidth==0.1.7
 whitenoise==5.0.1
 zipp==0.6.0
-pytoml==0.1.21
-schema==0.7.1

vulnerabilities/admin.py

@@ -25,11 +25,11 @@
 
 from vulnerabilities.models import (
     ImpactedPackage,
+    Importer,
     Package,
-    PackageReference,
     ResolvedPackage,
     Vulnerability,
-    VulnerabilityReference
+    VulnerabilityReference,
 )
 
 
@@ -58,6 +58,6 @@ class ResolvedPackageAdmin(admin.ModelAdmin):
     pass
 
 
-@admin.register(PackageReference)
-class PackageReferenceAdmin(admin.ModelAdmin):
+@admin.register(Importer)
+class ImporterAdmin(admin.ModelAdmin):
     pass

vulnerabilities/api.py

@@ -22,29 +22,15 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 from rest_framework import serializers
-from rest_framework import status
 from rest_framework import viewsets
-from rest_framework.response import Response
 
 from packageurl import PackageURL
 
 from vulnerabilities.models import Package
-from vulnerabilities.models import PackageReference
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
 
 
-class PackageReferenceSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = PackageReference
-        fields = [
-            'repository',
-            'platform',
-            'name',
-            'version',
-        ]
-
-
 class VulnerabilityReferenceSerializer(serializers.ModelSerializer):
     class Meta:
         model = VulnerabilityReference
@@ -69,7 +55,6 @@ class Meta:
 
 class PackageSerializer(serializers.ModelSerializer):
     vulnerabilities = VulnerabilitySerializer(many=True)
-    references = PackageReferenceSerializer(source='packagereference_set', many=True)
 
     class Meta:
         model = Package
@@ -78,7 +63,6 @@ class Meta:
             'version',
             'package_url',
             'vulnerabilities',
-            'references',
         ]
 
 

vulnerabilities/data_dump.py

@@ -23,57 +23,11 @@
 
 from vulnerabilities.models import ImpactedPackage
 from vulnerabilities.models import Package
-from vulnerabilities.models import PackageReference
 from vulnerabilities.models import ResolvedPackage
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
 
 
-def debian_dump(extract_data, base_release='jessie'):
-    """
-    Save data scraped from Debian' security tracker.
-    """
-    for data in extract_data:
-        vulnerability, _ = Vulnerability.objects.get_or_create(
-            cve_id=data['cve_id'],
-        )
-
-        pkg_name = data['package_name']
-        package = Package.objects.create(
-            name=pkg_name,
-            type='deb',
-            namespace='debian',
-            version=data.get('version', ''),
-            qualifiers=f'distro={base_release}',
-        )
-
-        if data['status'] == 'open':
-            ImpactedPackage.objects.create(
-                vulnerability=vulnerability,
-                package=package
-            )
-        else:
-            ResolvedPackage.objects.create(
-                vulnerability=vulnerability,
-                package=package
-            )
-
-            fixed_version = data.get('fixed_version')
-            if fixed_version:
-                package = Package.objects.create(
-                    name=pkg_name,
-                    type='deb',
-                    namespace='debian',
-                    version=fixed_version,
-                    qualifiers=f'distro={base_release}',
-                )
-
-                ResolvedPackage.objects.create(
-                    vulnerability=vulnerability,
-                    package=package
-                )
-
-
 def ubuntu_dump(html):
     """
     Dump data scraped from Ubuntu's security tracker.
@@ -93,68 +47,6 @@ def ubuntu_dump(html):
         )
 
 
-def archlinux_dump(extract_data):
-    """
-    Save data scraped from archlinux' security tracker.
-    """
-    base_url = 'https://security.archlinux.org'
-
-    for avg in extract_data:
-        affected_packages = []
-        fixed_packages = []
-
-        for package_name in avg['packages']:
-            ap, _ = Package.objects.get_or_create(
-                name=package_name,
-                type='pacman',
-                namespace='archlinux',
-                version=avg['affected'],
-            )
-            affected_packages.append(ap)
-
-            fp, _ = Package.objects.get_or_create(
-                name=package_name,
-                type='pacman',
-                namespace='archlinux',
-                version=avg['fixed'],
-            )
-            fixed_packages.append(fp)
-
-        for cve_id in avg['issues']:
-            vulnerability, _ = Vulnerability.objects.get_or_create(
-                cve_id=cve_id,
-            )
-            VulnerabilityReference.objects.create(
-                vulnerability=vulnerability,
-                url=f'{base_url}/{cve_id}',
-            )
-            avg_name = avg['name']
-            VulnerabilityReference.objects.create(
-                vulnerability=vulnerability,
-                reference_id=avg_name,
-                url=f'{base_url}/{avg_name}',
-            )
-
-            for asa in avg['advisories']:
-                VulnerabilityReference.objects.create(
-                    vulnerability=vulnerability,
-                    reference_id=asa,
-                    url=f'{base_url}/{asa}',
-                )
-
-            for ap in affected_packages:
-                ImpactedPackage.objects.get_or_create(
-                    vulnerability=vulnerability,
-                    package=ap,
-                )
-
-            for fp in fixed_packages:
-                ResolvedPackage.objects.get_or_create(
-                    vulnerability=vulnerability,
-                    package=fp,
-                )
-
-
 def npm_dump(extract_data):
     for data in extract_data:
         package_name = data['package_name']
@@ -229,43 +121,6 @@ def ruby_dump(extract_data):
             )
 
 
-def rust_dump(extract_data):
-
-    for package_data in extract_data:
-
-        vulnerability, _ = Vulnerability.objects.get_or_create(
-            summary=package_data['description']
-        )
-
-        VulnerabilityReference.objects.get_or_create(
-            vulnerability=vulnerability,
-            url=package_data['advisory'],
-            reference_id=package_data['vuln_id']
-        )
-
-        for version in package_data['affected_versions']:
-            affected_package = Package.objects.create(
-                name=package_data['package_name'],
-                type='cargo',
-                version=version
-            )
-            ImpactedPackage.objects.create(
-                vulnerability=vulnerability,
-                package=affected_package
-            )
-
-        for version in package_data['fixed_versions']:
-            unaffected_package = Package.objects.create(
-                name=package_data['package_name'],
-                type='cargo',
-                version=version
-            )
-            ResolvedPackage.objects.create(
-                vulnerability=vulnerability,
-                package=unaffected_package
-            )
-
-
 def safetydb_dump(extract_data):
     for package_data in extract_data:
         for cve_id in package_data['cve_id']:
@@ -300,33 +155,3 @@ def safetydb_dump(extract_data):
                 vulnerability=vulnerability,
                 package=unaffected_package
             )
-
-
-def alpine_linux_dump(data_dicts):
-    for package_data in data_dicts:
-        unaffected_package = Package.objects.create(
-            name=package_data['package_name'],
-            type='alpine',
-            version=package_data['fixed_version']
-        )
-
-        for vuln_groups in package_data['vuln_ids']:
-            if vuln_groups[0].startswith('CVE'):
-                vulnerability_obj, _ = Vulnerability.objects.get_or_create(
-                    cve_id=vuln_groups[0]
-                )
-                if len(vuln_groups) == 2:
-                    # TODO: Deal with  vulnerabilities without cves
-                    VulnerabilityReference.objects.get_or_create(
-                        vulnerability=vulnerability_obj,
-                        reference_id=vuln_groups[1]
-                    )
-
-            else:
-
-                continue
-
-            ResolvedPackage.objects.create(
-                vulnerability=vulnerability_obj,
-                package=unaffected_package
-            )