WIP

Signed-off-by: Haiko Schol <hs@haikoschol.com>

vulnerabilities/admin.py

@@ -25,11 +25,11 @@
 
 from vulnerabilities.models import (
     ImpactedPackage,
+    Importer,
     Package,
-    PackageReference,
     ResolvedPackage,
     Vulnerability,
-    VulnerabilityReference
+    VulnerabilityReference,
 )
 
 
@@ -58,6 +58,6 @@ class ResolvedPackageAdmin(admin.ModelAdmin):
     pass
 
 
-@admin.register(PackageReference)
-class PackageReferenceAdmin(admin.ModelAdmin):
+@admin.register(Importer)
+class ImporterAdmin(admin.ModelAdmin):
     pass

vulnerabilities/api.py

@@ -22,29 +22,15 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 from rest_framework import serializers
-from rest_framework import status
 from rest_framework import viewsets
-from rest_framework.response import Response
 
 from packageurl import PackageURL
 
 from vulnerabilities.models import Package
-from vulnerabilities.models import PackageReference
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
 
 
-class PackageReferenceSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = PackageReference
-        fields = [
-            'repository',
-            'platform',
-            'name',
-            'version',
-        ]
-
-
 class VulnerabilityReferenceSerializer(serializers.ModelSerializer):
     class Meta:
         model = VulnerabilityReference
@@ -69,7 +55,6 @@ class Meta:
 
 class PackageSerializer(serializers.ModelSerializer):
     vulnerabilities = VulnerabilitySerializer(many=True)
-    references = PackageReferenceSerializer(source='packagereference_set', many=True)
 
     class Meta:
         model = Package
@@ -78,7 +63,6 @@ class Meta:
             'version',
             'package_url',
             'vulnerabilities',
-            'references',
         ]
 
 

vulnerabilities/data_dump.py

@@ -23,7 +23,6 @@
 
 from vulnerabilities.models import ImpactedPackage
 from vulnerabilities.models import Package
-from vulnerabilities.models import PackageReference
 from vulnerabilities.models import ResolvedPackage
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference

vulnerabilities/data_source.py

@@ -20,61 +20,107 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-from attr import dataclass
 from datetime import datetime
 from typing import Any
 from typing import Mapping
+from typing import Optional
 from typing import Sequence
+from typing import Set
+import dataclasses
 
+from packageurl import PackageURL
 
-@dataclass
+
+@dataclasses.dataclass
 class DataSource:
     """
     TODO
     """
     batch_size: int
-    cutoff_date: datetime
-    config: Mapping[str, Any]
+    cutoff_date: Optional[datetime] = None
+    config: Optional[Mapping[str, Any]] = dataclasses.field(default_factory=dict)
 
     def __enter__(self):
         """
         Subclasses acquire per-run resources, such as network connections, file downloads, etc. here.
         """
-        pass
+        return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """
         Subclasses release per-run resources acquired in __enter__() here.
         """
         pass
 
-    def __next__(self):
+    def new_records(self):
         """
-        Subclasses return batch_size sized batches of VulnerabilityInfo objects
+        Subclasses return batch_size sized batches of Advisory objects that have been added to the data source
+        since self.cutoff_date.
         """
-        pass
+        raise StopIteration
+
+    def updated_records(self):
+        """
+        Subclasses return batch_size sized batches of Advisory objects that have been modified since
+        self.cutoff_date.
+
+        NOTE: Data sources that do now enable detection of changes to existing records vs added records must only
+              implement this method, not new_records(). The ImportRunner relies on this contract to decide between
+              insert and update operations.
+        """
+        raise StopIteration
 
 
 # The following data classes express the contract between data sources and the import runner.
-# Data sources are expected to be usable as context managers and generators, yielding
-# batches of VulnerabilityInfo sequences.
+# Data sources are expected to be usable as context managers and generators, yielding batches of Advisory
+# sequences.
 
-@dataclass
+# TODO Remove this class and just use PackageURL directly instead
+@dataclasses.dataclass(frozen=True)
 class Package:
     name: str
-    namespace: str
     type: str
     version: str
-    qualifiers: str
-    subpath: str
-    references: Sequence[str]
+    namespace: str = ''
+    qualifiers: str = ''
+    subpath: str = ''
+
+    @property
+    def package_url(self) -> str:
+        """
+        Return a compact Package URL "purl" string.
+        """
+        purl = PackageURL(
+            self.type, self.namespace, self.name,
+            self.version, self.qualifiers, self.subpath
+        )
+        return str(purl)
+
+    @staticmethod
+    def from_package_url(_, purl):
+        purl = PackageURL.from_string(purl)
+        return Package(purl.name, purl.type, purl.version, purl.namespace, purl.qualifiers, purl.subpath)
 
 
-@dataclass
-class VulnerabilityInfo:
-    cve_id: str
+@dataclasses.dataclass
+class Advisory:
     summary: str
-    affected_packages: Sequence[Package]
-    unaffected_packages: Sequence[Package]
-    fixed_packages: Sequence[Package]
-    references: Sequence[str]
+    impacted_packages: Sequence[Package]
+    resolved_packages: Sequence[Package] = dataclasses.field(default_factory=list)
+    references: Sequence[str] = dataclasses.field(default_factory=list)
+    cve_id: Optional[str] = None
+
+    _impacted_purls: Set[str] = dataclasses.field(init=False, compare=False, default=None)
+    _resolved_purls: Set[str] = dataclasses.field(init=False, compare=False, default=None)
+
+    @property
+    def impacted_package_urls(self):
+        if self._impacted_purls is None:
+            self._impacted_purls = {p.package_url for p in self.impacted_packages}
+        return self._impacted_purls
+
+    @property
+    def resolved_package_urls(self):
+        if self._resolved_purls is None:
+            self._resolved_purls = {p.package_url for p in self.resolved_packages}
+        return self._resolved_purls