Merge branch 'main' into 1214-fix-ver-range

CHANGELOG.rst

@@ -1,6 +1,17 @@
 Release notes
 =============
 
+Version (next)
+-------------------
+
+
+Version v34.0.2
+-------------------
+
+- Add management command to commit exported vulnerability data (#1600)
+- Fix API 500 error (#1603)
+
+
 Version v34.0.1
 -------------------
 

requirements.txt

@@ -21,6 +21,7 @@ click==8.1.2
 coreapi==2.3.3
 coreschema==0.0.4
 cryptography==43.0.1
+crispy-bootstrap4==2024.1
 cwe2==3.0.0
 dateparser==1.1.1
 decorator==5.1.1
@@ -35,8 +36,8 @@ djangorestframework==3.15.2
 doc8==0.11.1
 docopt==0.6.2
 docutils==0.17.1
-drf-spectacular==0.27.2
-drf-spectacular-sidecar==2024.7.1 
+drf-spectacular==0.24.2
+drf-spectacular-sidecar==2022.10.1
 executing==0.8.3
 fetchcode==0.3.0
 freezegun==1.2.1

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 34.0.1
+version = 34.0.2
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
@@ -62,11 +62,12 @@ install_requires =
     django-filter>=24.0
     django-widget-tweaks>=1.5.0
     django-crispy-forms>=2.3
+    crispy-bootstrap4>=2024.1
     django-environ>=0.11.0
     gunicorn>=23.0.0
 
     # for the API doc
-    drf-spectacular[sidecar]>=0.27.2
+    drf-spectacular[sidecar]>=0.24.2
 
     #essentials
     packageurl-python>=0.15

vulnerabilities/api.py

@@ -27,7 +27,7 @@
 from rest_framework.throttling import UserRateThrottle
 
 from vulnerabilities.models import Alias
-from vulnerabilities.models import Kev
+from vulnerabilities.models import Exploit
 from vulnerabilities.models import Package
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
@@ -175,10 +175,23 @@ def to_representation(self, instance):
         return representation
 
 
-class KEVSerializer(serializers.ModelSerializer):
+class ExploitSerializer(serializers.ModelSerializer):
     class Meta:
-        model = Kev
-        fields = ["date_added", "description", "required_action", "due_date", "resources_and_notes"]
+        model = Exploit
+        fields = [
+            "date_added",
+            "description",
+            "required_action",
+            "due_date",
+            "notes",
+            "known_ransomware_campaign_use",
+            "source_date_published",
+            "exploit_type",
+            "platform",
+            "source_date_updated",
+            "data_source",
+            "source_url",
+        ]
 
 
 class VulnerabilitySerializer(BaseResourceSerializer):
@@ -189,7 +202,7 @@ class VulnerabilitySerializer(BaseResourceSerializer):
 
     references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
     aliases = AliasSerializer(many=True, source="alias")
-    kev = KEVSerializer(read_only=True)
+    exploits = ExploitSerializer(many=True, read_only=True)
     weaknesses = WeaknessSerializer(many=True)
     severity_range_score = serializers.SerializerMethodField()
 
@@ -199,10 +212,6 @@ def to_representation(self, instance):
         weaknesses = data.get("weaknesses", [])
         data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None]
 
-        kev = data.get("kev", None)
-        if not kev:
-            data.pop("kev")
-
         return data
 
     def get_severity_range_score(self, instance):
@@ -240,7 +249,7 @@ class Meta:
             "affected_packages",
             "references",
             "weaknesses",
-            "kev",
+            "exploits",
             "severity_range_score",
         ]
 
@@ -676,14 +685,10 @@ def filter_alias(self, queryset, name, value):
         return self.queryset.filter(aliases__alias__icontains=alias)
 
 
-class AliasViewSet(viewsets.ReadOnlyModelViewSet):
+class AliasViewSet(VulnerabilityViewSet):
     """
     Lookup for vulnerabilities by vulnerability aliases such as a CVE
     (https://nvd.nist.gov/general/cve-process).
     """
 
-    queryset = Vulnerability.objects.all()
-    serializer_class = VulnerabilitySerializer
-    filter_backends = (filters.DjangoFilterBackend,)
     filterset_class = AliasFilterSet
-    throttle_classes = [StaffUserRateThrottle, AnonRateThrottle]

vulnerabilities/api_extension.py

@@ -26,7 +26,7 @@
 from rest_framework.throttling import AnonRateThrottle
 
 from vulnerabilities.api import BaseResourceSerializer
-from vulnerabilities.models import Kev
+from vulnerabilities.models import Exploit
 from vulnerabilities.models import Package
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
@@ -105,8 +105,21 @@ class Meta:
 
 class V2ExploitSerializer(ModelSerializer):
     class Meta:
-        model = Kev
-        fields = ("description", "required_action", "date_added", "due_date", "resources_and_notes")
+        model = Exploit
+        fields = [
+            "date_added",
+            "description",
+            "required_action",
+            "due_date",
+            "notes",
+            "known_ransomware_campaign_use",
+            "source_date_published",
+            "exploit_type",
+            "platform",
+            "source_date_updated",
+            "data_source",
+            "source_url",
+        ]
 
 
 class V2VulnerabilitySerializer(ModelSerializer):

vulnerabilities/importers/__init__.py

@@ -19,13 +19,9 @@
 from vulnerabilities.importers import epss
 from vulnerabilities.importers import fireeye
 from vulnerabilities.importers import gentoo
-from vulnerabilities.importers import github
 from vulnerabilities.importers import github_osv
-from vulnerabilities.importers import gitlab
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
-from vulnerabilities.importers import nginx
-from vulnerabilities.importers import nvd
 from vulnerabilities.importers import openssl
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
@@ -40,14 +36,14 @@
 from vulnerabilities.importers import vulnrichment
 from vulnerabilities.importers import xen
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
+from vulnerabilities.pipelines import github_importer
+from vulnerabilities.pipelines import gitlab_importer
+from vulnerabilities.pipelines import nginx_importer
 from vulnerabilities.pipelines import npm_importer
+from vulnerabilities.pipelines import nvd_importer
 from vulnerabilities.pipelines import pypa_importer
 
 IMPORTERS_REGISTRY = [
-    nvd.NVDImporter,
-    github.GitHubAPIImporter,
-    gitlab.GitLabAPIImporter,
-    nginx.NginxImporter,
     pysec.PyPIImporter,
     alpine_linux.AlpineImporter,
     openssl.OpensslImporter,
@@ -78,6 +74,10 @@
     vulnrichment.VulnrichImporter,
     pypa_importer.PyPaImporterPipeline,
     npm_importer.NpmImporterPipeline,
+    nginx_importer.NginxImporterPipeline,
+    gitlab_importer.GitLabImporterPipeline,
+    github_importer.GitHubAPIImporterPipeline,
+    nvd_importer.NVDImporterPipeline,
 ]
 
 IMPORTERS_REGISTRY = {

vulnerabilities/improvers/__init__.py

@@ -8,9 +8,11 @@
 #
 
 from vulnerabilities.improvers import valid_versions
-from vulnerabilities.improvers import vulnerability_kev
 from vulnerabilities.improvers import vulnerability_status
 from vulnerabilities.pipelines import VulnerableCodePipeline
+from vulnerabilities.pipelines import enhance_with_exploitdb
+from vulnerabilities.pipelines import enhance_with_kev
+from vulnerabilities.pipelines import enhance_with_metasploit
 from vulnerabilities.pipelines import flag_ghost_packages
 
 IMPROVERS_REGISTRY = [
@@ -31,8 +33,10 @@
     valid_versions.GithubOSVImprover,
     vulnerability_status.VulnerabilityStatusImprover,
     valid_versions.CurlImprover,
-    vulnerability_kev.VulnerabilityKevImprover,
     flag_ghost_packages.FlagGhostPackagePipeline,
+    enhance_with_kev.VulnerabilityKevPipeline,
+    enhance_with_metasploit.MetasploitImproverPipeline,
+    enhance_with_exploitdb.ExploitDBImproverPipeline,
 ]
 
 IMPROVERS_REGISTRY = {

vulnerabilities/improvers/valid_versions.py

@@ -12,7 +12,6 @@
 from datetime import datetime
 from typing import Iterable
 from typing import List
-from typing import Mapping
 from typing import Optional
 
 from django.db.models import Q
@@ -32,18 +31,19 @@
 from vulnerabilities.importers.debian import DebianImporter
 from vulnerabilities.importers.debian_oval import DebianOvalImporter
 from vulnerabilities.importers.elixir_security import ElixirSecurityImporter
-from vulnerabilities.importers.github import GitHubAPIImporter
 from vulnerabilities.importers.github_osv import GithubOSVImporter
-from vulnerabilities.importers.gitlab import GitLabAPIImporter
 from vulnerabilities.importers.istio import IstioImporter
-from vulnerabilities.importers.nginx import NginxImporter
 from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter
 from vulnerabilities.importers.ruby import RubyImporter
 from vulnerabilities.importers.ubuntu import UbuntuImporter
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
+from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline
+from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline
+from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline
 from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline
 from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
 from vulnerabilities.utils import clean_nginx_git_tag
@@ -63,6 +63,8 @@ class ValidVersionImprover(Improver):
 
     @property
     def interesting_advisories(self) -> QuerySet:
+        if issubclass(self.importer, VulnerableCodeBaseImporterPipeline):
+            return Advisory.objects.filter(Q(created_by=self.importer.pipeline_id)).paginated()
         return Advisory.objects.filter(Q(created_by=self.importer.qualified_name)).paginated()
 
     def get_package_versions(
@@ -220,7 +222,7 @@ class NginxBasicImprover(Improver):
 
     @property
     def interesting_advisories(self) -> QuerySet:
-        return Advisory.objects.filter(created_by=NginxImporter.qualified_name).paginated()
+        return Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).paginated()
 
     def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
         all_versions = list(self.fetch_nginx_version_from_git_tags())
@@ -364,12 +366,12 @@ class DebianBasicImprover(ValidVersionImprover):
 
 
 class GitLabBasicImprover(ValidVersionImprover):
-    importer = GitLabAPIImporter
+    importer = GitLabImporterPipeline
     ignorable_versions = []
 
 
 class GitHubBasicImprover(ValidVersionImprover):
-    importer = GitHubAPIImporter
+    importer = GitHubAPIImporterPipeline
     ignorable_versions = frozenset(
         [
             "0.1-bulbasaur",

vulnerabilities/improvers/vulnerability_status.py

@@ -14,14 +14,14 @@
 from django.db.models.query import QuerySet
 
 from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importers.nvd import NVDImporter
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
 from vulnerabilities.models import Alias
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityChangeLog
 from vulnerabilities.models import VulnerabilityStatusType
+from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline
 from vulnerabilities.utils import fetch_response
 from vulnerabilities.utils import get_item
 
@@ -38,7 +38,7 @@ class VulnerabilityStatusImprover(Improver):
     @property
     def interesting_advisories(self) -> QuerySet:
         return (
-            Advisory.objects.filter(Q(created_by=NVDImporter.qualified_name))
+            Advisory.objects.filter(Q(created_by=NVDImporterPipeline.pipeline_id))
             .distinct("aliases")
             .paginated()
         )