Merge branch 'main' into cvss-v-display

requirements.txt

@@ -35,7 +35,7 @@ executing==0.8.3
 freezegun==1.2.1
 frozenlist==1.3.0
 gitdb==4.0.9
-GitPython==3.1.37
+GitPython==3.1.41
 gunicorn==20.1.0
 idna==3.3
 imagesize==1.3.0
@@ -44,7 +44,7 @@ iniconfig==1.1.1
 ipython==8.10.0
 isort==5.10.1
 jedi==0.18.1
-Jinja2==3.1.1
+Jinja2==3.1.3
 jsonschema==3.2.0
 license-expression==21.6.14
 lxml==4.9.1
@@ -106,7 +106,7 @@ toml==0.10.2
 tomli==2.0.1
 traitlets==5.1.1
 typing_extensions==4.1.1
-univers==30.10.0
+univers==30.11.0
 urllib3==1.26.18
 wcwidth==0.2.5
 websocket-client==0.59.0

setup.cfg

@@ -71,7 +71,7 @@ install_requires =
 
     #essentials
     packageurl-python>=0.10.5rc1
-    univers>=30.10.0
+    univers>=30.11.0
     license-expression>=21.6.14
 
     # file and data formats

vulnerabilities/importers/fireeye.py

@@ -44,7 +44,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                 if Path(file).stem == "README":
                     continue
                 try:
-                    with open(file) as f:
+                    with open(file, encoding="utf-8-sig") as f:
                         yield parse_advisory_data(raw_data=f.read(), file=file, base_path=base_path)
                 except UnicodeError:
                     logger.error(f"Invalid file {file}")

vulnerabilities/importers/github.py

@@ -92,6 +92,7 @@
 class GitHubAPIImporter(Importer):
     spdx_license_expression = "CC-BY-4.0"
     importer_name = "GHSA Importer"
+    license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
         for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items():

vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    def remove_duped_changelogs(apps, schema_editor):
+        PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog")
+        VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog")
+
+        models_list = [PackageChangeLog, VulnerabilityChangeLog]
+
+        for model in models_list:
+        # Identify duplicate records based on actor_name, action_type, and source_url
+            duplicate_records = model.objects.values('actor_name', 'action_type', 'source_url').annotate(count=models.Count('id')).filter(count__gt=1)
+
+            to_be_deleted = list()
+
+            for duplicate_set in duplicate_records:
+                # Get the records for the current duplicate set
+                records_to_delete = model.objects.filter(
+                    actor_name=duplicate_set['actor_name'],
+                    action_type=duplicate_set['action_type'],
+                    source_url=duplicate_set['source_url']
+                ).order_by('-software_version')
+
+                # Keep the record with the older software version
+                record_to_keep = records_to_delete.last()
+
+                # Delete the records with the newer software version
+                to_be_deleted.extend(records_to_delete.exclude(id=record_to_keep.id))
+
+            to_be_deleted = list(set(to_be_deleted))
+            to_be_deleted = [rec.id for rec in to_be_deleted]
+            model.objects.filter(id__in = to_be_deleted).delete()
+
+    dependencies = [
+        ("vulnerabilities", "0054_alter_packagechangelog_software_version_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_duped_changelogs, reverse_code=migrations.RunPython.noop),
+    ]

vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.13 on 2024-01-22 09:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0055_remove_changelogs_with_same_data_different_software_version"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="packagechangelog",
+            unique_together={("action_time", "actor_name", "action_type", "source_url")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="vulnerabilitychangelog",
+            unique_together={("action_time", "actor_name", "action_type", "source_url")},
+        ),
+    ]

vulnerabilities/models.py

@@ -1145,6 +1145,7 @@ def get_iso_time(self):
     class Meta:
         abstract = True
         ordering = ("-action_time",)
+        unique_together = ("action_time", "actor_name", "action_type", "source_url")
 
 
 class VulnerabilityHistoryManager(models.Manager):

vulnerabilities/tests/test_data/fireeye/fireeye_test3.md

@@ -0,0 +1,31 @@
+# MNDT-2023-0017
+
+The IBM Personal Communications (PCOMM) application 13.0.0 and earlier caused a user's plaintext password to be written to the `C:\Temp\pcsnp_init.log` file when re-connection was made through a remote desktop protocol.
+
+## Common Weakness Enumeration
+CWE-312: Cleartext Storage of Sensitive Information
+
+## Impact
+High - An attacker with low-privilege access to a host with IBM PCOMM could recover the plaintext password of another user.
+
+## Exploitability
+Low - Exploitability varies depending on the environment in which IBM PCOMM is installed. Mandiant identified this vulnerability when conducting independent security research for a client that used Citrix to connect to shared Windows Server instances. In certain environments where remote desktop is used to connect to shared hosts with IBM PCOMM installed, the exploitability is greatly increased.
+
+## CVE Reference
+CVE-2016-0321 - scope expanded
+
+## Technical Details
+While conducting independent security research, Mandiant identified a plaintext Active Directory password stored within the `C:\Temp\pcsnp_init.log` file. The affected host had IBM PCOMM version 13.0.0 installed and was used by multiple users who connected with Citrix. Upon a user connecting, disconnecting, and connecting again, the user's plaintext password was stored in the `C:\Temp\pcsnp_init.log` file.
+
+## Discovery Credits
+- Adin Drabkin, Mandiant
+- Matthew Rotlevi, Mandiant
+
+## Disclosure Timeline
+- 2023-09-26 - Issue reported to the vendor.
+- 2023-11-03 - The vendor updated the security bulletin for CVE-2016-0321 to include all known affected and fixed versions.
+
+## References
+- [IBM Security Bulletin](https://www.ibm.com/support/pages/security-bulletin-ibm-personal-communications-could-allow-remote-user-obtain-sensitive-information-including-user-passwords-allowing-unauthorized-access-cve-2016-0321)
+- [IBM Personal Communications](https://www.ibm.com/support/pages/ibm-personal-communications)
+- [Mitre CVE-2016-0321](https://www.cve.org/CVERecord?id=CVE-2016-0321)

vulnerabilities/tests/test_data_migrations.py

@@ -610,3 +610,53 @@ def setUpBeforeMigration(self, apps):
     def test_removal_of_duped_purls(self):
         Package = apps.get_model("vulnerabilities", "Package")
         assert Package.objects.count() == 1
+
+
+class TestRemoveDupedChangeLogWithSameData(TestMigrations):
+    app_name = "vulnerabilities"
+    migrate_from = "0054_alter_packagechangelog_software_version_and_more"
+    migrate_to = "0055_remove_changelogs_with_same_data_different_software_version"
+
+    def setUpBeforeMigration(self, apps):
+        PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog")
+        VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog")
+        Package = apps.get_model("vulnerabilities", "Package")
+        Vulnerability = apps.get_model("vulnerabilities", "Vulnerability")
+        pkg1 = Package.objects.create(type="nginx", name="nginx", qualifiers={"os": "windows"})
+        vuln = Vulnerability.objects.create(summary="NEW")
+        PackageChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="1",
+            package=pkg1,
+            related_vulnerability=vuln,
+        )
+        PackageChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="2",
+            package=pkg1,
+            related_vulnerability=vuln,
+        )
+        VulnerabilityChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="2",
+            vulnerability=vuln,
+        )
+        VulnerabilityChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="1",
+            vulnerability=vuln,
+        )
+
+    def test_removal_of_changelog(self):
+        PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog")
+        VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog")
+        assert PackageChangeLog.objects.all().count() == 1
+        assert VulnerabilityChangeLog.objects.all().count() == 1

vulnerabilities/tests/test_fireeye.py

@@ -172,3 +172,48 @@ def test_parse_advisory_data_2(self):
         result = imported_data.to_dict()
 
         util_tests.check_results_against_json(result, expected_file)
+
+    def test_md_list_to_dict_2(self):
+        expected_output = {
+            "# MNDT-2023-0017\n": [
+                "\n",
+                "The IBM Personal Communications (PCOMM) application 13.0.0 and earlier caused a user's plaintext password to be written to the `C:\\Temp\\pcsnp_init.log` file when re-connection was made through a remote desktop protocol.\n",
+                "\n",
+            ],
+            "## Common Weakness Enumeration\n": [
+                "CWE-312: Cleartext Storage of Sensitive Information\n",
+                "\n",
+            ],
+            "## Impact\n": [
+                "High - An attacker with low-privilege access to a host with IBM PCOMM could recover the plaintext password of another user.\n",
+                "\n",
+            ],
+            "## Exploitability\n": [
+                "Low - Exploitability varies depending on the environment in which IBM PCOMM is installed. Mandiant identified this vulnerability when conducting independent security research for a client that used Citrix to connect to shared Windows Server instances. In certain environments where remote desktop is used to connect to shared hosts with IBM PCOMM installed, the exploitability is greatly increased.\n",
+                "\n",
+            ],
+            "## CVE Reference\n": ["CVE-2016-0321 - scope expanded\n", "\n"],
+            "## Technical Details\n": [
+                "While conducting independent security research, Mandiant identified a plaintext Active Directory password stored within the `C:\\Temp\\pcsnp_init.log` file. The affected host had IBM PCOMM version 13.0.0 installed and was used by multiple users who connected with Citrix. Upon a user connecting, disconnecting, and connecting again, the user's plaintext password was stored in the `C:\\Temp\\pcsnp_init.log` file.\n",
+                "\n",
+            ],
+            "## Discovery Credits\n": [
+                "- Adin Drabkin, Mandiant\n",
+                "- Matthew Rotlevi, Mandiant\n",
+                "\n",
+            ],
+            "## Disclosure Timeline\n": [
+                "- 2023-09-26 - Issue reported to the vendor.\n",
+                "- 2023-11-03 - The vendor updated the security bulletin for CVE-2016-0321 to include all known affected and fixed versions.\n",
+                "\n",
+            ],
+            "## References\n": [
+                "- [IBM Security Bulletin](https://www.ibm.com/support/pages/security-bulletin-ibm-personal-communications-could-allow-remote-user-obtain-sensitive-information-including-user-passwords-allowing-unauthorized-access-cve-2016-0321)\n",
+                "- [IBM Personal Communications](https://www.ibm.com/support/pages/ibm-personal-communications)\n",
+                "- [Mitre CVE-2016-0321](https://www.cve.org/CVERecord?id=CVE-2016-0321)\n",
+            ],
+        }
+        with open(os.path.join(TEST_DATA, "fireeye_test3.md"), encoding="utf-8-sig") as f:
+            md_list = f.readlines()
+            md_dict = md_list_to_dict(md_list)
+            assert md_dict == expected_output

vulntotal/datasources/deps.py

@@ -26,12 +26,21 @@ class DepsDataSource(DataSource):
 
     def fetch_json_response(self, url):
         response = requests.get(url)
-        if not response.status_code == 200 or response.text == "Not Found":
+        if response.status_code != 200 or response.text == "Not Found":
             logger.error(f"Error while fetching {url}")
             return
         return response.json()
 
     def datasource_advisory(self, purl) -> Iterable[VendorData]:
+        """
+        Fetch and parse advisories from a given purl.
+
+        Parameters:
+            purl: A string representing the package URL.
+
+        Returns:
+            A list of VendorData objects containing the advisory information.
+        """
         payload = generate_meta_payload(purl)
         response = self.fetch_json_response(payload)
         if response:
@@ -58,6 +67,16 @@ def supported_ecosystem(cls):
 
 
 def parse_advisory(advisory, purl) -> Iterable[VendorData]:
+    """
+    Parse an advisory into a VendorData object.
+
+    Parameters:
+        advisory: A dictionary representing the advisory data.
+        purl: PURL for the advisory.
+
+    Yields:
+        VendorData instance containing purl, aliases, affected_versions and fixed_versions.
+    """
     package = advisory["packages"][0]
     affected_versions = [event["version"] for event in package["versionsAffected"]]
     fixed_versions = [event["version"] for event in package["versionsUnaffected"]]
@@ -70,6 +89,15 @@ def parse_advisory(advisory, purl) -> Iterable[VendorData]:
 
 
 def parse_advisories_from_meta(advisories_metadata):
+    """
+    Parse advisories from a given metadata.
+
+    Parameters:
+        advisories_metadata: A dictionary representing the metadata of the advisories.
+
+    Returns:
+        A list of dictionaries, each representing an advisory.
+    """
     advisories = []
     dependencies = advisories_metadata.get("dependencies") or []
     for dependency in dependencies:
@@ -84,6 +112,15 @@ def generate_advisory_payload(advisory_meta):
 
 
 def generate_meta_payload(purl):
+    """
+    Generate a payload for fetching advisories metadata from a given purl.
+
+    Parameters:
+        purl: A PackageURL object representing the package URL.
+
+    Returns:
+        A string representing the payload for fetching advisories metadata. It should be a valid URL that contains the ecosystem, package name and package version of the dependency.
+    """
     url_advisories_meta = "https://deps.dev/_/s/{ecosystem}/p/{package}/v/{version}/dependencies"
     supported_ecosystem = DepsDataSource.supported_ecosystem()
     if purl.type in supported_ecosystem:

vulntotal/datasources/github.py

@@ -68,8 +68,8 @@ def datasource_advisory_from_cve(self, cve: str) -> Iterable[VendorData]:
             yield VendorData(
                 purl=purl,
                 aliases=sorted(list(set(advisory.get("identifiers", None)))),
-                affected_versions=sorted(list(set(advisory.get("firstPatchedVersion", None)))),
-                fixed_versions=sorted(list(set(advisory.get("vulnerableVersionRange", None)))),
+                affected_versions=sorted(list(set(advisory.get("vulnerableVersionRange", None)))),
+                fixed_versions=sorted(list(set(advisory.get("firstPatchedVersion", None)))),
             )
 
     @classmethod
@@ -101,7 +101,7 @@ def parse_advisory(interesting_edges, purl) -> Iterable[VendorData]:
     """
     for edge in interesting_edges:
         node = edge["node"]
-        aliases = [aliase["value"] for aliase in get_item(node, "advisory", "identifiers")]
+        aliases = [alias["value"] for alias in get_item(node, "advisory", "identifiers")]
         affected_versions = node["vulnerableVersionRange"].strip().replace(" ", "").split(",")
         parsed_fixed_versions = get_item(node, "firstPatchedVersion", "identifier")
         fixed_versions = [parsed_fixed_versions] if parsed_fixed_versions else []