Add support for all osv ecosystems (aboutcode-org#926)

Add a GithubOSVImporter to git_importer parametrize test
Refactor OSV ecosystem mapping
Fix the test
Update univers version and pass nuget test
Resolve merge conflict
Add a test for golang
Fix test by adding cwe to expected files
Resolve merge conflict

Signed-off-by: ziadhany <ziadhany2016@gmail.com>

vulnerabilities/importers/__init__.py

@@ -18,6 +18,7 @@
 from vulnerabilities.importers import fireeye
 from vulnerabilities.importers import gentoo
 from vulnerabilities.importers import github
+from vulnerabilities.importers import github_osv
 from vulnerabilities.importers import gitlab
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
@@ -69,6 +70,7 @@
     apache_kafka.ApacheKafkaImporter,
     oss_fuzz.OSSFuzzImporter,
     ruby.RubyImporter,
+    github_osv.GithubOSVImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/github_osv.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+import logging
+from pathlib import Path
+from typing import Iterable
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import Importer
+from vulnerabilities.importers.osv import parse_advisory_data
+from vulnerabilities.utils import get_advisory_url
+
+logger = logging.getLogger(__name__)
+
+
+class GithubOSVImporter(Importer):
+    license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
+    spdx_license_expression = "CC-BY-4.0"
+    repo_url = "git+https://github.com/github/advisory-database/"
+    importer_name = "GithubOSV Importer"
+
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        supported_ecosystems = [
+            "pypi",
+            "npm",
+            "maven",
+            "golang",
+            "composer",
+            "hex",
+            "gem",
+            "nuget",
+            "cargo",
+        ]
+        try:
+            self.clone(repo_url=self.repo_url)
+            base_path = Path(self.vcs_response.dest_dir)
+            # filter out non-github-reviewed files and only keep the files end-with .json
+            advisory_dirs = base_path / "advisories/github-reviewed"
+            for file in advisory_dirs.glob("**/*.json"):
+                advisory_url = get_advisory_url(
+                    file=file,
+                    base_path=base_path,
+                    url="https://github.com/github/advisory-database/blob/main/",
+                )
+                with open(file) as f:
+                    raw_data = json.load(f)
+                yield parse_advisory_data(raw_data, supported_ecosystems, advisory_url)
+        finally:
+            if self.vcs_response:
+                self.vcs_response.delete()

vulnerabilities/importers/oss_fuzz.py

@@ -40,7 +40,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                         url="https://github.com/pypa/advisory-database/blob/main/",
                     )
                     yield parse_advisory_data(
-                        yaml_data, supported_ecosystem="oss-fuzz", advisory_url=advisory_url
+                        yaml_data, supported_ecosystems=["oss-fuzz"], advisory_url=advisory_url
                     )
         finally:
             if self.vcs_response:

vulnerabilities/importers/osv.py

@@ -13,10 +13,10 @@
 from typing import Optional
 
 import dateparser
+from cvss.exceptions import CVSS3MalformedError
 from packageurl import PackageURL
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.versions import InvalidVersion
-from univers.versions import PypiVersion
 from univers.versions import SemverVersion
 from univers.versions import Version
 
@@ -31,9 +31,21 @@
 
 logger = logging.getLogger(__name__)
 
+PURL_TYPE_BY_OSV_ECOSYSTEM = {
+    "npm": "npm",
+    "pypi": "pypi",
+    "maven": "maven",
+    "nuget": "nuget",
+    "packagist": "composer",
+    "rubygems": "gem",
+    "go": "golang",
+    "hex": "hex",
+    "cargo": "cargo",
+}
+
 
 def parse_advisory_data(
-    raw_data: dict, supported_ecosystem, advisory_url: str
+    raw_data: dict, supported_ecosystems, advisory_url: str
 ) -> Optional[AdvisoryData]:
     """
     Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and
@@ -56,18 +68,21 @@ def parse_advisory_data(
 
     for affected_pkg in raw_data.get("affected") or []:
         purl = get_affected_purl(affected_pkg=affected_pkg, raw_id=raw_id)
-        if purl.type != supported_ecosystem:
-            logger.error(f"Unsupported package type: {purl!r} in OSV: {raw_id!r}")
+
+        if not purl or purl.type not in supported_ecosystems:
+            logger.error(f"Unsupported package type: {affected_pkg!r} in OSV: {raw_id!r}")
             continue
 
         affected_version_range = get_affected_version_range(
             affected_pkg=affected_pkg,
             raw_id=raw_id,
-            supported_ecosystem=supported_ecosystem,
+            supported_ecosystem=purl.type,
         )
 
         for fixed_range in affected_pkg.get("ranges") or []:
-            fixed_version = get_fixed_versions(fixed_range=fixed_range, raw_id=raw_id)
+            fixed_version = get_fixed_versions(
+                fixed_range=fixed_range, raw_id=raw_id, supported_ecosystem=purl.type
+            )
 
             for version in fixed_version:
                 affected_packages.append(
@@ -121,14 +136,22 @@ def get_severities(raw_data) -> Iterable[VulnerabilitySeverity]:
     """
     Yield VulnerabilitySeverity extracted from a mapping of OSV ``raw_data``
     """
-    for severity in raw_data.get("severity") or []:
-        if severity.get("type") == "CVSS_V3":
-            vector = severity["score"]
-            system = SCORING_SYSTEMS["cvssv3.1"]
-            score = system.compute(vector)
-            yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
-        else:
-            logger.error(f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}")
+    try:
+        for severity in raw_data.get("severity") or []:
+            if severity.get("type") == "CVSS_V3":
+                vector = severity.get("score")
+                # remove the / from the end of the vector if / exist
+                valid_vector = vector[:-1] if vector and vector[-1] == "/" else vector
+                system = SCORING_SYSTEMS["cvssv3.1"]
+                score = system.compute(valid_vector)
+                yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
+
+            else:
+                logger.error(
+                    f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}"
+                )
+    except CVSS3MalformedError as e:
+        logger.error(f"Invalid severity {e}")
 
     ecosystem_specific = raw_data.get("ecosystem_specific") or {}
     severity = ecosystem_specific.get("severity")
@@ -173,21 +196,31 @@ def get_affected_purl(affected_pkg, raw_id):
     purl = package.get("purl")
     if purl:
         try:
-            return PackageURL.from_string(purl)
+            purl = PackageURL.from_string(purl)
         except ValueError:
             logger.error(
                 f"Invalid PackageURL: {purl!r} for OSV "
                 f"affected_pkg {affected_pkg} and id: {raw_id}"
             )
-
-    ecosys = package.get("ecosystem")
-    name = package.get("name")
-    if ecosys and name:
-        return PackageURL(type=ecosys, name=name)
-
-    logger.error(
-        f"No PackageURL possible: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
-    )
+    else:
+        ecosys = package.get("ecosystem")
+        name = package.get("name")
+        if ecosys and name:
+            ecosys = ecosys.lower()
+            purl_type = PURL_TYPE_BY_OSV_ECOSYSTEM.get(ecosys)
+            if not purl_type:
+                return
+            namespace = ""
+            if purl_type == "maven":
+                namespace, _, name = name.partition(":")
+
+            purl = PackageURL(type=purl_type, namespace=namespace, name=name)
+        else:
+            logger.error(
+                f"No PackageURL possible: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
+            )
+            return
+    return PackageURL.from_string(str(purl))
 
 
 def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
@@ -206,18 +239,17 @@ def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
             )
 
 
-def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
+def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]:
     """
     Return a list of unique fixed univers Versions given a ``fixed_range``
     univers VersionRange and a ``raw_id``.
-
     For example::
-
-    >>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj")
+    >>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj", supported_ecosystem="pypi",)
     []
     >>> get_fixed_versions(
-    ...   fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}]},
-    ...   raw_id="GHSA-j3f7-7rmc-6wqj"
+    ...   fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}], },
+    ...   raw_id="GHSA-j3f7-7rmc-6wqj",
+    ...   supported_ecosystem="pypi",
     ... )
     [PypiVersion(string='1.7.0')]
     """
@@ -228,21 +260,27 @@ def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
 
     fixed_range_type = fixed_range["type"]
 
-    for version in extract_fixed_versions(fixed_range):
+    version_range_class = RANGE_CLASS_BY_SCHEMES.get(supported_ecosystem)
+    version_class = version_range_class.version_class if version_range_class else None
 
-        # FIXME: ECOSYSTEM does not imply PyPI!!!!
+    for version in extract_fixed_versions(fixed_range):
         if fixed_range_type == "ECOSYSTEM":
             try:
-                fixed_versions.append(PypiVersion(version))
+                if not version_class:
+                    raise InvalidVersion(
+                        f"Unsupported version for ecosystem: {supported_ecosystem}"
+                    )
+                fixed_versions.append(version_class(version))
             except InvalidVersion:
-                logger.error(f"Invalid PypiVersion: {version!r} for OSV id: {raw_id!r}")
+                logger.error(
+                    f"Invalid version class: {version_class} - {version!r} for OSV id: {raw_id!r}"
+                )
 
         elif fixed_range_type == "SEMVER":
             try:
                 fixed_versions.append(SemverVersion(version))
             except InvalidVersion:
                 logger.error(f"Invalid SemverVersion: {version!r} for OSV id: {raw_id!r}")
-
         else:
             logger.error(f"Unsupported fixed version type: {version!r} for OSV id: {raw_id!r}")
 

vulnerabilities/importers/pypa.py

@@ -34,7 +34,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
             for advisory_url, raw_data in fork_and_get_files(base_path=path):
                 yield parse_advisory_data(
                     raw_data=raw_data,
-                    supported_ecosystem="pypi",
+                    supported_ecosystems=["pypi"],
                     advisory_url=advisory_url,
                 )
         finally:

vulnerabilities/importers/pysec.py

@@ -40,5 +40,5 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                 with zip_file.open(file_name) as f:
                     vul_info = json.load(f)
                     yield parse_advisory_data(
-                        raw_data=vul_info, supported_ecosystem="pypi", advisory_url=url
+                        raw_data=vul_info, supported_ecosystems=["pypi"], advisory_url=url
                     )

vulnerabilities/improvers/__init__.py

@@ -25,6 +25,7 @@
     valid_versions.UbuntuOvalImprover,
     valid_versions.OSSFuzzImprover,
     valid_versions.RubyImprover,
+    valid_versions.GithubOSVImprover,
     vulnerability_status.VulnerabilityStatusImprover,
 ]
 

vulnerabilities/improvers/valid_versions.py

@@ -32,6 +32,7 @@
 from vulnerabilities.importers.debian_oval import DebianOvalImporter
 from vulnerabilities.importers.elixir_security import ElixirSecurityImporter
 from vulnerabilities.importers.github import GitHubAPIImporter
+from vulnerabilities.importers.github_osv import GithubOSVImporter
 from vulnerabilities.importers.gitlab import GitLabAPIImporter
 from vulnerabilities.importers.istio import IstioImporter
 from vulnerabilities.importers.nginx import NginxImporter
@@ -466,3 +467,8 @@ class OSSFuzzImprover(ValidVersionImprover):
 class RubyImprover(ValidVersionImprover):
     importer = RubyImporter
     ignorable_versions = []
+
+
+class GithubOSVImprover(ValidVersionImprover):
+    importer = GithubOSVImporter
+    ignorable_versions = []