Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for all osv ecosystems #926

Merged
merged 1 commit into from
Feb 13, 2024
Merged

Add support for all osv ecosystems #926

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions vulnerabilities/importers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from vulnerabilities.importers import fireeye
from vulnerabilities.importers import gentoo
from vulnerabilities.importers import github
from vulnerabilities.importers import github_osv
from vulnerabilities.importers import gitlab
from vulnerabilities.importers import istio
from vulnerabilities.importers import mozilla
Expand Down Expand Up @@ -69,6 +70,7 @@
apache_kafka.ApacheKafkaImporter,
oss_fuzz.OSSFuzzImporter,
ruby.RubyImporter,
github_osv.GithubOSVImporter,
]

IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}
56 changes: 56 additions & 0 deletions vulnerabilities/importers/github_osv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#
# Copyright (c) nexB Inc. and others. All rights reserved.
# VulnerableCode is a trademark of nexB Inc.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/nexB/vulnerablecode for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#
import json
import logging
from pathlib import Path
from typing import Iterable

from vulnerabilities.importer import AdvisoryData
from vulnerabilities.importer import Importer
from vulnerabilities.importers.osv import parse_advisory_data
from vulnerabilities.utils import get_advisory_url

logger = logging.getLogger(__name__)


class GithubOSVImporter(Importer):
ziadhany marked this conversation as resolved.
Show resolved Hide resolved
license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
spdx_license_expression = "CC-BY-4.0"
repo_url = "git+https://github.com/github/advisory-database/"
importer_name = "GithubOSV Importer"

def advisory_data(self) -> Iterable[AdvisoryData]:
supported_ecosystems = [
"pypi",
"npm",
"maven",
"golang",
"composer",
"hex",
"gem",
"nuget",
"cargo",
]
try:
self.clone(repo_url=self.repo_url)
base_path = Path(self.vcs_response.dest_dir)
# filter out non-github-reviewed files and only keep the files end-with .json
advisory_dirs = base_path / "advisories/github-reviewed"
for file in advisory_dirs.glob("**/*.json"):
advisory_url = get_advisory_url(
file=file,
base_path=base_path,
url="https://github.com/github/advisory-database/blob/main/",
)
with open(file) as f:
raw_data = json.load(f)
yield parse_advisory_data(raw_data, supported_ecosystems, advisory_url)
finally:
if self.vcs_response:
self.vcs_response.delete()
2 changes: 1 addition & 1 deletion vulnerabilities/importers/oss_fuzz.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
url="https://github.com/pypa/advisory-database/blob/main/",
)
yield parse_advisory_data(
yaml_data, supported_ecosystem="oss-fuzz", advisory_url=advisory_url
yaml_data, supported_ecosystems=["oss-fuzz"], advisory_url=advisory_url
)
finally:
if self.vcs_response:
Expand Down
108 changes: 73 additions & 35 deletions vulnerabilities/importers/osv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@
from typing import Optional

import dateparser
from cvss.exceptions import CVSS3MalformedError
from packageurl import PackageURL
from univers.version_range import RANGE_CLASS_BY_SCHEMES
from univers.versions import InvalidVersion
from univers.versions import PypiVersion
from univers.versions import SemverVersion
from univers.versions import Version

Expand All @@ -31,9 +31,21 @@

logger = logging.getLogger(__name__)

PURL_TYPE_BY_OSV_ECOSYSTEM = {
"npm": "npm",
"pypi": "pypi",
"maven": "maven",
"nuget": "nuget",
"packagist": "composer",
"rubygems": "gem",
"go": "golang",
"hex": "hex",
"cargo": "cargo",
}


def parse_advisory_data(
raw_data: dict, supported_ecosystem, advisory_url: str
raw_data: dict, supported_ecosystems, advisory_url: str
) -> Optional[AdvisoryData]:
"""
Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and
Expand All @@ -56,18 +68,21 @@ def parse_advisory_data(

for affected_pkg in raw_data.get("affected") or []:
purl = get_affected_purl(affected_pkg=affected_pkg, raw_id=raw_id)
if purl.type != supported_ecosystem:
logger.error(f"Unsupported package type: {purl!r} in OSV: {raw_id!r}")

if not purl or purl.type not in supported_ecosystems:
logger.error(f"Unsupported package type: {affected_pkg!r} in OSV: {raw_id!r}")
continue

affected_version_range = get_affected_version_range(
affected_pkg=affected_pkg,
raw_id=raw_id,
supported_ecosystem=supported_ecosystem,
supported_ecosystem=purl.type,
)

for fixed_range in affected_pkg.get("ranges") or []:
fixed_version = get_fixed_versions(fixed_range=fixed_range, raw_id=raw_id)
fixed_version = get_fixed_versions(
fixed_range=fixed_range, raw_id=raw_id, supported_ecosystem=purl.type
)

for version in fixed_version:
affected_packages.append(
Expand Down Expand Up @@ -121,14 +136,22 @@ def get_severities(raw_data) -> Iterable[VulnerabilitySeverity]:
"""
Yield VulnerabilitySeverity extracted from a mapping of OSV ``raw_data``
"""
for severity in raw_data.get("severity") or []:
if severity.get("type") == "CVSS_V3":
vector = severity["score"]
system = SCORING_SYSTEMS["cvssv3.1"]
score = system.compute(vector)
yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
else:
logger.error(f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}")
try:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why having try here? What are you catching exception here for ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to catch a malformed CVSS3 vector
ERROR:vulnerabilities.importers.osv:Invalid severity Malformed CVSS3 vector, trailing "/"
I think I should change the Exception to CVSS3MalformedError

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, please do!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you have an example of ERROR:vulnerabilities.importers.osv:Invalid severity Malformed CVSS3 vector, trailing "/"?
(with a link)

May be we can clean them up in some cases rather than failing at parsing them?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes , CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:C/C:H/I:H/A:N/

>>> from cvss import CVSS3
>>> vector = "CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:C/C:H/I:H/A:N/"
>>> c = CVSS3(vector) 
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/ziad/.local/lib/python3.10/site-packages/cvss/cvss3.py", line 114, in __init__
    self.parse_vector()
  File "/home/ziad/.local/lib/python3.10/site-packages/cvss/cvss3.py", line 133, in parse_vector
    raise CVSS3MalformedError('Malformed CVSS3 vector, trailing "/"')
cvss.exceptions.CVSS3MalformedError: Malformed CVSS3 vector, trailing "/"

All malformed vectors :

[{'type': 'CVSS_V3', 'score': 'CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:C/C:H/I:H/A:N/'}]
[{'type': 'CVSS_V3', 'score': 'CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H/'}]
[{'type': 'CVSS_V3', 'score': 'CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H/'}]
[{'type': 'CVSS_V3', 'score': 'CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:U/C:H/I:N/A:N/'}]
[{'type': 'CVSS_V3', 'score': 'CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H/'}]

I think I should remove the '/' from the end of the vector to parse it successfully

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I should remove the '/' from the end of the vector to parse it successfully

Go for it!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

for severity in raw_data.get("severity") or []:
if severity.get("type") == "CVSS_V3":
vector = severity.get("score")
# remove the / from the end of the vector if / exist
valid_vector = vector[:-1] if vector and vector[-1] == "/" else vector
system = SCORING_SYSTEMS["cvssv3.1"]
score = system.compute(valid_vector)
yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)

else:
logger.error(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there such a case? what other values?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to OSV schema CVSS_V2 but according to logs nothing

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pombredanne Should I add support for CVSS_V2 ?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ziadhany Nope :) we are trying to get away from it!

f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}"
)
except CVSS3MalformedError as e:
logger.error(f"Invalid severity {e}")

ecosystem_specific = raw_data.get("ecosystem_specific") or {}
severity = ecosystem_specific.get("severity")
Expand Down Expand Up @@ -173,21 +196,31 @@ def get_affected_purl(affected_pkg, raw_id):
purl = package.get("purl")
if purl:
try:
return PackageURL.from_string(purl)
purl = PackageURL.from_string(purl)
except ValueError:
logger.error(
f"Invalid PackageURL: {purl!r} for OSV "
f"affected_pkg {affected_pkg} and id: {raw_id}"
)

ecosys = package.get("ecosystem")
name = package.get("name")
if ecosys and name:
return PackageURL(type=ecosys, name=name)

logger.error(
f"No PackageURL possible: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
)
else:
ecosys = package.get("ecosystem")
name = package.get("name")
if ecosys and name:
ecosys = ecosys.lower()
purl_type = PURL_TYPE_BY_OSV_ECOSYSTEM.get(ecosys)
if not purl_type:
return
namespace = ""
if purl_type == "maven":
namespace, _, name = name.partition(":")

purl = PackageURL(type=purl_type, namespace=namespace, name=name)
else:
logger.error(
f"No PackageURL possible: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
)
return
return PackageURL.from_string(str(purl))


def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
Expand All @@ -206,18 +239,17 @@ def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
)


def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]:
"""
Return a list of unique fixed univers Versions given a ``fixed_range``
univers VersionRange and a ``raw_id``.

For example::

>>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj")
>>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj", supported_ecosystem="pypi",)
[]
>>> get_fixed_versions(
... fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}]},
... raw_id="GHSA-j3f7-7rmc-6wqj"
... fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}], },
... raw_id="GHSA-j3f7-7rmc-6wqj",
... supported_ecosystem="pypi",
... )
[PypiVersion(string='1.7.0')]
"""
Expand All @@ -228,21 +260,27 @@ def get_fixed_versions(fixed_range, raw_id) -> List[Version]:

fixed_range_type = fixed_range["type"]

for version in extract_fixed_versions(fixed_range):
version_range_class = RANGE_CLASS_BY_SCHEMES.get(supported_ecosystem)
version_class = version_range_class.version_class if version_range_class else None

# FIXME: ECOSYSTEM does not imply PyPI!!!!
for version in extract_fixed_versions(fixed_range):
if fixed_range_type == "ECOSYSTEM":
try:
fixed_versions.append(PypiVersion(version))
if not version_class:
raise InvalidVersion(
f"Unsupported version for ecosystem: {supported_ecosystem}"
)
fixed_versions.append(version_class(version))
except InvalidVersion:
logger.error(f"Invalid PypiVersion: {version!r} for OSV id: {raw_id!r}")
logger.error(
f"Invalid version class: {version_class} - {version!r} for OSV id: {raw_id!r}"
)

elif fixed_range_type == "SEMVER":
ziadhany marked this conversation as resolved.
Show resolved Hide resolved
try:
fixed_versions.append(SemverVersion(version))
except InvalidVersion:
logger.error(f"Invalid SemverVersion: {version!r} for OSV id: {raw_id!r}")

else:
logger.error(f"Unsupported fixed version type: {version!r} for OSV id: {raw_id!r}")

Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/importers/pypa.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
for advisory_url, raw_data in fork_and_get_files(base_path=path):
yield parse_advisory_data(
raw_data=raw_data,
supported_ecosystem="pypi",
supported_ecosystems=["pypi"],
advisory_url=advisory_url,
)
finally:
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/importers/pysec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,5 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
with zip_file.open(file_name) as f:
vul_info = json.load(f)
yield parse_advisory_data(
raw_data=vul_info, supported_ecosystem="pypi", advisory_url=url
raw_data=vul_info, supported_ecosystems=["pypi"], advisory_url=url
)
1 change: 1 addition & 0 deletions vulnerabilities/improvers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
valid_versions.UbuntuOvalImprover,
valid_versions.OSSFuzzImprover,
valid_versions.RubyImprover,
valid_versions.GithubOSVImprover,
vulnerability_status.VulnerabilityStatusImprover,
]

Expand Down
6 changes: 6 additions & 0 deletions vulnerabilities/improvers/valid_versions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from vulnerabilities.importers.debian_oval import DebianOvalImporter
from vulnerabilities.importers.elixir_security import ElixirSecurityImporter
from vulnerabilities.importers.github import GitHubAPIImporter
from vulnerabilities.importers.github_osv import GithubOSVImporter
from vulnerabilities.importers.gitlab import GitLabAPIImporter
from vulnerabilities.importers.istio import IstioImporter
from vulnerabilities.importers.nginx import NginxImporter
Expand Down Expand Up @@ -466,3 +467,8 @@ class OSSFuzzImprover(ValidVersionImprover):
class RubyImprover(ValidVersionImprover):
importer = RubyImporter
ignorable_versions = []


class GithubOSVImprover(ValidVersionImprover):
importer = GithubOSVImporter
ignorable_versions = []
Loading
Loading