Skip to content

Commit

Permalink
Improve naming and docs
Browse files Browse the repository at this point in the history
Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>
  • Loading branch information
sbs2001 committed Apr 22, 2021
1 parent efeea4f commit ff833e4
Show file tree
Hide file tree
Showing 39 changed files with 198 additions and 216 deletions.
16 changes: 6 additions & 10 deletions vulnerabilities/data_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
from vulnerabilities.severity_systems import ScoringSystem
from vulnerabilities.helpers import is_cve
from vulnerabilities.helpers import nearest_patched_package
from vulnerabilities.helpers import AffectedPackageWithPatchedPackage
from vulnerabilities.helpers import AffectedPackage

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -88,9 +88,7 @@ class Advisory:

summary: str
vulnerability_id: Optional[str] = None
affected_packages_with_patched_package: List[
AffectedPackageWithPatchedPackage
] = dataclasses.field(default_factory=list)
affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list)
references: List[Reference] = dataclasses.field(default_factory=list)

def __post_init__(self):
Expand All @@ -107,9 +105,7 @@ def normalized(self):
return Advisory(
summary=self.summary,
vulnerability_id=self.vulnerability_id,
affected_packages_with_patched_package=sorted(
self.affected_packages_with_patched_package
),
affected_packages=sorted(self.affected_packages),
references=references,
)

Expand Down Expand Up @@ -533,7 +529,7 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
vuln_id = definition_data["vuln_id"]
description = definition_data["description"]
references = [Reference(url=url) for url in definition_data["reference_urls"]]
affected_packages_with_patched_package = []
affected_packages = []
for test_data in definition_data["test_data"]:
for package_name in test_data["package_list"]:
if package_name and len(package_name) >= 50:
Expand Down Expand Up @@ -569,14 +565,14 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
else:
safe_purls.append(purl)

affected_packages_with_patched_package.extend(
affected_packages.extend(
nearest_patched_package(affected_purls, safe_purls),
)

all_adv.append(
Advisory(
summary=description,
affected_packages_with_patched_package=affected_packages_with_patched_package,
affected_packages=affected_packages,
vulnerability_id=vuln_id,
references=references,
)
Expand Down
24 changes: 18 additions & 6 deletions vulnerabilities/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import json
import re
from typing import Optional
from typing import List

import requests
import toml
Expand All @@ -39,7 +40,7 @@


@dataclasses.dataclass(order=True, frozen=True)
class AffectedPackageWithPatchedPackage:
class AffectedPackage:
vulnerable_package: PackageURL
patched_package: Optional[PackageURL] = None

Expand Down Expand Up @@ -118,11 +119,22 @@ def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5):
return session


def nearest_patched_package(vulnerable_packages, resolved_packages):
# This class is used to get around bisect module's lack of supplying custom
# compartor. Get rid of this once we use python 3.10 which supports this.
# See https://github.com/python/cpython/pull/20556
def nearest_patched_package(
vulnerable_packages: List[PackageURL], resolved_packages: List[PackageURL]
) -> List[AffectedPackage]:
"""
Parameters:
:vulnerable_packages(list)
:resolved_packages(list)
"""

class PackageURLWithVersionComparator:
"""
This class is used to get around bisect module's lack of supplying custom
compartor. Get rid of this once we use python 3.10 which supports this.
See https://github.com/python/cpython/pull/20556
"""

def __init__(self, package):
self.package = package
self.version_object = version_class_by_package_type[package.type](package.version)
Expand Down Expand Up @@ -150,7 +162,7 @@ def __lt__(self, other):
patched_package = resolved_packages[patched_package_index].package

affected_package_with_patched_package_objects.append(
AffectedPackageWithPatchedPackage(
AffectedPackage(
vulnerable_package=vulnerable_package.package, patched_package=patched_package
)
)
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/import_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def process_advisories(data_source: DataSource) -> None:
defaults={"value": str(score.value)},
)

for aff_pkg_with_patched_pkg in advisory.affected_packages_with_patched_package:
for aff_pkg_with_patched_pkg in advisory.affected_packages:
vulnerable_package, _ = _get_or_create_package(
aff_pkg_with_patched_pkg.vulnerable_package
)
Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/apache_kafka.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,7 @@ def to_advisory(self, advisory_page):
Advisory(
vulnerability_id=cve_id,
summary=cve_description_paragraph.text,
affected_packages_with_patched_package=nearest_patched_package(
affected_packages, fixed_packages
),
affected_packages=nearest_patched_package(affected_packages, fixed_packages),
references=[
Reference(url=ASF_PAGE_URL),
Reference(
Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/apache_tomcat.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,7 @@ def to_advisories(self, apache_tomcat_advisory_html):
advisories.append(
Advisory(
summary="",
affected_packages_with_patched_package=nearest_patched_package(
affected_packages, fixed_package
),
affected_packages=nearest_patched_package(affected_packages, fixed_package),
vulnerability_id=cve_id,
references=references,
)
Expand Down
8 changes: 3 additions & 5 deletions vulnerabilities/importers/archlinux.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def _parse(self, record) -> List[Advisory]:
advisories = []

for cve_id in record["issues"]:
affected_packages_with_patched_package = []
affected_packages = []
for name in record["packages"]:
impacted_purls, resolved_purls = [], []
impacted_purls.append(
Expand All @@ -109,9 +109,7 @@ def _parse(self, record) -> List[Advisory]:
version=record["fixed"],
)
)
affected_packages_with_patched_package.extend(
nearest_patched_package(impacted_purls, resolved_purls)
)
affected_packages.extend(nearest_patched_package(impacted_purls, resolved_purls))

references = []
references.append(
Expand All @@ -138,7 +136,7 @@ def _parse(self, record) -> List[Advisory]:
Advisory(
vulnerability_id=cve_id,
summary="",
affected_packages_with_patched_package=affected_packages_with_patched_package,
affected_packages=affected_packages,
references=references,
)
)
Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/debian.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,9 +154,7 @@ def _parse(self, pkg_name: str, records: Mapping[str, Any]) -> List[Advisory]:
advisories.append(
Advisory(
vulnerability_id=cve_id,
affected_packages_with_patched_package=nearest_patched_package(
impacted_purls, resolved_purls
),
affected_packages=nearest_patched_package(impacted_purls, resolved_purls),
summary=record.get("description", ""),
references=references,
)
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/importers/elixir_security.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def process_file(self, path):

return Advisory(
summary=yaml_file["description"],
affected_packages_with_patched_package=nearest_patched_package(vuln_purls, safe_purls),
affected_packages=nearest_patched_package(vuln_purls, safe_purls),
vulnerability_id=cve_id,
references=references,
)
2 changes: 1 addition & 1 deletion vulnerabilities/importers/gentoo.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def process_file(self, file):
advisory = Advisory(
vulnerability_id=cve,
summary=xml_data["description"],
affected_packages_with_patched_package=nearest_patched_package(
affected_packages=nearest_patched_package(
xml_data["affected_purls"], xml_data["unaffected_purls"]
),
references=vuln_reference,
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/importers/github.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ def process_response(self) -> List[Advisory]:
Advisory(
vulnerability_id=cve_id,
summary=vuln_desc,
affected_packages_with_patched_package=nearest_patched_package(
affected_packages=nearest_patched_package(
affected_purls, unaffected_purls
),
references=references,
Expand Down
12 changes: 4 additions & 8 deletions vulnerabilities/importers/istio.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ def process_file(self, path):
data["release_ranges"]
)

affected_packages_with_patched_package = []
affected_packages = []

safe_purls_golang = [
PackageURL(type="golang", name="istio", version=version)
Expand All @@ -174,9 +174,7 @@ def process_file(self, path):
for version in vuln_pkg_versions
]

affected_packages_with_patched_package.extend(
nearest_patched_package(vuln_purls_golang, safe_purls_golang)
)
affected_packages.extend(nearest_patched_package(vuln_purls_golang, safe_purls_golang))

safe_purls_github = [
PackageURL(type="github", name="istio", version=version)
Expand All @@ -188,15 +186,13 @@ def process_file(self, path):
for version in vuln_pkg_versions
]

affected_packages_with_patched_package.extend(
nearest_patched_package(vuln_purls_github, safe_purls_github)
)
affected_packages.extend(nearest_patched_package(vuln_purls_github, safe_purls_github))

advisories.append(
Advisory(
vulnerability_id=cve_id,
summary=data["description"],
affected_packages_with_patched_package=affected_packages_with_patched_package,
affected_packages=affected_packages,
)
)

Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/kaybee.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,6 @@ def yaml_file_to_advisory(yaml_path):
return Advisory(
vulnerability_id=vuln_id,
summary=summary,
affected_packages_with_patched_package=nearest_patched_package(
impacted_packages, resolved_packages
),
affected_packages=nearest_patched_package(impacted_packages, resolved_packages),
references=references,
)
4 changes: 1 addition & 3 deletions vulnerabilities/importers/nginx.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,7 @@ def to_advisories(self, data):
Advisory(
vulnerability_id=cve_id,
summary=summary,
affected_packages_with_patched_package=nearest_patched_package(
vulnerable_packages, fixed_packages
),
affected_packages=nearest_patched_package(vulnerable_packages, fixed_packages),
)
)

Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/npm.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,7 @@ def process_file(self, file) -> List[Advisory]:
Advisory(
summary=record.get("overview", ""),
vulnerability_id=cve_id,
affected_packages_with_patched_package=nearest_patched_package(
impacted_purls, resolved_purls
),
affected_packages=nearest_patched_package(impacted_purls, resolved_purls),
references=vuln_reference,
)
)
Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/openssl.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,7 @@ def to_advisories(xml_response: str) -> Set[Advisory]:
advisory = Advisory(
vulnerability_id=cve_id,
summary=summary,
affected_packages_with_patched_package=nearest_patched_package(
vuln_purls, safe_purls
),
affected_packages=nearest_patched_package(vuln_purls, safe_purls),
references=ref_urls,
)
advisories.append(advisory)
Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/postgresql.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,7 @@ def to_advisories(data):
vulnerability_id=cve_id,
summary=summary,
references=references,
affected_packages_with_patched_package=nearest_patched_package(
affected_packages, fixed_packages
),
affected_packages=nearest_patched_package(affected_packages, fixed_packages),
)
)

Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/importers/project_kb_msr2019.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def to_advisories(csv_reader):
advisories.append(
Advisory(
summary="",
affected_packages_with_patched_package=[],
affected_packages=[],
references=[reference],
vulnerability_id=vuln_id,
)
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/importers/redhat.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ def to_advisory(advisory_data):
return Advisory(
vulnerability_id=advisory_data["CVE"],
summary=advisory_data["bugzilla_description"],
affected_packages_with_patched_package=nearest_patched_package(affected_purls, []),
affected_packages=nearest_patched_package(affected_purls, []),
references=references,
)

Expand Down
10 changes: 5 additions & 5 deletions vulnerabilities/importers/retiredotnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from vulnerabilities.data_source import GitDataSource
from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import Reference
from vulnerabilities.helpers import AffectedPackageWithPatchedPackage
from vulnerabilities.helpers import AffectedPackage


class RetireDotnetDataSource(GitDataSource):
Expand Down Expand Up @@ -77,10 +77,10 @@ def process_file(self, path) -> List[Advisory]:
else:
return

affected_packages_with_patched_package = []
affected_packages = []
for pkg in json_doc["packages"]:
affected_packages_with_patched_package.append(
AffectedPackageWithPatchedPackage(
affected_packages.append(
AffectedPackage(
vulnerable_package=PackageURL(
name=pkg["id"], version=pkg["affected"], type="nuget"
),
Expand All @@ -99,6 +99,6 @@ def process_file(self, path) -> List[Advisory]:
return Advisory(
vulnerability_id=vuln_id,
summary=json_doc["description"],
affected_packages_with_patched_package=affected_packages_with_patched_package,
affected_packages=affected_packages,
references=vuln_reference,
)
4 changes: 1 addition & 3 deletions vulnerabilities/importers/ruby.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,7 @@ def process_file(self, path) -> List[Advisory]:

return Advisory(
summary=record.get("description", ""),
affected_packages_with_patched_package=nearest_patched_package(
impacted_purls, resolved_purls
),
affected_packages=nearest_patched_package(impacted_purls, resolved_purls),
references=references,
vulnerability_id=cve_id,
)
Expand Down
4 changes: 1 addition & 3 deletions vulnerabilities/importers/rust.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,7 @@ def _load_advisory(self, path: str) -> Optional[Advisory]:

return Advisory(
summary=advisory.get("description", ""),
affected_packages_with_patched_package=nearest_patched_package(
impacted_purls, resolved_purls
),
affected_packages=nearest_patched_package(impacted_purls, resolved_purls),
vulnerability_id=cve_id,
references=references,
)
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/importers/safety_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def updated_advisories(self) -> Set[Advisory]:
vulnerability_id=cve_id,
summary=advisory["advisory"],
references=reference,
affected_packages_with_patched_package=nearest_patched_package(
affected_packages=nearest_patched_package(
impacted_purls, resolved_purls
),
)
Expand Down
6 changes: 3 additions & 3 deletions vulnerabilities/tests/test_apache_kafka.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
from vulnerabilities.package_managers import GitHubTagsAPI
from vulnerabilities.importers.apache_kafka import ApacheKafkaDataSource
from vulnerabilities.importers.apache_kafka import to_version_ranges
from vulnerabilities.helpers import AffectedPackageWithPatchedPackage
from vulnerabilities.helpers import AffectedPackage

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_kafka", "cve-list.html")
Expand Down Expand Up @@ -68,8 +68,8 @@ def test_to_advisory(self):
Advisory(
summary="In Apache Kafka versions between 0.11.0.0 and 2.1.0, it is possible to manually\n craft a Produce request which bypasses transaction/idempotent ACL validation.\n Only authenticated clients with Write permission on the respective topics are\n able to exploit this vulnerability. Users should upgrade to 2.1.1 or later\n where this vulnerability has been fixed.",
vulnerability_id="CVE-2018-17196",
affected_packages_with_patched_package=[
AffectedPackageWithPatchedPackage(
affected_packages=[
AffectedPackage(
vulnerable_package=PackageURL(
type="apache",
namespace=None,
Expand Down
Loading

0 comments on commit ff833e4

Please sign in to comment.