From 734641109cb5368d821736c2cdf6086c15d756b3 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 29 Jul 2024 17:34:02 -0700 Subject: [PATCH 01/29] Add CSS for collapsed navbar, adjust vuln template nesting #1287 Reference: https://github.com/nexB/vulnerablecode/issues/1287 Signed-off-by: John M. Horan --- .../templates/vulnerability_details.html | 25 ++-- vulnerablecode/static/css/custom.css | 121 +++++++++++++++--- 2 files changed, 117 insertions(+), 29 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 2cfaacbd9..bb5d953ec 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -60,13 +60,15 @@ - {% if vulnerability.kev %}
  • + {% if vulnerability.kev %} +
  • Known Exploited Vulnerabilities -
    • {% endif %} + + {% endif %}
  • @@ -381,11 +383,12 @@ {% endfor %} - {% if vulnerability.kev %} + {# if vulnerability.kev #}
    Known Exploited Vulnerabilities
    + {% if vulnerability.kev %} @@ -397,7 +400,7 @@ - + {% if vulnerability.kev.description %} {% endif %} - + {% if vulnerability.kev.resources_and_notes %} {% endif %} - + {% if vulnerability.kev.due_date %} {% endif %} - +
    {{ vulnerability.kev.get_known_ransomware_campaign_use_type }}
    @@ -421,7 +424,7 @@ {{ vulnerability.kev.required_action }}
    @@ -433,7 +436,7 @@ {{ vulnerability.kev.resources_and_notes }}
    @@ -457,7 +460,7 @@ {{ vulnerability.kev.date_added }}
    {% endif %} @@ -500,6 +503,12 @@     {{log.source_url }} {{ log.software_version }} + {% empty %} + + + There are no relevant records. + + {% endfor %}
    @@ -528,4 +537,4 @@ -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index a04427957..6d8918a8f 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -187,12 +187,6 @@ code { border-color: #dbdbdb; } -/* 2023-08-28 Monday 14:55:42. Is this still needed or does wrap-strings take its place? Keep eyes peeled for any odd displays. */ -/* .table td { - word-wrap: break-word; -} */ - - .wrap-strings { word-break: break-word; } @@ -356,10 +350,9 @@ a.small_page_button { } .details-container { - border: solid 1px #e8e8e8; border: 0; border-radius: 6px; - box-shadow: 0 0.5em 1em -0.125em rgb(10 10 10 / 10%), 0 0px 0 1px rgb(10 10 10 / 2%); + box-shadow: 0 0.5em 1em -0.125em rgba(8, 8, 8, 0.1), 0 0px 0 1px rgba(10, 10, 10, 0.02); } .about-hover-div { @@ -373,7 +366,7 @@ a.small_page_button { } span.tag.custom { - margin: 0px 0px 6px 10px; + margin: 0px 0px 0px 10px; } /* CSS for dev fixed by headers */ @@ -428,14 +421,9 @@ span.tag.custom { border: solid 1px #dbdbdb; background-color: #ffffff; } -/* test bulleted list */ ul.fixed_by_bullet { list-style-type: disc; - /*margin-top: 2px; -margin-bottom: 10px;*/ - /*margin-left: -24px;*/ - /*margin-left: -30px;*/ margin-top: 0.25em; margin-left: 7px; margin-bottom: 0.25em; @@ -444,11 +432,8 @@ margin-bottom: 10px;*/ ul.fixed_by_bullet ul { list-style-type: disc; - /*margin-top: 10px;*/ - margin-top: 5px; margin-top: 0px; margin-bottom: 0px; - margin-left: 23px; margin-left: 18px; padding: 0; border: none; @@ -472,7 +457,6 @@ ul.fixed_by_bullet li:last-child { font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; font-size: 13px; font-weight: normal; - /*margin-bottom: 10px;*/ margin-bottom: 0px; } @@ -486,12 +470,11 @@ ul.fixed_by_bullet li li { color: #000000; } -/* 10/10/15 add 3rd-level bullets */ +/* 3rd-level bullets */ ul.fixed_by_bullet ul ul { list-style-type: disc; margin-top: 0px; margin-bottom: 0px; - margin-left: 50px; margin-left: 17px; padding: 0; border: none; @@ -569,10 +552,106 @@ ul.fixed_by_bullet li li li { /* Emphasis for not vulnerable. */ .emphasis-not-vulnerable { background-color: #e6ffe6; - /* background-color: #e6ffff; */ + word-wrap: break-word; word-break: break-all; + display: block; } /* Emphasis for vulnerable. */ .emphasis-vulnerable { background-color: #ffe6e6; } + +/* From https://github.com/jgthms/bulma/issues/2040#issuecomment-734507270 (the Bulma GH repo under the author's top-level GH entity). This helps display the responsive navbar dropdown properly < 1024px width. JMH: some added styles to remove box-shadow and make other adjustnments for the collapsed navbar. */ +@media screen and (max-width: 1024px) { + .navbar-menu { + align-items: stretch; + background-color: transparent; + font-size: 0.875rem; + display: flex; + flex-grow: 1; + flex-shrink: 0; + padding: 0; + + margin-right: 0px !important; + + .navbar-item.is-active .navbar-dropdown, + .navbar-item.is-hoverable:focus .navbar-dropdown, + .navbar-item.is-hoverable:focus-within .navbar-dropdown, + .navbar-item.is-hoverable:hover .navbar-dropdown { + display: block; + } + + .navbar-end { + justify-content: flex-end; + margin-left: auto; + align-items: stretch; + display: flex; + + .navbar-item.has-dropdown { + align-items: stretch; + } + + .navbar-item, + .navbar-link { + align-items: center; + display: flex; + } + + .navbar-dropdown { + border-bottom-left-radius: 6px; + border-bottom-right-radius: 6px; + border-top: 2px solid #dbdbdb; + box-shadow: 0 8px 8px rgba(10, 10, 10, 0.1); + display: none; + font-size: 0.875rem; + left: 0; + min-width: 100%; + position: absolute; + top: 100%; + z-index: 20; + } + } + } + + /* Make sure the 'About' navbar link hovering div appears left-aligned on narrower screens so it's visible rather than forced beyond the left-hand screen edge. */ + .dropdown.is-right .dropdown-menu { + left: 0; + right: auto; + } + + /* Remove the collapsed menu's right-hand dark bar, inherited because of how Bulma handles this process. */ + div.navbar-end.mr-3 { + margin-right: 0 !important; + } + + .navbar-item, + .navbar-item:active, + .navbar-item:focus, + .navbar-item:visited { + color: #ffffff; + background-color: transparent; + align-items: stretch; + display: flex; + } + + div.navbar-start { + width: 100%; + } + + a.navbar-item:focus { + background-color: transparent; + color: #ffffff; + } + + a.navbar-item:hover { + color: #ffffff; + background-color: #000000; + } + +} + +@media screen and (max-width: 1023px) { + .navbar-menu { + box-shadow: none; + } +} From e5ca080b06a0fae5d278f023bede40ed314d01a3 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 6 Aug 2024 17:30:20 -0700 Subject: [PATCH 02/29] Add matched affected/fixed-by Packages with correct sorting #1287 Reference: https://github.com/nexB/vulnerablecode/issues/1287 - Matched affected/fixed-by Package data added to Vulnerability details. - version_class-based sorting added to that data as well. - Note that version_class-based Packages sorting still needs to be added to Package search results. - Filter added to encode Package URLs. - Next: will add tests. Signed-off-by: John M. Horan --- .../templates/package_details.html | 40 ++-- vulnerabilities/templates/packages.html | 4 +- .../templates/vulnerabilities.html | 4 +- .../templates/vulnerability_details.html | 207 ++++++++---------- vulnerabilities/templatetags/url_filters.py | 10 + vulnerabilities/views.py | 42 +++- 6 files changed, 159 insertions(+), 148 deletions(-) create mode 100644 vulnerabilities/templatetags/url_filters.py diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 632790304..cf4ce7a54 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -2,6 +2,7 @@ {% load humanize %} {% load widget_tweaks %} {% load static %} +{% load url_filters %} {% block title %} VulnerableCode Package Details - {{ package.purl }} @@ -76,7 +77,7 @@ {% if fixed_package_details.next_non_vulnerable.version %} - {{ fixed_package_details.next_non_vulnerable.version }} {% else %} None. @@ -89,7 +90,7 @@ {% if fixed_package_details.latest_non_vulnerable.version %} - {{ fixed_package_details.latest_non_vulnerable.version }} {% else %} None. @@ -104,7 +105,7 @@
    - Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }}) + Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
    @@ -112,7 +113,7 @@ - + @@ -145,32 +146,28 @@ {% for vuln in value %} {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} {% if vuln.fixed_by_package_details is None %} - There are no reported fixed by versions. + There are no reported fixed by versions. {% else %} {% for fixed_pkg in vuln.fixed_by_package_details %}
    {% if fixed_pkg.fixed_by_purl_vulnerabilities|length == 0 %} - {{ fixed_pkg.fixed_by_purl.version }}
    - Affected - by 0 other vulnerabilities. + Affected by 0 other vulnerabilities. {% else %} - {{ fixed_pkg.fixed_by_purl.version }} {% if fixed_pkg.fixed_by_purl_vulnerabilities|length != 1 %}
    - Affected - by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. {% else %}
    - Affected - by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %} - +
    - This version is affected - by these other vulnerabilities: + This version is affected by these other vulnerabilities:
    {% for fixed_by_vuln in fixed_pkg.fixed_by_purl_vulnerabilities %}
    @@ -211,8 +207,7 @@ {% empty %}
    {% endfor %} @@ -222,7 +217,7 @@
    - Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }}) + Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }})
    Vulnerability SummaryFixed byFixed by
    - This package is not known to be affected by vulnerabilities. + This package is not known to be affected by vulnerabilities.
    @@ -258,8 +253,7 @@ {% empty %} {% endfor %} @@ -325,4 +319,4 @@ {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html index 2f91a5422..1f7687429 100644 --- a/vulnerabilities/templates/packages.html +++ b/vulnerabilities/templates/packages.html @@ -41,14 +41,14 @@ - Affected by vulnerabilities + Affected by vulnerabilities diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html index bdada6ee1..023d3f97f 100644 --- a/vulnerabilities/templates/vulnerabilities.html +++ b/vulnerabilities/templates/vulnerabilities.html @@ -32,8 +32,8 @@ - - + + diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index bb5d953ec..84ba8213e 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -3,6 +3,7 @@ {% load widget_tweaks %} {% load static %} {% load show_cvss %} +{% load url_filters %} {% block title %} VulnerableCode Vulnerability Details - {{ vulnerability.vulnerability_id }} @@ -32,17 +33,10 @@ Essentials -
  • +
  • - Fixed by packages ({{ fixed_by_packages|length }}) - - -
    • -
  • - - - Affected packages ({{ affected_packages|length }}) + Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }})
    • @@ -154,62 +148,52 @@
    - Fixed by packages ({{ fixed_by_packages|length }}) + Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }})
    - This package is not known to fix - vulnerabilities. + This package is not known to fix vulnerabilities.
    - Fixing vulnerabilities + Fixing vulnerabilities
    Vulnerability id AliasesAffected packagesFixed by packagesAffected packagesFixed by packages
    - {% for package in fixed_by_packages|slice:":3" %} - - - - {% empty %} - - - - {% endfor %} - {% if fixed_by_packages|length > 3 %} - - - - {% endif %} -
    - {{ package.purl }} -
    -
    - There are no known fixed by packages. -
    - See Fixed - by packages tab for more -
    -
    - -
    - Affected packages ({{ affected_packages|length }}) -
    -
    - - {% for package in affected_packages|slice:":3" %} - - - - {% empty %} - - - - {% endfor %} - {% if affected_packages|length > 3 %} - - - - {% endif %} + + + + + + + + {% for package in affected_packages|slice:":3" %} + + + + + {% empty %} + + + + {% endfor %} + {% if affected_packages|length > 3 %} + + + + {% endif %} +
    - {{ package.purl }} -
    -
    - There are no known affected packages. -
    - See Affected packages tab for more -
    AffectedFixed by
    + {{ package.purl }} + + {% for match in all_affected_fixed_by_matches %} + {% if match.affected_package == package %} + {% if match.matched_fixed_by_packages|length > 0 %} + {% for pkg in match.matched_fixed_by_packages %} + {{ pkg }} +
    + {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endif %} + {% endfor %} +
    + This vulnerability is not known to affect any packages. +
    + See Affected/Fixed by packages tab for more +
    @@ -241,43 +225,12 @@ -
    - - - - - - - - {% for ref in references %} - - {% if ref.reference_id %} - - {% else %} - - {% endif %} - - - {% empty %} - - - - {% endfor %} -
    Reference id URL
    {{ ref.reference_id }}{{ ref.url }}
    - There are no known references. -
    -
    - -
    +
    - + + @@ -286,10 +239,26 @@ + {% empty %} - @@ -298,34 +267,34 @@
    - Package URL - AffectedFixed by
    {{ package.purl }} + + {% for match in all_affected_fixed_by_matches %} + {% if match.affected_package == package %} + {% if match.matched_fixed_by_packages|length > 0 %} + {% for pkg in match.matched_fixed_by_packages %} + {{ pkg }} +
    + {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endif %} + {% endfor %} + +
    + This vulnerability is not known to affect any packages.
    -
    +
    - + + - - {% for package in fixed_by_packages %} - - - - {% empty %} - - - - {% endfor %} - + {% for ref in references %} + + {% if ref.reference_id %} + + {% else %} + + {% endif %} + + + {% empty %} + + + + {% endfor %}
    - Package URL - Reference id URL
    - {{package.purl }} -
    - This vulnerability is not known to be fixed by any packages. -
    {{ ref.reference_id }}{{ ref.url }}
    + There are no known references. +
    +
    {% for severity_vector in severity_vectors %} {% if severity_vector.version == '2.0' %} @@ -383,7 +352,7 @@ {% endfor %}
    - {# if vulnerability.kev #} +
    Known Exploited Vulnerabilities diff --git a/vulnerabilities/templatetags/url_filters.py b/vulnerabilities/templatetags/url_filters.py new file mode 100644 index 000000000..9a3f1a2f9 --- /dev/null +++ b/vulnerabilities/templatetags/url_filters.py @@ -0,0 +1,10 @@ +from urllib.parse import quote + +import packageurl +from django import template + +register = template.Library() + +@register.filter(name='url_quote') +def url_quote_filter(value): + return quote(str(value)) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 391c165e7..7d6f26ebf 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -22,6 +22,8 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView +from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.version_range import AlpineLinuxVersionRange from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm @@ -35,6 +37,18 @@ PAGE_SIZE = 20 +def purl_sort_key(purl: models.Package): + RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange + purl_version_class = RANGE_CLASS_BY_SCHEMES[purl.type].version_class + return (purl.type, purl.namespace, purl.name, purl_version_class(purl.version), purl.qualifiers, purl.subpath) + + +def get_purl_version_class(purl: models.Package): + RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange + purl_version_class = RANGE_CLASS_BY_SCHEMES[purl.type].version_class + return purl_version_class + + class PackageSearch(ListView): model = models.Package template_name = "packages.html" @@ -145,6 +159,29 @@ def get_context_data(self, **kwargs): except (CVSS2MalformedError, CVSS3MalformedError, NotImplementedError): logging.error(f"CVSSMalformedError for {s.scoring_elements}") + sorted_affected_packages = sorted(self.object.affected_packages.all(), key=purl_sort_key) + sorted_fixed_by_packages = sorted(self.object.fixed_by_packages.all(), key=purl_sort_key) + + all_affected_fixed_by_matches = [] + for sorted_affected_package in sorted_affected_packages: + affected_fixed_by_matches = {} + affected_fixed_by_matches["affected_package"] = sorted_affected_package + matched_fixed_by_packages = [] + for fixed_by_package in sorted_fixed_by_packages: + sorted_affected_version_class = get_purl_version_class(sorted_affected_package) + fixed_by_version_class = get_purl_version_class(fixed_by_package) + if ( + (fixed_by_package.type == sorted_affected_package.type) + and (fixed_by_package.namespace == sorted_affected_package.namespace) + and (fixed_by_package.name == sorted_affected_package.name) + and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) + and (fixed_by_package.subpath == sorted_affected_package.subpath) + and (fixed_by_version_class(fixed_by_package.version) > sorted_affected_version_class(sorted_affected_package.version)) + ): + matched_fixed_by_packages.append(fixed_by_package.purl) + affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages + all_affected_fixed_by_matches.append(affected_fixed_by_matches) + context.update( { "vulnerability": self.object, @@ -156,11 +193,12 @@ def get_context_data(self, **kwargs): "severity_vectors": severity_vectors, "references": self.object.references.all(), "aliases": self.object.aliases.all(), - "affected_packages": self.object.affected_packages.all(), - "fixed_by_packages": self.object.fixed_by_packages.all(), + "affected_packages": sorted_affected_packages, + "fixed_by_packages": sorted_fixed_by_packages, "weaknesses": weaknesses_present_in_db, "status": status, "history": self.object.history, + "all_affected_fixed_by_matches": all_affected_fixed_by_matches, } ) return context From 7e740a6459b9891c764f6c9cd9a600fc40c447fc Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 12 Aug 2024 13:21:49 -0700 Subject: [PATCH 03/29] Refactor sort and version_class functions and add tests #1287 Reference: https://github.com/nexB/vulnerablecode/issues/1287 Signed-off-by: John M. Horan --- .../test_data/package_sort/input_purls.txt | 103 ++++++++++++++ .../package_sort/purls_with_excel_sort.txt | 132 ++++++++++++++++++ .../test_data/package_sort/sorted_purls.txt | 103 ++++++++++++++ vulnerabilities/tests/test_view.py | 73 ++++++++++ vulnerabilities/views.py | 46 ++---- 5 files changed, 426 insertions(+), 31 deletions(-) create mode 100644 vulnerabilities/tests/test_data/package_sort/input_purls.txt create mode 100644 vulnerabilities/tests/test_data/package_sort/purls_with_excel_sort.txt create mode 100644 vulnerabilities/tests/test_data/package_sort/sorted_purls.txt diff --git a/vulnerabilities/tests/test_data/package_sort/input_purls.txt b/vulnerabilities/tests/test_data/package_sort/input_purls.txt new file mode 100644 index 000000000..9f0b214b1 --- /dev/null +++ b/vulnerabilities/tests/test_data/package_sort/input_purls.txt @@ -0,0 +1,103 @@ +pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64 +pkg:alpm/arch/pacman@6.0.1-1?arch=x86_64 +pkg:alpm/arch/python-pip@21.0-1?arch=any +pkg:cargo/clap@3.0.0 +pkg:cargo/clap@3.0.1 +pkg:cargo/clap@3.0.10 +pkg:cargo/clap@3.0.11 +pkg:cargo/clap@3.0.2 +pkg:cargo/clap@3.0.20 +pkg:cargo/rand@0.7.2 +pkg:cargo/structopt@0.3.11 +pkg:composer/bk2k/bootstrap-package@11.0.2 +pkg:composer/bk2k/bootstrap-package@11.0.3 +pkg:composer/bk2k/bootstrap-package@7.1.0 +pkg:composer/bk2k/bootstrap-package@7.1.1 +pkg:composer/bk2k/bootstrap-package@7.1.2 +pkg:conan/capnproto@0.15.0 +pkg:conan/capnproto@0.15.2 +pkg:conan/capnproto@0.7.0 +pkg:conan/capnproto@0.8.0 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch +pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed +pkg:deb/ubuntu/dpkg@1.13.21ubuntu1 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu11 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu12 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu2 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu3 +pkg:gem/actionpack@3.1.1 +pkg:gem/actionpack@3.1.10 +pkg:gem/actionpack@3.1.11 +pkg:gem/actionpack@3.1.2 +pkg:gem/webbynode@1.0.5.beta10 +pkg:gem/webbynode@1.0.5.beta2 +pkg:gem/webbynode@1.0.5.beta3 +pkg:generic/postgresql@10.19.0 +pkg:generic/postgresql@10.2.0 +pkg:generic/postgresql@10.21.0 +pkg:generic/postgresql@10.22.0 +pkg:generic/postgresql@10.3.0 +pkg:generic/postgresql@10.4.0 +pkg:github/istio/istio@0.2.0 +pkg:github/istio/istio@0.2.1 +pkg:github/istio/istio@0.2.10 +pkg:github/istio/istio@0.2.11 +pkg:github/istio/istio@0.2.12 +pkg:github/istio/istio@0.2.2 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.1-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.3.6 +pkg:golang/github.com/1Panel-dev/1Panel@1.4.3 +pkg:hex/pow@1.0.15 +pkg:hex/pow@1.0.16 +pkg:hex/pow@1.0.2 +pkg:hex/pow@1.0.3 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.10.0 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.2.1 +pkg:maven/io.netty/netty-codec-dns@4.1.101.Final +pkg:maven/io.netty/netty-codec-dns@4.1.97.Final +pkg:maven/io.netty/netty-codec-http@4.1.101.Final +pkg:maven/io.netty/netty-codec-http@4.1.97.Final +pkg:maven/io.netty/netty-codec-http2@4.1.101.Final +pkg:maven/io.netty/netty-codec-http2@4.1.97.Final +pkg:npm/%40budibase/bbui@1.2.44-alpha.1 +pkg:npm/%40budibase/bbui@1.2.44-alpha.10 +pkg:npm/%40budibase/bbui@1.2.44-alpha.11 +pkg:npm/%40budibase/bbui@1.2.44-alpha.2 +pkg:npm/%40budibase/bbui@1.2.44-alpha.3 +pkg:npm/bootstrap-select@1.13.5 +pkg:npm/bootstrap-select@1.13.6 +pkg:npm/bootstrap-select@1.6.2 +pkg:npm/bootstrap-select@1.6.3 +pkg:nuget/adplug@2.3.0-beta17 +pkg:nuget/adplug@2.3.0-beta172 +pkg:nuget/adplug@2.3.0-beta173 +pkg:nuget/adplug@2.3.0-beta18 +pkg:nuget/adplug@2.3.0-beta186 +pkg:nuget/adplug@2.3.0-beta19 +pkg:nuget/adplug@2.3.0-beta190 +pkg:pypi/jinja2@2.1 +pkg:pypi/jinja2@2.1.1 +pkg:pypi/jinja2@2.10 +pkg:pypi/jinja2@2.2 +pkg:pypi/jinja2@2.2.1 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 diff --git a/vulnerabilities/tests/test_data/package_sort/purls_with_excel_sort.txt b/vulnerabilities/tests/test_data/package_sort/purls_with_excel_sort.txt new file mode 100644 index 000000000..de9405797 --- /dev/null +++ b/vulnerabilities/tests/test_data/package_sort/purls_with_excel_sort.txt @@ -0,0 +1,132 @@ +pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64 +pkg:alpm/arch/pacman@6.0.1-1?arch=x86_64 +pkg:alpm/arch/python-pip@21.0-1?arch=any +pkg:apk/alpine/apk@2.12.9-r3?arch=x86 +pkg:apk/alpine/curl@7.83.0-r0?arch=x86 +pkg:bitbucket/birkenfeld/pygments-main@244fd47e07d1014f0aed9c +pkg:bitnami/wordpress?distro=debian-12 +pkg:bitnami/wordpress@6.2.0?arch=arm64&distro=debian-12 +pkg:bitnami/wordpress@6.2.0?arch=arm64&distro=photon-4 +pkg:bitnami/wordpress@6.2.0?distro=debian-12 +pkg:cargo/clap@3.0.0 +pkg:cargo/clap@3.0.1 +pkg:cargo/clap@3.0.10 +pkg:cargo/clap@3.0.11 +pkg:cargo/clap@3.0.2 +pkg:cargo/clap@3.0.20 +pkg:cargo/rand@0.7.2 +pkg:cargo/structopt@0.3.11 +pkg:cocoapods/AFNetworking@4.0.1 +pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib +pkg:cocoapods/MapsIndoors@3.24.0 +pkg:cocoapods/ShareKit@2.0#Twitter +pkg:composer/bk2k/bootstrap-package@11.0.2 +pkg:composer/bk2k/bootstrap-package@11.0.3 +pkg:composer/bk2k/bootstrap-package@7.1.0 +pkg:composer/bk2k/bootstrap-package@7.1.1 +pkg:composer/bk2k/bootstrap-package@7.1.2 +pkg:conan/capnproto@0.15.0 +pkg:conan/capnproto@0.15.2 +pkg:conan/capnproto@0.7.0 +pkg:conan/capnproto@0.8.0 +pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2 +pkg:conda/openssl@1.0.2l?channel=main&subdir=linux-64&build=h077ae2c_5&type=tar.bz2 +pkg:cpan/DROLSKY/DateTime@1.55 +pkg:cpan/DROLSKY/DateTime@1.56 +pkg:cpan/DROLSKY/DateTime@1.57 +pkg:cran/caret@6.0-88 +pkg:cran/caret@6.0-89 +pkg:cran/caret@6.0-90 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch +pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed +pkg:deb/ubuntu/dpkg@1.13.21ubuntu1 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu11 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu12 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu2 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu3 +pkg:docker/cassandra@latest +pkg:docker/customer/dockerimage@sha256%3A244fd47e07d10?repository_url=gcr.io +pkg:docker/smartentry/debian@dc437cc87d10 +pkg:gem/actionpack@3.1.1 +pkg:gem/actionpack@3.1.10 +pkg:gem/actionpack@3.1.11 +pkg:gem/actionpack@3.1.2 +pkg:gem/webbynode@1.0.5.beta10 +pkg:gem/webbynode@1.0.5.beta2 +pkg:gem/webbynode@1.0.5.beta3 +pkg:generic/postgresql@10.19.0 +pkg:generic/postgresql@10.2.0 +pkg:generic/postgresql@10.21.0 +pkg:generic/postgresql@10.22.0 +pkg:generic/postgresql@10.3.0 +pkg:generic/postgresql@10.4.0 +pkg:github/istio/istio@0.2.0 +pkg:github/istio/istio@0.2.1 +pkg:github/istio/istio@0.2.10 +pkg:github/istio/istio@0.2.11 +pkg:github/istio/istio@0.2.12 +pkg:github/istio/istio@0.2.2 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.1-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.3.6 +pkg:golang/github.com/1Panel-dev/1Panel@1.4.3 +pkg:hackage/Allure@0.11.0.0 +pkg:hackage/Allure@0.9.5.0 +pkg:hex/pow@1.0.15 +pkg:hex/pow@1.0.16 +pkg:hex/pow@1.0.2 +pkg:hex/pow@1.0.3 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.10.0 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.2.1 +pkg:maven/io.netty/netty-codec-dns@4.1.101.Final +pkg:maven/io.netty/netty-codec-dns@4.1.97.Final +pkg:maven/io.netty/netty-codec-http@4.1.101.Final +pkg:maven/io.netty/netty-codec-http@4.1.97.Final +pkg:maven/io.netty/netty-codec-http2@4.1.101.Final +pkg:maven/io.netty/netty-codec-http2@4.1.97.Final +pkg:npm/%40budibase/bbui@1.2.44-alpha.1 +pkg:npm/%40budibase/bbui@1.2.44-alpha.10 +pkg:npm/%40budibase/bbui@1.2.44-alpha.11 +pkg:npm/%40budibase/bbui@1.2.44-alpha.2 +pkg:npm/%40budibase/bbui@1.2.44-alpha.3 +pkg:npm/bootstrap-select@1.13.5 +pkg:npm/bootstrap-select@1.13.6 +pkg:npm/bootstrap-select@1.6.2 +pkg:npm/bootstrap-select@1.6.3 +pkg:nuget/adplug@2.3.0-beta17 +pkg:nuget/adplug@2.3.0-beta172 +pkg:nuget/adplug@2.3.0-beta173 +pkg:nuget/adplug@2.3.0-beta18 +pkg:nuget/adplug@2.3.0-beta186 +pkg:nuget/adplug@2.3.0-beta19 +pkg:nuget/adplug@2.3.0-beta190 +pkg:pypi/jinja2@2.1 +pkg:pypi/jinja2@2.1.1 +pkg:pypi/jinja2@2.10 +pkg:pypi/jinja2@2.2 +pkg:pypi/jinja2@2.2.1 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 +pkg:swid/Acme/example.com/Enterprise+Server@1.0.0?tag_id=75b8c285-fa7b-485b-b199-4745e3004d0d +pkg:swid/Adobe+Systems+Incorporated/Adobe+InDesign@CC?tag_id=CreativeCloud-CS6-Win-GM-MUL +pkg:swid/Fedora@29?tag_id=org.fedoraproject.Fedora-29 +pkg:swift/github.com/Alamofire/Alamofire@5.4.3 +pkg:swift/github.com/RxSwiftCommunity/RxFlow@2.12.4 diff --git a/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt new file mode 100644 index 000000000..886119bfd --- /dev/null +++ b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt @@ -0,0 +1,103 @@ +pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64 +pkg:alpm/arch/pacman@6.0.1-1?arch=x86_64 +pkg:alpm/arch/python-pip@21.0-1?arch=any +pkg:cargo/clap@3.0.0 +pkg:cargo/clap@3.0.1 +pkg:cargo/clap@3.0.2 +pkg:cargo/clap@3.0.10 +pkg:cargo/clap@3.0.11 +pkg:cargo/clap@3.0.20 +pkg:cargo/rand@0.7.2 +pkg:cargo/structopt@0.3.11 +pkg:composer/bk2k/bootstrap-package@7.1.0 +pkg:composer/bk2k/bootstrap-package@7.1.1 +pkg:composer/bk2k/bootstrap-package@7.1.2 +pkg:composer/bk2k/bootstrap-package@11.0.2 +pkg:composer/bk2k/bootstrap-package@11.0.3 +pkg:conan/capnproto@0.7.0 +pkg:conan/capnproto@0.8.0 +pkg:conan/capnproto@0.15.0 +pkg:conan/capnproto@0.15.2 +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch +pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 +pkg:deb/ubuntu/dpkg@1.13.21ubuntu1 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu2 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu3 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu11 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu12 +pkg:gem/actionpack@3.1.1 +pkg:gem/actionpack@3.1.2 +pkg:gem/actionpack@3.1.10 +pkg:gem/actionpack@3.1.11 +pkg:gem/webbynode@1.0.5.beta2 +pkg:gem/webbynode@1.0.5.beta3 +pkg:gem/webbynode@1.0.5.beta10 +pkg:generic/postgresql@10.2.0 +pkg:generic/postgresql@10.3.0 +pkg:generic/postgresql@10.4.0 +pkg:generic/postgresql@10.19.0 +pkg:generic/postgresql@10.21.0 +pkg:generic/postgresql@10.22.0 +pkg:github/istio/istio@0.2.0 +pkg:github/istio/istio@0.2.1 +pkg:github/istio/istio@0.2.2 +pkg:github/istio/istio@0.2.10 +pkg:github/istio/istio@0.2.11 +pkg:github/istio/istio@0.2.12 +pkg:golang/github.com/1Panel-dev/1Panel@1.3.6 +pkg:golang/github.com/1Panel-dev/1Panel@1.4.3 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.1-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3 +pkg:hex/pow@1.0.2 +pkg:hex/pow@1.0.3 +pkg:hex/pow@1.0.15 +pkg:hex/pow@1.0.16 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.2.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.10.0 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2.1 +pkg:maven/io.netty/netty-codec-dns@4.1.97.Final +pkg:maven/io.netty/netty-codec-dns@4.1.101.Final +pkg:maven/io.netty/netty-codec-http@4.1.97.Final +pkg:maven/io.netty/netty-codec-http@4.1.101.Final +pkg:maven/io.netty/netty-codec-http2@4.1.97.Final +pkg:maven/io.netty/netty-codec-http2@4.1.101.Final +pkg:npm/bootstrap-select@1.6.2 +pkg:npm/bootstrap-select@1.6.3 +pkg:npm/bootstrap-select@1.13.5 +pkg:npm/bootstrap-select@1.13.6 +pkg:npm/%40budibase/bbui@1.2.44-alpha.1 +pkg:npm/%40budibase/bbui@1.2.44-alpha.2 +pkg:npm/%40budibase/bbui@1.2.44-alpha.3 +pkg:npm/%40budibase/bbui@1.2.44-alpha.10 +pkg:npm/%40budibase/bbui@1.2.44-alpha.11 +pkg:nuget/adplug@2.3.0-beta17 +pkg:nuget/adplug@2.3.0-beta172 +pkg:nuget/adplug@2.3.0-beta173 +pkg:nuget/adplug@2.3.0-beta18 +pkg:nuget/adplug@2.3.0-beta186 +pkg:nuget/adplug@2.3.0-beta19 +pkg:nuget/adplug@2.3.0-beta190 +pkg:pypi/jinja2@2.1 +pkg:pypi/jinja2@2.1.1 +pkg:pypi/jinja2@2.2 +pkg:pypi/jinja2@2.2.1 +pkg:pypi/jinja2@2.10 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 20eb880af..8cb645680 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -7,15 +7,25 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import os + +import pytest from django.test import Client from django.test import TestCase from packageurl import PackageURL +from univers import versions from vulnerabilities.models import Alias from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability +from vulnerabilities.templatetags.url_filters import url_quote_filter from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch +from vulnerabilities.views import get_purl_version_class +from vulnerabilities.views import purl_sort_key + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DIR = os.path.join(BASE_DIR, "test_data/package_sort") class PackageSearchTestCase(TestCase): @@ -176,3 +186,66 @@ def test_robots_txt(self): assert response.status_code == 200 response = self.client.post("/robots.txt") assert response.status_code == 405 + + +class TestPackageSortTestCase(TestCase): + def setUp(self): + self.client = Client() + TEST_DATA = os.path.join(TEST_DIR, "input_purls.txt") + with open(TEST_DATA) as f: + input_purls = [l for l in f.readlines()] + self.input_purls = input_purls + for pkg in input_purls: + real_purl = PackageURL.from_string(pkg) + attrs = {k: v for k, v in real_purl.to_dict().items() if v} + Package.objects.create(**attrs) + + def test_sorted_queryset(self): + qs_all = Package.objects.all() + pkgs_qs_all = list(qs_all) + sorted_pkgs_qs_all = sorted(pkgs_qs_all, key=purl_sort_key) + + pkg_package_urls = [obj.package_url for obj in sorted_pkgs_qs_all] + sorted_purls = os.path.join(TEST_DIR, "sorted_purls.txt") + with open(sorted_purls, 'r') as f: + expected_content = f.read().splitlines() + assert pkg_package_urls == expected_content + + def test_get_purl_version_class(self): + test_cases = { + "pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64": versions.ArchLinuxVersion, + "pkg:cargo/clap@3.0.0": versions.SemverVersion, + "pkg:composer/bk2k/bootstrap-package@7.1.0": versions.ComposerVersion, + "pkg:conan/capnproto@0.7.0": versions.ConanVersion, + "pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch": versions.DebianVersion, + "pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed": versions.DebianVersion, + "pkg:gem/actionpack@3.1.1": versions.RubygemsVersion, + "pkg:generic/postgresql@10.2.0": versions.SemverVersion, + "pkg:github/istio/istio@0.2.0": versions.SemverVersion, + "pkg:golang/github.com/1Panel-dev/1Panel@1.3.6": versions.GolangVersion, + "pkg:hex/pow@1.0.2": versions.SemverVersion, + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1": versions.MavenVersion, + "pkg:npm/bootstrap-select@1.6.2": versions.SemverVersion, + "pkg:nuget/adplug@2.3.0-beta17": versions.NugetVersion, + "pkg:pypi/jinja2@2.1": versions.PypiVersion, + "pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11": versions.RpmVersion, + } + for k in test_cases: + pkg = Package.objects.get(package_url=k) + assert get_purl_version_class(pkg) == test_cases.get(k) + + +class TestCustomFilters: + @pytest.mark.parametrize("input_value, expected_output", [ + ("pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat", "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat"), + ("pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain"), + ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"), + ("pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1", "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1"), + ("pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1", "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1"), + ("pkg:golang/google.golang.org/genproto#googleapis/api/annotations", "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations"), + ("pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib", "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib"), + ("pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2", "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2"), + ]) + def test_url_quote_filter(self, input_value, expected_output): + filtered = url_quote_filter(input_value) + assert filtered == expected_output diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index f82e63ff3..fc22d1874 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -39,14 +39,24 @@ def purl_sort_key(purl: models.Package): - RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange - purl_version_class = RANGE_CLASS_BY_SCHEMES[purl.type].version_class - return (purl.type, purl.namespace, purl.name, purl_version_class(purl.version), purl.qualifiers, purl.subpath) - + """ + Return a sort key for the built-in sorted() function when sorting a list + of Package objects. If the Package ``type`` is supported by univers, apply + the univers version class to the Package ``version``, and otherwise use the + ``version`` attribute as is. + """ + purl_version_class = get_purl_version_class(purl) + purl_sort_version = purl.version + if purl_version_class: + purl_sort_version = purl_version_class(purl.version) + return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) def get_purl_version_class(purl: models.Package): RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange - purl_version_class = RANGE_CLASS_BY_SCHEMES[purl.type].version_class + purl_version_class = None + check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) + if check_version_class: + purl_version_class = check_version_class.version_class return purl_version_class @@ -163,32 +173,6 @@ def get_context_data(self, **kwargs): severity_vectors.append(vector_values) except (CVSS2MalformedError, CVSS3MalformedError, NotImplementedError): logging.error(f"CVSSMalformedError for {s.scoring_elements}") - if s.value: - severity_values.add(s.value) - - sorted_affected_packages = sorted(self.object.affected_packages.all(), key=purl_sort_key) - sorted_fixed_by_packages = sorted(self.object.fixed_by_packages.all(), key=purl_sort_key) - - all_affected_fixed_by_matches = [] - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = {} - affected_fixed_by_matches["affected_package"] = sorted_affected_package - matched_fixed_by_packages = [] - for fixed_by_package in sorted_fixed_by_packages: - sorted_affected_version_class = get_purl_version_class(sorted_affected_package) - fixed_by_version_class = get_purl_version_class(fixed_by_package) - if ( - (fixed_by_package.type == sorted_affected_package.type) - and (fixed_by_package.namespace == sorted_affected_package.namespace) - and (fixed_by_package.name == sorted_affected_package.name) - and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) - and (fixed_by_package.subpath == sorted_affected_package.subpath) - and (fixed_by_version_class(fixed_by_package.version) > sorted_affected_version_class(sorted_affected_package.version)) - ): - matched_fixed_by_packages.append(fixed_by_package.purl) - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) - if s.value: severity_values.add(s.value) From 76c5eda86377aba8bf8f98521759b5100aac80b0 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 13 Aug 2024 08:45:40 -0700 Subject: [PATCH 04/29] Run 'make valid' #1287 Reference: https://github.com/nexB/vulnerablecode/issues/1287 Signed-off-by: John M. Horan --- vulnerabilities/templatetags/url_filters.py | 3 +- vulnerabilities/tests/test_view.py | 46 ++++++++++++++++----- vulnerabilities/views.py | 6 ++- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/vulnerabilities/templatetags/url_filters.py b/vulnerabilities/templatetags/url_filters.py index 9a3f1a2f9..a6dda1dd8 100644 --- a/vulnerabilities/templatetags/url_filters.py +++ b/vulnerabilities/templatetags/url_filters.py @@ -5,6 +5,7 @@ register = template.Library() -@register.filter(name='url_quote') + +@register.filter(name="url_quote") def url_quote_filter(value): return quote(str(value)) diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 8cb645680..1ba488320 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -207,7 +207,7 @@ def test_sorted_queryset(self): pkg_package_urls = [obj.package_url for obj in sorted_pkgs_qs_all] sorted_purls = os.path.join(TEST_DIR, "sorted_purls.txt") - with open(sorted_purls, 'r') as f: + with open(sorted_purls, "r") as f: expected_content = f.read().splitlines() assert pkg_package_urls == expected_content @@ -236,16 +236,40 @@ def test_get_purl_version_class(self): class TestCustomFilters: - @pytest.mark.parametrize("input_value, expected_output", [ - ("pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat", "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat"), - ("pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain"), - ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"), - ("pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1", "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1"), - ("pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1", "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1"), - ("pkg:golang/google.golang.org/genproto#googleapis/api/annotations", "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations"), - ("pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib", "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib"), - ("pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2", "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2"), - ]) + @pytest.mark.parametrize( + "input_value, expected_output", + [ + ( + "pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat", + "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat", + ), + ( + "pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", + "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain", + ), + ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"), + ( + "pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1", + "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1", + ), + ( + "pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1", + "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1", + ), + ( + "pkg:golang/google.golang.org/genproto#googleapis/api/annotations", + "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations", + ), + ( + "pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib", + "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib", + ), + ( + "pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2", + "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2", + ), + ], + ) def test_url_quote_filter(self, input_value, expected_output): filtered = url_quote_filter(input_value) assert filtered == expected_output diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index fc22d1874..20241164e 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -51,6 +51,7 @@ def purl_sort_key(purl: models.Package): purl_sort_version = purl_version_class(purl.version) return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) + def get_purl_version_class(purl: models.Package): RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange purl_version_class = None @@ -194,7 +195,10 @@ def get_context_data(self, **kwargs): and (fixed_by_package.name == sorted_affected_package.name) and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) and (fixed_by_package.subpath == sorted_affected_package.subpath) - and (fixed_by_version_class(fixed_by_package.version) > sorted_affected_version_class(sorted_affected_package.version)) + and ( + fixed_by_version_class(fixed_by_package.version) + > sorted_affected_version_class(sorted_affected_package.version) + ) ): matched_fixed_by_packages.append(fixed_by_package.purl) affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages From 36ce5e1acf6c8efd44ebd4f415ac4def4759282d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 21 Aug 2024 17:12:54 +0200 Subject: [PATCH 05/29] Use correct regex for CVE Per the CVE JSON schema we had not the correct regex. Signed-off-by: Philippe Ombredanne --- vulnerabilities/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index c6874b7df..c8a09ad00 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) -cve_regex = re.compile(r"CVE-\d{4}-\d{4,7}", re.IGNORECASE) +cve_regex = re.compile(r"CVE-[0-9]{4}-[0-9]{4,19}", re.IGNORECASE) is_cve = cve_regex.match find_all_cve = cve_regex.findall From b0a6495ccd53e15ca15f9c6e2098e4fa6878200b Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 29 Aug 2024 00:33:03 +0530 Subject: [PATCH 06/29] Migrate Nginx importer to aboutcode pipeline Signed-off-by: Keshav Priyadarshi --- vulnerabilities/importers/__init__.py | 4 +- vulnerabilities/improvers/valid_versions.py | 4 +- vulnerabilities/models.py | 2 +- .../nginx.py => pipelines/nginx_importer.py} | 68 ++++++++-------- vulnerabilities/pipelines/pypa_importer.py | 2 - .../test_nginx_importer_pipeline.py} | 69 ++++++++-------- .../pipelines/test_pypa_importer_pipeline.py | 2 +- ...security_advisories-importer-expected.json | 78 +++++++++---------- 8 files changed, 116 insertions(+), 113 deletions(-) rename vulnerabilities/{importers/nginx.py => pipelines/nginx_importer.py} (77%) rename vulnerabilities/tests/{test_nginx.py => pipelines/test_nginx_importer_pipeline.py} (82%) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index c44ced245..efd8b71e9 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -24,7 +24,6 @@ from vulnerabilities.importers import gitlab from vulnerabilities.importers import istio from vulnerabilities.importers import mozilla -from vulnerabilities.importers import nginx from vulnerabilities.importers import nvd from vulnerabilities.importers import openssl from vulnerabilities.importers import oss_fuzz @@ -40,6 +39,7 @@ from vulnerabilities.importers import vulnrichment from vulnerabilities.importers import xen from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer from vulnerabilities.pipelines import pypa_importer @@ -47,7 +47,6 @@ nvd.NVDImporter, github.GitHubAPIImporter, gitlab.GitLabAPIImporter, - nginx.NginxImporter, pysec.PyPIImporter, alpine_linux.AlpineImporter, openssl.OpensslImporter, @@ -78,6 +77,7 @@ vulnrichment.VulnrichImporter, pypa_importer.PyPaImporterPipeline, npm_importer.NpmImporterPipeline, + nginx_importer.NginxImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index 32f3dfc35..9e6300bf3 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -36,7 +36,6 @@ from vulnerabilities.importers.github_osv import GithubOSVImporter from vulnerabilities.importers.gitlab import GitLabAPIImporter from vulnerabilities.importers.istio import IstioImporter -from vulnerabilities.importers.nginx import NginxImporter from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter from vulnerabilities.importers.ruby import RubyImporter from vulnerabilities.importers.ubuntu import UbuntuImporter @@ -44,6 +43,7 @@ from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage from vulnerabilities.utils import clean_nginx_git_tag @@ -220,7 +220,7 @@ class NginxBasicImprover(Improver): @property def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=NginxImporter.qualified_name).paginated() + return Advisory.objects.filter(created_by=NginxImporterPipeline.qualified_name).paginated() def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: all_versions = list(self.fetch_nginx_version_from_git_tags()) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index ada9bec54..cc3e920d9 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1103,7 +1103,7 @@ class Advisory(models.Model): max_length=100, help_text="Fully qualified name of the importer prefixed with the" "module name importing the advisory. Eg:" - "vulnerabilities.importers.nginx.NginxImporter", + "vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline", ) url = models.URLField( blank=True, diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/pipelines/nginx_importer.py similarity index 77% rename from vulnerabilities/importers/nginx.py rename to vulnerabilities/pipelines/nginx_importer.py index 4fe0ca6ae..c5e017033 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/pipelines/nginx_importer.py @@ -3,58 +3,62 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import logging from typing import Iterable -from typing import List from typing import NamedTuple import requests from bs4 import BeautifulSoup -from django.db.models.query import QuerySet from packageurl import PackageURL from univers.version_range import NginxVersionRange from univers.versions import NginxVersion from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.severity_systems import GENERIC -logger = logging.getLogger(__name__) +class NginxImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect Nginx security advisories.""" -class NginxImporter(Importer): - - url = "https://nginx.org/en/security_advisories.html" + pipeline_id = "nginx_importer" spdx_license_expression = "BSD-2-Clause" license_url = "https://nginx.org/LICENSE" + url = "https://nginx.org/en/security_advisories.html" importer_name = "Nginx Importer" - def advisory_data(self) -> Iterable[AdvisoryData]: - text = self.fetch() - yield from advisory_data_from_text(text) + @classmethod + def steps(cls): + return ( + cls.fetch, + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) def fetch(self): - return requests.get(self.url).content + self.log(f"Fetch `{self.url}`") + self.advisory_data = requests.get(self.url).text + def advisories_count(self): + return self.advisory_data.count("

  • ") -def advisory_data_from_text(text): - """ - Yield AdvisoryData from the ``text`` of the nginx security advisories HTML - web page. - """ - soup = BeautifulSoup(text, features="lxml") - vuln_list = soup.select("li p") - for vuln_info in vuln_list: - ngnix_adv = parse_advisory_data_from_paragraph(vuln_info) - yield to_advisory_data(ngnix_adv) + def collect_advisories(self) -> Iterable[AdvisoryData]: + """ + Yield AdvisoryData from nginx security advisories HTML + web page. + """ + soup = BeautifulSoup(self.advisory_data, features="lxml") + vulnerability_list = soup.select("li p") + for vulnerability_info in vulnerability_list: + ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info) + yield to_advisory_data(ngnix_advisory) class NginxAdvisory(NamedTuple): @@ -69,7 +73,7 @@ def to_dict(self): return self._asdict() -def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: +def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryData: """ Return AdvisoryData from an NginxAdvisory tuple. """ @@ -77,7 +81,7 @@ def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: package_type = "nginx" qualifiers = {} - _, _, affected_version_range = ngnx_adv.vulnerable.partition(":") + _, _, affected_version_range = nginx_adv.vulnerable.partition(":") if "nginx/Windows" in affected_version_range: qualifiers["os"] = "windows" affected_version_range = affected_version_range.replace("nginx/Windows", "") @@ -87,7 +91,7 @@ def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: affected_version_range = NginxVersionRange.from_native(affected_version_range) affected_packages = [] - _, _, fixed_versions = ngnx_adv.not_vulnerable.partition(":") + _, _, fixed_versions = nginx_adv.not_vulnerable.partition(":") for fixed_version in fixed_versions.split(","): fixed_version = fixed_version.rstrip("+") @@ -112,17 +116,17 @@ def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: ) return AdvisoryData( - aliases=ngnx_adv.aliases, - summary=ngnx_adv.summary, + aliases=nginx_adv.aliases, + summary=nginx_adv.summary, affected_packages=affected_packages, - references=ngnx_adv.references, + references=nginx_adv.references, url="https://nginx.org/en/security_advisories.html", ) -def parse_advisory_data_from_paragraph(vuln_info): +def parse_advisory_data_from_paragraph(vulnerability_info): """ - Return an NginxAdvisory from a ``vuln_info`` bs4 paragraph. + Return an NginxAdvisory from a ``vulnerability_info`` bs4 paragraph. An advisory paragraph, without html markup, looks like this: @@ -145,7 +149,7 @@ def parse_advisory_data_from_paragraph(vuln_info): # we iterate on the children to accumulate values in variables # FIXME: using an explicit xpath-like query could be simpler - for child in vuln_info.children: + for child in vulnerability_info.children: if is_first: summary = child is_first = False diff --git a/vulnerabilities/pipelines/pypa_importer.py b/vulnerabilities/pipelines/pypa_importer.py index 7a598de4d..29a1283fe 100644 --- a/vulnerabilities/pipelines/pypa_importer.py +++ b/vulnerabilities/pipelines/pypa_importer.py @@ -17,8 +17,6 @@ from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.utils import get_advisory_url -module_logger = logging.getLogger(__name__) - class PyPaImporterPipeline(VulnerableCodeBaseImporterPipeline): """Collect advisories from PyPA GitHub repository.""" diff --git a/vulnerabilities/tests/test_nginx.py b/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py similarity index 82% rename from vulnerabilities/tests/test_nginx.py rename to vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py index c27ef2d10..3e13d5d8a 100644 --- a/vulnerabilities/tests/test_nginx.py +++ b/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -15,16 +15,18 @@ from bs4 import BeautifulSoup from commoncode import testcase from django.db.models.query import QuerySet +from univers.version_range import NginxVersionRange from vulnerabilities import models from vulnerabilities import severity_systems -from vulnerabilities.import_runner import ImportRunner + +# from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.importers import nginx from vulnerabilities.improvers.valid_versions import NginxBasicImprover from vulnerabilities.models import Advisory +from vulnerabilities.pipelines import nginx_importer from vulnerabilities.tests import util_tests from vulnerabilities.utils import is_vulnerable_nginx_version @@ -40,14 +42,14 @@ class TestNginxImporterAndImprover(testcase.FileBasedTesting): - test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "nginx") + test_data_dir = Path(__file__).parent.parent / "test_data" / "nginx" def test_is_vulnerable(self): # Not vulnerable: 1.17.3+, 1.16.1+ # Vulnerable: 1.9.5-1.17.2 - vcls = nginx.NginxVersionRange.version_class - affected_version_range = nginx.NginxVersionRange.from_native("1.9.5-1.17.2") + vcls = NginxVersionRange.version_class + affected_version_range = NginxVersionRange.from_native("1.9.5-1.17.2") fixed_versions = [vcls("1.17.3"), vcls("1.16.1")] version = vcls("1.9.4") @@ -133,10 +135,10 @@ def test_parse_advisory_data_from_paragraph(self): ], } - result = nginx.parse_advisory_data_from_paragraph(vuln_info) + result = nginx_importer.parse_advisory_data_from_paragraph(vuln_info) assert result.to_dict() == expected - def test_advisory_data_from_text(self): + def test_collect_advisories(self): test_file = self.get_test_loc("security_advisories.html") with open(test_file) as tf: test_text = tf.read() @@ -145,52 +147,51 @@ def test_advisory_data_from_text(self): "security_advisories-advisory_data-expected.json", must_exist=False ) - results = [na.to_dict() for na in nginx.advisory_data_from_text(test_text)] + test_pipeline = nginx_importer.NginxImporterPipeline() + test_pipeline.advisory_data = test_text + results = [na.to_dict() for na in test_pipeline.collect_advisories()] util_tests.check_results_against_json(results, expected_file) @pytest.mark.django_db(transaction=True) def test_NginxImporter(self): + test_file = self.get_test_loc("security_advisories.html") + with open(test_file) as tf: + test_text = tf.read() + + test_pipeline = nginx_importer.NginxImporterPipeline() + test_pipeline.advisory_data = test_text expected_file = self.get_test_loc( "security_advisories-importer-expected.json", must_exist=False ) - results, _cls = self.run_import() - util_tests.check_results_against_json(results, expected_file) + test_pipeline.collect_and_store_advisories() + test_pipeline.import_new_advisories() - # run again as there should be no duplicates - results, _cls = self.run_import() + results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) util_tests.check_results_against_json(results, expected_file) - def run_import(self): - """ - Return a list of imported Advisory model objects and the MockImporter - used. - """ - - class MockImporter(nginx.NginxImporter): - """ - A mocked NginxImporter that loads content from a file rather than - making a network call. - """ - - def fetch(self): - with open(test_file) as tf: - return tf.read() - - test_file = self.get_test_loc("security_advisories.html") + # run again as there should be no duplicates + test_pipeline.collect_and_store_advisories() + test_pipeline.import_new_advisories() - ImportRunner(MockImporter).run() - return list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)), MockImporter + results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) + util_tests.check_results_against_json(results, expected_file) @pytest.mark.django_db(transaction=True) def test_NginxBasicImprover__interesting_advisories(self): - advisories, importer_class = self.run_import() + test_file = self.get_test_loc("security_advisories.html") + with open(test_file) as tf: + test_text = tf.read() + + test_pipeline = nginx_importer.NginxImporterPipeline() + test_pipeline.advisory_data = test_text + advisories = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) class MockNginxBasicImprover(NginxBasicImprover): @property def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=importer_class.qualified_name) + return Advisory.objects.filter(created_by=test_pipeline.pipeline_id) improver = MockNginxBasicImprover() interesting_advisories = list( diff --git a/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py index fa1360f1d..0bb631012 100644 --- a/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py @@ -16,7 +16,7 @@ from vulnerabilities.importers.osv import parse_advisory_data from vulnerabilities.tests import util_tests -TEST_DATA = data = Path(__file__).parent.parent / "test_data" / "pypa" +TEST_DATA = Path(__file__).parent.parent / "test_data" / "pypa" class TestPyPaImporterPipeline(TestCase): diff --git a/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json b/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json index 938e77249..6563f7085 100644 --- a/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json +++ b/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json @@ -1,6 +1,6 @@ [ { - "unique_content_id": "e06ef4fb12b1b0817736222cc219c5be", + "unique_content_id": "335a9c1c00513d109d5437afcc1d48c1", "aliases": [ "CORE-2010-0121" ], @@ -36,7 +36,7 @@ "weaknesses": [] }, { - "unique_content_id": "dab2e1aa4777dbcd579905643982aab1", + "unique_content_id": "403205c78453abb1a8562d7885adec25", "aliases": [ "CVE-2009-3896" ], @@ -115,7 +115,7 @@ "weaknesses": [] }, { - "unique_content_id": "91c6638b38a1e6e2ff4997eeefef8cf8", + "unique_content_id": "96a152603880c03608ed3160e5733267", "aliases": [ "CVE-2009-3898" ], @@ -158,7 +158,7 @@ "weaknesses": [] }, { - "unique_content_id": "31675b37fe392d1e36b77f7198b1d008", + "unique_content_id": "080bad63555f37591e5c2a51781de2ca", "aliases": [ "CVE-2009-4487" ], @@ -189,7 +189,7 @@ "weaknesses": [] }, { - "unique_content_id": "ef00adb6af6c2a00e81c8ec8de71eed6", + "unique_content_id": "3ca06ea9a54809cefa6656b38704b2ab", "aliases": [ "CVE-2010-2263" ], @@ -232,7 +232,7 @@ "weaknesses": [] }, { - "unique_content_id": "eb41c9a738129f7f76c5ff813d190621", + "unique_content_id": "b1f5ee46c793bc822dacec39d80d542d", "aliases": [ "CVE-2010-2266" ], @@ -275,7 +275,7 @@ "weaknesses": [] }, { - "unique_content_id": "d403898b9315a9ec88d9a401af5352fb", + "unique_content_id": "2cdace68cd16430e631ad7219d545825", "aliases": [ "CVE-2011-4315" ], @@ -318,7 +318,7 @@ "weaknesses": [] }, { - "unique_content_id": "96c2ffdeacca4901942abd83d54f33f5", + "unique_content_id": "91b8c592fdc630329d793fa1d44a8d74", "aliases": [ "CVE-2011-4963" ], @@ -373,7 +373,7 @@ "weaknesses": [] }, { - "unique_content_id": "ca72fb146fcd014ee284ef66f7fc1c08", + "unique_content_id": "9a75aa5a89cbbae37ca7538c369c0cc6", "aliases": [ "CVE-2012-1180" ], @@ -434,7 +434,7 @@ "weaknesses": [] }, { - "unique_content_id": "901e1dc04473ff40c6e503baec5e9bf6", + "unique_content_id": "5422572f0def3a030c6c840dfbd7845a", "aliases": [ "CVE-2012-2089" ], @@ -495,7 +495,7 @@ "weaknesses": [] }, { - "unique_content_id": "e74396e2dc204fb095c802fe54d4d176", + "unique_content_id": "7674897db3c7bef60d4bb82ab799b021", "aliases": [ "CVE-2013-2028" ], @@ -556,7 +556,7 @@ "weaknesses": [] }, { - "unique_content_id": "13592aaee15657bff9afca8c98edf8bf", + "unique_content_id": "5b6107b2a1ff968251c02fe3e1c1eb0d", "aliases": [ "CVE-2013-2070" ], @@ -647,7 +647,7 @@ "weaknesses": [] }, { - "unique_content_id": "0f21f4e3d88f4af06f0c46d096e90320", + "unique_content_id": "a838eb72eb823421cd94013f304bdb48", "aliases": [ "CVE-2013-4547" ], @@ -714,7 +714,7 @@ "weaknesses": [] }, { - "unique_content_id": "3430956de63de2b1188c3d1e50c3b0cd", + "unique_content_id": "efb0e82dd288eb9903dede418b07858f", "aliases": [ "CVE-2014-0088" ], @@ -763,7 +763,7 @@ "weaknesses": [] }, { - "unique_content_id": "db01da77157a7a773285dc98169416ec", + "unique_content_id": "8bb0e5c0fe7b13c7d53359f3f24d5c34", "aliases": [ "CVE-2014-0133" ], @@ -824,7 +824,7 @@ "weaknesses": [] }, { - "unique_content_id": "83d5fba07f12acd2e4947e68d233fbe5", + "unique_content_id": "d13aedf582d6b74b3932b8abd3e1ca12", "aliases": [ "CVE-2014-3556" ], @@ -891,7 +891,7 @@ "weaknesses": [] }, { - "unique_content_id": "ce87032bced3f187b1c0fbacc52b8c16", + "unique_content_id": "f84fa2467d4df52694234db6bf4c2b76", "aliases": [ "CVE-2014-3616" ], @@ -946,7 +946,7 @@ "weaknesses": [] }, { - "unique_content_id": "71c918b8f82b4de8cfa23fc96fa0d7a7", + "unique_content_id": "8617c237a937fa7ce6cfa99727325a0c", "aliases": [ "CVE-2016-0742" ], @@ -1001,7 +1001,7 @@ "weaknesses": [] }, { - "unique_content_id": "2ec9de991e2cb7a5a0ba79bed8556a41", + "unique_content_id": "dea85d9db96803b6f30ba88882bac004", "aliases": [ "CVE-2016-0746" ], @@ -1056,7 +1056,7 @@ "weaknesses": [] }, { - "unique_content_id": "925abc90d30273fe8cb404b7f3c8dfd3", + "unique_content_id": "d13a7fd387ef7873abceecfe5b8e06fd", "aliases": [ "CVE-2016-0747" ], @@ -1111,7 +1111,7 @@ "weaknesses": [] }, { - "unique_content_id": "04f5bc12ff49a95a29c459222379abe4", + "unique_content_id": "49c94274f8f7e56a9123477bd112e433", "aliases": [ "CVE-2016-4450" ], @@ -1190,7 +1190,7 @@ "weaknesses": [] }, { - "unique_content_id": "b3192a372fdac00b2cdf462b562cf73b", + "unique_content_id": "2aaefb239df5abb3994785bd40ab71f0", "aliases": [ "CVE-2017-7529" ], @@ -1257,7 +1257,7 @@ "weaknesses": [] }, { - "unique_content_id": "cb70875e6e02b2d41dd8876b4729bf84", + "unique_content_id": "0ee0797566ee9e12eb6ca17f8baee5e2", "aliases": [ "CVE-2018-16843" ], @@ -1312,7 +1312,7 @@ "weaknesses": [] }, { - "unique_content_id": "cf47abf58659080601c4cd87a119a769", + "unique_content_id": "6bb892f5f33e2cff305023723bd1e5a6", "aliases": [ "CVE-2018-16844" ], @@ -1367,7 +1367,7 @@ "weaknesses": [] }, { - "unique_content_id": "33d08a513ea5fef861e924f2601f7ac6", + "unique_content_id": "3c0e6f8b57cec78c5fb2b3eb5d1d75a5", "aliases": [ "CVE-2018-16845" ], @@ -1434,7 +1434,7 @@ "weaknesses": [] }, { - "unique_content_id": "8ca47577347bd9f2027e09e32bc74866", + "unique_content_id": "bdd084d7f43d425835f08d7ca6c98133", "aliases": [ "CVE-2019-9511" ], @@ -1489,7 +1489,7 @@ "weaknesses": [] }, { - "unique_content_id": "74ec3c647d544d6e6935492b7dceb572", + "unique_content_id": "df932a83ee1d6bd94395b599994807b5", "aliases": [ "CVE-2019-9513" ], @@ -1544,7 +1544,7 @@ "weaknesses": [] }, { - "unique_content_id": "2537fa6a9e8e84a3c06bb122fcbf468d", + "unique_content_id": "248fa20c4eb9cf8d32724ac84c6a761d", "aliases": [ "CVE-2019-9516" ], @@ -1599,7 +1599,7 @@ "weaknesses": [] }, { - "unique_content_id": "27612bc7cab82114b1549552f5ad48ff", + "unique_content_id": "0e4ece79727c13b9656b3bc760bd328d", "aliases": [ "CVE-2021-23017" ], @@ -1666,7 +1666,7 @@ "weaknesses": [] }, { - "unique_content_id": "dad2ebc242641f6a276b00769ef57efa", + "unique_content_id": "81545b256a26a3cccdb05e67b4f3ba9f", "aliases": [ "CVE-2022-41741" ], @@ -1733,7 +1733,7 @@ "weaknesses": [] }, { - "unique_content_id": "e17dde538a78c978602298541bcd29f0", + "unique_content_id": "c2e9b6ad436eb4c814337f903f5ec9cf", "aliases": [ "CVE-2022-41742" ], @@ -1800,7 +1800,7 @@ "weaknesses": [] }, { - "unique_content_id": "e4c6a0358264fb7523f6ee40f844854f", + "unique_content_id": "a9abcfac0befee9468fee011889a6a6b", "aliases": [ "CVE-2024-24989" ], @@ -1837,7 +1837,7 @@ "weaknesses": [] }, { - "unique_content_id": "f87492771be35866bf4dce017ea54dc8", + "unique_content_id": "8c31279d13d8300df1b3ab830e513911", "aliases": [ "CVE-2024-24990" ], @@ -1874,7 +1874,7 @@ "weaknesses": [] }, { - "unique_content_id": "79d9b38e6e89e3f3fc5ca4b2e64d0faa", + "unique_content_id": "59f5ec3eeb5b6b9e129ec45f9e386a77", "aliases": [ "CVE-2024-31079" ], @@ -1929,7 +1929,7 @@ "weaknesses": [] }, { - "unique_content_id": "b3d7627b206f561242cdd2eae0e3bbeb", + "unique_content_id": "86ef110ce63893b73d619a7dc1cc3ac6", "aliases": [ "CVE-2024-32760" ], @@ -1984,7 +1984,7 @@ "weaknesses": [] }, { - "unique_content_id": "43c2f41bb851164d3495f3c204a57f20", + "unique_content_id": "1553b9978e538bf5afdedaf10435783b", "aliases": [ "CVE-2024-34161" ], @@ -2039,7 +2039,7 @@ "weaknesses": [] }, { - "unique_content_id": "b72c609cd1be7c77f4432e1bc8c365f3", + "unique_content_id": "cb2e6f47e81c679c781d1ffa4bbb5b68", "aliases": [ "CVE-2024-35200" ], @@ -2094,7 +2094,7 @@ "weaknesses": [] }, { - "unique_content_id": "686399b9012be40d39b5366ec1695768", + "unique_content_id": "2e3ec84059d55863d2b80db5eeb90b1d", "aliases": [ "VU#120541", "CVE-2009-3555" @@ -2150,7 +2150,7 @@ "weaknesses": [] }, { - "unique_content_id": "c616b60f7fd802e88ca29fce6222654e", + "unique_content_id": "d5a14ef4e648d1a19b8f5ce8404490bc", "aliases": [ "VU#180065", "CVE-2009-2629" From 463c0a29e11bdac78fd42410ee77fadd4b44a562 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 4 Sep 2024 18:41:23 +0530 Subject: [PATCH 07/29] Test nginx advisory collection step Signed-off-by: Keshav Priyadarshi --- .../pipelines/test_nginx_importer_pipeline.py | 6 +- ...security_advisories-importer-expected.json | 78 +++++++++---------- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py index 3e13d5d8a..8a71a11fd 100644 --- a/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py @@ -19,8 +19,6 @@ from vulnerabilities import models from vulnerabilities import severity_systems - -# from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity @@ -153,7 +151,7 @@ def test_collect_advisories(self): util_tests.check_results_against_json(results, expected_file) @pytest.mark.django_db(transaction=True) - def test_NginxImporter(self): + def test_NginxImporterPipeline_collect_and_store_advisories(self): test_file = self.get_test_loc("security_advisories.html") with open(test_file) as tf: test_text = tf.read() @@ -166,14 +164,12 @@ def test_NginxImporter(self): ) test_pipeline.collect_and_store_advisories() - test_pipeline.import_new_advisories() results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) util_tests.check_results_against_json(results, expected_file) # run again as there should be no duplicates test_pipeline.collect_and_store_advisories() - test_pipeline.import_new_advisories() results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) util_tests.check_results_against_json(results, expected_file) diff --git a/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json b/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json index 6563f7085..938e77249 100644 --- a/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json +++ b/vulnerabilities/tests/test_data/nginx/security_advisories-importer-expected.json @@ -1,6 +1,6 @@ [ { - "unique_content_id": "335a9c1c00513d109d5437afcc1d48c1", + "unique_content_id": "e06ef4fb12b1b0817736222cc219c5be", "aliases": [ "CORE-2010-0121" ], @@ -36,7 +36,7 @@ "weaknesses": [] }, { - "unique_content_id": "403205c78453abb1a8562d7885adec25", + "unique_content_id": "dab2e1aa4777dbcd579905643982aab1", "aliases": [ "CVE-2009-3896" ], @@ -115,7 +115,7 @@ "weaknesses": [] }, { - "unique_content_id": "96a152603880c03608ed3160e5733267", + "unique_content_id": "91c6638b38a1e6e2ff4997eeefef8cf8", "aliases": [ "CVE-2009-3898" ], @@ -158,7 +158,7 @@ "weaknesses": [] }, { - "unique_content_id": "080bad63555f37591e5c2a51781de2ca", + "unique_content_id": "31675b37fe392d1e36b77f7198b1d008", "aliases": [ "CVE-2009-4487" ], @@ -189,7 +189,7 @@ "weaknesses": [] }, { - "unique_content_id": "3ca06ea9a54809cefa6656b38704b2ab", + "unique_content_id": "ef00adb6af6c2a00e81c8ec8de71eed6", "aliases": [ "CVE-2010-2263" ], @@ -232,7 +232,7 @@ "weaknesses": [] }, { - "unique_content_id": "b1f5ee46c793bc822dacec39d80d542d", + "unique_content_id": "eb41c9a738129f7f76c5ff813d190621", "aliases": [ "CVE-2010-2266" ], @@ -275,7 +275,7 @@ "weaknesses": [] }, { - "unique_content_id": "2cdace68cd16430e631ad7219d545825", + "unique_content_id": "d403898b9315a9ec88d9a401af5352fb", "aliases": [ "CVE-2011-4315" ], @@ -318,7 +318,7 @@ "weaknesses": [] }, { - "unique_content_id": "91b8c592fdc630329d793fa1d44a8d74", + "unique_content_id": "96c2ffdeacca4901942abd83d54f33f5", "aliases": [ "CVE-2011-4963" ], @@ -373,7 +373,7 @@ "weaknesses": [] }, { - "unique_content_id": "9a75aa5a89cbbae37ca7538c369c0cc6", + "unique_content_id": "ca72fb146fcd014ee284ef66f7fc1c08", "aliases": [ "CVE-2012-1180" ], @@ -434,7 +434,7 @@ "weaknesses": [] }, { - "unique_content_id": "5422572f0def3a030c6c840dfbd7845a", + "unique_content_id": "901e1dc04473ff40c6e503baec5e9bf6", "aliases": [ "CVE-2012-2089" ], @@ -495,7 +495,7 @@ "weaknesses": [] }, { - "unique_content_id": "7674897db3c7bef60d4bb82ab799b021", + "unique_content_id": "e74396e2dc204fb095c802fe54d4d176", "aliases": [ "CVE-2013-2028" ], @@ -556,7 +556,7 @@ "weaknesses": [] }, { - "unique_content_id": "5b6107b2a1ff968251c02fe3e1c1eb0d", + "unique_content_id": "13592aaee15657bff9afca8c98edf8bf", "aliases": [ "CVE-2013-2070" ], @@ -647,7 +647,7 @@ "weaknesses": [] }, { - "unique_content_id": "a838eb72eb823421cd94013f304bdb48", + "unique_content_id": "0f21f4e3d88f4af06f0c46d096e90320", "aliases": [ "CVE-2013-4547" ], @@ -714,7 +714,7 @@ "weaknesses": [] }, { - "unique_content_id": "efb0e82dd288eb9903dede418b07858f", + "unique_content_id": "3430956de63de2b1188c3d1e50c3b0cd", "aliases": [ "CVE-2014-0088" ], @@ -763,7 +763,7 @@ "weaknesses": [] }, { - "unique_content_id": "8bb0e5c0fe7b13c7d53359f3f24d5c34", + "unique_content_id": "db01da77157a7a773285dc98169416ec", "aliases": [ "CVE-2014-0133" ], @@ -824,7 +824,7 @@ "weaknesses": [] }, { - "unique_content_id": "d13aedf582d6b74b3932b8abd3e1ca12", + "unique_content_id": "83d5fba07f12acd2e4947e68d233fbe5", "aliases": [ "CVE-2014-3556" ], @@ -891,7 +891,7 @@ "weaknesses": [] }, { - "unique_content_id": "f84fa2467d4df52694234db6bf4c2b76", + "unique_content_id": "ce87032bced3f187b1c0fbacc52b8c16", "aliases": [ "CVE-2014-3616" ], @@ -946,7 +946,7 @@ "weaknesses": [] }, { - "unique_content_id": "8617c237a937fa7ce6cfa99727325a0c", + "unique_content_id": "71c918b8f82b4de8cfa23fc96fa0d7a7", "aliases": [ "CVE-2016-0742" ], @@ -1001,7 +1001,7 @@ "weaknesses": [] }, { - "unique_content_id": "dea85d9db96803b6f30ba88882bac004", + "unique_content_id": "2ec9de991e2cb7a5a0ba79bed8556a41", "aliases": [ "CVE-2016-0746" ], @@ -1056,7 +1056,7 @@ "weaknesses": [] }, { - "unique_content_id": "d13a7fd387ef7873abceecfe5b8e06fd", + "unique_content_id": "925abc90d30273fe8cb404b7f3c8dfd3", "aliases": [ "CVE-2016-0747" ], @@ -1111,7 +1111,7 @@ "weaknesses": [] }, { - "unique_content_id": "49c94274f8f7e56a9123477bd112e433", + "unique_content_id": "04f5bc12ff49a95a29c459222379abe4", "aliases": [ "CVE-2016-4450" ], @@ -1190,7 +1190,7 @@ "weaknesses": [] }, { - "unique_content_id": "2aaefb239df5abb3994785bd40ab71f0", + "unique_content_id": "b3192a372fdac00b2cdf462b562cf73b", "aliases": [ "CVE-2017-7529" ], @@ -1257,7 +1257,7 @@ "weaknesses": [] }, { - "unique_content_id": "0ee0797566ee9e12eb6ca17f8baee5e2", + "unique_content_id": "cb70875e6e02b2d41dd8876b4729bf84", "aliases": [ "CVE-2018-16843" ], @@ -1312,7 +1312,7 @@ "weaknesses": [] }, { - "unique_content_id": "6bb892f5f33e2cff305023723bd1e5a6", + "unique_content_id": "cf47abf58659080601c4cd87a119a769", "aliases": [ "CVE-2018-16844" ], @@ -1367,7 +1367,7 @@ "weaknesses": [] }, { - "unique_content_id": "3c0e6f8b57cec78c5fb2b3eb5d1d75a5", + "unique_content_id": "33d08a513ea5fef861e924f2601f7ac6", "aliases": [ "CVE-2018-16845" ], @@ -1434,7 +1434,7 @@ "weaknesses": [] }, { - "unique_content_id": "bdd084d7f43d425835f08d7ca6c98133", + "unique_content_id": "8ca47577347bd9f2027e09e32bc74866", "aliases": [ "CVE-2019-9511" ], @@ -1489,7 +1489,7 @@ "weaknesses": [] }, { - "unique_content_id": "df932a83ee1d6bd94395b599994807b5", + "unique_content_id": "74ec3c647d544d6e6935492b7dceb572", "aliases": [ "CVE-2019-9513" ], @@ -1544,7 +1544,7 @@ "weaknesses": [] }, { - "unique_content_id": "248fa20c4eb9cf8d32724ac84c6a761d", + "unique_content_id": "2537fa6a9e8e84a3c06bb122fcbf468d", "aliases": [ "CVE-2019-9516" ], @@ -1599,7 +1599,7 @@ "weaknesses": [] }, { - "unique_content_id": "0e4ece79727c13b9656b3bc760bd328d", + "unique_content_id": "27612bc7cab82114b1549552f5ad48ff", "aliases": [ "CVE-2021-23017" ], @@ -1666,7 +1666,7 @@ "weaknesses": [] }, { - "unique_content_id": "81545b256a26a3cccdb05e67b4f3ba9f", + "unique_content_id": "dad2ebc242641f6a276b00769ef57efa", "aliases": [ "CVE-2022-41741" ], @@ -1733,7 +1733,7 @@ "weaknesses": [] }, { - "unique_content_id": "c2e9b6ad436eb4c814337f903f5ec9cf", + "unique_content_id": "e17dde538a78c978602298541bcd29f0", "aliases": [ "CVE-2022-41742" ], @@ -1800,7 +1800,7 @@ "weaknesses": [] }, { - "unique_content_id": "a9abcfac0befee9468fee011889a6a6b", + "unique_content_id": "e4c6a0358264fb7523f6ee40f844854f", "aliases": [ "CVE-2024-24989" ], @@ -1837,7 +1837,7 @@ "weaknesses": [] }, { - "unique_content_id": "8c31279d13d8300df1b3ab830e513911", + "unique_content_id": "f87492771be35866bf4dce017ea54dc8", "aliases": [ "CVE-2024-24990" ], @@ -1874,7 +1874,7 @@ "weaknesses": [] }, { - "unique_content_id": "59f5ec3eeb5b6b9e129ec45f9e386a77", + "unique_content_id": "79d9b38e6e89e3f3fc5ca4b2e64d0faa", "aliases": [ "CVE-2024-31079" ], @@ -1929,7 +1929,7 @@ "weaknesses": [] }, { - "unique_content_id": "86ef110ce63893b73d619a7dc1cc3ac6", + "unique_content_id": "b3d7627b206f561242cdd2eae0e3bbeb", "aliases": [ "CVE-2024-32760" ], @@ -1984,7 +1984,7 @@ "weaknesses": [] }, { - "unique_content_id": "1553b9978e538bf5afdedaf10435783b", + "unique_content_id": "43c2f41bb851164d3495f3c204a57f20", "aliases": [ "CVE-2024-34161" ], @@ -2039,7 +2039,7 @@ "weaknesses": [] }, { - "unique_content_id": "cb2e6f47e81c679c781d1ffa4bbb5b68", + "unique_content_id": "b72c609cd1be7c77f4432e1bc8c365f3", "aliases": [ "CVE-2024-35200" ], @@ -2094,7 +2094,7 @@ "weaknesses": [] }, { - "unique_content_id": "2e3ec84059d55863d2b80db5eeb90b1d", + "unique_content_id": "686399b9012be40d39b5366ec1695768", "aliases": [ "VU#120541", "CVE-2009-3555" @@ -2150,7 +2150,7 @@ "weaknesses": [] }, { - "unique_content_id": "d5a14ef4e648d1a19b8f5ce8404490bc", + "unique_content_id": "c616b60f7fd802e88ca29fce6222654e", "aliases": [ "VU#180065", "CVE-2009-2629" From 495a8b5a4abaa0295ee568557ddd1d1b00795ef9 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 23 Sep 2024 19:03:25 +0530 Subject: [PATCH 08/29] Add data migration for nginx advisory Signed-off-by: Keshav Priyadarshi --- vulnerabilities/improvers/valid_versions.py | 2 +- .../0065_update_nginx_advisory_created_by.py | 38 ++++++++++++++++++ vulnerabilities/tests/test_data_migrations.py | 39 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index 9e6300bf3..ed9be12a3 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -220,7 +220,7 @@ class NginxBasicImprover(Improver): @property def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=NginxImporterPipeline.qualified_name).paginated() + return Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).paginated() def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: all_versions = list(self.fetch_nginx_version_from_git_tags()) diff --git a/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py b/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py new file mode 100644 index 000000000..80b43a954 --- /dev/null +++ b/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-23 13:06 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.nginx.NginxImporter").update( + created_by=NginxImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.nginx.NginxImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0064_update_npm_pypa_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 6e11bf367..31d05507d 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -681,3 +681,42 @@ def test_removal_of_duped_purls(self): assert adv.filter(created_by="vulnerabilities.importers.npm.NpmImporter").count() == 0 assert adv.filter(created_by="npm_importer").count() == 1 + + +class TestUpdateNginxAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0064_update_npm_pypa_advisory_created_by" + migrate_to = "0065_update_nginx_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="nginx", name="nginx"), + affected_version_range=VersionRange.from_string("vers:nginx/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.nginx.NginxImporter", + date_collected=timezone.now(), + ) + + def test_removal_of_duped_purls(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert adv.filter(created_by="vulnerabilities.importers.nginx.NginxImporter").count() == 0 + assert adv.filter(created_by="nginx_importer").count() == 1 From 6773d768a7489ab9917dfdbb0621bb85d424dbd5 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 25 Sep 2024 18:49:23 +0530 Subject: [PATCH 09/29] Use pipeline_id to get interesting_advisories Signed-off-by: Keshav Priyadarshi --- vulnerabilities/improvers/valid_versions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index ed9be12a3..ecbf2ddd3 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -43,6 +43,7 @@ from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage @@ -63,6 +64,8 @@ class ValidVersionImprover(Improver): @property def interesting_advisories(self) -> QuerySet: + if issubclass(self.importer, VulnerableCodeBaseImporterPipeline): + return Advisory.objects.filter(Q(created_by=self.importer.pipeline_id)).paginated() return Advisory.objects.filter(Q(created_by=self.importer.qualified_name)).paginated() def get_package_versions( From f8004999ce8aa4e1478ac2d6de29b731abee6988 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 2 Sep 2024 20:16:48 +0530 Subject: [PATCH 10/29] Migrate GitLab importer to aboutcode pipeline Signed-off-by: Keshav Priyadarshi --- vulnerabilities/importers/__init__.py | 4 +- vulnerabilities/improvers/valid_versions.py | 5 +- .../gitlab_importer.py} | 144 +++++++++++------- vulnerabilities/pipes/advisory.py | 2 +- .../test_gitlab_importer_pipeline.py} | 51 +++---- vulnerabilities/tests/test_data_source.py | 2 - 6 files changed, 117 insertions(+), 91 deletions(-) rename vulnerabilities/{importers/gitlab.py => pipelines/gitlab_importer.py} (68%) rename vulnerabilities/tests/{test_gitlab.py => pipelines/test_gitlab_importer_pipeline.py} (66%) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index efd8b71e9..75d9e8bed 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -21,7 +21,6 @@ from vulnerabilities.importers import gentoo from vulnerabilities.importers import github from vulnerabilities.importers import github_osv -from vulnerabilities.importers import gitlab from vulnerabilities.importers import istio from vulnerabilities.importers import mozilla from vulnerabilities.importers import nvd @@ -39,6 +38,7 @@ from vulnerabilities.importers import vulnrichment from vulnerabilities.importers import xen from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines import gitlab_importer from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer from vulnerabilities.pipelines import pypa_importer @@ -46,7 +46,6 @@ IMPORTERS_REGISTRY = [ nvd.NVDImporter, github.GitHubAPIImporter, - gitlab.GitLabAPIImporter, pysec.PyPIImporter, alpine_linux.AlpineImporter, openssl.OpensslImporter, @@ -78,6 +77,7 @@ pypa_importer.PyPaImporterPipeline, npm_importer.NpmImporterPipeline, nginx_importer.NginxImporterPipeline, + gitlab_importer.GitLabImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index ecbf2ddd3..e65b619ad 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -12,7 +12,6 @@ from datetime import datetime from typing import Iterable from typing import List -from typing import Mapping from typing import Optional from django.db.models import Q @@ -34,7 +33,6 @@ from vulnerabilities.importers.elixir_security import ElixirSecurityImporter from vulnerabilities.importers.github import GitHubAPIImporter from vulnerabilities.importers.github_osv import GithubOSVImporter -from vulnerabilities.importers.gitlab import GitLabAPIImporter from vulnerabilities.importers.istio import IstioImporter from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter from vulnerabilities.importers.ruby import RubyImporter @@ -44,6 +42,7 @@ from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage @@ -367,7 +366,7 @@ class DebianBasicImprover(ValidVersionImprover): class GitLabBasicImprover(ValidVersionImprover): - importer = GitLabAPIImporter + importer = GitLabImporterPipeline ignorable_versions = [] diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/pipelines/gitlab_importer.py similarity index 68% rename from vulnerabilities/importers/gitlab.py rename to vulnerabilities/pipelines/gitlab_importer.py index cd42b24ed..604ba7194 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/pipelines/gitlab_importer.py @@ -12,11 +12,12 @@ from pathlib import Path from typing import Iterable from typing import List -from typing import Optional +from typing import Tuple import pytz import saneyaml from dateutil import parser as dateparser +from fetchcode.vcs import fetch_via_vcs from packageurl import PackageURL from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange @@ -25,58 +26,84 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.utils import build_description from vulnerabilities.utils import get_advisory_url from vulnerabilities.utils import get_cwe_id -logger = logging.getLogger(__name__) -PURL_TYPE_BY_GITLAB_SCHEME = { - "conan": "conan", - "gem": "gem", - # Entering issue to parse go package names https://github.com/nexB/vulnerablecode/issues/742 - # "go": "golang", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "packagist": "composer", - "pypi": "pypi", -} - -GITLAB_SCHEME_BY_PURL_TYPE = {v: k for k, v in PURL_TYPE_BY_GITLAB_SCHEME.items()} - - -class GitLabAPIImporter(Importer): +class GitLabImporterPipeline(VulnerableCodeBaseImporterPipeline): spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" importer_name = "GitLab Importer" repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/" - def advisory_data(self, _keep_clone=False) -> Iterable[AdvisoryData]: - try: - self.clone(repo_url=self.repo_url) - base_path = Path(self.vcs_response.dest_dir) + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.import_new_advisories, + cls.clean_downloads, + ) - for file_path in base_path.glob("**/*.yml"): - gitlab_type, package_slug, vuln_id = parse_advisory_path( - base_path=base_path, - file_path=file_path, - ) + purl_type_by_gitlab_scheme = { + "conan": "conan", + "gem": "gem", + # Entering issue to parse go package names https://github.com/nexB/vulnerablecode/issues/742 + # "go": "golang", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "packagist": "composer", + "pypi": "pypi", + } + + gitlab_scheme_by_purl_type = {v: k for k, v in purl_type_by_gitlab_scheme.items()} + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + root = Path(self.vcs_response.dest_dir) + return sum(1 for _ in root.rglob("*.yml")) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + base_path = Path(self.vcs_response.dest_dir) + + for file_path in base_path.rglob("*.yml"): + if file_path.parent == base_path: + continue + + gitlab_type, _, _ = parse_advisory_path( + base_path=base_path, + file_path=file_path, + ) - if gitlab_type in PURL_TYPE_BY_GITLAB_SCHEME: - yield parse_gitlab_advisory(file=file_path, base_path=base_path) + if gitlab_type not in self.purl_type_by_gitlab_scheme: + # self.log( + # f"Unknown package type {gitlab_type!r} in {file_path!r}", + # level=logging.ERROR, + # ) + continue + + yield parse_gitlab_advisory( + file=file_path, + base_path=base_path, + gitlab_scheme_by_purl_type=self.gitlab_scheme_by_purl_type, + purl_type_by_gitlab_scheme=self.purl_type_by_gitlab_scheme, + logger=self.log, + ) - else: - logger.error(f"Unknow package type {gitlab_type!r} in {file_path!r}") - continue - finally: - if self.vcs_response and not _keep_clone: - self.vcs_response.delete() + def clean_downloads(self): + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() -def parse_advisory_path(base_path: Path, file_path: Path) -> Optional[AdvisoryData]: +def parse_advisory_path(base_path: Path, file_path: Path) -> Tuple[str, str, str]: """ Parse a gitlab advisory file and return a 3-tuple of: (gitlab_type, package_slug, vulnerability_id) @@ -96,21 +123,21 @@ def parse_advisory_path(base_path: Path, file_path: Path) -> Optional[AdvisoryDa >>> parse_advisory_path(base_path=base_path, file_path=file_path) ('npm', '@express/beego/beego/v2', 'CVE-2021-43831') """ - relative_path_segments = str(file_path.relative_to(base_path)).strip("/").split("/") + relative_path_segments = file_path.relative_to(base_path).parts gitlab_type = relative_path_segments[0] - vuln_id = relative_path_segments[-1].replace(".yml", "") + vuln_id = file_path.stem package_slug = "/".join(relative_path_segments[1:-1]) return gitlab_type, package_slug, vuln_id -def get_purl(package_slug): +def get_purl(package_slug, purl_type_by_gitlab_scheme, logger): """ Return a PackageURL object from a package slug """ parts = [p for p in package_slug.strip("/").split("/") if p] gitlab_scheme = parts[0] - purl_type = PURL_TYPE_BY_GITLAB_SCHEME[gitlab_scheme] + purl_type = purl_type_by_gitlab_scheme[gitlab_scheme] if gitlab_scheme == "go": name = "/".join(parts[1:]) return PackageURL(type=purl_type, namespace=None, name=name) @@ -125,7 +152,7 @@ def get_purl(package_slug): name = parts[-1] namespace = "/".join(parts[1:-1]) return PackageURL(type=purl_type, namespace=namespace, name=name) - logger.error(f"get_purl: package_slug can not be parsed: {package_slug!r}") + logger(f"get_purl: package_slug can not be parsed: {package_slug!r}", level=logging.ERROR) return @@ -140,7 +167,7 @@ def extract_affected_packages( In case of gitlab advisory data we get a list of fixed_versions and a affected_version_range. Since we can not determine which package fixes which range. We store the all the fixed_versions with the same affected_version_range in the advisory. - Later the advisory data is used to be infered in the GitLabBasicImprover. + Later the advisory data is used to be inferred in the GitLabBasicImprover. """ for fixed_version in fixed_versions: yield AffectedPackage( @@ -150,7 +177,9 @@ def extract_affected_packages( ) -def parse_gitlab_advisory(file, base_path): +def parse_gitlab_advisory( + file, base_path, gitlab_scheme_by_purl_type, purl_type_by_gitlab_scheme, logger +): """ Parse a Gitlab advisory file and return an AdvisoryData or None. These files are YAML. There is a JSON schema documented at @@ -177,8 +206,9 @@ def parse_gitlab_advisory(file, base_path): with open(file) as f: gitlab_advisory = saneyaml.load(f) if not isinstance(gitlab_advisory, dict): - logger.error( - f"parse_gitlab_advisory: unknown gitlab advisory format in {file!r} with data: {gitlab_advisory!r}" + logger( + f"parse_gitlab_advisory: unknown gitlab advisory format in {file!r} with data: {gitlab_advisory!r}", + level=logging.ERROR, ) return @@ -199,9 +229,15 @@ def parse_gitlab_advisory(file, base_path): base_path=base_path, url="https://gitlab.com/gitlab-org/advisories-community/-/blob/main/", ) - purl: PackageURL = get_purl(package_slug=package_slug) + purl: PackageURL = get_purl( + package_slug=package_slug, + purl_type_by_gitlab_scheme=purl_type_by_gitlab_scheme, + logger=logger, + ) if not purl: - logger.error(f"parse_yaml_file: purl is not valid: {file!r} {package_slug!r}") + logger( + f"parse_yaml_file: purl is not valid: {file!r} {package_slug!r}", level=logging.ERROR + ) return AdvisoryData( aliases=aliases, summary=summary, @@ -214,7 +250,7 @@ def parse_gitlab_advisory(file, base_path): affected_range = gitlab_advisory.get("affected_range") gitlab_native_schemes = set(["pypi", "gem", "npm", "go", "packagist", "conan"]) vrc: VersionRange = RANGE_CLASS_BY_SCHEMES[purl.type] - gitlab_scheme = GITLAB_SCHEME_BY_PURL_TYPE[purl.type] + gitlab_scheme = gitlab_scheme_by_purl_type[purl.type] try: if affected_range: if gitlab_scheme in gitlab_native_schemes: @@ -224,8 +260,9 @@ def parse_gitlab_advisory(file, base_path): else: affected_version_range = vrc.from_native(affected_range) except Exception as e: - logger.error( - f"parse_yaml_file: affected_range is not parsable: {affected_range!r} type:{purl.type!r} error: {e!r}\n {traceback.format_exc()}" + logger( + f"parse_yaml_file: affected_range is not parsable: {affected_range!r} for: {purl!s} error: {e!r}\n {traceback.format_exc()}", + level=logging.ERROR, ) parsed_fixed_versions = [] @@ -234,8 +271,9 @@ def parse_gitlab_advisory(file, base_path): fixed_version = vrc.version_class(fixed_version) parsed_fixed_versions.append(fixed_version) except Exception as e: - logger.error( - f"parse_yaml_file: fixed_version is not parsable`: {fixed_version!r} error: {e!r}\n {traceback.format_exc()}" + logger( + f"parse_yaml_file: fixed_version is not parsable`: {fixed_version!r} error: {e!r}\n {traceback.format_exc()}", + level=logging.ERROR, ) if parsed_fixed_versions: diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index ebc25f93b..f33eb4d2b 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -90,7 +90,7 @@ def import_advisory( if not vulnerability: if logger: - logger(f"Unable to get vulnerability for advisory: {advisory!r}", level=logging.WARNING) + logger(f"Unable to get vulnerability for advisory: {advisory!r}", level=logging.ERROR) return for ref in advisory_data.references: diff --git a/vulnerabilities/tests/test_gitlab.py b/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py similarity index 66% rename from vulnerabilities/tests/test_gitlab.py rename to vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py index bc2bfcaea..d10413a8b 100644 --- a/vulnerabilities/tests/test_gitlab.py +++ b/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py @@ -8,27 +8,34 @@ # import json -import os from pathlib import Path from unittest import mock import pytest from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.gitlab import parse_gitlab_advisory from vulnerabilities.improvers.default import DefaultImprover from vulnerabilities.improvers.valid_versions import GitLabBasicImprover +from vulnerabilities.pipelines import gitlab_importer from vulnerabilities.tests import util_tests +from vulnerabilities.tests.pipelines import TestLogger -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "gitlab") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "gitlab" @pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "composer", "pypi", "npm"]) def test_parse_yaml_file(pkg_type): - response_file = os.path.join(TEST_DATA, f"{pkg_type}.yaml") - expected_file = os.path.join(TEST_DATA, f"{pkg_type}-expected.json") - advisory = parse_gitlab_advisory(Path(response_file), Path(response_file).parent) + response_file = TEST_DATA / f"{pkg_type}.yaml" + expected_file = TEST_DATA / f"{pkg_type}-expected.json" + test_pipeline = gitlab_importer.GitLabImporterPipeline() + logger = TestLogger() + advisory = gitlab_importer.parse_gitlab_advisory( + response_file, + response_file.parent, + test_pipeline.gitlab_scheme_by_purl_type, + test_pipeline.purl_type_by_gitlab_scheme, + logger.write, + ) util_tests.check_results_against_json(advisory.to_dict(), expected_file) @@ -45,27 +52,11 @@ def valid_versions(pkg_type): "9.1.6", "10.0.0", ], - "gem": [ - "4.2.0.beta1", - "4.2.0.beta2", - "4.2.0.beta3", - ], - "golang": [ - "3.7.0", - "3.7.1", - ], + "gem": ["4.2.0.beta1", "4.2.0.beta2", "4.2.0.beta3"], + "golang": ["3.7.0", "3.7.1"], "nuget": ["1.11.0", "1.11.1", "1.11.2", "1.09.1"], - "npm": [ - "2.14.2", - "2.13.2", - "2.11.2", - ], - "pypi": [ - "1.0", - "0.9", - "0.8", - "1.1", - ], + "npm": ["2.14.2", "2.13.2", "2.11.2"], + "pypi": ["1.0", "0.9", "0.8", "1.1"], "composer": [], } return valid_versions_by_package_type[pkg_type] @@ -74,9 +65,9 @@ def valid_versions(pkg_type): @mock.patch("vulnerabilities.improvers.valid_versions.GitLabBasicImprover.get_package_versions") @pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "composer", "pypi", "npm"]) def test_gitlab_improver(mock_response, pkg_type): - advisory_file = os.path.join(TEST_DATA, f"{pkg_type}-expected.json") - expected_file = os.path.join(TEST_DATA, f"{pkg_type}-improver-expected.json") - with open(advisory_file) as exp: + advisory_file = TEST_DATA / f"{pkg_type}-expected.json" + expected_file = TEST_DATA / f"{pkg_type}-improver-expected.json" + with advisory_file.open() as exp: advisory = AdvisoryData.from_dict(json.load(exp)) mock_response.return_value = list(valid_versions(pkg_type)) improvers = [GitLabBasicImprover(), DefaultImprover()] diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py index 61cf56d46..b0baf5685 100644 --- a/vulnerabilities/tests/test_data_source.py +++ b/vulnerabilities/tests/test_data_source.py @@ -22,7 +22,6 @@ from vulnerabilities.importers.fireeye import FireyeImporter from vulnerabilities.importers.gentoo import GentooImporter from vulnerabilities.importers.github_osv import GithubOSVImporter -from vulnerabilities.importers.gitlab import GitLabAPIImporter from vulnerabilities.importers.istio import IstioImporter from vulnerabilities.importers.mozilla import MozillaImporter from vulnerabilities.importers.retiredotnet import RetireDotnetImporter @@ -117,7 +116,6 @@ def test_git_importer(mock_clone): ElixirSecurityImporter, FireyeImporter, GentooImporter, - GitLabAPIImporter, IstioImporter, MozillaImporter, RetireDotnetImporter, From 95ef0b507042ba9488ace13de2edce9827138812 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Sep 2024 19:20:33 +0530 Subject: [PATCH 11/29] Add pipeline_id to gitlab pipeline Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/gitlab_importer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vulnerabilities/pipelines/gitlab_importer.py b/vulnerabilities/pipelines/gitlab_importer.py index 604ba7194..0b76a31f2 100644 --- a/vulnerabilities/pipelines/gitlab_importer.py +++ b/vulnerabilities/pipelines/gitlab_importer.py @@ -34,6 +34,10 @@ class GitLabImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisory from GitLab Advisory Database (Open Source Edition).""" + + pipeline_id = "gitlab_importer" + spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" importer_name = "GitLab Importer" From 0e3ec68aaf43a3eea62c43a9bd663d06078396a7 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Sep 2024 19:21:37 +0530 Subject: [PATCH 12/29] Add data migration for gitlab advisory Signed-off-by: Keshav Priyadarshi --- .../0066_update_gitlab_advisory_created_by.py | 38 +++++++++++++++++ vulnerabilities/tests/test_data_migrations.py | 41 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py diff --git a/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py b/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py new file mode 100644 index 000000000..e72b0616b --- /dev/null +++ b/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-27 13:08 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter").update( + created_by=GitLabImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=GitLabImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0065_update_nginx_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 31d05507d..625e587ff 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -720,3 +720,44 @@ def test_removal_of_duped_purls(self): assert adv.filter(created_by="vulnerabilities.importers.nginx.NginxImporter").count() == 0 assert adv.filter(created_by="nginx_importer").count() == 1 + + +class TestUpdateGitLabAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0065_update_nginx_advisory_created_by" + migrate_to = "0066_update_gitlab_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter", + date_collected=timezone.now(), + ) + + def test_removal_of_duped_purls(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert ( + adv.filter(created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter").count() == 0 + ) + assert adv.filter(created_by="gitlab_importer").count() == 1 From bcd01d0df263d3f5d590ed5b50d9b19a3ec6e026 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 6 Sep 2024 14:26:05 +0530 Subject: [PATCH 13/29] Migrate GitHub importer to aboutcode pipeline Signed-off-by: Keshav Priyadarshi --- vulnerabilities/importers/__init__.py | 4 +- vulnerabilities/improvers/valid_versions.py | 4 +- vulnerabilities/pipelines/__init__.py | 7 +- .../github_importer.py} | 225 +++++++++++------- .../test_github_importer_pipeline.py} | 61 +++-- vulnerabilities/tests/test_upstream.py | 2 +- 6 files changed, 192 insertions(+), 111 deletions(-) rename vulnerabilities/{importers/github.py => pipelines/github_importer.py} (56%) rename vulnerabilities/tests/{test_github.py => pipelines/test_github_importer_pipeline.py} (84%) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 75d9e8bed..a69fe1629 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -19,7 +19,6 @@ from vulnerabilities.importers import epss from vulnerabilities.importers import fireeye from vulnerabilities.importers import gentoo -from vulnerabilities.importers import github from vulnerabilities.importers import github_osv from vulnerabilities.importers import istio from vulnerabilities.importers import mozilla @@ -38,6 +37,7 @@ from vulnerabilities.importers import vulnrichment from vulnerabilities.importers import xen from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines import github_importer from vulnerabilities.pipelines import gitlab_importer from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer @@ -45,7 +45,6 @@ IMPORTERS_REGISTRY = [ nvd.NVDImporter, - github.GitHubAPIImporter, pysec.PyPIImporter, alpine_linux.AlpineImporter, openssl.OpensslImporter, @@ -78,6 +77,7 @@ npm_importer.NpmImporterPipeline, nginx_importer.NginxImporterPipeline, gitlab_importer.GitLabImporterPipeline, + github_importer.GitHubAPIImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index e65b619ad..5d1e087ec 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -31,7 +31,6 @@ from vulnerabilities.importers.debian import DebianImporter from vulnerabilities.importers.debian_oval import DebianOvalImporter from vulnerabilities.importers.elixir_security import ElixirSecurityImporter -from vulnerabilities.importers.github import GitHubAPIImporter from vulnerabilities.importers.github_osv import GithubOSVImporter from vulnerabilities.importers.istio import IstioImporter from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter @@ -42,6 +41,7 @@ from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline @@ -371,7 +371,7 @@ class GitLabBasicImprover(ValidVersionImprover): class GitHubBasicImprover(ValidVersionImprover): - importer = GitHubAPIImporter + importer = GitHubAPIImporterPipeline ignorable_versions = frozenset( [ "0.1-bulbasaur", diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index aa3d59d83..0d3589b67 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -89,7 +89,12 @@ def advisories_count(self) -> int: def collect_and_store_advisories(self): collected_advisory_count = 0 - progress = LoopProgress(total_iterations=self.advisories_count(), logger=self.log) + estimated_advisory_count = self.advisories_count() + + if estimated_advisory_count > 0: + self.log(f"Collecting {estimated_advisory_count:,d} advisories") + + progress = LoopProgress(total_iterations=estimated_advisory_count, logger=self.log) for advisory in progress.iter(self.collect_advisories()): if _obj := insert_advisory( advisory=advisory, diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/pipelines/github_importer.py similarity index 56% rename from vulnerabilities/importers/github.py rename to vulnerabilities/pipelines/github_importer.py index c12c43044..d5df390b4 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/pipelines/github_importer.py @@ -8,9 +8,14 @@ # import logging +from traceback import format_exc as traceback_format_exc +from typing import Callable from typing import Iterable +from typing import List from typing import Optional +import requests +from bs4 import BeautifulSoup from cwe2.database import Database from dateutil import parser as dateparser from packageurl import PackageURL @@ -21,85 +26,120 @@ from vulnerabilities import utils from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.utils import dedupe from vulnerabilities.utils import get_cwe_id from vulnerabilities.utils import get_item -logger = logging.getLogger(__name__) - -PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM = { - "MAVEN": "maven", - "NUGET": "nuget", - "COMPOSER": "composer", - "PIP": "pypi", - "RUBYGEMS": "gem", - "NPM": "npm", - "RUST": "cargo", - # "GO": "golang", -} - -GITHUB_ECOSYSTEM_BY_PACKAGE_TYPE = { - value: key for (key, value) in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items() -} - -# TODO: We will try to gather more info from GH API -# Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885 -# Check https://github.com/nexB/vulnerablecode/issues/645 -# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM', 'RUST'} -# second '%s' is interesting, it will have the value '' for the first request, -GRAPHQL_QUERY_TEMPLATE = """ -query{ - securityVulnerabilities(first: 100, ecosystem: %s, %s) { - edges { - node { - advisory { - identifiers { - type - value - } - summary - references { - url - } - severity - cwes(first: 10){ - nodes { - cweId + +class GitHubAPIImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect GitHub advisories.""" + + spdx_license_expression = "CC-BY-4.0" + license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" + importer_name = "GHSA Importer" + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + package_type_by_github_ecosystem = { + "MAVEN": "maven", + "NUGET": "nuget", + "COMPOSER": "composer", + "PIP": "pypi", + "RUBYGEMS": "gem", + "NPM": "npm", + "RUST": "cargo", + # "GO": "golang", + } + + github_ecosystem_by_package_type = { + value: key for (key, value) in package_type_by_github_ecosystem.items() + } + + def advisories_count(self): + normalized_github_ecosystems = [ + k.lower() for k in self.package_type_by_github_ecosystem.keys() + ] + + try: + response = requests.get("https://github.com/advisories") + response.raise_for_status() + except requests.HTTPError as http_err: + self.log( + f"HTTP error occurred: {http_err} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + return 0 + + soup = BeautifulSoup(response.text, "html.parser") + advisory_counts = 0 + for li in soup.select("ul.filter-list li") or []: + if link := li.find("a", class_="filter-item"): + ecosystem, _, _ = link.text.strip().rpartition(" ") + if count_span := li.find("span", class_="count"): + count = int(count_span.text.strip().replace(",", "")) + ecosystem = ecosystem.strip().lower() + if ecosystem in normalized_github_ecosystems: + advisory_counts += count + + return advisory_counts + + def collect_advisories(self) -> Iterable[AdvisoryData]: + + # TODO: We will try to gather more info from GH API + # Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885 + # Check https://github.com/nexB/vulnerablecode/issues/645 + # set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM', 'RUST'} + # second '%s' is interesting, it will have the value '' for the first request, + advisory_query = """ + query{ + securityVulnerabilities(first: 100, ecosystem: %s, %s) { + edges { + node { + advisory { + identifiers { + type + value + } + summary + references { + url + } + severity + cwes(first: 10){ + nodes { + cweId + } + } + publishedAt } + firstPatchedVersion{ + identifier + } + package { + name + } + vulnerableVersionRange } - publishedAt } - firstPatchedVersion{ - identifier + pageInfo { + hasNextPage + endCursor } - package { - name - } - vulnerableVersionRange } } - pageInfo { - hasNextPage - endCursor - } - } -} -""" - - -class GitHubAPIImporter(Importer): - spdx_license_expression = "CC-BY-4.0" - importer_name = "GHSA Importer" - license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" - - def advisory_data(self) -> Iterable[AdvisoryData]: - for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items(): + """ + for ecosystem, package_type in self.package_type_by_github_ecosystem.items(): end_cursor_exp = "" while True: - graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)} + graphql_query = {"query": advisory_query % (ecosystem, end_cursor_exp)} response = utils.fetch_github_graphql_query(graphql_query) page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo") @@ -114,7 +154,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]: break -def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]: +def get_purl(pkg_type: str, github_name: str, logger: Callable = None) -> Optional[PackageURL]: """ Return a PackageURL by splitting the `github_name` using the `pkg_type` convention. Return None and log an error if we can not split or it is an @@ -129,7 +169,8 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]: """ if pkg_type == "maven": if ":" not in github_name: - logger.error(f"get_purl: Invalid maven package name {github_name}") + if logger: + logger(f"get_purl: Invalid maven package name {github_name}", level=logging.ERROR) return ns, _, name = github_name.partition(":") return PackageURL(type=pkg_type, namespace=ns, name=name) @@ -143,18 +184,23 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]: if pkg_type in ("nuget", "pypi", "gem", "golang", "npm", "cargo"): return PackageURL(type=pkg_type, name=github_name) - logger.error(f"get_purl: Unknown package type {pkg_type}") + if logger: + logger(f"get_purl: Unknown package type {pkg_type}", level=logging.ERROR) -def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: +def process_response( + resp: dict, package_type: str, logger: Callable = None +) -> Iterable[AdvisoryData]: """ Yield `AdvisoryData` by taking `resp` and `ecosystem` as input """ vulnerabilities = get_item(resp, "data", "securityVulnerabilities", "edges") or [] if not vulnerabilities: - logger.error( - f"No vulnerabilities found for package_type: {package_type!r} in response: {resp!r}" - ) + if logger: + logger( + f"No vulnerabilities found for package_type: {package_type!r} in response: {resp!r}", + level=logging.ERROR, + ) return for vulnerability in vulnerabilities: @@ -162,12 +208,14 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: affected_packages = [] github_advisory = get_item(vulnerability, "node") if not github_advisory: - logger.error(f"No node found in {vulnerability!r}") + if logger: + logger(f"No node found in {vulnerability!r}", level=logging.ERROR) continue advisory = get_item(github_advisory, "advisory") if not advisory: - logger.error(f"No advisory found in {github_advisory!r}") + if logger: + logger(f"No advisory found in {github_advisory!r}", level=logging.ERROR) continue summary = get_item(advisory, "summary") or "" @@ -183,7 +231,7 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: name = get_item(github_advisory, "package", "name") if name: - purl = get_purl(pkg_type=package_type, github_name=name) + purl = get_purl(pkg_type=package_type, github_name=name, logger=logger) if purl: affected_range = get_item(github_advisory, "vulnerableVersionRange") fixed_version = get_item(github_advisory, "firstPatchedVersion", "identifier") @@ -193,7 +241,11 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: package_type, affected_range ) except Exception as e: - logger.error(f"Could not parse affected range {affected_range!r} {e!r}") + if logger: + logger( + f"Could not parse affected range {affected_range!r} {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) affected_range = None if fixed_version: try: @@ -201,7 +253,11 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: fixed_version ) except Exception as e: - logger.error(f"Invalid fixed version {fixed_version!r} {e!r}") + if logger: + logger( + f"Invalid fixed version {fixed_version!r} {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) fixed_version = None if affected_range or fixed_version: affected_packages.append( @@ -236,9 +292,13 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: elif identifier_type == "CVE": pass else: - logger.error(f"Unknown identifier type {identifier_type!r} and value {value!r}") + if logger: + logger( + f"Unknown identifier type {identifier_type!r} and value {value!r}", + level=logging.ERROR, + ) - weaknesses = get_cwes_from_github_advisory(advisory) + weaknesses = get_cwes_from_github_advisory(advisory, logger) yield AdvisoryData( aliases=sorted(dedupe(aliases)), @@ -251,7 +311,7 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: ) -def get_cwes_from_github_advisory(advisory) -> [int]: +def get_cwes_from_github_advisory(advisory, logger=None) -> List[int]: """ Return the cwe-id list from advisory ex: [ 522 ] by extracting the cwe_list from advisory ex: [{'cweId': 'CWE-522'}] @@ -267,6 +327,7 @@ def get_cwes_from_github_advisory(advisory) -> [int]: try: db.get(cwe_id) weaknesses.append(cwe_id) - except Exception: - logger.error("Invalid CWE id") + except Exception as e: + if logger: + logger(f"Invalid CWE id {e!r} \n {traceback_format_exc()}", level=logging.ERROR) return weaknesses diff --git a/vulnerabilities/tests/test_github.py b/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py similarity index 84% rename from vulnerabilities/tests/test_github.py rename to vulnerabilities/tests/pipelines/test_github_importer_pipeline.py index 2b5593137..d46e3ef19 100644 --- a/vulnerabilities/tests/test_github.py +++ b/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py @@ -10,6 +10,7 @@ import json import os from datetime import datetime +from pathlib import Path from unittest import mock import pytest @@ -23,23 +24,22 @@ from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.importers.github import GitHubAPIImporter -from vulnerabilities.importers.github import get_cwes_from_github_advisory -from vulnerabilities.importers.github import process_response from vulnerabilities.improvers.valid_versions import GitHubBasicImprover +from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline +from vulnerabilities.pipelines.github_importer import get_cwes_from_github_advisory +from vulnerabilities.pipelines.github_importer import process_response +from vulnerabilities.tests.pipelines import TestLogger from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN -from vulnerabilities.utils import GitHubTokenError -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "github_api") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "github_api" @pytest.mark.parametrize( "pkg_type", ["maven", "nuget", "gem", "golang", "composer", "pypi", "npm", "cargo"] ) def test_process_response_github_importer(pkg_type, regen=REGEN): - response_file = os.path.join(TEST_DATA, f"{pkg_type}.json") - expected_file = os.path.join(TEST_DATA, f"{pkg_type}-expected.json") + response_file = TEST_DATA / f"{pkg_type}.json" + expected_file = TEST_DATA / f"{pkg_type}-expected.json" with open(response_file) as f: response = json.load(f) @@ -56,34 +56,49 @@ def test_process_response_github_importer(pkg_type, regen=REGEN): assert result == expected -def test_process_response_with_empty_vulnaribilities(caplog): - list(process_response({"data": {"securityVulnerabilities": {"edges": []}}}, "maven")) - assert "No vulnerabilities found for package_type: 'maven'" in caplog.text +def test_process_response_with_empty_vulnaribilities(): + logger = TestLogger() + list( + process_response( + {"data": {"securityVulnerabilities": {"edges": []}}}, + "maven", + logger=logger.write, + ) + ) + assert "No vulnerabilities found for package_type: 'maven'" in logger.getvalue() -def test_process_response_with_empty_vulnaribilities_2(caplog): +def test_process_response_with_empty_vulnaribilities_2(): + logger = TestLogger() list( process_response( - {"data": {"securityVulnerabilities": {"edges": [{"node": {}}, None]}}}, "maven" + {"data": {"securityVulnerabilities": {"edges": [{"node": {}}, None]}}}, + "maven", + logger=logger.write, ) ) - assert "No node found" in caplog.text + assert "No node found" in logger.getvalue() def test_github_importer_with_missing_credentials(): - with pytest.raises(GitHubTokenError) as e: - with mock.patch.dict(os.environ, {}, clear=True): - importer = GitHubAPIImporter() - list(importer.advisory_data()) + with mock.patch.dict(os.environ, {}, clear=True): + github_pipeline = GitHubAPIImporterPipeline() + status, error = github_pipeline.execute() + assert 1 == status + assert ( + "Cannot call GitHub API without a token set in the GH_TOKEN environment variable." + in error + ) @mock.patch("vulnerabilities.utils._get_gh_response") def test_github_importer_with_missing_credentials_2(mock_response): mock_response.return_value = {"message": "Bad credentials"} - with pytest.raises(GitHubTokenError) as e: - with mock.patch.dict(os.environ, {"GH_TOKEN": "FOOD"}, clear=True): - importer = GitHubAPIImporter() - list(importer.advisory_data()) + with mock.patch.dict(os.environ, {"GH_TOKEN": "FOOD"}, clear=True): + github_pipeline = GitHubAPIImporterPipeline() + status, error = github_pipeline.execute() + assert 1 == status + assert "Invalid GitHub token: Bad credentials" in error def valid_versions(): @@ -283,7 +298,7 @@ def test_github_improver(mock_response, regen=REGEN): @mock.patch("fetchcode.package_versions.get_response") def test_get_package_versions(mock_response): - with open(os.path.join(BASE_DIR, "test_data", "package_manager_data", "pypi.json"), "r") as f: + with open(TEST_DATA.parent / "package_manager_data" / "pypi.json", "r") as f: mock_response.return_value = json.load(f) improver = GitHubBasicImprover() diff --git a/vulnerabilities/tests/test_upstream.py b/vulnerabilities/tests/test_upstream.py index 925d28d80..ad5f50113 100644 --- a/vulnerabilities/tests/test_upstream.py +++ b/vulnerabilities/tests/test_upstream.py @@ -19,7 +19,7 @@ ) def test_updated_advisories(importer_name, importer_class): # FIXME: why are we doing this? - if importer_name.endswith("GitHubAPIImporter"): + if importer_name.endswith("GitHubAPIImporterPipeline"): return advisory_datas = importer_class().advisory_data() From 5386adcbf9f0c0e7f8ca4a229558e0613d6ff4fa Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Sep 2024 20:01:32 +0530 Subject: [PATCH 14/29] Add pipeline_id to github pipeline Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/github_importer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vulnerabilities/pipelines/github_importer.py b/vulnerabilities/pipelines/github_importer.py index d5df390b4..748674a58 100644 --- a/vulnerabilities/pipelines/github_importer.py +++ b/vulnerabilities/pipelines/github_importer.py @@ -37,6 +37,8 @@ class GitHubAPIImporterPipeline(VulnerableCodeBaseImporterPipeline): """Collect GitHub advisories.""" + pipeline_id = "github_importer" + spdx_license_expression = "CC-BY-4.0" license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" importer_name = "GHSA Importer" From b6651a443a21e5d079b9736d3cd39259e075e220 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Sep 2024 20:19:04 +0530 Subject: [PATCH 15/29] Add data migration for github advisory Signed-off-by: Keshav Priyadarshi --- .../0067_update_github_advisory_created_by.py | 38 +++++++++++++++++ vulnerabilities/tests/test_data_migrations.py | 41 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 vulnerabilities/migrations/0067_update_github_advisory_created_by.py diff --git a/vulnerabilities/migrations/0067_update_github_advisory_created_by.py b/vulnerabilities/migrations/0067_update_github_advisory_created_by.py new file mode 100644 index 000000000..4b9bb8485 --- /dev/null +++ b/vulnerabilities/migrations/0067_update_github_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-27 14:31 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").update( + created_by=GitHubAPIImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=GitHubAPIImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.github.GitHubAPIImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0066_update_gitlab_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 625e587ff..fcad0a1d4 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -761,3 +761,44 @@ def test_removal_of_duped_purls(self): adv.filter(created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter").count() == 0 ) assert adv.filter(created_by="gitlab_importer").count() == 1 + + +class TestUpdateGitHubAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0066_update_gitlab_advisory_created_by" + migrate_to = "0067_update_github_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.github.GitHubAPIImporter", + date_collected=timezone.now(), + ) + + def test_removal_of_duped_purls(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert ( + adv.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").count() == 0 + ) + assert adv.filter(created_by="github_importer").count() == 1 From 1d3da91680ee68cb82c31690e1e8191515d521fd Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Sep 2024 22:06:31 +0530 Subject: [PATCH 16/29] Use GraphQL to get the advisories_count Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/github_importer.py | 41 ++++++-------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/vulnerabilities/pipelines/github_importer.py b/vulnerabilities/pipelines/github_importer.py index 748674a58..4603b939a 100644 --- a/vulnerabilities/pipelines/github_importer.py +++ b/vulnerabilities/pipelines/github_importer.py @@ -14,8 +14,6 @@ from typing import List from typing import Optional -import requests -from bs4 import BeautifulSoup from cwe2.database import Database from dateutil import parser as dateparser from packageurl import PackageURL @@ -61,36 +59,19 @@ def steps(cls): # "GO": "golang", } - github_ecosystem_by_package_type = { - value: key for (key, value) in package_type_by_github_ecosystem.items() - } - def advisories_count(self): - normalized_github_ecosystems = [ - k.lower() for k in self.package_type_by_github_ecosystem.keys() - ] - - try: - response = requests.get("https://github.com/advisories") - response.raise_for_status() - except requests.HTTPError as http_err: - self.log( - f"HTTP error occurred: {http_err} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - return 0 - - soup = BeautifulSoup(response.text, "html.parser") + advisory_query = """ + query{ + securityVulnerabilities(first: 0, ecosystem: %s) { + totalCount + } + } + """ advisory_counts = 0 - for li in soup.select("ul.filter-list li") or []: - if link := li.find("a", class_="filter-item"): - ecosystem, _, _ = link.text.strip().rpartition(" ") - if count_span := li.find("span", class_="count"): - count = int(count_span.text.strip().replace(",", "")) - ecosystem = ecosystem.strip().lower() - if ecosystem in normalized_github_ecosystems: - advisory_counts += count - + for ecosystem in self.package_type_by_github_ecosystem.keys(): + graphql_query = {"query": advisory_query % (ecosystem)} + response = utils.fetch_github_graphql_query(graphql_query) + advisory_counts += get_item(response, "data", "securityVulnerabilities", "totalCount") return advisory_counts def collect_advisories(self) -> Iterable[AdvisoryData]: From 69739c1d896629cc1341a4d72429151c4ca05bc6 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 11 Sep 2024 15:08:01 +0530 Subject: [PATCH 17/29] Migrate NVD importer to aboutcode pipeline Signed-off-by: Keshav Priyadarshi --- vulnerabilities/importers/__init__.py | 4 +- .../improvers/vulnerability_status.py | 4 +- .../nvd.py => pipelines/nvd_importer.py} | 46 ++++++++++++++++--- .../test_nvd_importer_pipeline.py} | 36 ++++++++------- .../test_vulnerability_status_improver.py | 9 ++-- 5 files changed, 66 insertions(+), 33 deletions(-) rename vulnerabilities/{importers/nvd.py => pipelines/nvd_importer.py} (88%) rename vulnerabilities/tests/{test_nvd.py => pipelines/test_nvd_importer_pipeline.py} (84%) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index a69fe1629..be1e838b0 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -22,7 +22,6 @@ from vulnerabilities.importers import github_osv from vulnerabilities.importers import istio from vulnerabilities.importers import mozilla -from vulnerabilities.importers import nvd from vulnerabilities.importers import openssl from vulnerabilities.importers import oss_fuzz from vulnerabilities.importers import postgresql @@ -41,10 +40,10 @@ from vulnerabilities.pipelines import gitlab_importer from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer +from vulnerabilities.pipelines import nvd_importer from vulnerabilities.pipelines import pypa_importer IMPORTERS_REGISTRY = [ - nvd.NVDImporter, pysec.PyPIImporter, alpine_linux.AlpineImporter, openssl.OpensslImporter, @@ -78,6 +77,7 @@ nginx_importer.NginxImporterPipeline, gitlab_importer.GitLabImporterPipeline, github_importer.GitHubAPIImporterPipeline, + nvd_importer.NVDImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/improvers/vulnerability_status.py b/vulnerabilities/improvers/vulnerability_status.py index 0157db557..7287e483f 100644 --- a/vulnerabilities/improvers/vulnerability_status.py +++ b/vulnerabilities/improvers/vulnerability_status.py @@ -14,7 +14,6 @@ from django.db.models.query import QuerySet from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.nvd import NVDImporter from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory @@ -22,6 +21,7 @@ from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityChangeLog from vulnerabilities.models import VulnerabilityStatusType +from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline from vulnerabilities.utils import fetch_response from vulnerabilities.utils import get_item @@ -38,7 +38,7 @@ class VulnerabilityStatusImprover(Improver): @property def interesting_advisories(self) -> QuerySet: return ( - Advisory.objects.filter(Q(created_by=NVDImporter.qualified_name)) + Advisory.objects.filter(Q(created_by=NVDImporterPipeline.qualified_name)) .distinct("aliases") .paginated() ) diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/pipelines/nvd_importer.py similarity index 88% rename from vulnerabilities/importers/nvd.py rename to vulnerabilities/pipelines/nvd_importer.py index 1a6048dfd..3ccb254f1 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/pipelines/nvd_importer.py @@ -9,7 +9,10 @@ import gzip import json +import logging from datetime import date +from traceback import format_exc as traceback_format_exc +from typing import Iterable import attr import requests @@ -17,14 +20,16 @@ from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.utils import get_cwe_id from vulnerabilities.utils import get_item -class NVDImporter(Importer): +class NVDImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories from NVD.""" + # See https://github.com/nexB/vulnerablecode/issues/665 for follow up spdx_license_expression = ( "LicenseRef-scancode-us-govt-public-domain AND LicenseRef-scancode-cve-tou" @@ -61,19 +66,46 @@ class NVDImporter(Importer): """ importer_name = "NVD Importer" - def advisory_data(self): - for _year, cve_data in fetch_cve_data_1_1(): + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def advisories_count(self): + url = "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=1" + + advisory_count = 0 + try: + response = requests.get(url) + response.raise_for_status() + data = response.json() + except requests.HTTPError as http_err: + self.log( + f"HTTP error occurred: {http_err} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + return advisory_count + + advisory_count = data.get("totalResults", 0) + return advisory_count + + def collect_advisories(self) -> Iterable[AdvisoryData]: + for _year, cve_data in fetch_cve_data_1_1(logger=self.log): yield from to_advisories(cve_data=cve_data) # Isolating network calls for simplicity of testing -def fetch(url): +def fetch(url, logger=None): + if logger: + logger(f"Fetching `{url}`") gz_file = requests.get(url) data = gzip.decompress(gz_file.content) return json.loads(data) -def fetch_cve_data_1_1(starting_year=2002): +def fetch_cve_data_1_1(starting_year=2002, logger=None): """ Yield tuples of (year, lists of CVE mappings) from the NVD, one for each year since ``starting_year`` defaulting to 2002. @@ -82,7 +114,7 @@ def fetch_cve_data_1_1(starting_year=2002): # NVD json feeds start from 2002. for year in range(starting_year, current_year + 1): download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" - yield year, fetch(url=download_url) + yield year, fetch(url=download_url, logger=logger) def to_advisories(cve_data): diff --git a/vulnerabilities/tests/test_nvd.py b/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py similarity index 84% rename from vulnerabilities/tests/test_nvd.py rename to vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py index 702faa7f4..549a269bb 100644 --- a/vulnerabilities/tests/test_nvd.py +++ b/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py @@ -8,14 +8,12 @@ # import json -import os +from pathlib import Path -from vulnerabilities.importers import nvd +from vulnerabilities.pipelines import nvd_importer from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/nvd/nvd_test.json") -REJECTED_CVE = os.path.join(BASE_DIR, "test_data/nvd/rejected_nvd.json") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "nvd" def load_test_data(file): @@ -37,10 +35,11 @@ def sorted_advisory_data(advisory_data): def test_to_advisories_skips_hardware(regen=REGEN): - expected_file = os.path.join(BASE_DIR, "test_data/nvd/nvd-expected.json") + expected_file = TEST_DATA / "nvd-expected.json" - test_data = load_test_data(file=TEST_DATA) - result = [data.to_dict() for data in nvd.to_advisories(test_data)] + test_file = TEST_DATA / "nvd_test.json" + test_data = load_test_data(file=test_file) + result = [data.to_dict() for data in nvd_importer.to_advisories(test_data)] result = sorted_advisory_data(result) if regen: @@ -56,10 +55,11 @@ def test_to_advisories_skips_hardware(regen=REGEN): def test_to_advisories_marks_rejected_cve(regen=REGEN): - expected_file = os.path.join(BASE_DIR, "test_data/nvd/nvd-rejected-expected.json") + expected_file = TEST_DATA / "nvd-rejected-expected.json" - test_data = load_test_data(file=REJECTED_CVE) - result = [data.to_dict() for data in nvd.to_advisories(test_data)] + test_file = TEST_DATA / "rejected_nvd.json" + test_data = load_test_data(file=test_file) + result = [data.to_dict() for data in nvd_importer.to_advisories(test_data)] result = sorted_advisory_data(result) if regen: @@ -168,14 +168,16 @@ def test_CveItem_cpes(): "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", ] - found_cpes = nvd.CveItem(cve_item=get_test_cve_item()).cpes + found_cpes = nvd_importer.CveItem(cve_item=get_test_cve_item()).cpes assert found_cpes == expected_cpes def test_is_related_to_hardware(): - assert nvd.is_related_to_hardware("cpe:2.3:h:csilvers:gperftools:0.2:*:*:*:*:*:*:*") - assert not nvd.is_related_to_hardware("cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*") - assert not nvd.is_related_to_hardware("cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*") + assert nvd_importer.is_related_to_hardware("cpe:2.3:h:csilvers:gperftools:0.2:*:*:*:*:*:*:*") + assert not nvd_importer.is_related_to_hardware( + "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*" + ) + assert not nvd_importer.is_related_to_hardware("cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*") def test_CveItem_summary_with_single_summary(): @@ -186,7 +188,7 @@ def test_CveItem_summary_with_single_summary(): "be allocated than expected." ) - assert nvd.CveItem(cve_item=get_test_cve_item()).summary == expected_summary + assert nvd_importer.CveItem(cve_item=get_test_cve_item()).summary == expected_summary def test_CveItem_reference_urls(): @@ -195,4 +197,4 @@ def test_CveItem_reference_urls(): "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", ] - assert nvd.CveItem(cve_item=get_test_cve_item()).reference_urls == expected_urls + assert nvd_importer.CveItem(cve_item=get_test_cve_item()).reference_urls == expected_urls diff --git a/vulnerabilities/tests/test_vulnerability_status_improver.py b/vulnerabilities/tests/test_vulnerability_status_improver.py index 5bad2f498..8ea00215d 100644 --- a/vulnerabilities/tests/test_vulnerability_status_improver.py +++ b/vulnerabilities/tests/test_vulnerability_status_improver.py @@ -13,13 +13,12 @@ import pytest -from vulnerabilities.importers.nvd import NVDImporter from vulnerabilities.improvers.vulnerability_status import VulnerabilityStatusImprover -from vulnerabilities.improvers.vulnerability_status import get_status_from_api from vulnerabilities.models import Advisory from vulnerabilities.models import Alias from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityStatusType +from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -34,13 +33,13 @@ def test_interesting_advisories(): Advisory.objects.create( aliases=["CVE-1"], - created_by=NVDImporter.qualified_name, + created_by=NVDImporterPipeline.qualified_name, summary="1", date_collected=datetime.now(), ) Advisory.objects.create( aliases=["CVE-1"], - created_by=NVDImporter.qualified_name, + created_by=NVDImporterPipeline.qualified_name, summary="2", date_collected=datetime.now(), ) @@ -55,7 +54,7 @@ def test_improver_end_to_end(mock_response): mock_response.return_value = response adv = Advisory.objects.create( aliases=["CVE-2023-35866"], - created_by=NVDImporter.qualified_name, + created_by=NVDImporterPipeline.qualified_name, summary="1", date_collected=datetime.now(), ) From ee3edcbd5cdd46a3d2f8776fdbb6e9aa66e029df Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Sat, 28 Sep 2024 01:06:05 +0530 Subject: [PATCH 18/29] Add pipeline_id to nvd pipeline Signed-off-by: Keshav Priyadarshi --- vulnerabilities/improvers/vulnerability_status.py | 2 +- vulnerabilities/pipelines/nvd_importer.py | 2 ++ vulnerabilities/tests/test_vulnerability_status_improver.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/improvers/vulnerability_status.py b/vulnerabilities/improvers/vulnerability_status.py index 7287e483f..353cca54c 100644 --- a/vulnerabilities/improvers/vulnerability_status.py +++ b/vulnerabilities/improvers/vulnerability_status.py @@ -38,7 +38,7 @@ class VulnerabilityStatusImprover(Improver): @property def interesting_advisories(self) -> QuerySet: return ( - Advisory.objects.filter(Q(created_by=NVDImporterPipeline.qualified_name)) + Advisory.objects.filter(Q(created_by=NVDImporterPipeline.pipeline_id)) .distinct("aliases") .paginated() ) diff --git a/vulnerabilities/pipelines/nvd_importer.py b/vulnerabilities/pipelines/nvd_importer.py index 3ccb254f1..38800eb62 100644 --- a/vulnerabilities/pipelines/nvd_importer.py +++ b/vulnerabilities/pipelines/nvd_importer.py @@ -30,6 +30,8 @@ class NVDImporterPipeline(VulnerableCodeBaseImporterPipeline): """Collect advisories from NVD.""" + pipeline_id = "nvd_importer" + # See https://github.com/nexB/vulnerablecode/issues/665 for follow up spdx_license_expression = ( "LicenseRef-scancode-us-govt-public-domain AND LicenseRef-scancode-cve-tou" diff --git a/vulnerabilities/tests/test_vulnerability_status_improver.py b/vulnerabilities/tests/test_vulnerability_status_improver.py index 8ea00215d..f2eb5ce0f 100644 --- a/vulnerabilities/tests/test_vulnerability_status_improver.py +++ b/vulnerabilities/tests/test_vulnerability_status_improver.py @@ -33,13 +33,13 @@ def test_interesting_advisories(): Advisory.objects.create( aliases=["CVE-1"], - created_by=NVDImporterPipeline.qualified_name, + created_by=NVDImporterPipeline.pipeline_id, summary="1", date_collected=datetime.now(), ) Advisory.objects.create( aliases=["CVE-1"], - created_by=NVDImporterPipeline.qualified_name, + created_by=NVDImporterPipeline.pipeline_id, summary="2", date_collected=datetime.now(), ) @@ -54,7 +54,7 @@ def test_improver_end_to_end(mock_response): mock_response.return_value = response adv = Advisory.objects.create( aliases=["CVE-2023-35866"], - created_by=NVDImporterPipeline.qualified_name, + created_by=NVDImporterPipeline.pipeline_id, summary="1", date_collected=datetime.now(), ) From 2c2dfff58cdf266ad48ef789de71edc7f274764f Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Sat, 28 Sep 2024 01:17:09 +0530 Subject: [PATCH 19/29] Add data migration for old nvd advisory Signed-off-by: Keshav Priyadarshi --- .../0068_update_nvd_advisory_created_by.py | 38 ++++++++++++++++++ vulnerabilities/tests/test_data_migrations.py | 39 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py diff --git a/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py b/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py new file mode 100644 index 000000000..2a91f55ee --- /dev/null +++ b/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-27 19:38 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").update( + created_by=NVDImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=NVDImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.nvd.NVDImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0067_update_github_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index fcad0a1d4..d43755980 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -802,3 +802,42 @@ def test_removal_of_duped_purls(self): adv.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").count() == 0 ) assert adv.filter(created_by="github_importer").count() == 1 + + +class TestUpdateNVDAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0067_update_github_advisory_created_by" + migrate_to = "0068_update_nvd_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.nvd.NVDImporter", + date_collected=timezone.now(), + ) + + def test_removal_of_duped_purls(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert adv.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").count() == 0 + assert adv.filter(created_by="nvd_importer").count() == 1 From 6c5e7757009e1ce4b9dcf6cf13c76cf3bfd712e5 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 25 Sep 2024 15:24:48 +0530 Subject: [PATCH 20/29] Add management command to commit exported data Signed-off-by: Keshav Priyadarshi --- .../management/commands/commit_export.py | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 vulnerabilities/management/commands/commit_export.py diff --git a/vulnerabilities/management/commands/commit_export.py b/vulnerabilities/management/commands/commit_export.py new file mode 100644 index 000000000..75098689d --- /dev/null +++ b/vulnerabilities/management/commands/commit_export.py @@ -0,0 +1,146 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +import os +import shutil +import tempfile +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import requests +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError +from git import Repo + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = """Commit the exported vulnerability and package in backing git repository""" + + def add_arguments(self, parser): + parser.add_argument( + "path", + help="Path to exported data.", + ) + + def handle(self, *args, **options): + if path := options["path"]: + base_path = Path(path) + + if not path or not base_path.is_dir(): + raise CommandError("Enter a valid directory path") + + export_repo_url = os.environ.get("VULNERABLECODE_EXPORT_REPO_URL", None) + github_service_token = os.environ.get("GITHUB_SERVICE_TOKEN", None) + github_service_name = os.environ.get("GITHUB_SERVICE_NAME", None) + github_service_email = os.environ.get("GITHUB_SERVICE_EMAIL", None) + + local_dir = tempfile.mkdtemp() + current_date = datetime.now().strftime("%Y-%m-%d") + + branch_name = f"export-update-{current_date}" + commit_message = f"Update package and vulnerability data\nSigned-off-by: {github_service_name} <{github_service_email}>" + pr_title = "Update package and vulnerability" + pr_body = "" + + self.stdout.write("Committing vulnerablecode Package and Vulnerability data.") + repo = self.clone_repository( + repo_url=export_repo_url, + local_path=local_dir, + token=github_service_token, + ) + + repo.config_writer().set_value("user", "name", github_service_name).release() + repo.config_writer().set_value("user", "email", github_service_email).release() + + self.add_changes(repo=repo, content_path=path) + + if self.commit_and_push_changes( + repo=repo, + branch=branch_name, + commit_message=commit_message, + ): + self.create_pull_request( + repo_url=export_repo_url, + branch=branch_name, + title=pr_title, + body=pr_body, + token=github_service_token, + ) + shutil.rmtree(local_dir) + + def clone_repository(self, repo_url, local_path, token): + """Clone repository to local_path.""" + if os.path.exists(local_path): + shutil.rmtree(local_path) + + authenticated_repo_url = repo_url.replace("https://", f"https://{token}@") + return Repo.clone_from(authenticated_repo_url, local_path) + + def add_changes(self, repo, content_path): + """Copy changes from the ``content_path`` to ``repo``.""" + + source_path = Path(content_path) + destination_path = Path(repo.working_dir) + + for item in source_path.iterdir(): + if not item.is_dir(): + continue + target_item = destination_path / item.name + if target_item.exists(): + shutil.rmtree(target_item) + shutil.copytree(item, target_item) + + def commit_and_push_changes(self, repo, branch, commit_message, remote_name="origin"): + """Commit changes and push to remote repository, return name of changed files.""" + + repo.git.checkout("HEAD", b=branch) + files_changed = repo.git.diff("HEAD", name_only=True) + + if not files_changed: + self.stderr.write(self.style.SUCCESS("No changes to commit.")) + return + + repo.git.add(A=True) + repo.index.commit(commit_message) + repo.git.push(remote_name, branch) + return files_changed + + def create_pull_request(self, repo_url, branch, title, body, token): + """Create a pull request in the GitHub repository.""" + + url_parts = urlparse(repo_url).path + path_parts = url_parts.strip("/").rstrip(".git").split("/") + + if len(path_parts) >= 2: + repo_owner = path_parts[0] + repo_name = path_parts[1] + else: + raise ValueError("Invalid GitHub repo URL") + + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/pulls" + headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"} + data = {"title": title, "head": branch, "base": "main", "body": body} + + response = requests.post(url, headers=headers, json=data) + + if response.status_code == 201: + pr_response = response.json() + self.stdout.write( + self.style.SUCCESS( + f"Pull request created successfully: {pr_response.get('html_url')}." + ) + ) + else: + self.stderr.write( + self.style.ERROR(f"Failed to create pull request: {response.content}") + ) From a1b0eb09043fe0ccb83e313fc0b4551bd7a3b496 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 26 Sep 2024 16:05:56 +0530 Subject: [PATCH 21/29] Add tool details in the generated commit message Signed-off-by: Keshav Priyadarshi --- .../management/commands/commit_export.py | 71 ++++++++++++++----- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/vulnerabilities/management/commands/commit_export.py b/vulnerabilities/management/commands/commit_export.py index 75098689d..9d47904f3 100644 --- a/vulnerabilities/management/commands/commit_export.py +++ b/vulnerabilities/management/commands/commit_export.py @@ -11,6 +11,7 @@ import os import shutil import tempfile +import textwrap from datetime import datetime from pathlib import Path from urllib.parse import urlparse @@ -20,11 +21,18 @@ from django.core.management.base import CommandError from git import Repo +from vulnerablecode.settings import ALLOWED_HOSTS +from vulnerablecode.settings import VULNERABLECODE_VERSION + logger = logging.getLogger(__name__) class Command(BaseCommand): - help = """Commit the exported vulnerability and package in backing git repository""" + help = """Commit the exported vulnerability data in the backing GitHub repository. + + This command takes the path to the exported vulnerability data and creates a pull + request in the backing GitHub repository with the changes. + """ def add_arguments(self, parser): parser.add_argument( @@ -37,49 +45,74 @@ def handle(self, *args, **options): base_path = Path(path) if not path or not base_path.is_dir(): - raise CommandError("Enter a valid directory path") - - export_repo_url = os.environ.get("VULNERABLECODE_EXPORT_REPO_URL", None) - github_service_token = os.environ.get("GITHUB_SERVICE_TOKEN", None) - github_service_name = os.environ.get("GITHUB_SERVICE_NAME", None) - github_service_email = os.environ.get("GITHUB_SERVICE_EMAIL", None) + raise CommandError("Enter a valid directory path to the exported data.") + + vcio_export_repo_url = os.environ.get("VULNERABLECODE_EXPORT_REPO_URL") + vcio_github_service_token = os.environ.get("VULNERABLECODE_GITHUB_SERVICE_TOKEN") + vcio_github_service_name = os.environ.get("VULNERABLECODE_GITHUB_SERVICE_NAME") + vcio_github_service_email = os.environ.get("VULNERABLECODE_GITHUB_SERVICE_EMAIL") + + # Check for missing environment variables + missing_vars = [] + if not vcio_export_repo_url: + missing_vars.append("VULNERABLECODE_EXPORT_REPO_URL") + if not vcio_github_service_token: + missing_vars.append("VULNERABLECODE_GITHUB_SERVICE_TOKEN") + if not vcio_github_service_name: + missing_vars.append("VULNERABLECODE_GITHUB_SERVICE_NAME") + if not vcio_github_service_email: + missing_vars.append("VULNERABLECODE_GITHUB_SERVICE_EMAIL") + + if missing_vars: + raise CommandError(f'Missing environment variables: {", ".join(missing_vars)}') local_dir = tempfile.mkdtemp() current_date = datetime.now().strftime("%Y-%m-%d") branch_name = f"export-update-{current_date}" - commit_message = f"Update package and vulnerability data\nSigned-off-by: {github_service_name} <{github_service_email}>" - pr_title = "Update package and vulnerability" - pr_body = "" + pr_title = "Update package vulnerabilities from VulnerableCode" + pr_body = f"""\ + Tool: pkg:github/aboutcode-org/vulnerablecode@v{VULNERABLECODE_VERSION} + Reference: https://{ALLOWED_HOSTS[0]}/ + """ + commit_message = f"""\ + Update package vulnerabilities from VulnerableCode - self.stdout.write("Committing vulnerablecode Package and Vulnerability data.") + Tool: pkg:github/aboutcode-org/vulnerablecode@v{VULNERABLECODE_VERSION} + Reference: https://{ALLOWED_HOSTS[0]}/ + + Signed-off-by: {vcio_github_service_name} <{vcio_github_service_email}> + """ + + self.stdout.write("Committing VulnerableCode package and vulnerability data.") repo = self.clone_repository( - repo_url=export_repo_url, + repo_url=vcio_export_repo_url, local_path=local_dir, - token=github_service_token, + token=vcio_github_service_token, ) - repo.config_writer().set_value("user", "name", github_service_name).release() - repo.config_writer().set_value("user", "email", github_service_email).release() + repo.config_writer().set_value("user", "name", vcio_github_service_name).release() + repo.config_writer().set_value("user", "email", vcio_github_service_email).release() self.add_changes(repo=repo, content_path=path) if self.commit_and_push_changes( repo=repo, branch=branch_name, - commit_message=commit_message, + commit_message=textwrap.dedent(commit_message), ): self.create_pull_request( - repo_url=export_repo_url, + repo_url=vcio_export_repo_url, branch=branch_name, title=pr_title, - body=pr_body, - token=github_service_token, + body=textwrap.dedent(pr_body), + token=vcio_github_service_token, ) shutil.rmtree(local_dir) def clone_repository(self, repo_url, local_path, token): """Clone repository to local_path.""" + if os.path.exists(local_path): shutil.rmtree(local_path) From 3718965c829b8f389a9a56fc88a812851ec6d4b8 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 30 Sep 2024 13:40:52 +0530 Subject: [PATCH 22/29] Update CHANGELOG.rst Signed-off-by: Keshav Priyadarshi --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0fe4b6d10..93beb0ff4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Release notes ============= + +Version (next) +------------------- + +- Add management command to commit exported vulnerability data (#1600) + + Version v34.0.1 ------------------- From e273c67e7b48e09337de00367cabd23ceb566604 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 30 Sep 2024 15:08:31 +0300 Subject: [PATCH 23/29] Add support for Exploit model (#1562) * Migrate ( metasploit, exploit-db, kev ) to aboutcode pipeline. Set data_source as the header for the exploit table. Squash the migration files into a single file. Add test for exploit-db , metasploit Add a missing migration file Rename resources_and_notes to notes Fix Api test Refactor metasploit , exploitdb , kev improver Rename Kev tab to exploit tab Add support for exploitdb , metasploit, kev Signed-off-by: ziadhany * Implement the appropriate LoopProgress progress bar. Refactor the error handling logic in the code. Signed-off-by: ziadhany * Resolve migration conflicts. Address the exploit in the API extension. Signed-off-by: ziadhany * Add any missing logs message Remove unused logging module Signed-off-by: ziadhany * Migrate ( metasploit, exploit-db, kev ) to aboutcode pipeline. Set data_source as the header for the exploit table. Squash the migration files into a single file. Add test for exploit-db , metasploit Add a missing migration file Rename resources_and_notes to notes Fix Api test Refactor metasploit , exploitdb , kev improver Rename Kev tab to exploit tab Add support for exploitdb , metasploit, kev Signed-off-by: ziadhany * Implement the appropriate LoopProgress progress bar. Refactor the error handling logic in the code. Signed-off-by: ziadhany * Resolve migration conflicts. Address the exploit in the API extension. Signed-off-by: ziadhany * Add any missing logs message Remove unused logging module Signed-off-by: ziadhany * Fix migration conflict Add pipeline_id for ( kev, metasploit, exploit-db ) Signed-off-by: ziadhany * Remove unwanted migration file Signed-off-by: ziadhany * Add log traceback for all the errors. Add missing logs Handle cases of one exploit for multiple vulnerabilities. Signed-off-by: ziadhany * Skip empty aliases Remove empty vulnerability_kev.py file Signed-off-by: ziadhany * Replace references log with interesting_references Signed-off-by: ziadhany * Use proper labels in vulnerability details Signed-off-by: Keshav Priyadarshi * Display Known/Unknown for ransomware campaign use Signed-off-by: Keshav Priyadarshi --------- Signed-off-by: ziadhany Signed-off-by: Keshav Priyadarshi Co-authored-by: Keshav Priyadarshi --- vulnerabilities/api.py | 29 +-- vulnerabilities/api_extension.py | 19 +- vulnerabilities/improvers/__init__.py | 8 +- .../improvers/vulnerability_kev.py | 66 ------- .../migrations/0069_exploit_delete_kev.py | 131 ++++++++++++++ vulnerabilities/models.py | 73 ++++++-- .../pipelines/enhance_with_exploitdb.py | 158 ++++++++++++++++ vulnerabilities/pipelines/enhance_with_kev.py | 89 ++++++++++ .../pipelines/enhance_with_metasploit.py | 111 ++++++++++++ .../templates/vulnerability_details.html | 168 +++++++++++++----- .../tests/pipelines/test_exploitdb.py | 38 ++++ .../test_kev.py} | 27 +-- .../tests/pipelines/test_metasploit.py | 35 ++++ vulnerabilities/tests/test_api.py | 2 + .../exploitdb_improver/files_exploits.csv | 2 + .../modules_metadata_base.json | 93 ++++++++++ 16 files changed, 885 insertions(+), 164 deletions(-) delete mode 100644 vulnerabilities/improvers/vulnerability_kev.py create mode 100644 vulnerabilities/migrations/0069_exploit_delete_kev.py create mode 100644 vulnerabilities/pipelines/enhance_with_exploitdb.py create mode 100644 vulnerabilities/pipelines/enhance_with_kev.py create mode 100644 vulnerabilities/pipelines/enhance_with_metasploit.py create mode 100644 vulnerabilities/tests/pipelines/test_exploitdb.py rename vulnerabilities/tests/{test_kev_improver.py => pipelines/test_kev.py} (50%) create mode 100644 vulnerabilities/tests/pipelines/test_metasploit.py create mode 100644 vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv create mode 100644 vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 278ed636c..0b3dc2b8e 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -27,7 +27,7 @@ from rest_framework.throttling import UserRateThrottle from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -175,10 +175,23 @@ def to_representation(self, instance): return representation -class KEVSerializer(serializers.ModelSerializer): +class ExploitSerializer(serializers.ModelSerializer): class Meta: - model = Kev - fields = ["date_added", "description", "required_action", "due_date", "resources_and_notes"] + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] class VulnerabilitySerializer(BaseResourceSerializer): @@ -189,7 +202,7 @@ class VulnerabilitySerializer(BaseResourceSerializer): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") - kev = KEVSerializer(read_only=True) + exploits = ExploitSerializer(many=True, read_only=True) weaknesses = WeaknessSerializer(many=True) severity_range_score = serializers.SerializerMethodField() @@ -199,10 +212,6 @@ def to_representation(self, instance): weaknesses = data.get("weaknesses", []) data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None] - kev = data.get("kev", None) - if not kev: - data.pop("kev") - return data def get_severity_range_score(self, instance): @@ -240,7 +249,7 @@ class Meta: "affected_packages", "references", "weaknesses", - "kev", + "exploits", "severity_range_score", ] diff --git a/vulnerabilities/api_extension.py b/vulnerabilities/api_extension.py index a974f0796..4b9211c76 100644 --- a/vulnerabilities/api_extension.py +++ b/vulnerabilities/api_extension.py @@ -26,7 +26,7 @@ from rest_framework.throttling import AnonRateThrottle from vulnerabilities.api import BaseResourceSerializer -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -105,8 +105,21 @@ class Meta: class V2ExploitSerializer(ModelSerializer): class Meta: - model = Kev - fields = ("description", "required_action", "date_added", "due_date", "resources_and_notes") + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] class V2VulnerabilitySerializer(ModelSerializer): diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index d15504166..6e9c24b38 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -8,9 +8,11 @@ # from vulnerabilities.improvers import valid_versions -from vulnerabilities.improvers import vulnerability_kev from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipelines import enhance_with_exploitdb +from vulnerabilities.pipelines import enhance_with_kev +from vulnerabilities.pipelines import enhance_with_metasploit from vulnerabilities.pipelines import flag_ghost_packages IMPROVERS_REGISTRY = [ @@ -31,8 +33,10 @@ valid_versions.GithubOSVImprover, vulnerability_status.VulnerabilityStatusImprover, valid_versions.CurlImprover, - vulnerability_kev.VulnerabilityKevImprover, flag_ghost_packages.FlagGhostPackagePipeline, + enhance_with_kev.VulnerabilityKevPipeline, + enhance_with_metasploit.MetasploitImproverPipeline, + enhance_with_exploitdb.ExploitDBImproverPipeline, ] IMPROVERS_REGISTRY = { diff --git a/vulnerabilities/improvers/vulnerability_kev.py b/vulnerabilities/improvers/vulnerability_kev.py deleted file mode 100644 index 3ca3291bc..000000000 --- a/vulnerabilities/improvers/vulnerability_kev.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -from typing import Iterable - -import requests -from django.db.models import QuerySet - -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Kev - -logger = logging.getLogger(__name__) - - -class VulnerabilityKevImprover(Improver): - """ - Known Exploited Vulnerabilities Improver - """ - - @property - def interesting_advisories(self) -> QuerySet: - # TODO Modify KEV improver to iterate over the vulnerabilities alias, not the advisory - return [Advisory.objects.first()] - - def get_inferences(self, advisory_data) -> Iterable[Inference]: - """ - Fetch Kev data, iterate over it to find the vulnerability with the specified alias, and create or update - the Kev instance accordingly. - """ - - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - kev_data = response.json() - if response.status_code != 200: - logger.error( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return [] - - for kev_vul in kev_data.get("vulnerabilities", []): - alias = Alias.objects.get_or_none(alias=kev_vul["cveID"]) - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Kev.objects.update_or_create( - vulnerability=vul, - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "resources_and_notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) - return [] diff --git a/vulnerabilities/migrations/0069_exploit_delete_kev.py b/vulnerabilities/migrations/0069_exploit_delete_kev.py new file mode 100644 index 000000000..5c06911eb --- /dev/null +++ b/vulnerabilities/migrations/0069_exploit_delete_kev.py @@ -0,0 +1,131 @@ +# Generated by Django 4.2.15 on 2024-09-21 15:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0068_update_nvd_advisory_created_by"), + ] + + operations = [ + migrations.CreateModel( + name="Exploit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "date_added", + models.DateField( + blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", + null=True, + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", + null=True, + ), + ), + ( + "required_action", + models.TextField( + blank=True, + help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", + null=True, + ), + ), + ( + "due_date", + models.DateField( + blank=True, + help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", + null=True, + ), + ), + ( + "notes", + models.TextField( + blank=True, + help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", + null=True, + ), + ), + ( + "known_ransomware_campaign_use", + models.BooleanField( + default=False, + help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", + ), + ), + ( + "source_date_published", + models.DateField( + blank=True, + help_text="The date that the exploit was published or disclosed.", + null=True, + ), + ), + ( + "exploit_type", + models.TextField( + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "platform", + models.TextField( + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "source_date_updated", + models.DateField( + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + null=True, + ), + ), + ( + "data_source", + models.TextField( + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + null=True, + ), + ), + ( + "source_url", + models.URLField( + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", + null=True, + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="exploits", + to="vulnerabilities.vulnerability", + ), + ), + ], + ), + migrations.DeleteModel( + name="Kev", + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index cc3e920d9..fc5eb5e3c 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1389,49 +1389,90 @@ def log_fixing(cls, package, importer, source_url, related_vulnerability): ) -class Kev(models.Model): +class Exploit(models.Model): """ - Known Exploited Vulnerabilities + A vulnerability exploit is code used to + take advantage of a security flaw for unauthorized access or malicious activity. """ - vulnerability = models.OneToOneField( + vulnerability = models.ForeignKey( Vulnerability, + related_name="exploits", on_delete=models.CASCADE, - related_name="kev", ) date_added = models.DateField( - help_text="The date the vulnerability was added to the Known Exploited Vulnerabilities" - " (KEV) catalog in the format YYYY-MM-DD.", null=True, blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", ) description = models.TextField( - help_text="Description of the vulnerability in the Known Exploited Vulnerabilities" - " (KEV) catalog, usually a refinement of the original CVE description" + null=True, + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", ) required_action = models.TextField( + null=True, + blank=True, help_text="The required action to address the vulnerability, typically to " - "apply vendor updates or apply vendor mitigations or to discontinue use." + "apply vendor updates or apply vendor mitigations or to discontinue use.", ) due_date = models.DateField( - help_text="The date the required action is due in the format YYYY-MM-DD," - "which applies to all USA federal civilian executive branch (FCEB) agencies," - "but all organizations are strongly encouraged to execute the required action." + null=True, + blank=True, + help_text="The date the required action is due, which applies" + " to all USA federal civilian executive branch (FCEB) agencies, " + "but all organizations are strongly encouraged to execute the required action", ) - resources_and_notes = models.TextField( + notes = models.TextField( + null=True, + blank=True, help_text="Additional notes and resources about the vulnerability," - " often a URL to vendor instructions." + " often a URL to vendor instructions.", ) known_ransomware_campaign_use = models.BooleanField( default=False, - help_text="""Known if this vulnerability is known to have been leveraged as part of a ransomware campaign; - or 'Unknown' if CISA lacks confirmation that the vulnerability has been utilized for ransomware.""", + help_text="""Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; + or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.""", + ) + + source_date_published = models.DateField( + null=True, blank=True, help_text="The date that the exploit was published or disclosed." + ) + + exploit_type = models.TextField( + null=True, + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + ) + + platform = models.TextField( + null=True, + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + ) + + source_date_updated = models.DateField( + null=True, + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + ) + + data_source = models.TextField( + null=True, + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + ) + + source_url = models.URLField( + null=True, + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", ) @property diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py new file mode 100644 index 000000000..54554f951 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -0,0 +1,158 @@ +import csv +import io +import logging +from traceback import format_exc as traceback_format_exc + +import requests +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser +from django.db import DataError + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + pipeline_id = "enhance_with_exploitdb" + spdx_license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + self.log(f"Fetching {exploit_db_url}") + + try: + response = requests.get(exploit_db_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Exploit-DB Exploits: {exploit_db_url} with error {http_err!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + raise + + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + + csvreader = csv.DictReader(self.exploit_data) + + raw_data = list(csvreader) + fetched_exploit_count = len(raw_data) + + vulnerability_exploit_count = 0 + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for row in progress.iter(raw_data): + vulnerability_exploit_count += add_vulnerability_exploit(row, self.log) + + self.log( + f"Successfully added {vulnerability_exploit_count:,d} exploit-db vulnerability exploit" + ) + + +def add_vulnerability_exploit(row, logger): + vulnerabilities = set() + + aliases = row["codes"].split(";") if row["codes"] else [] + + if not aliases: + return 0 + + for raw_alias in aliases: + try: + if alias := Alias.objects.get(alias=raw_alias): + vulnerabilities.add(alias.vulnerability) + except Alias.DoesNotExist: + continue + + if not vulnerabilities: + logger(f"No vulnerability found for aliases {aliases}") + return 0 + + date_added = parse_date(row["date_added"]) + source_date_published = parse_date(row["date_published"]) + source_date_updated = parse_date(row["date_updated"]) + + for vulnerability in vulnerabilities: + add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger) + try: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Exploit-DB", + defaults={ + "date_added": date_added, + "description": row["description"], + "known_ransomware_campaign_use": row["verified"], + "source_date_published": source_date_published, + "exploit_type": row["type"], + "platform": row["platform"], + "source_date_updated": source_date_updated, + "source_url": row["source_url"], + }, + ) + except DataError as e: + logger( + f"Failed to Create the Vulnerability Exploit-DB with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + return 1 + + +def add_exploit_references(ref_id, direct_url, path, vul, logger): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + try: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) + + except DataError as e: + logger( + f"Failed to Create the Vulnerability Reference For Exploit-DB with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + + +def parse_date(date_string): + if date_string: + try: + date_obj = dateparser.parse(date_string).date() + return date_obj.strftime("%Y-%m-%d") + except (ValueError, TypeError, Exception) as e: + logging.error( + f"Error while parsing ExploitDB date '{date_string}' with error {e!r}:\n{traceback_format_exc()}" + ) + return diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py new file mode 100644 index 000000000..6372bd3b0 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -0,0 +1,89 @@ +import logging +from traceback import format_exc as traceback_format_exc + +import requests +from aboutcode.pipeline import LoopProgress + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + pipeline_id = "enhance_with_kev" + license_expression = None + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = ( + "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + ) + self.log(f"Fetching {kev_url}") + + try: + response = requests.get(kev_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the KEV Exploits: {kev_url} with error {http_err!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + raise + self.kev_data = response.json() + + def add_exploits(self): + fetched_exploit_count = self.kev_data.get("count") + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for record in progress.iter(self.kev_data.get("vulnerabilities", [])): + vulnerability_exploit_count += add_vulnerability_exploit( + kev_vul=record, + logger=self.log, + ) + + self.log(f"Successfully added {vulnerability_exploit_count:,d} kev exploit") + + +def add_vulnerability_exploit(kev_vul, logger): + cve_id = kev_vul.get("cveID") + + if not cve_id: + return 0 + + vulnerability = None + try: + if alias := Alias.objects.get(alias=cve_id): + vulnerability = alias.vulnerability + except Alias.DoesNotExist: + logger(f"No vulnerability found for aliases {cve_id}") + return 0 + + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) + return 1 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py new file mode 100644 index 000000000..72897abd0 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -0,0 +1,111 @@ +import logging +from traceback import format_exc as traceback_format_exc + +import requests +import saneyaml +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + pipeline_id = "enhance_with_metasploit" + spdx_license_expression = "BSD-3-clause" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_vulnerability_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + self.log(f"Fetching {url}") + try: + response = requests.get(url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Metasploit Exploits: {url} with error {http_err!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + raise + + self.metasploit_data = response.json() + + def add_vulnerability_exploits(self): + fetched_exploit_count = len(self.metasploit_data) + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + for _, record in progress.iter(self.metasploit_data.items()): + vulnerability_exploit_count += add_vulnerability_exploit( + record=record, + logger=self.log, + ) + self.log(f"Successfully added {vulnerability_exploit_count:,d} vulnerability exploit") + + +def add_vulnerability_exploit(record, logger): + vulnerabilities = set() + references = record.get("references", []) + + interesting_references = [ + ref for ref in references if not ref.startswith("OSVDB") and not ref.startswith("URL-") + ] + + if not interesting_references: + return 0 + + for ref in interesting_references: + try: + if alias := Alias.objects.get(alias=ref): + vulnerabilities.add(alias.vulnerability) + except Alias.DoesNotExist: + continue + + if not vulnerabilities: + logger(f"No vulnerability found for aliases {interesting_references}") + return 0 + + description = record.get("description", "") + notes = record.get("notes", {}) + platform = record.get("platform") + + source_url = "" + if path := record.get("path"): + source_url = f"https://github.com/rapid7/metasploit-framework/tree/master{path}" + source_date_published = None + + if disclosure_date := record.get("disclosure_date"): + try: + source_date_published = dateparser.parse(disclosure_date).date() + except ValueError as e: + logger( + f"Error while parsing date {disclosure_date} with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + + for vulnerability in vulnerabilities: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) + return 1 diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index d12748d0c..d1f2fb6de 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -55,11 +55,11 @@

    • - {% if vulnerability.kev %} -
  • + {% if vulnerability.exploits %} +
  • - Known Exploited Vulnerabilities + Exploits ({{ vulnerability.exploits.count }})
    • @@ -71,7 +71,7 @@ EPSS - +
  • @@ -408,88 +408,158 @@ {% endfor %} - {% if vulnerability.kev %} -
    -
    - Known Exploited Vulnerabilities -
    - - - - - - - {% if vulnerability.kev.description %} + +
    + {% for exploit in vulnerability.exploits.all %} +
    - - Known Ransomware Campaign Use: - - {{ vulnerability.kev.get_known_ransomware_campaign_use_type }}
    + + + + + + + {% if exploit.date_added %} - + {% endif %} - {% if vulnerability.kev.required_action %} + {% if exploit.description %} - + {% endif %} - - {% if vulnerability.kev.resources_and_notes %} + {% if exploit.required_action %} - + {% endif %} - - {% if vulnerability.kev.due_date %} + {% if exploit.due_date %} - + {% endif %} - {% if vulnerability.kev.date_added %} + {% if exploit.notes %} + + + + + {% endif %} + {% if exploit.known_ransomware_campaign_use is not None %} + + + + + {% endif %} + {% if exploit.source_date_published %} + + + + + {% endif %} + {% if exploit.exploit_type %} + + + + + {% endif %} + {% if exploit.platform %} + + + + + {% endif %} + {% if exploit.source_date_updated %} - + {% endif %} + {% if exploit.source_url %} + + + + + {% endif %} -
    Data source {{ exploit.data_source }}
    - Description: + data-tooltip="The date the vulnerability was added to an exploit catalog."> + Date added {{ vulnerability.kev.description }}{{ exploit.date_added }}
    - Required Action: + data-tooltip="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description"> + Description {{ vulnerability.kev.required_action }}{{ exploit.description }}
    - Notes: + data-tooltip="The required action to address the vulnerability, + typically to apply vendor updates or apply vendor mitigations or to discontinue use."> + Required action {{ vulnerability.kev.resources_and_notes }}{{ exploit.required_action }}
    - Due Date: + data-tooltip="The date the required action is due in the format YYYY-MM-DD, + which applies to all USA federal civilian executive branch (FCEB) agencies, + but all organizations are strongly encouraged to execute the required action."> + Due date {{ vulnerability.kev.due_date }}{{ exploit.due_date }}
    + + Note + + 
    {{ exploit.notes }}
    + + Ransomware campaign use + + {{ exploit.known_ransomware_campaign_use|yesno:"Known,Unknown" }}
    + + Source publication date + + {{ exploit.source_date_published }}
    + + Exploit type + + {{ exploit.exploit_type }}
    + + Platform + + {{ exploit.platform }}
    - Date Added: + data-tooltip="The date the exploit was updated in the original upstream data source."> + Source update date {{ vulnerability.kev.date_added }}{{ exploit.source_date_updated }}
    + + Source URL + + {{ exploit.source_url }}
    -
    - {% endif %} + + {% empty %} + + + No exploits are available. + + + {% endfor %} + + {% for severity in severities %} {% if severity.scoring_system == 'epss' %}
    @@ -502,7 +572,7 @@ - Percentile: + Percentile {{ severity.scoring_elements }} @@ -512,7 +582,7 @@ - EPSS score: + EPSS score {{ severity.value }} @@ -524,7 +594,7 @@ - Published at: + Published at {{ severity.published_at }} diff --git a/vulnerabilities/tests/pipelines/test_exploitdb.py b/vulnerabilities/tests/pipelines/test_exploitdb.py new file mode 100644 index 000000000..f08f7fec0 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_exploitdb.py @@ -0,0 +1,38 @@ +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.enhance_with_exploitdb import ExploitDBImproverPipeline + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_exploit_db_improver(mock_get): + mock_response = Mock(status_code=200) + with open(TEST_DATA, "r") as f: + mock_response.text = f.read() + mock_get.return_value = mock_response + + improver = ExploitDBImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + v1.save() + + Alias.objects.create(alias="CVE-2009-3699", vulnerability=v1) + + # Run Exploit-DB Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/test_kev_improver.py b/vulnerabilities/tests/pipelines/test_kev.py similarity index 50% rename from vulnerabilities/tests/test_kev_improver.py rename to vulnerabilities/tests/pipelines/test_kev.py index d0b1c981a..71583a617 100644 --- a/vulnerabilities/tests/test_kev_improver.py +++ b/vulnerabilities/tests/pipelines/test_kev.py @@ -1,41 +1,32 @@ import os -from datetime import datetime from unittest import mock from unittest.mock import Mock import pytest -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improvers.vulnerability_kev import VulnerabilityKevImprover from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.enhance_with_kev import VulnerabilityKevPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "kev_data.json") +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "kev_data.json") @pytest.mark.django_db @mock.patch("requests.get") def test_kev_improver(mock_get): - advisory_data = AdvisoryData( - aliases=["CVE-2022-21831"], - summary="Possible code injection vulnerability in Rails / Active Storage", - affected_packages=[], - references=[], - date_published=datetime.now(), - ) # to just run the improver - mock_response = Mock(status_code=200) mock_response.json.return_value = load_json(TEST_DATA) mock_get.return_value = mock_response - improver = VulnerabilityKevImprover() + improver = VulnerabilityKevPipeline() # Run the improver when there is no matching aliases - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 0 + improver.execute() + + assert Exploit.objects.count() == 0 v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") v1.save() @@ -43,5 +34,5 @@ def test_kev_improver(mock_get): Alias.objects.create(alias="CVE-2021-38647", vulnerability=v1) # Run Kev Improver again when there are matching aliases. - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 1 + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/pipelines/test_metasploit.py b/vulnerabilities/tests/pipelines/test_metasploit.py new file mode 100644 index 000000000..1116950d2 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_metasploit.py @@ -0,0 +1,35 @@ +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.enhance_with_metasploit import MetasploitImproverPipeline +from vulnerabilities.utils import load_json + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "metasploit_improver/modules_metadata_base.json") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_metasploit_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + improver = MetasploitImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + Alias.objects.create(alias="CVE-2007-4387", vulnerability=v1) + + # Run metasploit Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 8fb50243a..18807a6d7 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -296,6 +296,7 @@ def test_api_with_single_vulnerability(self): "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], } def test_api_with_single_vulnerability_with_filters(self): @@ -341,6 +342,7 @@ def test_api_with_single_vulnerability_with_filters(self): "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], } diff --git a/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv new file mode 100644 index 000000000..a63701d8c --- /dev/null +++ b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv @@ -0,0 +1,2 @@ +id,file,description,date_published,author,type,platform,port,date_added,date_updated,verified,codes,tags,aliases,screenshot_url,application_url,source_url +16929,exploits/aix/dos/16929.rb,"AIX Calendar Manager Service Daemon (rpc.cmsd) Opcode 21 - Buffer Overflow (Metasploit)",2010-11-11,Metasploit,dos,aix,,2010-11-11,2011-03-06,1,CVE-2009-3699;OSVDB-58726,"Metasploit Framework (MSF)",,,,http://aix.software.ibm.com/aix/efixes/security/cmsd_advisory.asc diff --git a/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json new file mode 100644 index 000000000..e9351a1df --- /dev/null +++ b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json @@ -0,0 +1,93 @@ +{ + "auxiliary_admin/2wire/xslt_password_reset": { + "name": "2Wire Cross-Site Request Forgery Password Reset Vulnerability", + "fullname": "auxiliary/admin/2wire/xslt_password_reset", + "aliases": [ + ], + "rank": 300, + "disclosure_date": "2007-08-15", + "type": "auxiliary", + "author": [ + "hkm ", + "Travis Phillips" + ], + "description": "This module will reset the admin password on a 2Wire wireless router. This is\n done by using the /xslt page where authentication is not required, thus allowing\n configuration changes (such as resetting the password) as administrators.", + "references": [ + "CVE-2007-4387", + "OSVDB-37667", + "BID-36075", + "URL-https://seclists.org/bugtraq/2007/Aug/225" + ], + "platform": "", + "arch": "", + "rport": 80, + "autofilter_ports": [ + 80, + 8080, + 443, + 8000, + 8888, + 8880, + 8008, + 3000, + 8443 + ], + "autofilter_services": [ + "http", + "https" + ], + "targets": null, + "mod_time": "2020-10-02 17:38:06 +0000", + "path": "/modules/auxiliary/admin/2wire/xslt_password_reset.rb", + "is_install_path": true, + "ref_name": "admin/2wire/xslt_password_reset", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": false, + "needs_cleanup": false, + "actions": [ + ] + }, + "post_firefox/manage/webcam_chat": { + "name": "Firefox Webcam Chat on Privileged Javascript Shell", + "fullname": "post/firefox/manage/webcam_chat", + "aliases": [ + + ], + "rank": 300, + "disclosure_date": "2014-05-13", + "type": "post", + "author": [ + "joev " + ], + "description": "This module allows streaming a webcam from a privileged Firefox Javascript shell.", + "references": [ + "URL-http://www.rapid7.com/db/modules/exploit/firefox/local/exec_shellcode" + ], + "platform": "", + "arch": "", + "rport": null, + "autofilter_ports": null, + "autofilter_services": null, + "targets": null, + "mod_time": "2023-02-08 13:47:34 +0000", + "path": "/modules/post/firefox/manage/webcam_chat.rb", + "is_install_path": true, + "ref_name": "firefox/manage/webcam_chat", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": [ + + ], + "needs_cleanup": null, + "actions": [ + + ] + } +} \ No newline at end of file From a6ce64c40c789d5a51f35571bf5ae8f90e4177a8 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 2 Oct 2024 15:10:10 +0530 Subject: [PATCH 24/29] Properly migrate crispy_forms to 2.x - Crispy removed all Bootstrap template packs. - These template packs are now available as standalone packages. - See https://github.com/django-crispy-forms/django-crispy-forms/releases/tag/2.0 - Fixes https://github.com/aboutcode-org/vulnerablecode/issues/1602 Signed-off-by: Keshav Priyadarshi --- requirements.txt | 1 + setup.cfg | 1 + vulnerablecode/settings.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index 994f9ccf4..885130eab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ click==8.1.2 coreapi==2.3.3 coreschema==0.0.4 cryptography==43.0.1 +crispy-bootstrap4==2024.1 cwe2==3.0.0 dateparser==1.1.1 decorator==5.1.1 diff --git a/setup.cfg b/setup.cfg index b46f23cdf..531212bde 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,7 @@ install_requires = django-filter>=24.0 django-widget-tweaks>=1.5.0 django-crispy-forms>=2.3 + crispy-bootstrap4>=2024.1 django-environ>=0.11.0 gunicorn>=23.0.0 diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index d3d302d2e..f2f612098 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -78,6 +78,7 @@ "rest_framework.authtoken", "widget_tweaks", "crispy_forms", + "crispy_bootstrap4", # for API doc "drf_spectacular", # required for Django collectstatic discovery @@ -196,6 +197,7 @@ str(PROJECT_DIR / "static"), ] +CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap4" CRISPY_TEMPLATE_PACK = "bootstrap4" From cf2e6c83c1a8b5a025588121f8efab6c53ce2c75 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 2 Oct 2024 18:16:46 +0530 Subject: [PATCH 25/29] Fix Swagger API documentation - Revert drf-spectacular to 0.24.2 Signed-off-by: Keshav Priyadarshi --- requirements.txt | 4 ++-- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 885130eab..0326622b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,8 +36,8 @@ djangorestframework==3.15.2 doc8==0.11.1 docopt==0.6.2 docutils==0.17.1 -drf-spectacular==0.27.2 -drf-spectacular-sidecar==2024.7.1 +drf-spectacular==0.24.2 +drf-spectacular-sidecar==2022.10.1 executing==0.8.3 fetchcode==0.3.0 freezegun==1.2.1 diff --git a/setup.cfg b/setup.cfg index 531212bde..ec64dcfd4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ install_requires = gunicorn>=23.0.0 # for the API doc - drf-spectacular[sidecar]>=0.27.2 + drf-spectacular[sidecar]>=0.24.2 #essentials packageurl-python>=0.15 From 447d1bfca73b4bd5c1e43c1e9e6f400365069b4b Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 2 Oct 2024 18:39:15 +0530 Subject: [PATCH 26/29] Use queryset with prefetch in /api/aliases endpoint Signed-off-by: Keshav Priyadarshi --- vulnerabilities/api.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 0b3dc2b8e..5d953db9b 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -685,14 +685,10 @@ def filter_alias(self, queryset, name, value): return self.queryset.filter(aliases__alias__icontains=alias) -class AliasViewSet(viewsets.ReadOnlyModelViewSet): +class AliasViewSet(VulnerabilityViewSet): """ Lookup for vulnerabilities by vulnerability aliases such as a CVE (https://nvd.nist.gov/general/cve-process). """ - queryset = Vulnerability.objects.all() - serializer_class = VulnerabilitySerializer - filter_backends = (filters.DjangoFilterBackend,) filterset_class = AliasFilterSet - throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] From 518c3e1cbf6904c58731739fc2f02d9fdef34342 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 7 Oct 2024 17:53:29 +0530 Subject: [PATCH 27/29] Add changelog Signed-off-by: Tushar Goel --- CHANGELOG.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 93beb0ff4..5edefef5a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,11 +1,14 @@ Release notes ============= - Version (next) ------------------- +Version v34.0.2 +------------------- + - Add management command to commit exported vulnerability data (#1600) +- Fix API 500 error (#1603) Version v34.0.1 From f6310f87e7b678eac331a494e312f80b658aeaa5 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 7 Oct 2024 17:53:44 +0530 Subject: [PATCH 28/29] Add changelog Signed-off-by: Tushar Goel --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5edefef5a..7caadf836 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,7 @@ Release notes Version (next) ------------------- + Version v34.0.2 ------------------- From 954110daa67ff7efed863a40d8c68c8103f452f9 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 7 Oct 2024 17:59:34 +0530 Subject: [PATCH 29/29] Bump VCIO version Signed-off-by: Tushar Goel --- setup.cfg | 2 +- ...0070_alter_advisory_created_by_and_more.py | 39 +++++++++++++++++++ vulnerablecode/__init__.py | 2 +- 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py diff --git a/setup.cfg b/setup.cfg index ec64dcfd4..a2fae91ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = vulnerablecode -version = 34.0.1 +version = 34.0.2 license = Apache-2.0 AND CC-BY-SA-4.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py b/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py new file mode 100644 index 000000000..41294f20a --- /dev/null +++ b/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.15 on 2024-10-07 12:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0069_exploit_delete_kev"), + ] + + operations = [ + migrations.AlterField( + model_name="advisory", + name="created_by", + field=models.CharField( + help_text="Fully qualified name of the importer prefixed with themodule name importing the advisory. Eg:vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline", + max_length=100, + ), + ), + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.0.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.0.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerablecode/__init__.py b/vulnerablecode/__init__.py index c8e069baf..797c0871b 100644 --- a/vulnerablecode/__init__.py +++ b/vulnerablecode/__init__.py @@ -12,7 +12,7 @@ import warnings from pathlib import Path -__version__ = "34.0.1" +__version__ = "34.0.2" def command_line():