diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 567755c3c..8fe372646 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -22,218 +22,221 @@ from vulnerabilities.models import Importer +# TODO: This entire registry needs to go away in favor of a registry similar to +# improvers. +# See ./improvers/__init__.py IMPORTER_REGISTRY = [ { - "name": "rust", - "license": "cc0-1.0", - "last_run": None, - "data_source": "RustDataSource", - "data_source_cfg": { - "branch": None, - "repository_url": "https://github.com/RustSec/advisory-db", - }, - }, - { - "name": "alpine", + "name": "nginx", "license": "", "last_run": None, - "data_source": "AlpineDataSource", - "data_source_cfg": {}, - }, - { - "name": "archlinux", - "license": "mit", - "last_run": None, - "data_source": "ArchlinuxDataSource", - "data_source_cfg": {"archlinux_tracker_url": "https://security.archlinux.org/json"}, - }, - { - "name": "debian", - "license": "mit", - "last_run": None, - "data_source": "DebianDataSource", - "data_source_cfg": { - "debian_tracker_url": "https://security-tracker.debian.org/tracker/data/json" - }, + "data_source": "NginxDataSource", + "data_source_cfg": {"etags": {}}, }, # { - # "name": "safetydb", - # "license": "cc-by-nc-4.0", + # "name": "rust", + # "license": "cc0-1.0", + # "last_run": None, + # "data_source": "RustDataSource", + # "data_source_cfg": { + # "branch": None, + # "repository_url": "https://github.com/RustSec/advisory-db", + # }, + # }, + # { + # "name": "alpine", + # "license": "", + # "last_run": None, + # "data_source": "AlpineDataSource", + # "data_source_cfg": {}, + # }, + # { + # "name": "archlinux", + # "license": "mit", + # "last_run": None, + # "data_source": "ArchlinuxDataSource", + # "data_source_cfg": {"archlinux_tracker_url": "https://security.archlinux.org/json"}, + # }, + # { + # "name": "debian", + # "license": "mit", # "last_run": None, - # "data_source": "SafetyDbDataSource", + # "data_source": "DebianDataSource", + # "data_source_cfg": { + # "debian_tracker_url": "https://security-tracker.debian.org/tracker/data/json" + # }, + # }, + # # { + # # "name": "safetydb", + # # "license": "cc-by-nc-4.0", + # # "last_run": None, + # # "data_source": "SafetyDbDataSource", + # # "data_source_cfg": { + # # "url": "https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json", + # # "etags": {}, + # # }, + # # }, + # { + # "name": "npm", + # "license": "mit", + # "last_run": None, + # "data_source": "NpmDataSource", + # "data_source_cfg": {"repository_url": "https://github.com/nodejs/security-wg.git"}, + # }, + # { + # "name": "ruby", + # "license": "", + # "last_run": None, + # "data_source": "RubyDataSource", + # "data_source_cfg": {"repository_url": "https://github.com/rubysec/ruby-advisory-db.git"}, + # }, + # { + # "name": "ubuntu", + # "license": "gpl-2.0", + # "last_run": None, + # "data_source": "UbuntuDataSource", # "data_source_cfg": { - # "url": "https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json", # "etags": {}, + # "releases": ["bionic", "trusty", "focal", "eoan", "xenial"], # }, # }, - { - "name": "npm", - "license": "mit", - "last_run": None, - "data_source": "NpmDataSource", - "data_source_cfg": {"repository_url": "https://github.com/nodejs/security-wg.git"}, - }, - { - "name": "ruby", - "license": "", - "last_run": None, - "data_source": "RubyDataSource", - "data_source_cfg": {"repository_url": "https://github.com/rubysec/ruby-advisory-db.git"}, - }, - { - "name": "ubuntu", - "license": "gpl-2.0", - "last_run": None, - "data_source": "UbuntuDataSource", - "data_source_cfg": { - "etags": {}, - "releases": ["bionic", "trusty", "focal", "eoan", "xenial"], - }, - }, - { - "name": "retiredotnet", - "license": "mit", - "last_run": None, - "data_source": "RetireDotnetDataSource", - "data_source_cfg": {"repository_url": "https://github.com/RetireNet/Packages.git"}, - }, # { - # "name": "suse_backports", + # "name": "retiredotnet", + # "license": "mit", + # "last_run": None, + # "data_source": "RetireDotnetDataSource", + # "data_source_cfg": {"repository_url": "https://github.com/RetireNet/Packages.git"}, + # }, + # # { + # # "name": "suse_backports", + # # "license": "", + # # "last_run": None, + # # "data_source": "SUSEBackportsDataSource", + # # "data_source_cfg": {"url": "http://ftp.suse.com/pub/projects/security/yaml/", "etags": {}}, + # # }, + # { + # "name": "suse_scores", # "license": "", # "last_run": None, - # "data_source": "SUSEBackportsDataSource", - # "data_source_cfg": {"url": "http://ftp.suse.com/pub/projects/security/yaml/", "etags": {}}, + # "data_source": "SUSESeverityScoreDataSource", + # "data_source_cfg": {}, + # }, + # { + # "name": "debian_oval", + # "license": "", + # "last_run": None, + # "data_source": "DebianOvalDataSource", + # "data_source_cfg": {"etags": {}, "releases": ["wheezy", "stretch", "jessie", "buster"]}, + # }, + # { + # "name": "redhat", + # "license": "cc-by-4.0", + # "last_run": None, + # "data_source": "RedhatDataSource", + # "data_source_cfg": {}, + # }, + # { + # "name": "nvd", + # "license": "", + # "last_run": None, + # "data_source": "NVDDataSource", + # "data_source_cfg": {"etags": {}}, + # }, + # { + # "name": "gentoo", + # "license": "", + # "last_run": None, + # "data_source": "GentooDataSource", + # "data_source_cfg": {"repository_url": "https://anongit.gentoo.org/git/data/glsa.git"}, + # }, + # { + # "name": "openssl", + # "license": "", + # "last_run": None, + # "data_source": "OpenSSLDataSource", + # "data_source_cfg": {"etags": {}}, + # }, + # { + # "name": "ubuntu_usn", + # "license": "gpl-2.0", + # "last_run": None, + # "data_source": "UbuntuUSNDataSource", + # "data_source_cfg": { + # "etags": {}, + # "db_url": "https://usn.ubuntu.com/usn-db/database-all.json.bz2", + # }, + # }, + # { + # "name": "github", + # "license": "", + # "last_run": None, + # "data_source": "GitHubAPIDataSource", + # "data_source_cfg": { + # "endpoint": "https://api.github.com/graphql", + # "ecosystems": ["MAVEN", "NUGET", "COMPOSER", "PIP", "RUBYGEMS"], + # }, + # }, + # { + # "name": "msr2019", + # "license": "apache-2.0", + # "last_run": None, + # "data_source": "ProjectKBMSRDataSource", + # "data_source_cfg": {"etags": {}}, + # }, + # { + # "name": "apache_httpd", + # "license": "", + # "last_run": None, + # "data_source": "ApacheHTTPDDataSource", + # "data_source_cfg": {"etags": {}}, + # }, + # { + # "name": "kaybee", + # "license": "apache-2.0", + # "last_run": None, + # "data_source": "KaybeeDataSource", + # "data_source_cfg": { + # "repository_url": "https://github.com/SAP/project-kb.git", + # "branch": "vulnerability-data", + # }, + # }, + # { + # "name": "postgresql", + # "license": "", + # "last_run": None, + # "data_source": "PostgreSQLDataSource", + # "data_source_cfg": {}, + # }, + # { + # "name": "elixir_security", + # "license": "cc0-1.0", + # "last_run": None, + # "data_source": "ElixirSecurityDataSource", + # "data_source_cfg": { + # "repository_url": "https://github.com/dependabot/elixir-security-advisories" + # }, + # }, + # { + # "name": "apache_tomcat", + # "license": "", + # "last_run": None, + # "data_source": "ApacheTomcatDataSource", + # "data_source_cfg": {"etags": {}}, + # }, + # { + # "name": "apache_kafka", + # "license": "", + # "last_run": None, + # "data_source": "ApacheKafkaDataSource", + # "data_source_cfg": {}, + # }, + # { + # "name": "istio", + # "license": "apache-2.0", + # "last_run": None, + # "data_source": "IstioDataSource", + # "data_source_cfg": {"repository_url": "https://github.com/istio/istio.io"}, # }, - { - "name": "suse_scores", - "license": "", - "last_run": None, - "data_source": "SUSESeverityScoreDataSource", - "data_source_cfg": {}, - }, - { - "name": "debian_oval", - "license": "", - "last_run": None, - "data_source": "DebianOvalDataSource", - "data_source_cfg": {"etags": {}, "releases": ["wheezy", "stretch", "jessie", "buster"]}, - }, - { - "name": "redhat", - "license": "cc-by-4.0", - "last_run": None, - "data_source": "RedhatDataSource", - "data_source_cfg": {}, - }, - { - "name": "nvd", - "license": "", - "last_run": None, - "data_source": "NVDDataSource", - "data_source_cfg": {"etags": {}}, - }, - { - "name": "gentoo", - "license": "", - "last_run": None, - "data_source": "GentooDataSource", - "data_source_cfg": {"repository_url": "https://anongit.gentoo.org/git/data/glsa.git"}, - }, - { - "name": "openssl", - "license": "", - "last_run": None, - "data_source": "OpenSSLDataSource", - "data_source_cfg": {"etags": {}}, - }, - { - "name": "ubuntu_usn", - "license": "gpl-2.0", - "last_run": None, - "data_source": "UbuntuUSNDataSource", - "data_source_cfg": { - "etags": {}, - "db_url": "https://usn.ubuntu.com/usn-db/database-all.json.bz2", - }, - }, - { - "name": "github", - "license": "", - "last_run": None, - "data_source": "GitHubAPIDataSource", - "data_source_cfg": { - "endpoint": "https://api.github.com/graphql", - "ecosystems": ["MAVEN", "NUGET", "COMPOSER", "PIP", "RUBYGEMS"], - }, - }, - { - "name": "msr2019", - "license": "apache-2.0", - "last_run": None, - "data_source": "ProjectKBMSRDataSource", - "data_source_cfg": {"etags": {}}, - }, - { - "name": "apache_httpd", - "license": "", - "last_run": None, - "data_source": "ApacheHTTPDDataSource", - "data_source_cfg": {"etags": {}}, - }, - { - "name": "kaybee", - "license": "apache-2.0", - "last_run": None, - "data_source": "KaybeeDataSource", - "data_source_cfg": { - "repository_url": "https://github.com/SAP/project-kb.git", - "branch": "vulnerability-data", - }, - }, - { - "name": "nginx", - "license": "", - "last_run": None, - "data_source": "NginxDataSource", - "data_source_cfg": {"etags": {}}, - }, - { - "name": "postgresql", - "license": "", - "last_run": None, - "data_source": "PostgreSQLDataSource", - "data_source_cfg": {}, - }, - { - "name": "elixir_security", - "license": "cc0-1.0", - "last_run": None, - "data_source": "ElixirSecurityDataSource", - "data_source_cfg": { - "repository_url": "https://github.com/dependabot/elixir-security-advisories" - }, - }, - { - "name": "apache_tomcat", - "license": "", - "last_run": None, - "data_source": "ApacheTomcatDataSource", - "data_source_cfg": {"etags": {}}, - }, - { - "name": "apache_kafka", - "license": "", - "last_run": None, - "data_source": "ApacheKafkaDataSource", - "data_source_cfg": {}, - }, - { - "name": "istio", - "license": "apache-2.0", - "last_run": None, - "data_source": "IstioDataSource", - "data_source_cfg": {"repository_url": "https://github.com/istio/istio.io"}, - }, ] diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index da333dcdd..d45d4983e 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -28,6 +28,7 @@ from django.core.management.base import CommandError from vulnerabilities.models import Importer +from vulnerabilities.importer_yielder import IMPORTER_REGISTRY from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importer_yielder import load_importers @@ -73,9 +74,9 @@ def handle(self, *args, **options): self.import_data(sources, options["cutoff_date"]) def list_sources(self): - importers = Importer.objects.all() + importers = IMPORTER_REGISTRY self.stdout.write("Vulnerability data can be imported from the following sources:") - self.stdout.write(", ".join([i.name for i in importers])) + self.stdout.write(", ".join([i["name"] for i in importers])) def import_data(self, names, cutoff_date): importers = []