From 28d1e63275538cf8f3c3a5ccb5cf8f87ccf2aee7 Mon Sep 17 00:00:00 2001 From: Andrew Pollock Date: Fri, 24 May 2024 14:36:49 +1000 Subject: [PATCH] Further validate repos by only accepting ones with tags (#2233) Tags are necessary for version resolution, and a repo without them is useless to us, and many of the repos in the current denylist do not have any tags. This enables a radical simplification of the repo denylist and largely removes ongoing maintenance burden. Latest run in Production: ``` nvdcve-2.0-2024.json Metrics: {TotalCVEs:11389 CVEsForApplications:1581 CVEsForKnownRepos:2364 OSVRecordsGenerated:1093 Outcomes:map[]} ``` Local test run: ``` nvdcve-2.0-2024.json Metrics: {TotalCVEs:11511 CVEsForApplications:1581 CVEsForKnownRepos:1651 OSVRecordsGenerated:1047 Outcomes:map[]} ``` A fabulous improvement in CVEsForKnownRepos, a much (durably) firmer looking denominator for conversion metrics. --- vulnfeeds/cves/versions.go | 340 +------------------------------ vulnfeeds/cves/versions_test.go | 18 -- vulnfeeds/git/repository.go | 42 ++-- vulnfeeds/git/repository_test.go | 23 +++ 4 files changed, 56 insertions(+), 367 deletions(-) diff --git a/vulnfeeds/cves/versions.go b/vulnfeeds/cves/versions.go index d812fa28131..765fdced870 100644 --- a/vulnfeeds/cves/versions.go +++ b/vulnfeeds/cves/versions.go @@ -226,353 +226,19 @@ type CPE struct { } var ( - // TODO(apollock): read this from an external file InvalidRepos = []string{ - "https://github.com/0day1/g1ory", - "https://github.com/0x14dli/ffos-SQL-injection-vulnerability-exists", - "https://github.com/0xdea/exploits", - "https://github.com/0xQRx/VulnerabilityResearch", - "https://github.com/0xxtoby/Vuldb", - "https://github.com/10cks/inkdropPoc", - "https://github.com/10cksyiqiyinhangzhoutechnology/elf-parser_segments_poc", - "https://github.com/1MurasaKi/Eyewear_Shop_XSS", - "https://github.com/1MurasaKi/PboostCMS_XSS", - "https://github.com/1MurasaKi/PizzeXSS_Report", - "https://github.com/1MurasaKi/STMS_CSRF", - "https://github.com/1s1and123/Vulnerabilities", - "https://github.com/1security/Vulnerability", - "https://github.com/202ecommerce/security-advisories", - "https://github.com/594238758/mycve", - "https://github.com/777erp/cms", - "https://github.com/A-TGAO/MxsDocVul", - "https://github.com/abcdefg-png/IoT-vulnerable", - "https://github.com/abhiunix/goo-blog-App-CVE", - "https://github.com/Accenture/AARO-Bugs", - "https://github.com/active-labs/Advisories", - "https://github.com/ae6e361b/online-job-portal-forget", - "https://github.com/agadient/SERVEEZ-CVE", - "https://github.com/Airrudder/vuls", - "https://github.com/AlwaysHereFight/YZMCMSxss", - "https://github.com/alwentiu/COVIDSafe-CVE-2020-12856", - "https://github.com/anhdq201/rukovoditel", - "https://github.com/anhdq201/webtareas", - "https://github.com/anvilsecure/garmin-ciq-app-research", - "https://github.com/Anza2001/IOT_VULN", - "https://github.com/apriorit/pentesting", - "https://github.com/ArianeBlow/Axelor_Stored_XSS", - "https://github.com/atredispartners/advisories", - "https://github.com/awillix/research", - "https://github.com/b17fr13nds/MPlayer_cve_poc", - "https://github.com/badboycxcc/Student-Admission-Sqlinjection", - "https://github.com/badboycxcc/Student-Admission-Xss", - "https://github.com/beicheng-maker/vulns", - "https://github.com/benjaminpsinclair/netdisco-2023-advisory", - "https://github.com/biantaibao/mldong_RCE", - "https://github.com/biantaibao/octopus_SQL", - "https://github.com/biantaibao/octopus_XSS", - "https://github.com/biantaibao/zhglxt_xss", - "https://github.com/BigTiger2020/2022", - "https://github.com/BigTiger2020/2023-1", - "https://github.com/BigTiger2020/2023", - "https://github.com/BigTiger2020/74CMS", - "https://github.com/BigTiger2020/Fantastic-Blog-CMS-", - "https://github.com/BigTiger2020/Theme-Park-Ticketing-System", - "https://github.com/BigTiger2020/UCMS", - "https://github.com/BlackFan/client-side-prototype-pollution", - "https://github.com/BLL-l/vulnerability_wiki", - "https://github.com/blockomat2100/PoCs", - "https://github.com/bosslabdcu/Vulnerability-Reporting", - "https://github.com/BurakSevben/2024_Math_Game_XSS", - "https://github.com/BurakSevben/2024_Online_Food_Menu_XSS", - "https://github.com/BurakSevben/2024_Product_Inventory_with_Export_to_Excel_XSS", - "https://github.com/BurakSevben/Daily_Habit_Tracker_App_SQL_Injection", - "https://github.com/BurakSevben/Login_System_with_Email_Verification_SQL_Injection", - "https://github.com/BurakSevben/School-Task-Manager-System-SQLi-1", - "https://github.com/ByteHackr/unzip_poc", - "https://github.com/capgeminicisredteam/disclosure", - "https://github.com/CapgeminiCisRedTeam/Disclosure", - "https://github.com/ch0ing/vul", - "https://github.com/Ch0pin/security-advisories", - "https://github.com/chenan224/webchess_sqli_poc", - "https://github.com/Chu1z1/Chuizi", - "https://github.com/ciph0x01/poc", - "https://github.com/ciph0x01/Simple-Exam-Reviewer-Management-System-CVE", - "https://github.com/cloudflare/advisories", - "https://github.com/Coalfire-Research/WinAPRS-Exploits", "https://github.com/ComparedArray/printix-CVE-2022-25089", - "https://github.com/cribdragg3r/offensive_research", - "https://github.com/ctflearner/Vulnerability", - "https://github.com/cvdyfbwa/IoT_LBT_Router", - "https://github.com/CVEProject/cvelist", // Heavily in Advisory URLs, sometimes shows up elsewhere - "https://github.com/Cvjark/Poc", - "https://github.com/cxaqhq/Loan-Management-System-Sqlinjection", - "https://github.com/cxcxcxcxcxcxcxc/cxcxcxcxcxcxcxc", - "https://github.com/cybersecurityworks/disclosed", - "https://github.com/D4rkP0w4r/AeroCMS-Add_Posts-Stored_XSS-Poc", - "https://github.com/D4rkP0w4r/AeroCMS-Comment-Stored_XSS-Poc", - "https://github.com/D4rkP0w4r/AeroCMS-Unrestricted-File-Upload-POC", - "https://github.com/D4rkP0w4r/Full-Ecommece-Website-Add_Product-Unrestricted-File-Upload-RCE-POC", - "https://github.com/D4rkP0w4r/Full-Ecommece-Website-Add_User-Stored-XSS-POC", - "https://github.com/D4rkP0w4r/Full-Ecommece-Website-Slides-Unrestricted-File-Upload-RCE-POC", - "https://github.com/D4rkP0w4r/sms-Add_Student-Stored_XSS-POC", - "https://github.com/D4rkP0w4r/sms-Unrestricted-File-Upload-RCE-POC", - "https://github.com/dhabaleshwar/Open-Source-Vulnerabilities", - "https://github.com/dhammon/pfBlockerNg-CVE-2022-40624", - "https://github.com/dhammon/pfBlockerNg-RCE", - "https://github.com/Dheeraj-Deshmukh/Hospital-s-patient-management-system", - "https://github.com/Dheeraj-Deshmukh/stored-xss-in-Hospital-s-Patient-Records-Management-System", - "https://github.com/digitemis/advisory", - "https://github.com/DiliLearngent/BugReport", - "https://github.com/Dir0x/Multiple-SQLi-in-Simple-Subscription-Company", - "https://github.com/Dir0x/SQLi-exploit---Simple-Client-Management-System", - "https://github.com/DisguisedRoot/Exploit", - "https://github.com/Don-H50/wp-vul", - "https://github.com/dota-st/Vulnerability", - "https://github.com/draco1725/POC", - "https://github.com/draco1725/Stored-XSS", - "https://github.com/Durian1546/vul", - "https://github.com/Dyrandy/BugBounty", - "https://github.com/E1CHO/water_cve", - "https://github.com/Edubr2020/RealPlayer_G2_RCE", - "https://github.com/Edubr2020/RP_DCP_Code_Exec", - "https://github.com/Edubr2020/RP_Import_RCE", - "https://github.com/enesozeser/Vulnerabilities", - "https://github.com/Ephemeral1y/Vulnerability", - "https://github.com/erengozaydin/College-Management-System-course_code-SQL-Injection-Authenticated", - "https://github.com/erengozaydin/Microfinance-Management-System-V1.0-SQL-Injection-Vulnerability-Unauthenticated", - "https://github.com/erengozaydin/Royal-Event-Management-System-todate-SQL-Injection-Authenticated", - "https://github.com/esp0xdeadbeef/rce_webmin", - "https://github.com/etn0tw/cmscve_test", - "https://github.com/f4cky0u/security-vulnerabilities", - "https://github.com/FCncdn/Appsmith-Js-Injection-POC", - "https://github.com/Filiplain/LFI-to-RCE-SE-Suite-2.0", - "https://github.com/fireeye/Vulnerability-Disclosures", - "https://github.com/frame84/vulns", - "https://github.com/Frank-Z7/z-vulnerabilitys", - "https://github.com/friends-of-presta/security-advisories", - "https://github.com/funny-mud-peee/IoT-vuls", - "https://github.com/FusionAuth/fusionauth-issues", - "https://github.com/g1an123/poc", - "https://github.com/gdianq/Gym-Management-Exercises-Sqlinjection", - "https://github.com/gdianq/Gym-Management-System-loginpage-Sqlinjection", - "https://github.com/gdianq/Gym-Management-System-Sqlinjection", - "https://github.com/gdianq/Sparkz-Hotel-Management-loginpage-Sqlinjection", - "https://github.com/github/cvelist", // Fork of https://github.com/CVEProject/cvelist + "https://github.com/CVEProject/cvelist", + "https://github.com/github/cvelist", // Heavily in Advisory URLs, sometimes shows up elsewhere "https://github.com/github/securitylab", - "https://github.com/gitlabhq/gitlabhq", // GitHub mirror, not canonical - "https://github.com/google/oss-fuzz-vulns", // 8^) - "https://github.com/gou-web/Parking-management-systemXSS-", - "https://github.com/Gr4y21/My-CVE-IDs", - "https://github.com/grafana/bugbounty", - "https://github.com/guyinatuxedo/sqlite3_record_leaking", - "https://github.com/H4rk3nz0/PenTesting", - "https://github.com/hackerzyq/mycve", - "https://github.com/haile01/perl_spreadsheet_excel_rce_poc", - "https://github.com/Hakcoder/Simple-Online-Public-Access-Catalog-OPAC---SQL-injection", - "https://github.com/Hanfu-l/POC-Exp", - "https://github.com/hashicorp/terraform-enterprise-release-notes", - "https://github.com/haxpunk1337/Enterprise-Survey-Software", - "https://github.com/haxpunk1337/MDaemon-", - "https://github.com/Hckwzh/cms", - "https://github.com/HH1F/KbaseDoc-v1.0-Arbitrary-file-deletion-vulnerability", - "https://github.com/hkerma/opa-gatekeeper-concurrency-issue", - "https://github.com/hmsec/advisories", - "https://github.com/hnsecurity/vulns", - "https://github.com/hubenlab/hubenvullist", - "https://github.com/Hyperkopite/Roothub_vulns", - "https://github.com/i3umi3iei3ii/CentOS-Control-Web-Panel-CVE", - "https://github.com/ianxtianxt/gitbook-xss", - "https://github.com/imsebao/404team", - "https://github.com/InfoSecWarrior/Offensive-Payloads", - "https://github.com/IthacaLabs/DevExpress", - "https://github.com/IthacaLabs/Parallels", - "https://github.com/IthacaLabs/Vsourz-Digital", - "https://github.com/itodaro/doorGets_cve", - "https://github.com/Jaarden/AlphaInnotec-Password-Vulnerability", - "https://github.com/jacky-y/vuls", - "https://github.com/JackyG0/Online-Accreditation-Management-System-v1.0-SQLi", - "https://github.com/Jamison2022/Company-Website-CMS", - "https://github.com/Jamison2022/Wedding-Hall-Booking-System", - "https://github.com/jcarabantes/Bus-Vulnerabilities", - "https://github.com/jingping911/exshopbug", - "https://github.com/jiy2020/bugReport", - "https://github.com/jlleitschuh/security-research", - "https://github.com/jmrcsnchz/ClinicQueueingSystem_RCE", - "https://github.com/joinia/webray.com.cn", - "https://github.com/jomskiller/Employee-Management-System---Stored-XSS", - "https://github.com/jomskiller/Employee-Managemet-System---Broken-Access-Control", - "https://github.com/JunyanYip/itsourcecode_justines_xss_vul", - "https://github.com/jusstSahil/CSRF-", - "https://github.com/jvz/test-cvelist", - "https://github.com/k0xx11/vul-wiki", - "https://github.com/k0xx11/Vulscve", - "https://github.com/kaoudis/advisories", - "https://github.com/keru6k/Online-Admission-System-RCE-PoC", - "https://github.com/Keyvanhardani/Exploit-eShop-Multipurpose-Ecommerce-Store-Website-3.0.4-Cross-Site-Scripting-XSS", - "https://github.com/killmonday/isic.lk-RCE", - "https://github.com/KingBridgeSS/Online_Driving_School_Project_In_PHP_With_Source_Code_Vulnerabilities", - "https://github.com/Kitsun3Sec/exploits", - "https://github.com/kk98kk0/exploit", - "https://github.com/KLSEHB/vulnerability-report", - "https://github.com/kmkz/exploit", - "https://github.com/kyrie403/Vuln", - "https://github.com/L1917/Fast-Food-Ordering-System", - "https://github.com/l1nk3rlin/php_code_audit_project", - "https://github.com/lakshaya0557/POCs", - "https://github.com/laoquanshi/BILLING-SOFTWARE-SQL-injection-vulnerability", - "https://github.com/laoquanshi/PHPGurukul-Hospital-Management-System", - "https://github.com/laotun-s/POC", - "https://github.com/Lemon4044/Fast-Food-Ordering-System", - "https://github.com/lohyt/Persistent-Cross-Site-Scripting-found-in-Online-Jewellery-Store-from-Sourcecodester-website.", - "https://github.com/lohyt/web-shell-via-file-upload-in-hocms", - "https://github.com/luelueking/ruoyi-4.7.5-vuln-poc", - "https://github.com/lukaszstu/SmartAsset-CORS-CVE-2020-26527", - "https://github.com/ly1g3/Mailcow-CVE-2022-31138", - "https://github.com/mandiant/Vulnerability-Disclosures", - "https://github.com/Matrix07ksa/ALLMediaServer-1.6-Buffer-Overflow", - "https://github.com/mclab-hbrs/BBB-POC", - "https://github.com/metaredteam/external-disclosures", - "https://github.com/metaStor/Vuls", - "https://github.com/mi2acle/forucmsvuln", - "https://github.com/mikeccltt/0525", - "https://github.com/mikeccltt/0724", - "https://github.com/mikeccltt/automotive", - "https://github.com/mikeccltt/badminton-center-management-system", - "https://github.com/mikeccltt/chatbot", - "https://github.com/mikeccltt/wbms_bug_report", - "https://github.com/mikeisastar/counter-strike-arbitrary-file-read", - "https://github.com/Mirantis/security", - "https://github.com/mirchr/security-research", - "https://github.com/Mr-Secure-Code/My-CVE", - "https://github.com/mrojz/rconfig-exploit", - "https://github.com/MrTuxracer/advisories", + "https://github.com/gitlabhq/gitlabhq", // GitHub mirror, not canonical "https://github.com/n0Sleeper/bosscmsVuln", - "https://github.com/N1ce759/74cmsSE-Arbitrary-File-Reading", - "https://github.com/nam3lum/msi-central_privesc", - "https://github.com/Netflix/security-bulletins", - "https://github.com/nextcloud/security-advisories", - "https://github.com/novysodope/vulreq", - "https://github.com/nsparker1337/OpenSource", - "https://github.com/offsecin/bugsdisclose", - "https://github.com/orangecertcc/security-research", - "https://github.com/Ozozuz/Qlik-View-Stored-XSS", - "https://github.com/PabloMK7/ENLBufferPwn", - "https://github.com/palantir/security-bulletins", - "https://github.com/passtheticket/vulnerability-research", - "https://github.com/Peanut886/Vulnerability", - "https://github.com/piuppi/proof-of-concepts", - "https://github.com/playZG/Exploit-", - "https://github.com/PostalBlab/Vulnerabilities", - "https://github.com/prismbreak/vulnerabilities", - "https://github.com/purplededa/EasyoneCRM-5.50.02-SQLinjection", - "https://github.com/PurplePetrus/MxCC_Credential-Storage_issue", - "https://github.com/qqqyc/vlun1", - "https://github.com/Ramansh123454/POCs", - "https://github.com/rand0midas/randomideas", "https://github.com/rapid7/metasploit-framework", - "https://github.com/riteshgohil/My_CVE_References", - "https://github.com/rohit0x5/poc", - "https://github.com/rsrahulsingh05/POC", - "https://github.com/rtcrowley/poc", - "https://github.com/rumble773/sec-research", - "https://github.com/Ryan0lb/EC-cloud-e-commerce-system-CVE-application", - "https://github.com/s1kr10s/EasyChatServer-DOS", - "https://github.com/saitamang/POC-DUMP", - "https://github.com/sartlabs/0days", - "https://github.com/SaumyajeetDas/POC-of-CVE-2022-36271", - "https://github.com/SaumyajeetDas/Vulnerability", - "https://github.com/secf0ra11/secf0ra11.github.io", - "https://github.com/Security-AVS/-CVE-2021-26904", - "https://github.com/seizer-zyx/Vulnerability", - "https://github.com/seqred-s-a/gxdlmsdirector-cve", - "https://github.com/Serces-X/vul_report", - "https://github.com/shellshok3/Cross-Site-Scripting-XSS", - "https://github.com/sickcodes/security", - "https://github.com/silence-silence/xxl-job-lateral-privilege-escalation-vulnerability-", - "https://github.com/sinemsahn/POC", - "https://github.com/sleepyvv/vul_report", - "https://github.com/smurf-reigz/security", - "https://github.com/Snakinya/Vuln", - "https://github.com/snyk/zip-slip-vulnerability", - "https://github.com/soheilsamanabadi/vulnerability", - "https://github.com/soheilsamanabadi/vulnerabilitys", - "https://github.com/Sospiro014/zday1", - "https://github.com/soundarkutty/stored-xss", - "https://github.com/souravkr529/CSRF-in-Cold-Storage-Management-System", - "https://github.com/spwpun/ntp-4.2.8p15-cves", - "https://github.com/sromanhu/Cmsmadesimple-CMS-Stored-XSS", - "https://github.com/sromanhu/CMSmadesimple-File-Upload--XSS---File-Manager", - "https://github.com/sromanhu/CSZ-CMS-Stored-XSS---Pages-Content", - "https://github.com/sromanhu/e107-CMS-Stored-XSS---Manage", - "https://github.com/sromanhu/RiteCMS-Stored-XSS---Home", "https://github.com/starnightcyber/miscellaneous", - "https://github.com/strangebeecorp/security", - "https://github.com/strik3r0x1/Vulns", - "https://github.com/sunset-move/EasyImages2.0-arbitrary-file-download-vulnerability", - "https://github.com/SunshineOtaku/Report-CVE", - "https://github.com/superkojiman/vulnerabilities", - "https://github.com/sweatxi/BugHub", - "https://github.com/TCSWT/Baby-Care-System", - "https://github.com/thehackingverse/Stored-xss-", - "https://github.com/theyiyibest/Reflected-XSS-on-SockJS", - "https://github.com/thisissuperann/Vul", - "https://github.com/TimeSeg/IOT_CVE", - "https://github.com/TishaManandhar/Superstore-sql-poc", - "https://github.com/toyydsBT123/One_of_my_take_on_SourceCodester", - "https://github.com/transcendent-group/advisories", - "https://github.com/tremwil/ds3-nrssr-rce", - "https://github.com/trinity-syt-security/xss_vuln_issue", - "https://github.com/Trinity-SYT-SECURITY/XSS_vuln_issue", - "https://github.com/uBlockOrigin/uBlock-issues", - "https://github.com/umarfarook882/avast_multiple_vulnerability_disclosure", - "https://github.com/v2ish1yan/mycve", - "https://github.com/V3geD4g/cmseasy_vul", - "https://github.com/verf1sh/Poc", - "https://github.com/versprite/research", - "https://github.com/VistaAX/vulnerablility", - "https://github.com/vQAQv/Request-CVE-ID-PoC", - "https://github.com/vulnerabilities-cve/vulnerabilities", - "https://github.com/vuls/vuls", - "https://github.com/wagnerdracha/ProofOfConcept", - "https://github.com/wandera/public-disclosures", - "https://github.com/Wh04m1001/ZoneAlarmEoP", - "https://github.com/whiex/c2Rhc2Rhc2Q-", - "https://github.com/whitehatl/Vulnerability", - "https://github.com/wind-cyber/LJCMS-UserTraversal-Vulnerability", - "https://github.com/wkeyi0x1/vul-report", - "https://github.com/wsummerhill/BSA-Radar_CVE-Vulnerabilities", - "https://github.com/xcodeOn1/xcode0x-CVEs", - "https://github.com/XiLitter/CMS_vulnerability-discovery", - "https://github.com/xnobody12/jaws-cms-rce", - "https://github.com/Xor-Gerke/webray.com.cn", - "https://github.com/xuanluansec/vul", - "https://github.com/xunyang1/my-vulnerability", - "https://github.com/xxhzz1/74cmsSE-Arbitrary-file-upload-vulnerability", - "https://github.com/y1s3m0/vulnfind", - "https://github.com/yasinyildiz26/Badminton-Center-Management-System", - "https://github.com/YavuzSahbaz/Limbas-4.3.36.1319-is-vulnerable-to-Cross-Site-Scripting-XSS-", - "https://github.com/YavuzSahbaz/Red-Planet-Laundry-Management-System-1.0-is-vulnerable-to-SQL", - "https://github.com/ycdxsb/Vuln", - "https://github.com/ykosan1/Simple-Task-Scheduling-System-id-SQL-Injection-Unauthenticated", - "https://github.com/YLoiK/74cmsSE-Arbitrary-file-upload-vulnerability", - "https://github.com/Yu1e/vuls", - "https://github.com/YZLCQX/Mailbox-remote-command-execution", - "https://github.com/z00z00z00/Safenet_SAC_CVE-2021-42056", - "https://github.com/zerrr0/Zerrr0_Vulnerability", - "https://github.com/Zeyad-Azima/Issabel-stored-XSS", - "https://github.com/ZhuoNiBa/Delta-DIAEnergie-XSS", - "https://github.com/ZJQcicadawings/VulSql", - "https://github.com/Zoe0427/YJCMS", - "https://github.com/zzh-newlearner/record", - "https://gitlab.com/-/snippets/1937042", - "https://gitlab.com/FallFur/exploiting-unprotected-admin-funcionalities-on-besder-ip-cameras", "https://gitlab.com/gitlab-org/gitlab-ce", // redirects to gitlab-foss "https://gitlab.com/gitlab-org/gitlab-ee", // redirects to gitlab "https://gitlab.com/gitlab-org/gitlab-foss", // not the canonical source "https://gitlab.com/gitlab-org/omnibus-gitlab", // not the source - "https://gitlab.com/gitlab-org/release", // not the source - "https://gitlab.com/kop316/vvm-disclosure", - "https://gitlab.com/yongchuank/avast-aswsnx-ioctl-82ac0060-oob-write", } InvalidRepoRegex = `(?i)/(?:(?:CVEs?)|(?:CVE-\d{4}-\d{4,})(?:/?.*)?|bug_report(?:/.*)?|GitHubAssessments/.*)` ) diff --git a/vulnfeeds/cves/versions_test.go b/vulnfeeds/cves/versions_test.go index 6b96efe0da4..e73bac0b132 100644 --- a/vulnfeeds/cves/versions_test.go +++ b/vulnfeeds/cves/versions_test.go @@ -278,12 +278,6 @@ func TestRepo(t *testing.T) { expectedRepoURL: "https://bitbucket.org/snakeyaml/snakeyaml", expectedOk: true, }, - { - description: "Valid URL but not wanted (by denylist)", - inputLink: "https://github.com/orangecertcc/security-research/security/advisories/GHSA-px2c-q384-5wxc", - expectedRepoURL: "", - expectedOk: false, - }, { description: "Valid URL but not wanted (by deny regexp)", inputLink: "https://github.com/Ko-kn3t/CVE-2020-29156", @@ -404,24 +398,12 @@ func TestRepo(t *testing.T) { expectedRepoURL: "https://git.ffmpeg.org/ffmpeg.git", expectedOk: true, }, - { - description: "Undesired researcher repo (by denylist)", - inputLink: "https://github.com/chenan224/webchess_sqli_poc", - expectedRepoURL: "", - expectedOk: false, - }, { description: "Undesired researcher repo (by deny regex)", inputLink: "https://github.com/bigzooooz/CVE-2023-26692#readme", expectedRepoURL: "", expectedOk: false, }, - { - description: "Undesired repo (by deny regex)", - inputLink: "https://gitlab.com/gitlab-org/cves/-/blob/master/2023/CVE-2023-0413.json", - expectedRepoURL: "", - expectedOk: false, - }, { description: "GNU glibc GitWeb repo (with no distinguishing marks)", inputLink: "https://sourceware.org/git/?p=glibc.git", diff --git a/vulnfeeds/git/repository.go b/vulnfeeds/git/repository.go index adedb29186c..f98dcc7f00c 100644 --- a/vulnfeeds/git/repository.go +++ b/vulnfeeds/git/repository.go @@ -124,10 +124,7 @@ func RepoTags(repoURL string, repoTagsCache RepoTagsCache) (tags Tags, e error) return tags, err } tagsMap := make(map[string]Tag) - for _, ref := range refs { - if !ref.Name().IsTag() { - continue - } + for _, ref := range RefTags(refs) { // This is used for caching and direct lookup by tag name. tagsMap[ref.Name().Short()] = Tag{Tag: ref.Name().Short(), Commit: ref.Hash().String()} } @@ -218,16 +215,30 @@ func NormalizeRepoTags(repoURL string, repoTagsCache RepoTagsCache) (NormalizedT return NormalizedTags, nil } +// Return a list of just the references that are tags. +func RefTags(refs []*plumbing.Reference) (tags []*plumbing.Reference) { + for _, ref := range refs { + if ref.Name().IsTag() { + tags = append(tags, ref) + } + } + return tags +} + +// Return a list of just the references that are branches. +func RefBranches(refs []*plumbing.Reference) (branches []*plumbing.Reference) { + for _, ref := range refs { + if ref.Name().IsBranch() { + branches = append(branches, ref) + } + } + return branches +} + // Validate the repo by attempting to query it's references. +// Repos that don't have any tags are not valid. func ValidRepo(repoURL string) (valid bool) { - remoteConfig := &config.RemoteConfig{ - Name: "source", - URLs: []string{ - repoURL, - }, - } - r := git.NewRemote(memory.NewStorage(), remoteConfig) - _, err := r.List(&git.ListOptions{}) + refs, err := RemoteRepoRefsWithRetry(repoURL, 3) if err != nil && err == transport.ErrAuthenticationRequired { // somewhat strangely, we get an authentication prompt via Git on non-existent repos. return false @@ -235,5 +246,12 @@ func ValidRepo(repoURL string) (valid bool) { if err != nil { return false } + if len(refs) == 0 { + return false + } + // Repos with no tags aren't useful. + if len(RefTags(refs)) == 0 { + return false + } return true } diff --git a/vulnfeeds/git/repository_test.go b/vulnfeeds/git/repository_test.go index 0065975ab85..92028d514c8 100644 --- a/vulnfeeds/git/repository_test.go +++ b/vulnfeeds/git/repository_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/osv/vulnfeeds/cves" "golang.org/x/exp/maps" ) @@ -303,6 +304,16 @@ func TestValidRepo(t *testing.T) { repoURL: "https://github.com/andrewpollock/mybogusrepo", expectedResult: false, }, + { + description: "Legitimate repository with no tags and two branches", + repoURL: "https://github.com/202ecommerce/security-advisories", + expectedResult: false, + }, + { + description: "Legitimate repository with no tags and one branch", + repoURL: "https://github.com/active-labs/Advisories", + expectedResult: false, + }, } for _, tc := range tests { got := ValidRepo(tc.repoURL) @@ -311,3 +322,15 @@ func TestValidRepo(t *testing.T) { } } } + +func TestInvalidRepos(t *testing.T) { + redundantRepos := []string{} + for _, repo := range cves.InvalidRepos { + if !ValidRepo(repo) { + redundantRepos = append(redundantRepos, repo) + } + } + if diff := cmp.Diff([]string{}, redundantRepos); diff != "" { + t.Errorf("These redundant repos are in InvalidRepos: %s", diff) + } +}