From a36bb34e0afaf51d67a4b63d25b8e01988db2aef Mon Sep 17 00:00:00 2001 From: Daniel Scanteianu Date: Mon, 6 May 2024 22:11:03 +0100 Subject: [PATCH 01/18] rename ci, add vdr creation yml --- .github/workflows/{github-actions.yml => ci.yml} | 0 .github/workflows/vdr-creation.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{github-actions.yml => ci.yml} (100%) create mode 100644 .github/workflows/vdr-creation.yml diff --git a/.github/workflows/github-actions.yml b/.github/workflows/ci.yml similarity index 100% rename from .github/workflows/github-actions.yml rename to .github/workflows/ci.yml diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml new file mode 100644 index 0000000..e69de29 From 5f07c11503d70eea9dadcf26b23a0e2ebfc8a084 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Fri, 10 May 2024 13:15:29 +0100 Subject: [PATCH 02/18] try to run the downloader and upload partial --- .github/workflows/vdr-creation.yml | 34 ++++++++++++++++++++++++++++++ ojvg_download.py | 3 ++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index e69de29..2b83d16 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -0,0 +1,34 @@ +name: VDR Creation + +on: + workflow_dispatch: + branches: [ main ] + push: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + + - name: Set up Python 3.10 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Download ojvg + run: python3 ojvg_download.py + - name: Upload data directory + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 + with: + name: openjvg_summary.json + path: data/openjvg_summary.json diff --git a/ojvg_download.py b/ojvg_download.py index e9d5bce..4eeea3d 100644 --- a/ojvg_download.py +++ b/ojvg_download.py @@ -7,7 +7,8 @@ It downloads all the vulnerability reports as html files to the `data` directory and saves the relevant data in `data/ojvg_summary.json` """ start_date = date(2019, 1, 1) -end_date = date.today() +end_date = date(2019, 12, 31) +#end_date = date.today() current_date = start_date responses = [] while current_date < end_date: From ff1702c6764692f59ef2d1caf64a378fdf509327 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Fri, 10 May 2024 13:20:16 +0100 Subject: [PATCH 03/18] add trigger on PR --- .github/workflows/vdr-creation.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 2b83d16..92cc32b 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -2,9 +2,12 @@ name: VDR Creation on: workflow_dispatch: - branches: [ main ] + branches: [ main, ] push: branches: [ main ] + pull_request: # todo: do we want it to run on all PRs? + branches: [ "main" ] + jobs: build: From d80fc1622c62620e752a020b5db558dcca49e777 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Fri, 10 May 2024 13:23:22 +0100 Subject: [PATCH 04/18] make data dir --- .github/workflows/vdr-creation.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 92cc32b..4d6fdf1 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -29,7 +29,9 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Download ojvg - run: python3 ojvg_download.py + run: | + mkdir -p data + python3 ojvg_download.py - name: Upload data directory uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 with: From b287aa49fee2d003b200bc6642099fa253d6634c Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Fri, 10 May 2024 13:28:35 +0100 Subject: [PATCH 05/18] run the conversion script --- .github/workflows/vdr-creation.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 4d6fdf1..20f56e5 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -32,8 +32,9 @@ jobs: run: | mkdir -p data python3 ojvg_download.py + python3 ojvg_convert.py - name: Upload data directory uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 with: - name: openjvg_summary.json - path: data/openjvg_summary.json + name: data directory + path: data From 41a7c8079f9c726bbceffa5973fd25487631527e Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Fri, 10 May 2024 13:38:00 +0100 Subject: [PATCH 06/18] run vdr creation for the whole vdr period --- .github/workflows/vdr-creation.yml | 7 ++++++- ojvg_download.py | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 20f56e5..c3cd674 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -33,8 +33,13 @@ jobs: mkdir -p data python3 ojvg_download.py python3 ojvg_convert.py - - name: Upload data directory + - name: Upload data directory (for debugging/introspection) uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 with: name: data directory path: data + - name: Upload final vdr + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 + with: + name: data directory + path: data/vdr.json diff --git a/ojvg_download.py b/ojvg_download.py index 4eeea3d..e9d5bce 100644 --- a/ojvg_download.py +++ b/ojvg_download.py @@ -7,8 +7,7 @@ It downloads all the vulnerability reports as html files to the `data` directory and saves the relevant data in `data/ojvg_summary.json` """ start_date = date(2019, 1, 1) -end_date = date(2019, 12, 31) -#end_date = date.today() +end_date = date.today() current_date = start_date responses = [] while current_date < end_date: From b1307bc33e7151b6e3c01cd77362bec1a9b463f1 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Fri, 10 May 2024 13:45:17 +0100 Subject: [PATCH 07/18] fix vdr artifact name --- .github/workflows/vdr-creation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index c3cd674..3dbb2af 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -41,5 +41,5 @@ jobs: - name: Upload final vdr uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 with: - name: data directory + name: final vdr path: data/vdr.json From a10c87326b29e3b9a22fdc6156a1093cda7f9c21 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Mon, 13 May 2024 15:15:10 +0100 Subject: [PATCH 08/18] try to make nist calls use api key --- cvereporter/nist_enhance.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cvereporter/nist_enhance.py b/cvereporter/nist_enhance.py index 7a8a9fc..98f8490 100644 --- a/cvereporter/nist_enhance.py +++ b/cvereporter/nist_enhance.py @@ -6,6 +6,8 @@ ) import requests import json +import time +import os """ this file has the utilities for downloading data about cves from NIST and updating Vulnerability objects with the data @@ -14,7 +16,16 @@ def fetch_nist(url: str, id: str) -> dict: data = None - nist_resp = requests.get(url) + nist_resp = None + if "NIST_NVD_TOKEN" in os.environ: + print("making call to NIST using api key!") + time.sleep(1) # stay well within 50 requests/30 seconds + nist_resp = requests.get(url, headers= {"apiKey": os.environ["NIST_NVD_TOKEN"]}) + else: + print("making call to NIST without using api key!") + time.sleep(10) # stay well within 5 requests/30 seconds + nist_resp = requests.get(url) + time.sleep(sleep_time) if nist_resp.status_code != 200: print( "error fetching {}; status code: {}; text: {}".format( From 9ee0d94f87f67264b74b6e2b3f7dc872f26e9d82 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Mon, 13 May 2024 15:22:13 +0100 Subject: [PATCH 09/18] fix the build --- cvereporter/nist_enhance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cvereporter/nist_enhance.py b/cvereporter/nist_enhance.py index 98f8490..ae1a13e 100644 --- a/cvereporter/nist_enhance.py +++ b/cvereporter/nist_enhance.py @@ -25,7 +25,6 @@ def fetch_nist(url: str, id: str) -> dict: print("making call to NIST without using api key!") time.sleep(10) # stay well within 5 requests/30 seconds nist_resp = requests.get(url) - time.sleep(sleep_time) if nist_resp.status_code != 200: print( "error fetching {}; status code: {}; text: {}".format( From 772b48329b387e74d2afb3e93ecb60445f15dfea Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Mon, 13 May 2024 16:22:07 +0100 Subject: [PATCH 10/18] add flush --- cvereporter/nist_enhance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cvereporter/nist_enhance.py b/cvereporter/nist_enhance.py index ae1a13e..2c4dede 100644 --- a/cvereporter/nist_enhance.py +++ b/cvereporter/nist_enhance.py @@ -18,11 +18,11 @@ def fetch_nist(url: str, id: str) -> dict: data = None nist_resp = None if "NIST_NVD_TOKEN" in os.environ: - print("making call to NIST using api key!") + print("making call to NIST using api key! "+url, flush=True) time.sleep(1) # stay well within 50 requests/30 seconds nist_resp = requests.get(url, headers= {"apiKey": os.environ["NIST_NVD_TOKEN"]}) else: - print("making call to NIST without using api key!") + print("making call to NIST without using api key! "+url, flush=True) time.sleep(10) # stay well within 5 requests/30 seconds nist_resp = requests.get(url) if nist_resp.status_code != 200: From 9f29bfb0b948d7cc6537a8485720e9605c43ab57 Mon Sep 17 00:00:00 2001 From: Daniel Scanteianu Date: Mon, 13 May 2024 16:47:41 +0100 Subject: [PATCH 11/18] Apply suggestions from gadams Co-authored-by: George Adams --- .github/workflows/vdr-creation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 3dbb2af..4f8a076 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -6,7 +6,7 @@ on: push: branches: [ main ] pull_request: # todo: do we want it to run on all PRs? - branches: [ "main" ] + branches: [ main ] jobs: @@ -34,12 +34,12 @@ jobs: python3 ojvg_download.py python3 ojvg_convert.py - name: Upload data directory (for debugging/introspection) - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: data directory path: data - name: Upload final vdr - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: final vdr path: data/vdr.json From 2058d6de58e98abb072b946b4d7460571c2ea418 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Mon, 13 May 2024 17:16:24 +0100 Subject: [PATCH 12/18] add the api key secret, remove branches from workflows --- .github/workflows/vdr-creation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 4f8a076..64b0e18 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -2,7 +2,6 @@ name: VDR Creation on: workflow_dispatch: - branches: [ main, ] push: branches: [ main ] pull_request: # todo: do we want it to run on all PRs? @@ -29,6 +28,8 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Download ojvg + env: + NIST_NVD_TOKEN: ${{ secrets.NIST_NVD_TOKEN }} run: | mkdir -p data python3 ojvg_download.py From d76a5072dcb6eec17284725c7acec10c9d031c3b Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Mon, 13 May 2024 17:30:24 +0100 Subject: [PATCH 13/18] check api key is actually there --- cvereporter/nist_enhance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvereporter/nist_enhance.py b/cvereporter/nist_enhance.py index 2c4dede..ca84639 100644 --- a/cvereporter/nist_enhance.py +++ b/cvereporter/nist_enhance.py @@ -17,7 +17,7 @@ def fetch_nist(url: str, id: str) -> dict: data = None nist_resp = None - if "NIST_NVD_TOKEN" in os.environ: + if "NIST_NVD_TOKEN" in os.environ and os.environ["NIST_NVD_TOKEN"]: # check not empty print("making call to NIST using api key! "+url, flush=True) time.sleep(1) # stay well within 50 requests/30 seconds nist_resp = requests.get(url, headers= {"apiKey": os.environ["NIST_NVD_TOKEN"]}) From 3a18942a6ceb4a8cf6e526dd20ff715b31d2bb8f Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Tue, 14 May 2024 12:01:40 +0100 Subject: [PATCH 14/18] remove pip and flake8 from the installs --- .github/workflows/vdr-creation.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 64b0e18..9059ed4 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -24,7 +24,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Download ojvg From d270e5a9e669c78336cd3531e6870a9fd5610c09 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Tue, 14 May 2024 12:03:16 +0100 Subject: [PATCH 15/18] update comments --- .github/workflows/vdr-creation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 9059ed4..b891b2b 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: push: branches: [ main ] - pull_request: # todo: do we want it to run on all PRs? + pull_request: # prs don't get secrets, but the API works (albeit 10x slower) without the api key branches: [ main ] From 1d3198d635d3f7fffafb7dfb9884093325bc94f8 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Fri, 17 May 2024 13:43:48 +0100 Subject: [PATCH 16/18] rm newlines --- .github/workflows/vdr-creation.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index b891b2b..6b5cecd 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -10,9 +10,7 @@ on: jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 From 0d345320cf2b32917cedff6f531661c7b2da9395 Mon Sep 17 00:00:00 2001 From: dscanteianu Date: Thu, 23 May 2024 10:59:42 +0100 Subject: [PATCH 17/18] fix flaky inability to find risk matrix --- cvereporter/fetch_vulnerabilities.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cvereporter/fetch_vulnerabilities.py b/cvereporter/fetch_vulnerabilities.py index f832469..a7b5b34 100644 --- a/cvereporter/fetch_vulnerabilities.py +++ b/cvereporter/fetch_vulnerabilities.py @@ -102,7 +102,9 @@ def parse_to_dict(resp_text: str, date: str) -> list[dict]: # find the table with the CVEs table = soup.find("table", attrs={"class": "risk-matrix"}) - + if table is None: + print("unable to find risk matrix for "+date) + return None # find all the rows in the table rows = table.find_all("tr") dicts = [] @@ -180,8 +182,8 @@ def dict_to_vulns(dicts: list[dict]) -> list[Vulnerability]: """ We assume the text for the affected versions is in a block like: -"The following vulnerabilities in OpenJDK source code were fixed in this release. -The affected versions are 12, 11.0.2, 8u202, 7u211, and earlier. +"The following vulnerabilities in OpenJDK source code were fixed in this release. +The affected versions are 12, 11.0.2, 8u202, 7u211, and earlier. We recommend that you upgrade as soon as possible." """ From 6bdcd3428dcad46f4ca74663a60ba4885b46eed2 Mon Sep 17 00:00:00 2001 From: George Adams Date: Tue, 28 May 2024 10:52:57 +0100 Subject: [PATCH 18/18] Update .github/workflows/vdr-creation.yml --- .github/workflows/vdr-creation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml index 6b5cecd..f592052 100644 --- a/.github/workflows/vdr-creation.yml +++ b/.github/workflows/vdr-creation.yml @@ -26,7 +26,7 @@ jobs: - name: Download ojvg env: - NIST_NVD_TOKEN: ${{ secrets.NIST_NVD_TOKEN }} + NIST_NVD_TOKEN: ${{ secrets.NIST_NVD_TOKEN }} run: | mkdir -p data python3 ojvg_download.py