-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce a script to generate a database usable by osv-offline
This script largely builds on top of PR #89 with some necessary modifications and additions. The finalized script downloads and extracts the upstream OSV database and generates additional files for container and RPM vulnerabilities. Few modifications to the original scripts were necessary: - Add a retry mechanism for getting the CSAF VEX files. Transient networking errors were common during testing and retries shold resolve them. - Add internal ID (_id) to the OSV schema and set it to a random string. Every object in the nedb database must have this property. - Split how the affectedList is generated between RPMs and containers. - Duplicate affectedList entries for containers to also have registry registry.access.redhat.com. This is necessary because the CSAF VEX data only tracks images with registry.redhat.io. This script is expected to be run periodically by a cronjob and regenerate the DB files onto a persistent volume. Add the new script to the mintmaker container, so that the cronjob can use the same image. WARNING: The script now gets the list of advisories from changes.csv. This is incorrect and should be changed to releases.csv when it becomes available. The current state is for testing purposes only.
- Loading branch information
Showing
12 changed files
with
1,038 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "flag" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| osv_downloader "github.com/konflux-ci/mintmaker/tools/osv-downloader" | ||
| osv_generator "github.com/konflux-ci/mintmaker/tools/osv-generator" | ||
| ) | ||
|
|
||
| func main() { | ||
| dockerFilename := flag.String("docker-filename", "docker.nedb", "Filename for the Docker DB file") | ||
| rpmFilename := flag.String("rpm-filename", "rpm.nedb", "Filename for the RPM DB file") | ||
| destDir := flag.String("destination-dir", "/tmp/osv-offline", "Destination directory for the OSV DB files") | ||
| days := flag.Int("days", 90, "Only advisories created in the last X days are included") | ||
|
|
||
| flag.Parse() | ||
| err := os.MkdirAll(*destDir, 0755) | ||
| if err != nil { | ||
| fmt.Println("failed to create destination path: ", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| err = osv_downloader.DownloadOsvDb(*destDir) | ||
| if err != nil { | ||
| fmt.Println("Downloading the OSV database has failed: ", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| osv_generator.GenerateOSV(filepath.Join(*destDir, *dockerFilename), true, *days) | ||
| if err != nil { | ||
| fmt.Println("Generating the container OSV database has failed: ", err) | ||
| os.Exit(1) | ||
| } | ||
| osv_generator.GenerateOSV(filepath.Join(*destDir, *rpmFilename), false, *days) | ||
| if err != nil { | ||
| fmt.Println("Generating the RPM OSV database has failed: ", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| package osv_downloader | ||
|
|
||
| import ( | ||
| "archive/zip" | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/google/go-github/v45/github" | ||
| ) | ||
|
|
||
| // Download and extract the upstream OSV database | ||
| func DownloadOsvDb(path string) error { | ||
| fmt.Println("getting osv-offline.zip URL from Github") | ||
| client := github.NewClient(nil) | ||
| release, _, err := client.Repositories.GetLatestRelease(context.Background(), "renovatebot", "osv-offline") | ||
| if err != nil { | ||
| return fmt.Errorf("error when accessing latest osv-offline release: %s", err) | ||
| } | ||
|
|
||
| var zipped_db *github.ReleaseAsset | ||
| for _, asset := range release.Assets { | ||
| if *asset.Name == "osv-offline.zip" { | ||
| zipped_db = asset | ||
| break | ||
| } | ||
| } | ||
| if zipped_db == nil { | ||
| return fmt.Errorf("osv-offline.zip asset couldn't be found in the latest release") | ||
| } | ||
| archive_path := filepath.Join(path, "osv-offline.zip") | ||
| err = downloadFile(*zipped_db.BrowserDownloadURL, archive_path) | ||
| if err != nil { | ||
| return fmt.Errorf("error when downloading the osv-offline file: %s", err) | ||
| } | ||
| err = unzipFile(archive_path, path) | ||
| if err != nil { | ||
| return fmt.Errorf("error when unzipping the osv-offline file: %s", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func downloadFile(url string, filepath string) error { | ||
| fmt.Println("downloading osv-offline database") | ||
| resp, err := http.Get(url) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| out, err := os.Create(filepath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer out.Close() | ||
|
|
||
| _, err = io.Copy(out, resp.Body) | ||
| return err | ||
| } | ||
|
|
||
| func unzipFile(archive_path string, destination string) error { | ||
| fmt.Println("unzipping osv-offline database") | ||
| archive, err := zip.OpenReader(archive_path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer archive.Close() | ||
|
|
||
| for _, file := range archive.File { | ||
| filePath := filepath.Join(destination, file.Name) | ||
| if file.FileInfo().IsDir() { | ||
| os.MkdirAll(filePath, os.ModePerm) | ||
| continue | ||
| } | ||
| if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { | ||
| return err | ||
| } | ||
| destFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| fileInArchive, err := file.Open() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if _, err := io.Copy(destFile, fileInArchive); err != nil { | ||
| return err | ||
| } | ||
| destFile.Close() | ||
| fileInArchive.Close() | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package osv_generator | ||
|
|
||
| type VEX struct { | ||
| Document struct { | ||
| AggregateSeverity struct { | ||
| Text string `json:"text"` | ||
| } `json:"aggregate_severity"` | ||
| } `json:"document"` | ||
| Vulnerabilities []*Vulnerability `json:"vulnerabilities"` | ||
| ProductTree struct { | ||
| Branches []struct { | ||
| Branches []struct { | ||
| Category string `json:"category"` | ||
| Branches []struct { | ||
| Product struct { | ||
| ProductIdentificationHelper struct { | ||
| Purl string `json:"purl"` | ||
| } `json:"product_identification_helper"` | ||
| } `json:"product"` | ||
| } `json:"branches"` | ||
| } `json:"branches"` | ||
| } `json:"branches"` | ||
| } `json:"product_tree"` | ||
| } | ||
|
|
||
| type Vulnerability struct { | ||
| Cve string `json:"cve"` | ||
| DiscoveryDate string `json:"discovery_date"` | ||
| Cwe struct { | ||
| Id string `json:"id"` | ||
| } `json:"cwe"` | ||
| References []struct { | ||
| Category string `json:"category"` | ||
| Url string `json:"url"` | ||
| } `json:"references"` | ||
| Notes []struct { | ||
| Category string `json:"category"` | ||
| Text string `json:"text"` | ||
| } `json:"notes"` | ||
| } |
Oops, something went wrong.