Skip to content

Commit

Permalink
Get advisories based on changes.csv
Browse files Browse the repository at this point in the history
  • Loading branch information
nvtnlucie committed Dec 4, 2024
1 parent 52443ea commit 56e84e6
Show file tree
Hide file tree
Showing 4 changed files with 170 additions and 4 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/jarcoal/httpmock v1.3.1
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
Expand All @@ -95,6 +97,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down
46 changes: 42 additions & 4 deletions tools/osv-generator/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,17 @@ import (
"encoding/csv"
"fmt"
"net/http"
"sort"
"strings"
"time"
)

const URL = "https://security.access.redhat.com/data/csaf/v2/advisories"

// Get list of new advisories from the last `days`
func getAdvisoryList(days int) ([]string, error) {
response, err := http.Get(fmt.Sprintf("%s/%s", URL, "changes.csv"))
// Get list of new advisories published in the last `days`
// TODO: Do not use until published.csv is periodically generated
func getAdvisoryListByPublished(days int) ([]string, error) {
response, err := http.Get(fmt.Sprintf("%s/%s", URL, "published.csv"))
if err != nil {
fmt.Println("Error downloading file:", err)
return nil, err
Expand Down Expand Up @@ -40,13 +43,48 @@ func getAdvisoryList(days int) ([]string, error) {
advisories = append(advisories, record[0])

if advisoryDate.Before(dateThreshold) {
// All recently modified advisories were found
fmt.Printf("Found %d new advisories\n", len(advisories))
break
}
}

return advisories, nil
}

// Get list of new advisories sorted by name, returns maximum of `limit` advisories
func getAdvisoryListByModified(limit int) ([]string, error) {
response, err := http.Get(fmt.Sprintf("%s/%s", URL, "changes.csv"))
if err != nil {
fmt.Println("Error downloading file:", err)
return nil, err
}
defer response.Body.Close()

// Load all advisories from the document
csvReader := csv.NewReader(response.Body)
records, err := csvReader.ReadAll()
if err != nil {
fmt.Println("Error parsing CSV:", err)
return nil, err
}

// Sort records by name in descending order to get newest advisories first
sort.Slice(records, func(i, j int) bool {
return records[i][0] > records[j][0]
})

var advisories []string
for _, record := range records {
// Add only RHSA advisories for now
if strings.Contains(record[0], "rhsa-") {
advisories = append(advisories, record[0])
if len(advisories) == limit {
break
}
}
}
return advisories, nil
}

// Extract advisory data from the given URL, store as list of OSV objects
Expand All @@ -62,7 +100,7 @@ func extractAdvisory(advisory string) []OSV {
// Generate OSV vulnerabilities from CSAF VEX data and store to a file
func GenerateOSV(filename string) error {
var osvList []OSV
advisories, err := getAdvisoryList(1) // Advisories created in the last 24 hours
advisories, err := getAdvisoryListByModified(200)
if err != nil {
return err
}
Expand Down
123 changes: 123 additions & 0 deletions tools/osv-generator/generator_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package osv_generator

import (
"os"
"regexp"
"strings"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/jarcoal/httpmock"
)

var sampleCSV = []byte(`2023/rhsa-2023_6919.json,2024-12-02T08:22:15+00:00
2023/rhba-2023_6330.json,2024-12-02T08:22:03+00:00
2022/rhba-2022_5476.json,2024-12-02T07:52:26+00:00
2022/rhsa-2022_5249.json,2024-12-02T07:52:17+00:00
2024/rhsa-2024_5439.json,2024-12-02T07:52:10+00:00`)

var sampleOSVResult = `{"schema_version":"1.6.0","id":"CVE","database_specific":{"severity":"Moderate","cwe_ids":["CWE"]},"published":"2024-08-20T10:54:54Z","summary":"summary","details":"description","affected":[{"package":{"ecosystem":"Red Hat","name":"redhat/openstack","purl":"pkg:rpm/redhat/openstack@2.0.0"},"ranges":[{"type":"ECOSYSTEM","events":[{"introduced":"0.0.0"},{"fixed":"2.0.0"}]}]}],"references":[{"type":"WEB","url":"fake-url"}]}`

func TestGetAdvisoryListByModified(t *testing.T) {
// Create a mock
httpmock.Activate()
t.Cleanup(httpmock.DeactivateAndReset)
httpmock.RegisterResponder("GET", "https://security.access.redhat.com/data/csaf/v2/advisories/changes.csv",
httpmock.NewBytesResponder(200, sampleCSV))

advisories, err := getAdvisoryListByModified(10)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}

expected := []string{"2024/rhsa-2024_5439.json", "2023/rhsa-2023_6919.json", "2022/rhsa-2022_5249.json"}
if !cmp.Equal(advisories, expected) {
t.Fatalf("Different result: %v", cmp.Diff(expected, advisories))
}
}

// E2E test of the whole functionality of OSV module
func TestGenerateOSV(t *testing.T) {
// Create a mock
httpmock.Activate()
t.Cleanup(httpmock.DeactivateAndReset)
httpmock.RegisterResponder("GET", "https://security.access.redhat.com/data/csaf/v2/advisories/changes.csv",
httpmock.NewBytesResponder(200, []byte(`test-rhsa-advisory.json,2024-12-02T07:52:10+00:00`)))
// Set up a fake advisory
httpmock.RegisterResponder("GET", "https://security.access.redhat.com/data/csaf/v2/advisories/test-rhsa-advisory.json",
httpmock.NewBytesResponder(200, []byte(`{
"document": {
"aggregate_severity": {
"text": "Moderate"
}
},
"vulnerabilities": [
{
"cve": "CVE",
"discovery_date": "2024-08-20T10:54:54.042000+00:00",
"cwe": {
"id": "CWE"
},
"references": [
{
"category": "REPORT",
"url": "fake-url"
}
],
"notes": [
{
"category": "summary",
"text": "summary"
},
{
"category": "description",
"text": "description"
}
]
}
],
"product_tree": {
"branches": [
{
"branches": [
{
"category": "architecture",
"branches": [
{
"product": {
"product_identification_helper": {
"purl": "pkg:rpm/redhat/openstack@2.0.0?arch=src"
}
}
}
]
}
]
}
]
}
}`)))

// Create a test file
err := GenerateOSV("testfile")
defer os.Remove("testfile")
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
createdFile, err := os.ReadFile("testfile")
if err != nil {
t.Fatalf("Expected no error reading the file, got %v", err)
}

// Remove the modified field since it is set to current time
re := regexp.MustCompile(`"modified":"[^"]*",`)
result := re.ReplaceAllString(string(createdFile), "")
// Remove newlines
result = strings.ReplaceAll(result, "\n", "")

// Compare the results
if result != sampleOSVResult {
t.Fatalf("Different result: %v", cmp.Diff(string(sampleOSVResult), string(result)))
}

}

0 comments on commit 56e84e6

Please sign in to comment.