Skip to content

Commit

Permalink
Refactor integ test to table test (#1390)
Browse files Browse the repository at this point in the history 
To make it easier to see which tests fail if there's a failure.

Signed-off-by: Will Murphy <will.murphy@anchore.com>
  • Loading branch information
@willmurphyscode
willmurphyscode authored Jul 18, 2023
1 parent 43bcf30 commit e347e03
Showing 1 changed file with 184 additions and 46 deletions.
230 changes: 184 additions & 46 deletions test/integration/compare_sbom_input_vs_lib_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,6 @@ func getListingURL() string {
}

func TestCompareSBOMInputToLibResults(t *testing.T) {
formats := []sbom.FormatID{
syft.JSONFormatID,
syft.SPDXJSONFormatID,
syft.SPDXTagValueFormatID,
}

// get a grype DB
store, _, closer, err := grype.LoadVulnerabilityDB(db.Config{
DBRootDir: "test-fixtures/grype-db",
Expand Down Expand Up @@ -77,51 +71,195 @@ func TestCompareSBOMInputToLibResults(t *testing.T) {
string(syftPkg.Rpkg),
)
observedPkgTypes := strset.New()
testCases := []struct {
name string
image string
format sbom.FormatID
}{
{
image: "anchore/test_images:vulnerabilities-alpine",
format: syft.JSONFormatID,
name: "alpine-syft-json",
},

{
image: "anchore/test_images:vulnerabilities-alpine",
format: syft.SPDXJSONFormatID,
name: "alpine-spdx-json",
},

{
image: "anchore/test_images:vulnerabilities-alpine",
format: syft.SPDXTagValueFormatID,
name: "alpine-spdx-tag-value",
},

{
image: "anchore/test_images:gems",
format: syft.JSONFormatID,
name: "gems-syft-json",
},

{
image: "anchore/test_images:gems",
format: syft.SPDXJSONFormatID,
name: "gems-spdx-json",
},

{
image: "anchore/test_images:gems",
format: syft.SPDXTagValueFormatID,
name: "gems-spdx-tag-value",
},

{
image: "anchore/test_images:vulnerabilities-debian",
format: syft.JSONFormatID,
name: "debian-syft-json",
},

{
image: "anchore/test_images:vulnerabilities-debian",
format: syft.SPDXJSONFormatID,
name: "debian-spdx-json",
},

{
image: "anchore/test_images:vulnerabilities-debian",
format: syft.SPDXTagValueFormatID,
name: "debian-spdx-tag-value",
},

{
image: "anchore/test_images:vulnerabilities-centos",
format: syft.JSONFormatID,
name: "centos-syft-json",
},

{
image: "anchore/test_images:vulnerabilities-centos",
format: syft.SPDXJSONFormatID,
name: "centos-spdx-json",
},

{
image: "anchore/test_images:vulnerabilities-centos",
format: syft.SPDXTagValueFormatID,
name: "centos-spdx-tag-value",
},

{
image: "anchore/test_images:npm",
format: syft.JSONFormatID,
name: "npm-syft-json",
},

{
image: "anchore/test_images:npm",
format: syft.SPDXJSONFormatID,
name: "npm-spdx-json",
},

{
image: "anchore/test_images:npm",
format: syft.SPDXTagValueFormatID,
name: "npm-spdx-tag-value",
},

{
image: "anchore/test_images:java",
format: syft.JSONFormatID,
name: "java-syft-json",
},

for _, image := range imagesWithVulnerabilities {
imageArchive := PullThroughImageCache(t, image)
{
image: "anchore/test_images:java",
format: syft.SPDXJSONFormatID,
name: "java-spdx-json",
},

{
image: "anchore/test_images:java",
format: syft.SPDXTagValueFormatID,
name: "java-spdx-tag-value",
},

{
image: "anchore/test_images:golang-56d52bc",
format: syft.JSONFormatID,
name: "go-syft-json",
},

{
image: "anchore/test_images:golang-56d52bc",
format: syft.SPDXJSONFormatID,
name: "go-spdx-json",
},

{
image: "anchore/test_images:golang-56d52bc",
format: syft.SPDXTagValueFormatID,
name: "go-spdx-tag-value",
},

{
image: "anchore/test_images:arch",
format: syft.JSONFormatID,
name: "arch-syft-json",
},

{
image: "anchore/test_images:arch",
format: syft.SPDXJSONFormatID,
name: "arch-spdx-json",
},

{
image: "anchore/test_images:arch",
format: syft.SPDXTagValueFormatID,
name: "arch-spdx-tag-value",
},
}
for _, tc := range testCases {
imageArchive := PullThroughImageCache(t, tc.image)
imageSource := fmt.Sprintf("docker-archive:%s", imageArchive)
f := syft.FormatByID(tc.format)
if f == nil {
t.Errorf("Invalid formatID: %s", tc.format)
}
t.Run(tc.name, func(t *testing.T) {
// get SBOM from syft, write to temp file
sbomBytes := getSyftSBOM(t, imageSource, f)
sbomFile, err := os.CreateTemp("", "")
assert.NoError(t, err)
t.Cleanup(func() {
assert.NoError(t, os.Remove(sbomFile.Name()))
})
_, err = sbomFile.WriteString(sbomBytes)
assert.NoError(t, err)
assert.NoError(t, sbomFile.Close())

// get vulns (sbom)
matchesFromSbom, _, pkgsFromSbom, err := grype.FindVulnerabilities(*store, fmt.Sprintf("sbom:%s", sbomFile.Name()), source.SquashedScope, nil)
assert.NoError(t, err)

// get vulns (image)
matchesFromImage, _, _, err := grype.FindVulnerabilities(*store, imageSource, source.SquashedScope, nil)
assert.NoError(t, err)

for _, formatID := range formats {
f := syft.FormatByID(formatID)
if f == nil {
t.Errorf("Invalid formatID: %s", formatID)
// compare packages (shallow)
matchSetFromSbom := getMatchSet(matchesFromSbom)
matchSetFromImage := getMatchSet(matchesFromImage)

assert.Empty(t, strset.Difference(matchSetFromSbom, matchSetFromImage).List(), "vulnerabilities present only in results when using sbom as input")
assert.Empty(t, strset.Difference(matchSetFromImage, matchSetFromSbom).List(), "vulnerabilities present only in results when using image as input")

// track all covered package types (for use after the test)
for _, p := range pkgsFromSbom {
observedPkgTypes.Add(string(p.Type))
}
t.Run(fmt.Sprintf("%s/%s", image, formatID), func(t *testing.T) {

// get SBOM from syft, write to temp file
sbomBytes := getSyftSBOM(t, imageSource, f)
sbomFile, err := os.CreateTemp("", "")
assert.NoError(t, err)
t.Cleanup(func() {
assert.NoError(t, os.Remove(sbomFile.Name()))
})
_, err = sbomFile.WriteString(sbomBytes)
assert.NoError(t, err)
assert.NoError(t, sbomFile.Close())

// get vulns (sbom)
matchesFromSbom, _, pkgsFromSbom, err := grype.FindVulnerabilities(*store, fmt.Sprintf("sbom:%s", sbomFile.Name()), source.SquashedScope, nil)
assert.NoError(t, err)

// get vulns (image)
matchesFromImage, _, _, err := grype.FindVulnerabilities(*store, imageSource, source.SquashedScope, nil)
assert.NoError(t, err)

// compare packages (shallow)
matchSetFromSbom := getMatchSet(matchesFromSbom)
matchSetFromImage := getMatchSet(matchesFromImage)

assert.Empty(t, strset.Difference(matchSetFromSbom, matchSetFromImage).List(), "vulnerabilities present only in results when using sbom as input")
assert.Empty(t, strset.Difference(matchSetFromImage, matchSetFromSbom).List(), "vulnerabilities present only in results when using image as input")

// track all covered package types (for use after the test)
for _, p := range pkgsFromSbom {
observedPkgTypes.Add(string(p.Type))
}

})
}
})
}

// ensure we've covered all package types (-rust, -kb)
Expand Down

0 comments on commit e347e03

Please sign in to comment.