From e347e03f4d7a370550d3ca16c736a740cf3c5651 Mon Sep 17 00:00:00 2001 From: William Murphy Date: Tue, 18 Jul 2023 11:27:46 -0400 Subject: [PATCH] Refactor integ test to table test (#1390) To make it easier to see which tests fail if there's a failure. Signed-off-by: Will Murphy --- .../compare_sbom_input_vs_lib_test.go | 230 ++++++++++++++---- 1 file changed, 184 insertions(+), 46 deletions(-) diff --git a/test/integration/compare_sbom_input_vs_lib_test.go b/test/integration/compare_sbom_input_vs_lib_test.go index 0caed5ac565..c903f48a496 100644 --- a/test/integration/compare_sbom_input_vs_lib_test.go +++ b/test/integration/compare_sbom_input_vs_lib_test.go @@ -36,12 +36,6 @@ func getListingURL() string { } func TestCompareSBOMInputToLibResults(t *testing.T) { - formats := []sbom.FormatID{ - syft.JSONFormatID, - syft.SPDXJSONFormatID, - syft.SPDXTagValueFormatID, - } - // get a grype DB store, _, closer, err := grype.LoadVulnerabilityDB(db.Config{ DBRootDir: "test-fixtures/grype-db", @@ -77,51 +71,195 @@ func TestCompareSBOMInputToLibResults(t *testing.T) { string(syftPkg.Rpkg), ) observedPkgTypes := strset.New() + testCases := []struct { + name string + image string + format sbom.FormatID + }{ + { + image: "anchore/test_images:vulnerabilities-alpine", + format: syft.JSONFormatID, + name: "alpine-syft-json", + }, + + { + image: "anchore/test_images:vulnerabilities-alpine", + format: syft.SPDXJSONFormatID, + name: "alpine-spdx-json", + }, + + { + image: "anchore/test_images:vulnerabilities-alpine", + format: syft.SPDXTagValueFormatID, + name: "alpine-spdx-tag-value", + }, + + { + image: "anchore/test_images:gems", + format: syft.JSONFormatID, + name: "gems-syft-json", + }, + + { + image: "anchore/test_images:gems", + format: syft.SPDXJSONFormatID, + name: "gems-spdx-json", + }, + + { + image: "anchore/test_images:gems", + format: syft.SPDXTagValueFormatID, + name: "gems-spdx-tag-value", + }, + + { + image: "anchore/test_images:vulnerabilities-debian", + format: syft.JSONFormatID, + name: "debian-syft-json", + }, + + { + image: "anchore/test_images:vulnerabilities-debian", + format: syft.SPDXJSONFormatID, + name: "debian-spdx-json", + }, + + { + image: "anchore/test_images:vulnerabilities-debian", + format: syft.SPDXTagValueFormatID, + name: "debian-spdx-tag-value", + }, + + { + image: "anchore/test_images:vulnerabilities-centos", + format: syft.JSONFormatID, + name: "centos-syft-json", + }, + + { + image: "anchore/test_images:vulnerabilities-centos", + format: syft.SPDXJSONFormatID, + name: "centos-spdx-json", + }, + + { + image: "anchore/test_images:vulnerabilities-centos", + format: syft.SPDXTagValueFormatID, + name: "centos-spdx-tag-value", + }, + + { + image: "anchore/test_images:npm", + format: syft.JSONFormatID, + name: "npm-syft-json", + }, + + { + image: "anchore/test_images:npm", + format: syft.SPDXJSONFormatID, + name: "npm-spdx-json", + }, + + { + image: "anchore/test_images:npm", + format: syft.SPDXTagValueFormatID, + name: "npm-spdx-tag-value", + }, + + { + image: "anchore/test_images:java", + format: syft.JSONFormatID, + name: "java-syft-json", + }, - for _, image := range imagesWithVulnerabilities { - imageArchive := PullThroughImageCache(t, image) + { + image: "anchore/test_images:java", + format: syft.SPDXJSONFormatID, + name: "java-spdx-json", + }, + + { + image: "anchore/test_images:java", + format: syft.SPDXTagValueFormatID, + name: "java-spdx-tag-value", + }, + + { + image: "anchore/test_images:golang-56d52bc", + format: syft.JSONFormatID, + name: "go-syft-json", + }, + + { + image: "anchore/test_images:golang-56d52bc", + format: syft.SPDXJSONFormatID, + name: "go-spdx-json", + }, + + { + image: "anchore/test_images:golang-56d52bc", + format: syft.SPDXTagValueFormatID, + name: "go-spdx-tag-value", + }, + + { + image: "anchore/test_images:arch", + format: syft.JSONFormatID, + name: "arch-syft-json", + }, + + { + image: "anchore/test_images:arch", + format: syft.SPDXJSONFormatID, + name: "arch-spdx-json", + }, + + { + image: "anchore/test_images:arch", + format: syft.SPDXTagValueFormatID, + name: "arch-spdx-tag-value", + }, + } + for _, tc := range testCases { + imageArchive := PullThroughImageCache(t, tc.image) imageSource := fmt.Sprintf("docker-archive:%s", imageArchive) + f := syft.FormatByID(tc.format) + if f == nil { + t.Errorf("Invalid formatID: %s", tc.format) + } + t.Run(tc.name, func(t *testing.T) { + // get SBOM from syft, write to temp file + sbomBytes := getSyftSBOM(t, imageSource, f) + sbomFile, err := os.CreateTemp("", "") + assert.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, os.Remove(sbomFile.Name())) + }) + _, err = sbomFile.WriteString(sbomBytes) + assert.NoError(t, err) + assert.NoError(t, sbomFile.Close()) + + // get vulns (sbom) + matchesFromSbom, _, pkgsFromSbom, err := grype.FindVulnerabilities(*store, fmt.Sprintf("sbom:%s", sbomFile.Name()), source.SquashedScope, nil) + assert.NoError(t, err) + + // get vulns (image) + matchesFromImage, _, _, err := grype.FindVulnerabilities(*store, imageSource, source.SquashedScope, nil) + assert.NoError(t, err) - for _, formatID := range formats { - f := syft.FormatByID(formatID) - if f == nil { - t.Errorf("Invalid formatID: %s", formatID) + // compare packages (shallow) + matchSetFromSbom := getMatchSet(matchesFromSbom) + matchSetFromImage := getMatchSet(matchesFromImage) + + assert.Empty(t, strset.Difference(matchSetFromSbom, matchSetFromImage).List(), "vulnerabilities present only in results when using sbom as input") + assert.Empty(t, strset.Difference(matchSetFromImage, matchSetFromSbom).List(), "vulnerabilities present only in results when using image as input") + + // track all covered package types (for use after the test) + for _, p := range pkgsFromSbom { + observedPkgTypes.Add(string(p.Type)) } - t.Run(fmt.Sprintf("%s/%s", image, formatID), func(t *testing.T) { - - // get SBOM from syft, write to temp file - sbomBytes := getSyftSBOM(t, imageSource, f) - sbomFile, err := os.CreateTemp("", "") - assert.NoError(t, err) - t.Cleanup(func() { - assert.NoError(t, os.Remove(sbomFile.Name())) - }) - _, err = sbomFile.WriteString(sbomBytes) - assert.NoError(t, err) - assert.NoError(t, sbomFile.Close()) - - // get vulns (sbom) - matchesFromSbom, _, pkgsFromSbom, err := grype.FindVulnerabilities(*store, fmt.Sprintf("sbom:%s", sbomFile.Name()), source.SquashedScope, nil) - assert.NoError(t, err) - - // get vulns (image) - matchesFromImage, _, _, err := grype.FindVulnerabilities(*store, imageSource, source.SquashedScope, nil) - assert.NoError(t, err) - - // compare packages (shallow) - matchSetFromSbom := getMatchSet(matchesFromSbom) - matchSetFromImage := getMatchSet(matchesFromImage) - - assert.Empty(t, strset.Difference(matchSetFromSbom, matchSetFromImage).List(), "vulnerabilities present only in results when using sbom as input") - assert.Empty(t, strset.Difference(matchSetFromImage, matchSetFromSbom).List(), "vulnerabilities present only in results when using image as input") - - // track all covered package types (for use after the test) - for _, p := range pkgsFromSbom { - observedPkgTypes.Add(string(p.Type)) - } - }) - } + }) } // ensure we've covered all package types (-rust, -kb)