Merge pull request #69 from puerco/micro-sboms

Generate purls from read sources

go.mod

@@ -55,6 +55,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/package-url/packageurl-go v0.1.1-0.20220203205134-d70459300c8a
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.3.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect

go.sum

@@ -696,6 +696,8 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/package-url/packageurl-go v0.1.1-0.20220203205134-d70459300c8a h1:tkTSd1nhioPqi5Whu3CQ79UjPtaGOytqyNnSCVOqzHM=
+github.com/package-url/packageurl-go v0.1.1-0.20220203205134-d70459300c8a/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

pkg/osinfo/container_scanner.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"strings"
 
+	purl "github.com/package-url/packageurl-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -48,10 +49,22 @@ func (ct *ContainerScanner) ReadOSPackages(layers []string) (
 		}
 	}
 
-	if osKind == OSDebian {
-		return ct.ReadDebianPackages(layers)
+	purlType := ""
+
+	switch osKind {
+	case OSDebian, OSUbuntu:
+		layerNum, packages, err = ct.ReadDebianPackages(layers)
+		purlType = purl.TypeDebian
+	default:
+		return 0, nil, nil
+	}
+
+	for i := range *packages {
+		(*packages)[i].Type = purlType
+		(*packages)[i].Namespace = osKind
 	}
-	return 0, nil, nil
+
+	return layerNum, packages, err
 }
 
 // ReadDebianPackages scans through a set of container layers looking for the
@@ -89,11 +102,41 @@ func (ct *ContainerScanner) ReadDebianPackages(layers []string) (layer int, pk *
 }
 
 type PackageDBEntry struct {
-	Package      string
-	Version      string
-	Architecture string
+	Package         string
+	Version         string
+	Architecture    string
+	Type            string // purl package type (ref: https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst)
+	Namespace       string // purl namespace
+	MaintainerName  string
+	MaintainerEmail string
+	HomePage        string
+}
+
+// PackageURL returns a purl representing the db entry. If the entry
+// does not have enough data to generate the purl, it will return an
+// empty string
+func (e *PackageDBEntry) PackageURL() string {
+	// We require type, package, namespace and version at the very
+	// least to generate a purl
+	if e.Package == "" || e.Version == "" || e.Namespace == "" || e.Type == "" {
+		return ""
+	}
+
+	qualifiersMap := map[string]string{}
+
+	// Add the architecture
+	// TODO(puerco): Support adding the distro
+	if e.Architecture != "" {
+		qualifiersMap["arch"] = e.Architecture
+	}
+	return purl.NewPackageURL(
+		e.Type, e.Namespace, e.Package,
+		e.Version, purl.QualifiersFromMap(qualifiersMap), "",
+	).ToString()
 }
 
+// parseDpkgDB reads a dpks database and populates a slice of PackageDBEntry
+// with information from the packages found
 func (ct *ContainerScanner) parseDpkgDB(dbPath string) (*[]PackageDBEntry, error) {
 	file, err := os.Open(dbPath)
 	if err != nil {
@@ -105,28 +148,39 @@ func (ct *ContainerScanner) parseDpkgDB(dbPath string) (*[]PackageDBEntry, error
 	scanner := bufio.NewScanner(file)
 	var curPkg *PackageDBEntry
 	for scanner.Scan() {
-		if strings.HasPrefix(scanner.Text(), "Package:") {
+		parts := strings.SplitN(scanner.Text(), ":", 2)
+		if len(parts) < 2 {
+			continue
+		}
+
+		switch parts[0] {
+		case "Package":
 			if curPkg != nil {
 				db = append(db, *curPkg)
 			}
 			curPkg = &PackageDBEntry{
-				Package: strings.TrimSpace(strings.TrimPrefix(scanner.Text(), "Package:")),
+				Package: strings.TrimSpace(parts[1]),
+				Type:    purl.TypeDebian,
 			}
-		}
-
-		if strings.HasPrefix(scanner.Text(), "Architecture:") {
+		case "Architecture":
 			if curPkg != nil {
-				curPkg.Architecture = strings.TrimSpace(
-					strings.TrimPrefix(scanner.Text(), "Architecture:"),
-				)
+				curPkg.Architecture = strings.TrimSpace(parts[1])
 			}
-		}
-
-		if strings.HasPrefix(scanner.Text(), "Version:") {
+		case "Version":
+			if curPkg != nil {
+				curPkg.Version = strings.TrimSpace(parts[1])
+			}
+		case "Homepage":
+			if curPkg != nil {
+				curPkg.HomePage = strings.TrimSpace(parts[1])
+			}
+		case "Maintainer":
 			if curPkg != nil {
-				curPkg.Version = strings.TrimSpace(
-					strings.TrimPrefix(scanner.Text(), "Version:"),
-				)
+				mparts := strings.SplitN(parts[1], "<", 2)
+				if len(mparts) == 2 {
+					curPkg.MaintainerName = strings.TrimSpace(mparts[0])
+					curPkg.MaintainerEmail = strings.TrimSuffix(strings.TrimSpace(mparts[1]), ">")
+				}
 			}
 		}
 	}

pkg/osinfo/container_scanner_test.go

@@ -17,8 +17,12 @@ limitations under the License.
 package osinfo
 
 import (
+	"fmt"
+	"net/url"
+	"strings"
 	"testing"
 
+	purl "github.com/package-url/packageurl-go"
 	"github.com/stretchr/testify/require"
 )
 
@@ -78,3 +82,64 @@ func TestReadOSPackages(t *testing.T) {
 	_, _, err = ct.ReadOSPackages([]string{"testdata/nonexistent"})
 	require.Error(t, err)
 }
+
+func TestPackageURL(t *testing.T) {
+	for _, tc := range []struct {
+		dbe      PackageDBEntry
+		expected string
+	}{
+		{
+			// Emtpty db entry
+			dbe:      PackageDBEntry{},
+			expected: "",
+		},
+		{
+			// Only package
+			dbe:      PackageDBEntry{Package: "test"},
+			expected: "",
+		},
+		{
+			// Emtpty db entry
+			dbe: PackageDBEntry{
+				Package: "test", Namespace: "osname",
+			},
+			expected: "",
+		},
+		{
+			// Tyoe missing
+			dbe: PackageDBEntry{
+				Package: "test", Version: "v1.0.0", Namespace: "osname",
+			},
+			expected: "",
+		},
+		{
+			// Minimum elements
+			dbe: PackageDBEntry{
+				Package: "test", Version: "v1.0.0", Type: purl.TypeDebian, Namespace: "osname",
+			},
+			expected: "pkg:deb/osname/test@v1.0.0",
+		},
+		{
+			// All but type
+			dbe: PackageDBEntry{
+				Package: "test", Version: "v1.0.0", Architecture: "amd64",
+				Type: purl.TypeDebian, Namespace: "osname",
+			},
+			expected: "pkg:deb/osname/test@v1.0.0?arch=amd64",
+		},
+	} {
+		p := tc.dbe.PackageURL()
+		require.Equal(t, tc.expected, p)
+		if p == "" {
+			continue
+		}
+		parsed, err := url.Parse(p)
+		require.NoError(t, err)
+		require.Equal(t, "pkg", parsed.Scheme)
+		require.True(t, strings.HasPrefix(p, fmt.Sprintf(
+			"pkg:%s/%s/%s@%s", tc.dbe.Type, tc.dbe.Namespace,
+			tc.dbe.Package, tc.dbe.Version,
+		)))
+		require.Equal(t, tc.dbe.Architecture, parsed.Query().Get("arch"))
+	}
+}

pkg/osinfo/layer_scanner.go

@@ -30,7 +30,10 @@ import (
 
 const (
 	OSDebian     = "debian"
+	OSUbuntu     = "ubuntu"
 	OSFedora     = "fedora"
+	OSCentos     = "centos"
+	OSRHEL       = "rhel"
 	OSAlpine     = "alpine"
 	OSDistroless = "distroless"
 )
@@ -60,13 +63,21 @@ func (loss *LayerScanner) OSType(layerPath string) (ostype string, err error) {
 	}
 
 	if strings.Contains(osrelease, "NAME=\"Ubuntu\"") {
-		return OSDebian, nil
+		return OSUbuntu, nil
 	}
 
 	if strings.Contains(osrelease, "NAME=\"Fedora Linux\"") {
 		return OSFedora, nil
 	}
 
+	if strings.Contains(osrelease, "NAME=\"CentOS Linux\"") {
+		return OSCentos, nil
+	}
+
+	if strings.Contains(osrelease, "NAME=\"Red Hat Enterprise Linux\"") {
+		return OSRHEL, nil
+	}
+
 	if strings.Contains(osrelease, "NAME=\"Alpine Linux\"") {
 		return OSAlpine, nil
 	}

pkg/spdx/gomod.go

@@ -18,13 +18,15 @@ package spdx
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
 
 	"github.com/nozzle/throttler"
+	purl "github.com/package-url/packageurl-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/mod/modfile"
@@ -100,13 +102,49 @@ func (pkg *GoPackage) ToSPDXPackage() (*Package, error) {
 		spdxPackage.Name += "@" + strings.TrimSuffix(pkg.Revision, "+incompatible")
 	}
 	spdxPackage.BuildID()
-	spdxPackage.DownloadLocation = repo.Repo
+	if strings.Contains(pkg.Revision, "+incompatible") {
+		spdxPackage.DownloadLocation = repo.VCS.Scheme[0] + "+" + repo.Repo
+	} else {
+		spdxPackage.DownloadLocation = fmt.Sprintf(
+			"https://proxy.golang.org/%s/@v/%s.zip", pkg.ImportPath,
+			strings.TrimSuffix(pkg.Revision, "+incompatible"),
+		)
+	}
 	spdxPackage.LicenseConcluded = pkg.LicenseID
 	spdxPackage.Version = strings.TrimSuffix(pkg.Revision, "+incompatible")
 	spdxPackage.CopyrightText = pkg.CopyrightText
+	if packageurl := pkg.PackageURL(); packageurl != "" {
+		spdxPackage.ExternalRefs = append(spdxPackage.ExternalRefs, ExternalRef{
+			Category: "PACKAGE-MANAGER",
+			Type:     "purl",
+			Locator:  packageurl,
+		})
+	}
 	return spdxPackage, nil
 }
 
+// PackageURL returns a purl if the go package has enough data to generate
+// one. If data is missing, it will return an empty string
+func (pkg *GoPackage) PackageURL() string {
+	parts := strings.Split(pkg.ImportPath, "/")
+	if len(parts) < 2 {
+		return ""
+	}
+	pname := parts[len(parts)-1]
+	namespace := strings.TrimSuffix(parts[0], "/"+pname)
+
+	// We require type, package, namespace and version at the very
+	// least to generate a purl
+	if pname == "" || pkg.Revision == "" || namespace == "" {
+		return ""
+	}
+
+	return purl.NewPackageURL(
+		purl.TypeGolang, namespace, pname,
+		strings.TrimSuffix(pkg.Revision, "+incompatible"), nil, "",
+	).ToString()
+}
+
 type GoModImplementation interface {
 	OpenModule(*GoModuleOptions) (*modfile.File, error)
 	BuildPackageList(*modfile.File) ([]*GoPackage, error)

pkg/spdx/gomod_test.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package spdx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPackageURL(t *testing.T) {
+	for _, tc := range []struct {
+		pkg      GoPackage
+		expected string
+	}{
+		// No error
+		{GoPackage{ImportPath: "package/name", Revision: "v1.0.0"}, "pkg:golang/package/name@v1.0.0"},
+		// No import path
+		{GoPackage{ImportPath: "", Revision: "v1.0.0"}, ""},
+		// Incomplete import path
+		{GoPackage{ImportPath: "package", Revision: "v1.0.0"}, ""},
+		// No revision
+		{GoPackage{ImportPath: "package/name", Revision: ""}, ""},
+	} {
+		require.Equal(t, tc.expected, tc.pkg.PackageURL())
+	}
+}