support for scanning license files in golang packages

Signed-off-by: Avi Deitcher <avi@deitcher.net>

README.md

@@ -504,6 +504,14 @@ golang:
    # SYFT_GOLANG_LOCAL_MOD_CACHE_DIR env var
    local-mod-cache-dir: ""
 
+   # search for go package licences by retrieving the package from the Internet
+   # SYFT_GOLANG_SEARCH_INTERNET_LICENSES env var
+   search-internet-licenses: false
+
+   # remote proxy to use when retrieving go packages from the Internet
+   # SYFT_GOLANG_REMOTE_PROXY env var
+   remote-proxy: https://proxy.golang.org
+
 # cataloging file contents is exposed through the power-user subcommand
 file-contents:
   cataloger:

cmd/syft/cli/options/packages.go

@@ -22,6 +22,8 @@ type PackagesOptions struct {
 	Exclude            []string
 	Catalogers         []string
 	Name               string
+	GoFetchPackages    bool
+	GoProxy            string
 }
 
 var _ Interface = (*PackagesOptions)(nil)
@@ -51,6 +53,12 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
 	cmd.Flags().StringVarP(&o.Name, "name", "", "",
 		"set the name of the target being analyzed")
 
+	cmd.Flags().BoolVarP(&o.GoFetchPackages, "go-fetch", "", false,
+		"enable fetching of Go packages from the internet for license analysis, otherwise will look only in local")
+
+	cmd.Flags().StringVarP(&o.GoProxy, "go-proxy", "", "https://proxy.golang.org",
+		"proxy to use when fetching Go packages from the internet for license analysis; used only if --go-fetch is set")
+
 	return bindPackageConfigOptions(cmd.Flags(), v)
 }
 
@@ -90,5 +98,13 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 		return err
 	}
 
+	if err := v.BindPFlag("go-fetch", flags.Lookup("go-fetch")); err != nil {
+		return err
+	}
+
+	if err := v.BindPFlag("go-proxy", flags.Lookup("go-proxy")); err != nil {
+		return err
+	}
+
 	return nil
 }

go.mod

@@ -54,7 +54,7 @@ require (
 	github.com/anchore/go-logger v0.0.0-20220728155337-03b66a5207d8
 	github.com/anchore/stereoscope v0.0.0-20230317134707-7928713c391e
 	github.com/docker/docker v23.0.1+incompatible
-	github.com/google/go-containerregistry v0.14.0
+	github.com/google/go-containerregistry v0.13.0
 	github.com/google/licensecheck v0.3.1
 	github.com/invopop/jsonschema v0.7.0
 	github.com/knqyf263/go-rpmdb v0.0.0-20221030135625-4082a22221ce

go.sum

@@ -261,8 +261,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
-github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
+github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k=
+github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/licensecheck v0.3.1 h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs=
 github.com/google/licensecheck v0.3.1/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY=

internal/config/application.go

@@ -60,6 +60,8 @@ type Application struct {
 	Platform           string             `yaml:"platform" json:"platform" mapstructure:"platform"`
 	Name               string             `yaml:"name" json:"name" mapstructure:"name"`
 	Parallelism        int                `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
+	GoFetch            bool               `yaml:"go-fetch" json:"go-fetch" mapstructure:"go-fetch"`
+	GoProxy            string             `yaml:"go-proxy]" json:"go-proxy" mapstructure:"go-proxy"`
 }
 
 func (cfg Application) ToCatalogerConfig() cataloger.Config {
@@ -73,7 +75,9 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config {
 		Parallelism: cfg.Parallelism,
 		Golang: golangCataloger.GoCatalogerOpts{
 			SearchLocalModCacheLicenses: cfg.Golang.SearchLocalModCacheLicenses,
+			SearchRemoteLicenses:        cfg.Golang.SearchInternetLicenses,
 			LocalModCacheDir:            cfg.Golang.LocalModCacheDir,
+			RemoteProxy:                 cfg.Golang.RemoteProxy,
 		},
 	}
 }

internal/config/golang.go

@@ -5,9 +5,13 @@ import "github.com/spf13/viper"
 type golang struct {
 	SearchLocalModCacheLicenses bool   `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
 	LocalModCacheDir            string `json:"local-mod-cache-dir" yaml:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"`
+	SearchInternetLicenses      bool   `json:"search-internet-licenses" yaml:"search-internet-licenses" mapstructure:"search-internet-licenses"`
+	RemoteProxy                 string `json:"remote-proxy" yaml:"remote-proxy" mapstructure:"remote-proxy"`
 }
 
 func (cfg golang) loadDefaultValues(v *viper.Viper) {
 	v.SetDefault("golang.search-local-mod-cache-licenses", false)
+	v.SetDefault("golang.search-internet-licenses", false)
 	v.SetDefault("golang.local-mod-cache-dir", "")
+	v.SetDefault("golang.remote-proxy", "https://proxy.golang.org")
 }

syft/pkg/cataloger/golang/cataloger.go

@@ -10,7 +10,9 @@ import (
 
 type GoCatalogerOpts struct {
 	SearchLocalModCacheLicenses bool
+	SearchRemoteLicenses        bool
 	LocalModCacheDir            string
+	RemoteProxy                 string
 }
 
 // NewGoModFileCataloger returns a new Go module cataloger object.

syft/pkg/cataloger/golang/licenses.go

@@ -1,9 +1,15 @@
 package golang
 
 import (
+	"archive/zip"
+	"bytes"
 	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -16,13 +22,17 @@ import (
 
 type goLicenses struct {
 	searchLocalModCacheLicenses bool
+	searchRemoteLicenses        bool
 	localModCacheResolver       source.FileResolver
+	remoteProxy                 string
 }
 
 func newGoLicenses(opts GoCatalogerOpts) goLicenses {
 	return goLicenses{
 		searchLocalModCacheLicenses: opts.SearchLocalModCacheLicenses,
 		localModCacheResolver:       modCacheResolver(opts.LocalModCacheDir),
+		searchRemoteLicenses:        opts.SearchRemoteLicenses,
+		remoteProxy:                 opts.RemoteProxy,
 	}
 }
 
@@ -94,6 +104,15 @@ func (c *goLicenses) getLicenses(resolver source.FileResolver, moduleName, modul
 		)
 	}
 
+	// if we did not find it yet, and remote searching was enabled, then use that
+	if c.searchRemoteLicenses && err == nil && len(licenses) == 0 {
+		var fsys fs.FS
+		fsys, err = getModule(moduleName, moduleVersion, c.remoteProxy)
+		if err == nil {
+			licenses, err = findLicensesFS(fsys)
+		}
+	}
+
 	// always return a non-nil slice
 	if licenses == nil {
 		licenses = []string{}
@@ -131,10 +150,72 @@ func findLicenses(resolver source.FileResolver, globMatch string) (out []string,
 	return
 }
 
+func findLicensesFS(fsys fs.FS) (out []string, err error) {
+	if fsys == nil {
+		return
+	}
+	err = fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		// ignore git directory
+		if path == ".git" || strings.HasPrefix(path, ".git/") {
+			return nil
+		}
+		if d.IsDir() {
+			return nil
+		}
+		basename := filepath.Base(path)
+		if !licenses.FileNameSet.Contains(basename) {
+			return nil
+		}
+		f, err := fsys.Open(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		parsed, err := licenses.Parse(f)
+		if err != nil {
+			return err
+		}
+		out = append(out, parsed...)
+		return nil
+	})
+
+	return
+}
+
 var capReplacer = regexp.MustCompile("[A-Z]")
 
 func processCaps(s string) string {
 	return capReplacer.ReplaceAllStringFunc(s, func(s string) string {
 		return "!" + strings.ToLower(s)
 	})
 }
+
+func getModule(moduleName, moduleVersion, proxy string) (fs.FS, error) {
+	// get the module zip
+	resp, err := http.Get(fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		// try lowercasing it; some packages have mixed casing that really messes up the proxy
+		respLC, errLC := http.Get(fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion))
+		if errLC != nil {
+			return nil, err
+		}
+		defer respLC.Body.Close()
+		if respLC.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("failed to get module zip: %s", resp.Status)
+		}
+		resp = respLC
+	}
+	// read the zip
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return zip.NewReader(bytes.NewReader(b), resp.ContentLength)
+}

syft/pkg/cataloger/golang/parse_go_binary_test.go

@@ -223,6 +223,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					Name:     "github.com/adrg/xdg",
 					Version:  "v0.2.1",
 					PURL:     "pkg:golang/github.com/adrg/xdg@v0.2.1",
+					Licenses: []string{"MIT"},
 					Language: pkg.Go,
 					Type:     pkg.GoModulePkg,
 					Locations: source.NewLocationSet(
@@ -372,6 +373,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					Name:     "github.com/adrg/xdg",
 					Version:  "v0.2.1",
 					PURL:     "pkg:golang/github.com/adrg/xdg@v0.2.1",
+					Licenses: []string{"MIT"},
 					Language: pkg.Go,
 					Type:     pkg.GoModulePkg,
 					Locations: source.NewLocationSet(
@@ -394,6 +396,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					Name:     "github.com/anchore/client-go",
 					Version:  "v0.0.0-20210222170800-9c70f9b80bcf",
 					PURL:     "pkg:golang/github.com/anchore/client-go@v0.0.0-20210222170800-9c70f9b80bcf",
+					Licenses: []string{"Apache-2.0"},
 					Language: pkg.Go,
 					Type:     pkg.GoModulePkg,
 					Locations: source.NewLocationSet(
@@ -449,6 +452,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					Name:     "golang.org/x/sys",
 					Version:  "v0.0.0-20211006194710-c8a6f5223071",
 					PURL:     "pkg:golang/golang.org/x/sys@v0.0.0-20211006194710-c8a6f5223071",
+					Licenses: []string{"BSD-3-Clause"},
 					Language: pkg.Go,
 					Type:     pkg.GoModulePkg,
 					Locations: source.NewLocationSet(
@@ -470,6 +474,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					Name:     "golang.org/x/term",
 					Version:  "v0.0.0-20210916214954-140adaaadfaf",
 					PURL:     "pkg:golang/golang.org/x/term@v0.0.0-20210916214954-140adaaadfaf",
+					Licenses: []string{"BSD-3-Clause"},
 					Language: pkg.Go,
 					Type:     pkg.GoModulePkg,
 					Locations: source.NewLocationSet(