Enhance Go dependency resolver to resolve built-in and non-standard G…

…o modules

assets/lcs-templates/BSD-2-Clause.txt

@@ -0,0 +1,16 @@
+Copyright (c) <year> <owner>. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
+disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

assets/lcs-templates/ISC.txt

@@ -0,0 +1,5 @@
+Copyright [year] [owner]
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

assets/lcs-templates/WTFPL.txt

@@ -0,0 +1,11 @@
+DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+Version 2, December 2004
+
+Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+Everyone is permitted to copy and distribute verbatim or modified copies of this license document, and changing it is allowed as long as the name is changed.
+
+DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. You just DO WHAT THE FUCK YOU WANT TO.

commands/deps_resolve.go

@@ -45,8 +45,8 @@ var ResolveCommand = &cobra.Command{
 				pkgs[i] = s.Dependency
 			}
 			return fmt.Errorf(
-				"failed to identify the licenses of following packages:\n%s",
-				strings.Join(pkgs, "\n"),
+				"failed to identify the licenses of following packages (%d):\n%s",
+				len(pkgs), strings.Join(pkgs, "\n"),
 			)
 		}
 

go.mod

@@ -4,11 +4,11 @@ go 1.16
 
 require (
 	github.com/bmatcuk/doublestar/v2 v2.0.4
+	github.com/google/go-cmp v0.3.0 // indirect
 	github.com/google/go-github/v33 v33.0.0
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.1.1
-	golang.org/x/mod v0.4.0
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e
+	golang.org/x/tools v0.1.5
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )

pkg/deps/golang.go

@@ -18,20 +18,20 @@
 package deps
 
 import (
-	"context"
-	"fmt"
+	"bytes"
+	"encoding/json"
 	"go/build"
+	"io"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
-	"strings"
-
-	"golang.org/x/mod/modfile"
-	"golang.org/x/tools/go/packages"
 
 	"github.com/apache/skywalking-eyes/license-eye/internal/logger"
 	"github.com/apache/skywalking-eyes/license-eye/pkg/license"
+
+	"golang.org/x/tools/go/packages"
 )
 
 type GoModResolver struct {
@@ -46,98 +46,68 @@ func (resolver *GoModResolver) CanResolve(file string) bool {
 
 // Resolve resolves licenses of all dependencies declared in the go.mod file.
 func (resolver *GoModResolver) Resolve(goModFile string, report *Report) error {
-	content, err := ioutil.ReadFile(goModFile)
-	if err != nil {
+	if err := os.Chdir(filepath.Dir(goModFile)); err != nil {
 		return err
 	}
 
-	file, err := modfile.Parse(goModFile, content, nil)
-	if err != nil {
+	goModDownload := exec.Command("go", "mod", "download")
+	logger.Log.Debugf("Run command: %v, please wait", goModDownload.String())
+	goModDownload.Stdout = os.Stdout
+	goModDownload.Stderr = os.Stderr
+	if err := goModDownload.Run(); err != nil {
 		return err
 	}
 
-	logger.Log.Debugln("Resolving module:", file.Module.Mod)
-
-	if err := os.Chdir(filepath.Dir(goModFile)); err != nil {
+	output, err := exec.Command("go", "list", "-m", "-json", "all").Output()
+	if err != nil {
 		return err
 	}
 
-	requiredPkgNames := make([]string, len(file.Require))
-	for i, require := range file.Require {
-		requiredPkgNames[i] = require.Mod.Path
+	modules := make([]*packages.Module, 0)
+	decoder := json.NewDecoder(bytes.NewReader(output))
+	for {
+		var m packages.Module
+		if err := decoder.Decode(&m); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		modules = append(modules, &m)
 	}
 
-	logger.Log.Debugln("Required packages:", requiredPkgNames)
+	logger.Log.Debugln("Module size:", len(modules))
 
-	if err := resolver.ResolvePackages(requiredPkgNames, report); err != nil {
+	if err := resolver.ResolvePackages(modules, report); err != nil {
 		return err
 	}
 
 	return nil
 }
 
 // ResolvePackages resolves the licenses of the given packages.
-func (resolver *GoModResolver) ResolvePackages(pkgNames []string, report *Report) error {
-	requiredPkgs, err := packages.Load(&packages.Config{
-		Context: context.Background(),
-		Mode:    packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedDeps,
-	}, pkgNames...)
-
-	if err != nil {
-		return err
-	}
-
-	packages.Visit(requiredPkgs, func(p *packages.Package) bool {
-		if isBuiltIn(p) {
-			logger.Log.Debugln("Built-in package doesn't require license check:", p.PkgPath)
-			return false
-		}
-
-		if len(p.Errors) > 0 {
-			logger.Log.Warnln("Failed to visit package:", p.PkgPath, p.Errors)
-			report.Skip(&Result{
-				Dependency:    p.PkgPath,
-				LicenseSpdxID: Unknown,
-			})
-			return true
-		}
-		err := resolver.ResolvePackageLicense(p, report)
+func (resolver *GoModResolver) ResolvePackages(modules []*packages.Module, report *Report) error {
+	for _, module := range modules {
+		err := resolver.ResolvePackageLicense(module, report)
 		if err != nil {
-			logger.Log.Warnln("Failed to resolve the license of dependency:", p.PkgPath, err)
+			logger.Log.Warnln("Failed to resolve the license of dependency:", module.Path, err)
 			report.Skip(&Result{
-				Dependency:    p.PkgPath,
+				Dependency:    module.Path,
 				LicenseSpdxID: Unknown,
 			})
 		}
-		return true
-	}, nil)
+	}
 
 	return nil
 }
 
 var possibleLicenseFileName = regexp.MustCompile(`(?i)^LICENSE|LICENCE(\.txt)?$`)
 
-func (resolver *GoModResolver) ResolvePackageLicense(p *packages.Package, report *Report) error {
-	var filesInPkg []string
-	if len(p.GoFiles) > 0 {
-		filesInPkg = p.GoFiles
-	} else if len(p.CompiledGoFiles) > 0 {
-		filesInPkg = p.CompiledGoFiles
-	} else if len(p.OtherFiles) > 0 {
-		filesInPkg = p.OtherFiles
-	}
-
-	if len(filesInPkg) == 0 {
-		return fmt.Errorf("empty package")
-	}
-
-	absPath, err := filepath.Abs(filesInPkg[0])
-	if err != nil {
-		return err
-	}
-	dir := filepath.Dir(absPath)
+func (resolver *GoModResolver) ResolvePackageLicense(module *packages.Module, report *Report) error {
+	dir := module.Dir
 
 	for {
+		logger.Log.Debugf("Directory of %+v is %+v", module.Path, dir)
 		files, err := ioutil.ReadDir(dir)
 		if err != nil {
 			return err
@@ -151,12 +121,12 @@ func (resolver *GoModResolver) ResolvePackageLicense(p *packages.Package, report
 			if err != nil {
 				return err
 			}
-			identifier, err := license.Identify(p.PkgPath, string(content))
+			identifier, err := license.Identify(module.Path, string(content))
 			if err != nil {
 				return err
 			}
 			report.Resolve(&Result{
-				Dependency:      p.PkgPath,
+				Dependency:      module.Path,
 				LicenseFilePath: licenseFilePath,
 				LicenseContent:  string(content),
 				LicenseSpdxID:   identifier,
@@ -172,14 +142,5 @@ func (resolver *GoModResolver) ResolvePackageLicense(p *packages.Package, report
 }
 
 func (resolver *GoModResolver) shouldStopAt(dir string) bool {
-	for _, srcDir := range build.Default.SrcDirs() {
-		if srcDir == dir {
-			return true
-		}
-	}
-	return false
-}
-
-func isBuiltIn(pkg *packages.Package) bool {
-	return len(pkg.GoFiles) > 0 && strings.HasPrefix(pkg.GoFiles[0], build.Default.GOROOT)
+	return dir == build.Default.GOPATH
 }

pkg/license/identifier.go

@@ -19,14 +19,20 @@ package license
 
 import (
 	"fmt"
+	"io/fs"
 	"path/filepath"
 	"regexp"
 	"strings"
 
 	"github.com/apache/skywalking-eyes/license-eye/assets"
+	"github.com/apache/skywalking-eyes/license-eye/internal/logger"
 )
 
-const templatesDir = "lcs-templates"
+var templatesDirs = []string{
+	"lcs-templates",
+	// Some projects simply use the header text as their LICENSE content...
+	"header-templates",
+}
 
 var dualLicensePatterns = []*regexp.Regexp{
 	regexp.MustCompile(`(?i)This project is covered by two different licenses: (?P<license>[^.]+)`),
@@ -45,11 +51,23 @@ func Identify(pkgPath, content string) (string, error) {
 
 	content = Normalize(content)
 
-	templates, err := assets.AssetDir(templatesDir)
-	if err != nil {
-		return "", err
+	for _, dir := range templatesDirs {
+		templates, err := assets.AssetDir(dir)
+		if err != nil {
+			return "", err
+		}
+
+		if s, err := identify(dir, templates, content); err == nil {
+			return s, err
+		}
 	}
 
+	logger.Log.Debugf("Normalized content for %+v:\n%+v\n", pkgPath, content)
+
+	return "", fmt.Errorf("cannot identify license content")
+}
+
+func identify(templatesDir string, templates []fs.DirEntry, content string) (string, error) {
 	for _, template := range templates {
 		templateName := template.Name()
 		t, err := assets.Asset(filepath.Join(templatesDir, templateName))
@@ -58,10 +76,9 @@ func Identify(pkgPath, content string) (string, error) {
 		}
 		license := string(t)
 		license = Normalize(license)
-		if license == content {
+		if strings.Contains(content, license) {
 			return strings.TrimSuffix(templateName, filepath.Ext(templateName)), nil
 		}
 	}
-
 	return "", fmt.Errorf("cannot identify license content")
 }