Tool for identifying and complying with licenses for Go packages

It has two subcommands:
- "csv": Prints a report of the licenses that apply to all dependencies
- "save": Saves licenses, copyright notices and source code (when required) to a directory

csv.go

@@ -0,0 +1,86 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/csv"
+	"os"
+
+	"github.com/golang/glog"
+	"github.com/google/trillian/scripts/licenses/licenses"
+	"github.com/spf13/cobra"
+)
+
+var (
+	csvCmd = &cobra.Command{
+		Use:   "csv <package>",
+		Short: "Prints all licenses that apply to a Go package and its dependencies",
+		Args:  cobra.ExactArgs(1),
+		RunE:  csvMain,
+	}
+
+	gitRemotes []string
+)
+
+func init() {
+	csvCmd.Flags().StringArrayVar(&gitRemotes, "git_remote", []string{"origin", "upstream"}, "Remote Git repositories to try")
+
+	rootCmd.AddCommand(csvCmd)
+}
+
+func csvMain(_ *cobra.Command, args []string) error {
+	importPath := args[0]
+	writer := csv.NewWriter(os.Stdout)
+
+	classifier, err := licenses.NewClassifier(confidenceThreshold)
+	if err != nil {
+		return err
+	}
+
+	libs, err := libraries(importPath)
+	if err != nil {
+		return err
+	}
+	for _, lib := range libs {
+		licenseURL := "Unknown"
+		licenseName := "Unknown"
+		if lib.LicensePath != "" {
+			// Find a URL for the license file, based on the URL of a remote for the Git repository.
+			var errs []error
+			for _, remote := range gitRemotes {
+				url, err := licenses.GitFileURL(lib.LicensePath, remote)
+				if err != nil {
+					errs = append(errs, err)
+					continue
+				}
+				licenseURL = url.String()
+				break
+			}
+			if licenseURL == "Unknown" {
+				glog.Errorf("Error discovering URL for %q: %v", lib.LicensePath, errs)
+			}
+			licenseName, _, err = classifier.Identify(lib.LicensePath)
+			if err != nil {
+				return err
+			}
+		}
+		// Remove the "*/vendor/" prefix from the library name for conciseness.
+		if err := writer.Write([]string{unvendor(lib.Name()), licenseURL, licenseName}); err != nil {
+			return err
+		}
+	}
+	writer.Flush()
+	return writer.Error()
+}

licenses/classifier.go

@@ -0,0 +1,97 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package licenses
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/google/licenseclassifier"
+)
+
+// Type identifies a class of software license.
+type Type string
+
+// License types
+const (
+	// Unknown license type.
+	Unknown = Type("")
+	// Restricted licenses require mandatory source distribution if we ship a
+	// product that includes third-party code protected by such a license.
+	Restricted = Type("restricted")
+	// Reciprocal licenses allow usage of software made available under such
+	// licenses freely in *unmodified* form. If the third-party source code is
+	// modified in any way these modifications to the original third-party
+	// source code must be made available.
+	Reciprocal = Type("reciprocal")
+	// Notice licenses contain few restrictions, allowing original or modified
+	// third-party software to be shipped in any product without endangering or
+	// encumbering our source code. All of the licenses in this category do,
+	// however, have an "original Copyright notice" or "advertising clause",
+	// wherein any external distributions must include the notice or clause
+	// specified in the license.
+	Notice = Type("notice")
+	// Permissive licenses are even more lenient than a 'notice' license.
+	// Not even a copyright notice is required for license compliance.
+	Permissive = Type("permissive")
+	// Unencumbered covers licenses that basically declare that the code is "free for any use".
+	Unencumbered = Type("unencumbered")
+	// Forbidden licenses are forbidden to be used.
+	Forbidden = Type("FORBIDDEN")
+)
+
+func (t Type) String() string {
+	switch t {
+	case Unknown:
+		// licenseclassifier uses an empty string to indicate an unknown license
+		// type, which is unclear to users when printed as a string.
+		return "unknown"
+	default:
+		return string(t)
+	}
+}
+
+// Classifier can detect the type of a software license.
+type Classifier struct {
+	classifier *licenseclassifier.License
+}
+
+// NewClassifier creates a classifier that requires a specified confidence threshold
+// in order to return a positive license classification.
+func NewClassifier(confidenceThreshold float64) (*Classifier, error) {
+	c, err := licenseclassifier.New(confidenceThreshold)
+	if err != nil {
+		return nil, err
+	}
+	return &Classifier{classifier: c}, nil
+}
+
+// Identify returns the name and type of a license, given its file path.
+// An empty license path results in an empty name and Unknown type.
+func (c *Classifier) Identify(licensePath string) (string, Type, error) {
+	if licensePath == "" {
+		return "", Unknown, nil
+	}
+	content, err := ioutil.ReadFile(licensePath)
+	if err != nil {
+		return "", "", err
+	}
+	matches := c.classifier.MultipleMatch(string(content), true)
+	if len(matches) == 0 {
+		return "", "", fmt.Errorf("unknown license")
+	}
+	licenseName := matches[0].Name
+	return licenseName, Type(licenseclassifier.LicenseType(licenseName)), nil
+}

licenses/classifier_test.go

@@ -0,0 +1,67 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package licenses
+
+import (
+	"testing"
+)
+
+func TestIdentify(t *testing.T) {
+	for _, test := range []struct {
+		desc        string
+		file        string
+		confidence  float64
+		wantLicense string
+		wantType    Type
+		wantErr     bool
+	}{
+		{
+			desc:        "Apache 2.0 license",
+			file:        "../../../LICENSE",
+			confidence:  1,
+			wantLicense: "Apache-2.0",
+			wantType:    Notice,
+		},
+		{
+			desc:       "non-existent file",
+			file:       "non-existent-file",
+			confidence: 1,
+			wantErr:    true,
+		},
+		{
+			desc:        "empty file path",
+			file:        "",
+			confidence:  1,
+			wantLicense: "",
+			wantType:    Unknown,
+		},
+	} {
+		t.Run(test.desc, func(t *testing.T) {
+			c, err := NewClassifier(test.confidence)
+			if err != nil {
+				t.Fatalf("NewClassifier(%v) = (_, %q), want (_, nil)", test.confidence, err)
+			}
+			gotLicense, gotType, err := c.Identify(test.file)
+			if gotErr := err != nil; gotErr != test.wantErr {
+				t.Fatalf("c.Identify(%q) = (_, _, %q), want err? %t", test.file, err, test.wantErr)
+			} else if gotErr {
+				return
+			}
+			if gotLicense != test.wantLicense || gotType != test.wantType {
+				t.Fatalf("c.Identify(%q) = (%q, %q, %v), want (%q, %q, <nil>)", test.file, gotLicense, gotType, err, test.wantLicense, test.wantType)
+			}
+		})
+	}
+}

licenses/find.go

@@ -0,0 +1,69 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package licenses
+
+import (
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+)
+
+var (
+	licenseRegexp = regexp.MustCompile(`^LICENSE(\.(txt|md))?$`)
+	srcDirRegexps = func() []*regexp.Regexp {
+		var rs []*regexp.Regexp
+		for _, s := range build.Default.SrcDirs() {
+			rs = append(rs, regexp.MustCompile("^"+regexp.QuoteMeta(s)+"$"))
+		}
+		return rs
+	}()
+	vendorRegexp = regexp.MustCompile(`.+/vendor(/)?$`)
+)
+
+// Find returns the file path of the license for this package.
+func Find(pkg *build.Package) (string, error) {
+	var stopAt []*regexp.Regexp
+	stopAt = append(stopAt, srcDirRegexps...)
+	stopAt = append(stopAt, vendorRegexp)
+	return findUpwards(pkg.Dir, licenseRegexp, stopAt)
+}
+
+func findUpwards(dir string, r *regexp.Regexp, stopAt []*regexp.Regexp) (string, error) {
+	start := dir
+	for !matchAny(stopAt, dir) {
+		files, err := ioutil.ReadDir(dir)
+		if err != nil {
+			return "", err
+		}
+		for _, f := range files {
+			if r.MatchString(f.Name()) {
+				return filepath.Join(dir, f.Name()), nil
+			}
+		}
+		dir = filepath.Dir(dir)
+	}
+	return "", fmt.Errorf("no file matching %q found for %s", r, start)
+}
+
+func matchAny(patterns []*regexp.Regexp, s string) bool {
+	for _, p := range patterns {
+		if p.MatchString(s) {
+			return true
+		}
+	}
+	return false
+}

licenses/find_test.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package licenses
+
+import (
+	"go/build"
+	"path/filepath"
+	"testing"
+)
+
+func TestFind(t *testing.T) {
+	for _, test := range []struct {
+		desc            string
+		importPath      string
+		workingDir      string
+		importMode      build.ImportMode
+		wantLicensePath string
+	}{
+		{
+			desc:            "Trillian license",
+			importPath:      "github.com/google/trillian/scripts/licenses/licenses",
+			wantLicensePath: filepath.Join(build.Default.GOPATH, "src/github.com/google/trillian/LICENSE"),
+		},
+	} {
+		t.Run(test.desc, func(t *testing.T) {
+			pkg, err := build.Import(test.importPath, test.workingDir, test.importMode)
+			if err != nil {
+				t.Fatalf("build.Import(%q, %q, %v) = (_, %q), want (_, nil)", test.importPath, test.workingDir, test.importMode, err)
+			}
+			licensePath, err := Find(pkg)
+			if err != nil || licensePath != test.wantLicensePath {
+				t.Fatalf("Find(%v) = (%#v, %q), want (%q, nil)", pkg, licensePath, err, test.wantLicensePath)
+			}
+		})
+	}
+}