feat: support for Go modules to imports [needs unit test review]

.github/workflows/main.yml

@@ -103,6 +103,9 @@ jobs:
       - name: Setup GOPATH
         run: go env -w GOPATH=${{ github.workspace }}/go
 
+      - name: Setup GOCACHE
+        run: go env -w GOCACHE=${{ github.workspace }}/.cache/go-build
+
       - name: Build
         run: go build -v ./...
 

README.md

@@ -15,6 +15,7 @@ It powers executable Go scripts and plugins, in embedded interpreters or interac
 * Complete support of [Go specification][specs]
 * Written in pure Go, using only the standard library
 * Simple interpreter API: `New()`, `Eval()`, `Use()`
+* Supports Go Modules
 * Works everywhere Go works
 * All Go & runtime resources accessible from script (with control)
 * Security: `unsafe` and `syscall` packages neither used nor exported by default
@@ -114,6 +115,10 @@ func main() {
 
 [Go Playground](https://play.golang.org/p/WvwH4JqrU-p)
 
+### Use Go Modules
+
+TODO
+
 ### As a command-line interpreter
 
 The Yaegi command can run an interactive Read-Eval-Print-Loop:

go.mod

@@ -1,3 +1,10 @@
 module github.com/traefik/yaegi
 
 go 1.18
+
+require golang.org/x/tools v0.1.12
+
+require (
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+)

go.sum

@@ -0,0 +1,6 @@
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

interp/doc.go

@@ -9,8 +9,7 @@ Importing packages
 Packages can be imported in source or binary form, using the standard
 Go import statement. In source form, packages are searched first in the
 vendor directory, the preferred way to store source dependencies. If not
-found in vendor, sources modules will be searched in GOPATH. Go modules
-are not supported yet by yaegi.
+found in vendor, sources modules will be searched in GOPATH.
 
 Binary form packages are compiled and linked with the interpreter
 executable, and exposed to scripts with the Use method. The extract

interp/interp.go

@@ -328,7 +328,9 @@ type Options struct {
 // New returns a new interpreter.
 func New(options Options) *Interpreter {
 	i := Interpreter{
-		opt:      opt{context: build.Default, filesystem: &realFS{}, env: map[string]string{}},
+		opt: opt{
+			context: build.Default, filesystem: &realFS{}, env: make(map[string]string),
+		},
 		frame:    newFrame(nil, 0, 0),
 		fset:     token.NewFileSet(),
 		universe: initUniverse(),

interp/src.go

@@ -3,12 +3,13 @@ package interp
 import (
 	"fmt"
 	"io/fs"
-	"os"
 	"path/filepath"
 	"strings"
+
+	"golang.org/x/tools/go/packages"
 )
 
-// importSrc calls gta on the source code for the package identified by
+// importSrc calls global tag analysis on the source code for the package identified by
 // importPath. rPath is the relative path to the directory containing the source
 // code for the package. It can also be "main" as a special value.
 func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (string, error) {
@@ -23,24 +24,9 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s
 		return name, nil
 	}
 
-	// For relative import paths in the form "./xxx" or "../xxx", the initial
-	// base path is the directory of the interpreter input file, or "." if no file
-	// was provided.
-	// In all other cases, absolute import paths are resolved from the GOPATH
-	// and the nested "vendor" directories.
-	if isPathRelative(importPath) {
-		if rPath == mainID {
-			rPath = "."
-		}
-		dir = filepath.Join(filepath.Dir(interp.name), rPath, importPath)
-	} else if dir, rPath, err = interp.pkgDir(interp.context.GOPATH, rPath, importPath); err != nil {
-		// Try again, assuming a root dir at the source location.
-		if rPath, err = interp.rootFromSourceLocation(); err != nil {
-			return "", err
-		}
-		if dir, rPath, err = interp.pkgDir(interp.context.GOPATH, rPath, importPath); err != nil {
-			return "", err
-		}
+	// resolve relative and absolute import paths.
+	if dir, err = interp.getPackageDir(importPath); err != nil {
+		return "", err
 	}
 
 	if interp.rdir[importPath] {
@@ -171,119 +157,41 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s
 	return pkgName, nil
 }
 
-// rootFromSourceLocation returns the path to the directory containing the input
-// Go file given to the interpreter, relative to $GOPATH/src.
-// It is meant to be called in the case when the initial input is a main package.
-func (interp *Interpreter) rootFromSourceLocation() (string, error) {
-	sourceFile := interp.name
-	if sourceFile == DefaultSourceName {
-		return "", nil
-	}
-	wd, err := os.Getwd()
+// getPackageDir uses the provided Go module environment variables to find the absolute path of an import path.
+func (interp *Interpreter) getPackageDir(importPath string) (string, error) {
+	// ensure that an absolute GOPATH is used.
+	absGoPath, err := filepath.Abs(interp.context.GOPATH)
 	if err != nil {
-		return "", err
-	}
-	pkgDir := filepath.Join(wd, filepath.Dir(sourceFile))
-	root := strings.TrimPrefix(pkgDir, filepath.Join(interp.context.GOPATH, "src")+"/")
-	if root == wd {
-		return "", fmt.Errorf("package location %s not in GOPATH", pkgDir)
-	}
-	return root, nil
-}
-
-// pkgDir returns the absolute path in filesystem for a package given its import path
-// and the root of the subtree dependencies.
-func (interp *Interpreter) pkgDir(goPath string, root, importPath string) (string, string, error) {
-	rPath := filepath.Join(root, "vendor")
-	dir := filepath.Join(goPath, "src", rPath, importPath)
-
-	if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil {
-		return dir, rPath, nil // found!
+		return "", fmt.Errorf("an error occurred determining the absolute path of a GOPATH %q: %w", interp.context.GOPATH, err)
 	}
 
-	dir = filepath.Join(goPath, "src", effectivePkg(root, importPath))
-
-	if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil {
-		return dir, root, nil // found!
-	}
-
-	if len(root) == 0 {
-		if interp.context.GOPATH == "" {
-			return "", "", fmt.Errorf("unable to find source related to: %q. Either the GOPATH environment variable, or the Interpreter.Options.GoPath needs to be set", importPath)
-		}
-		return "", "", fmt.Errorf("unable to find source related to: %q", importPath)
+	// load the package.
+	config := packages.Config{
+		Env: []string{
+			"GOPATH=" + absGoPath,
+			"GOCACHE=" + interp.opt.env["GOCACHE"],
+			"GOROOT=" + interp.context.GOROOT,
+			"GOPRIVATE=" + interp.opt.env["GOPRIVATE"],
+			"GOMODCACHE=" + interp.opt.env["GOMODCACHE"],
+			"GO111MODULE=" + interp.opt.env["GO111MODULE"],
+		},
 	}
 
-	rootPath := filepath.Join(goPath, "src", root)
-	prevRoot, err := previousRoot(interp.opt.filesystem, rootPath, root)
+	pkgs, err := packages.Load(&config, importPath)
 	if err != nil {
-		return "", "", err
+		return "", fmt.Errorf("an error occurred retrieving a package: %v\n%v\nIf Access is denied, run in administrator", importPath, err)
 	}
 
-	return interp.pkgDir(goPath, prevRoot, importPath)
-}
-
-const vendor = "vendor"
-
-// Find the previous source root (vendor > vendor > ... > GOPATH).
-func previousRoot(filesystem fs.FS, rootPath, root string) (string, error) {
-	rootPath = filepath.Clean(rootPath)
-	parent, final := filepath.Split(rootPath)
-	parent = filepath.Clean(parent)
-
-	// TODO(mpl): maybe it works for the special case main, but can't be bothered for now.
-	if root != mainID && final != vendor {
-		root = strings.TrimSuffix(root, string(filepath.Separator))
-		prefix := strings.TrimSuffix(strings.TrimSuffix(rootPath, root), string(filepath.Separator))
-
-		// look for the closest vendor in one of our direct ancestors, as it takes priority.
-		var vendored string
-		for {
-			fi, err := fs.Stat(filesystem, filepath.Join(parent, vendor))
-			if err == nil && fi.IsDir() {
-				vendored = strings.TrimPrefix(strings.TrimPrefix(parent, prefix), string(filepath.Separator))
-				break
-			}
-			if !os.IsNotExist(err) {
-				return "", err
-			}
-
-			// stop when we reach GOPATH/src/blah
-			parent = filepath.Dir(parent)
-			if parent == prefix {
-				break
+	// confirm the import path is found.
+	for _, pkg := range pkgs {
+		for _, goFile := range pkg.GoFiles {
+			if strings.Contains(filepath.Dir(goFile), pkg.Name) {
+				return filepath.Dir(goFile), nil
 			}
-
-			// just an additional failsafe, stop if we reach the filesystem root, or dot (if
-			// we are dealing with relative paths).
-			// TODO(mpl): It should probably be a critical error actually,
-			// as we shouldn't have gone that high up in the tree.
-			if parent == string(filepath.Separator) || parent == "." {
-				break
-			}
-		}
-
-		if vendored != "" {
-			return vendored, nil
 		}
 	}
 
-	// TODO(mpl): the algorithm below might be redundant with the one above,
-	// but keeping it for now. Investigate/simplify/remove later.
-	splitRoot := strings.Split(root, string(filepath.Separator))
-	var index int
-	for i := len(splitRoot) - 1; i >= 0; i-- {
-		if splitRoot[i] == "vendor" {
-			index = i
-			break
-		}
-	}
-
-	if index == 0 {
-		return "", nil
-	}
-
-	return filepath.Join(splitRoot[:index]...), nil
+	return "", fmt.Errorf("an import source could not be found for %q", importPath)
 }
 
 func effectivePkg(root, path string) string {
@@ -313,9 +221,3 @@ func effectivePkg(root, path string) string {
 
 	return filepath.Join(root, frag)
 }
-
-// isPathRelative returns true if path starts with "./" or "../".
-// It is intended for use on import paths, where "/" is always the directory separator.
-func isPathRelative(s string) bool {
-	return strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../")
-}

interp/src_test.go

@@ -160,11 +160,9 @@ func Test_pkgDir(t *testing.T) {
 		},
 	}
 
-	interp := &Interpreter{
-		opt: opt{
-			filesystem: &realFS{},
-		},
-	}
+	interp := New(Options{
+		Env: []string{"GOCACHE=" + os.Getenv("GOCACHE")},
+	})
 
 	for _, test := range testCases {
 		test := test
@@ -183,7 +181,7 @@ func Test_pkgDir(t *testing.T) {
 				}
 			}
 
-			dir, rPath, err := interp.pkgDir(goPath, test.root, test.path)
+			dir, err := interp.getPackageDir(test.path)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -192,71 +190,8 @@ func Test_pkgDir(t *testing.T) {
 				t.Errorf("[dir] got: %s, want: %s", dir, test.expected.dir)
 			}
 
-			if rPath != test.expected.rpath {
-				t.Errorf(" [rpath] got: %s, want: %s", rPath, test.expected.rpath)
-			}
-		})
-	}
-}
-
-func Test_previousRoot(t *testing.T) {
-	testCases := []struct {
-		desc           string
-		root           string
-		rootPathSuffix string
-		expected       string
-	}{
-		{
-			desc:     "GOPATH",
-			root:     "github.com/foo/pkg/",
-			expected: "",
-		},
-		{
-			desc:     "vendor level 1",
-			root:     "github.com/foo/pkg/vendor/guthib.com/traefik/fromage",
-			expected: "github.com/foo/pkg",
-		},
-		{
-			desc:     "vendor level 2",
-			root:     "github.com/foo/pkg/vendor/guthib.com/traefik/fromage/vendor/guthib.com/traefik/fuu",
-			expected: "github.com/foo/pkg/vendor/guthib.com/traefik/fromage",
-		},
-		{
-			desc:           "vendor is sibling",
-			root:           "github.com/foo/bar",
-			rootPathSuffix: "testdata/src/github.com/foo/bar",
-			expected:       "github.com/foo",
-		},
-		{
-			desc:           "vendor is uncle",
-			root:           "github.com/foo/bar/baz",
-			rootPathSuffix: "testdata/src/github.com/foo/bar/baz",
-			expected:       "github.com/foo",
-		},
-	}
-
-	for _, test := range testCases {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			t.Parallel()
-
-			var rootPath string
-			if test.rootPathSuffix != "" {
-				wd, err := os.Getwd()
-				if err != nil {
-					t.Fatal(err)
-				}
-				rootPath = filepath.Join(wd, test.rootPathSuffix)
-			} else {
-				rootPath = vendor
-			}
-			p, err := previousRoot(&realFS{}, rootPath, test.root)
-			if err != nil {
-				t.Error(err)
-			}
-
-			if p != test.expected {
-				t.Errorf("got: %s, want: %s", p, test.expected)
+			if test.root != test.expected.rpath {
+				t.Errorf(" [rpath] got: %s, want: %s", test.root, test.expected.rpath)
 			}
 		})
 	}