diff --git a/gps/pkgtree/gomod.go b/gps/pkgtree/gomod.go new file mode 100644 index 0000000000..2cb9906752 --- /dev/null +++ b/gps/pkgtree/gomod.go @@ -0,0 +1,73 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgtree + +import ( + "bytes" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "sync" +) + +// goModPath was taken, nearly, verbatim from: go1.10.3/src/cmd/go/internal/load/pkg.go + +var ( + modulePrefix = []byte("\nmodule ") + goModPathCache = make(map[string]string) + goModPathCacheLock sync.RWMutex +) + +// goModPath returns the module path in the go.mod in dir, if any. +func goModPath(dir string) (path string) { + goModPathCacheLock.RLock() + path, ok := goModPathCache[dir] + goModPathCacheLock.RUnlock() + if ok { + return path + } + defer func() { + goModPathCacheLock.Lock() + goModPathCache[dir] = path + goModPathCacheLock.Unlock() + }() + + data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + return "" + } + var i int + if bytes.HasPrefix(data, modulePrefix[1:]) { + i = 0 + } else { + i = bytes.Index(data, modulePrefix) + if i < 0 { + return "" + } + i++ + } + line := data[i:] + + // Cut line at \n, drop trailing \r if present. + if j := bytes.IndexByte(line, '\n'); j >= 0 { + line = line[:j] + } + if line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + line = line[len("module "):] + + // If quoted, unquote. + path = strings.TrimSpace(string(line)) + if path != "" && path[0] == '"' { + s, err := strconv.Unquote(path) + if err != nil { + return "" + } + path = s + } + return path +} diff --git a/gps/pkgtree/pkgtree.go b/gps/pkgtree/pkgtree.go index 55f2af46ff..4ae5c07385 100644 --- a/gps/pkgtree/pkgtree.go +++ b/gps/pkgtree/pkgtree.go @@ -25,6 +25,8 @@ import ( type Package struct { Name string // Package name, as declared in the package statement ImportPath string // Full import path, including the prefix provided to ListPackages() + RelModPath string // Location of go.mod file for package, relative to project root + Module string // The name (import path) of the go module CommentPath string // Import path given in the comment on the package statement Imports []string // Imports from all go and cgo files TestImports []string // Imports from all go test files (in go/build parlance: both TestImports and XTestImports) @@ -126,10 +128,38 @@ func ListPackages(fileRoot, importRoot string) (PackageTree, error) { f.Close() } + ip := filepath.Join(importRoot, strings.TrimPrefix(wp, fileRoot)) + var modpath, module string + + // walk back up the filesystem to the import root, inclusive, to check + // to see if a go.mod file requires an adjustment to the import path. + for i := len(wp); i >= len(fileRoot); i-- { + if i < len(wp) && wp[i] != filepath.Separator { + continue + } + + var gomod string + if gomod = goModPath(wp[:i]); gomod == "" { + continue + } + + // there was a go.mod file in this directory which states that the + // module, from this point in the file tree, should be imported as gomod + ip = filepath.Join(gomod, wp[i:]) + modpath = strings.TrimPrefix(wp[:i], fileRoot) + module = gomod + + if modpath == "" { + modpath = "." + } + + break + } + // Compute the import path. Run the result through ToSlash(), so that // windows file paths are normalized to slashes, as is expected of // import paths. - ip := filepath.ToSlash(filepath.Join(importRoot, strings.TrimPrefix(wp, fileRoot))) + ip = filepath.ToSlash(ip) // Find all the imports, across all os/arch combos p := &build.Package{ @@ -154,6 +184,8 @@ func ListPackages(fileRoot, importRoot string) (PackageTree, error) { pkg := Package{ ImportPath: ip, + RelModPath: modpath, + Module: module, CommentPath: p.ImportComment, Name: p.Name, Imports: p.Imports, diff --git a/gps/pkgtree/pkgtree_test.go b/gps/pkgtree/pkgtree_test.go index a2b9a9a2e9..3ef0b79926 100644 --- a/gps/pkgtree/pkgtree_test.go +++ b/gps/pkgtree/pkgtree_test.go @@ -2247,6 +2247,8 @@ func TestCanaryPackageTreeCopy(t *testing.T) { packageFields := []string{ "Name", "ImportPath", + "RelModPath", + "Module", "CommentPath", "Imports", "TestImports", diff --git a/gps/source.go b/gps/source.go index 1a1a0456a0..1a63512fbe 100644 --- a/gps/source.go +++ b/gps/source.go @@ -9,6 +9,9 @@ import ( "context" "fmt" "log" + "os" + "path/filepath" + "strings" "sync" "github.com/golang/dep/gps/pkgtree" @@ -322,11 +325,66 @@ func (sg *sourceGateway) existsUpstream(ctx context.Context) error { return err } -func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to string) error { +func moveModules(to string, projectRoot ProjectRoot, pkgs pkgtree.PackageTree) error { + basedir := strings.TrimSuffix(to, string(projectRoot)) + modules := map[string]struct{}{} + for _, pkg := range pkgs.Packages { + if pkg.P.RelModPath == "" { + continue + } + + // the filesystem path containing go.mod + modpath := filepath.FromSlash(filepath.Join(string(projectRoot), pkg.P.RelModPath)) + + // the filesystem path of the module (before renaming) + godir := filepath.FromSlash(filepath.Join(basedir, modpath)) + + // where the module will have to be moved + moddir := filepath.FromSlash(filepath.Join(basedir, pkg.P.Module)) + + // ensure we have to move the directory + if godir == moddir { + continue + } + + // only do this once per modpath + if _, ok := modules[modpath]; ok { + continue + } + + modules[modpath] = struct{}{} + + tmppath := filepath.Join(filepath.Dir(godir), "_"+filepath.Base(godir)) + + // move the dir to a temp dir + if err := os.Rename(godir, tmppath); err != nil { + return errors.Wrap(err, "error moving module to tempdir") + } + + // create the dir containing the module + if err := os.MkdirAll(filepath.Dir(moddir), 0755); err != nil { + return errors.Wrap(err, "error making module dir") + } + + // move the module into the proper place + if err := os.Rename(tmppath, moddir); err != nil { + return errors.Wrap(err, "error moving module from tempdir") + } + } + + return nil +} + +func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to string, projectRoot ProjectRoot) error { + pkgs, err := sg.listPackages(ctx, projectRoot, v) + if err != nil { + return err + } + sg.mu.Lock() defer sg.mu.Unlock() - err := sg.require(ctx, sourceExistsLocally) + err = sg.require(ctx, sourceExistsLocally) if err != nil { return err } @@ -337,7 +395,11 @@ func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to stri } err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error { - return sg.src.exportRevisionTo(ctx, r, to) + if eerr := sg.src.exportRevisionTo(ctx, r, to); eerr != nil { + return eerr + } + + return moveModules(to, projectRoot, pkgs) }) // It's possible (in git) that we may have tried this against a version that @@ -349,7 +411,11 @@ func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to stri if err != nil && sg.srcState&sourceHasLatestLocally == 0 { if err = sg.require(ctx, sourceHasLatestLocally); err == nil { err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error { - return sg.src.exportRevisionTo(ctx, r, to) + if eerr := sg.src.exportRevisionTo(ctx, r, to); eerr != nil { + return eerr + } + + return moveModules(to, projectRoot, pkgs) }) } } @@ -358,10 +424,15 @@ func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to stri } func (sg *sourceGateway) exportPrunedVersionTo(ctx context.Context, lp LockedProject, prune PruneOptions, to string) error { + pkgs, err := sg.listPackages(ctx, lp.Ident().ProjectRoot, lp.Version()) + if err != nil { + return err + } + sg.mu.Lock() defer sg.mu.Unlock() - err := sg.require(ctx, sourceExistsLocally) + err = sg.require(ctx, sourceExistsLocally) if err != nil { return err } @@ -378,7 +449,11 @@ func (sg *sourceGateway) exportPrunedVersionTo(ctx context.Context, lp LockedPro } if err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error { - return sg.src.exportRevisionTo(ctx, r, to) + if eerr := sg.src.exportRevisionTo(ctx, r, to); eerr != nil { + return eerr + } + + return moveModules(to, lp.Ident().ProjectRoot, pkgs) }); err != nil { return err } diff --git a/gps/source_cache_bolt_encode.go b/gps/source_cache_bolt_encode.go index 5b6a903cf6..23f5466761 100644 --- a/gps/source_cache_bolt_encode.go +++ b/gps/source_cache_bolt_encode.go @@ -18,6 +18,8 @@ import ( ) var ( + cacheKeyModule = []byte("d") + cacheKeyModPath = []byte("a") cacheKeyComment = []byte("c") cacheKeyConstraint = cacheKeyComment cacheKeyError = []byte("e") @@ -372,6 +374,18 @@ func cachePutPackageOrErr(b *bolt.Bucket, poe pkgtree.PackageOrErr) error { err := b.Put(cacheKeyError, []byte(poe.Err.Error())) return errors.Wrapf(err, "failed to put error: %v", poe.Err) } + if len(poe.P.RelModPath) > 0 { + err := b.Put(cacheKeyModPath, []byte(poe.P.RelModPath)) + if err != nil { + return errors.Wrapf(err, "failed to put package: %v", poe.P) + } + } + if len(poe.P.Module) > 0 { + err := b.Put(cacheKeyModule, []byte(poe.P.Module)) + if err != nil { + return errors.Wrapf(err, "failed to put package: %v", poe.P) + } + } if len(poe.P.CommentPath) > 0 { err := b.Put(cacheKeyComment, []byte(poe.P.CommentPath)) if err != nil { @@ -427,6 +441,8 @@ func cacheGetPackageOrErr(b *bolt.Bucket) (pkgtree.PackageOrErr, error) { } var p pkgtree.Package + p.Module = string(b.Get(cacheKeyModule)) + p.RelModPath = string(b.Get(cacheKeyModPath)) p.CommentPath = string(b.Get(cacheKeyComment)) if ip := b.Bucket(cacheKeyImport); ip != nil { err := ip.ForEach(func(_, v []byte) error { diff --git a/gps/source_manager.go b/gps/source_manager.go index 16c3f4816d..7fb4279874 100644 --- a/gps/source_manager.go +++ b/gps/source_manager.go @@ -556,7 +556,7 @@ func (sm *SourceMgr) ExportProject(ctx context.Context, id ProjectIdentifier, v return err } - return srcg.exportVersionTo(ctx, v, to) + return srcg.exportVersionTo(ctx, v, to, id.ProjectRoot) } // ExportPrunedProject writes out a tree of the provided LockedProject, applying diff --git a/gps/source_test.go b/gps/source_test.go index 6debaee639..f0b34ae2f7 100644 --- a/gps/source_test.go +++ b/gps/source_test.go @@ -151,7 +151,7 @@ func testSourceGateway(t *testing.T) { t.Fatalf("wanted nonexistent err when passing bad version, got: %s", err) } - err = sg.exportVersionTo(ctx, badver, cachedir) + err = sg.exportVersionTo(ctx, badver, cachedir, ProjectRoot("github.com/sdboyer/deptest")) if err == nil { t.Fatal("wanted err on nonexistent version") } else if err.Error() != wanterr.Error() {