From 214b30ed34ea48b5f57aa3bd4cedaf2eb7ab4a0f Mon Sep 17 00:00:00 2001 From: HakanSunay Date: Mon, 9 May 2022 19:29:33 +0300 Subject: [PATCH 1/7] Implement very minimalistic support go workspaces With this change gazelle can invoke `update-repos` by processing `go.work` files. The basis of the change is a plain copy-paste of `update.go` into `work.go`. However, since `go list` does not show module sums, the whole process is delegated to `go mod download`. https://github.com/golang/go/issues/52792 was created to track the above. The standard go tooling in 1.18 can work with `go.work` files so not many changes were actually necessary. A smoke test is also added to `update_import_test.go` to verify that the latest version of govmomi and rest is always picked. How to invoke: $ gazelle update-repos -from_file=go.work -to_macro=deps.bzl%go_deps -prune=True --- language/go/update.go | 1 + language/go/update_import_test.go | 312 ++++++++++++++++++++++++++++++ language/go/work.go | 142 ++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 language/go/work.go diff --git a/language/go/update.go b/language/go/update.go index dbddef7fd..a66bf3730 100644 --- a/language/go/update.go +++ b/language/go/update.go @@ -62,6 +62,7 @@ var repoImportFuncs = map[string]func(args language.ImportReposArgs) language.Im "Gopkg.lock": importReposFromDep, "go.mod": importReposFromModules, "Godeps.json": importReposFromGodep, + "go.work": importReposFromWork, } func (*goLang) CanImport(path string) bool { diff --git a/language/go/update_import_test.go b/language/go/update_import_test.go index 4014f7384..3d644a365 100644 --- a/language/go/update_import_test.go +++ b/language/go/update_import_test.go @@ -431,6 +431,318 @@ go_repository( wantErr: "", stubGoModDownload: nil, }, + { + desc: "work", + files: []testtools.FileSpec{ + { + Path: "go.work", + Content: ` +go 1.18 + +use ( + ./project1 + ./project2 + ./project3 +) +`, + }, + { + Path: "project1/go.mod", + Content: ` +module project1 + +go 1.18 + +require github.com/vmware/govmomi v0.21.0 // indirect +`, + }, + { + Path: "project1/go.sum", + Content: ` +github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/vmware/govmomi v0.21.0 h1:jc8uMuxpcV2xMAA/cnEDlnsIjvqcMra5Y8onh/U3VuY= +github.com/vmware/govmomi v0.21.0/go.mod h1:zbnFoBQ9GIjs2RVETy8CNEpb+L+Lwkjs3XZUL0B3/m0= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +`, + }, + { + Path: "project2/go.mod", + Content: ` +module project2 + +go 1.18 + +require github.com/vmware/govmomi v0.24.0 // indirect +`, + }, + { + Path: "project2/go.sum", + Content: ` +github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/vmware/govmomi v0.24.0 h1:G7YFF6unMTG3OY25Dh278fsomVTKs46m2ENlEFSbmbs= +github.com/vmware/govmomi v0.24.0/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +`, + }, + { + Path: "project3/go.mod", + Content: ` +module project3 + +go 1.18 + +require github.com/vmware/govmomi v0.27.0 // indirect +`, + }, + { + Path: "project3/go.sum", + Content: ` +github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg= +github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/vmware/govmomi v0.27.0 h1:KoQ8IsLAa7V78s5d7dgpZA8d039GBM83cVxgAq9uWuw= +github.com/vmware/govmomi v0.27.0/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +`, + }, + }, + want: ` +go_repository( + name = "com_github_a8m_tree", + importpath = "github.com/a8m/tree", + sum = "h1:4E8RufAN3UQ/weB6AnQ4y5miZCO0Yco8ZdGId41WuQs=", + version = "v0.0.0-20210115125333-10a5fd5b637d", +) + +go_repository( + name = "com_github_davecgh_go_xdr", + importpath = "github.com/davecgh/go-xdr", + sum = "h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o=", + version = "v0.0.0-20161123171359-e6a2ba005892", +) + +go_repository( + name = "com_github_google_uuid", + importpath = "github.com/google/uuid", + sum = "h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=", + version = "v1.2.0", +) + +go_repository( + name = "com_github_kr_pretty", + importpath = "github.com/kr/pretty", + sum = "h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=", + version = "v0.1.0", +) + +go_repository( + name = "com_github_kr_pty", + importpath = "github.com/kr/pty", + sum = "h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=", + version = "v1.1.1", +) + +go_repository( + name = "com_github_kr_text", + importpath = "github.com/kr/text", + sum = "h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=", + version = "v0.1.0", +) + +go_repository( + name = "com_github_vmware_govmomi", + importpath = "github.com/vmware/govmomi", + sum = "h1:KoQ8IsLAa7V78s5d7dgpZA8d039GBM83cVxgAq9uWuw=", + version = "v0.27.0", +) + +go_repository( + name = "com_github_vmware_vmw_guestinfo", + importpath = "github.com/vmware/vmw-guestinfo", + sum = "h1:sH9mEk+flyDxiUa5BuPiuhDETMbzrt9A20I2wktMvRQ=", + version = "v0.0.0-20170707015358-25eff159a728", +) +`, + wantErr: "", + stubGoModDownload: func(s string, i []string) ([]byte, error) { + return []byte(` +{ + "Path": "github.com/a8m/tree", + "Version": "v0.0.0-20210115125333-10a5fd5b637d", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/a8m/tree/@v/v0.0.0-20210115125333-10a5fd5b637d.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/a8m/tree/@v/v0.0.0-20210115125333-10a5fd5b637d.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/a8m/tree/@v/v0.0.0-20210115125333-10a5fd5b637d.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/a8m/tree@v0.0.0-20210115125333-10a5fd5b637d", + "Sum": "h1:4E8RufAN3UQ/weB6AnQ4y5miZCO0Yco8ZdGId41WuQs=", + "GoModSum": "h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=" +} +{ + "Path": "github.com/davecgh/go-xdr", + "Version": "v0.0.0-20161123171359-e6a2ba005892", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/davecgh/go-xdr/@v/v0.0.0-20161123171359-e6a2ba005892.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/davecgh/go-xdr/@v/v0.0.0-20161123171359-e6a2ba005892.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/davecgh/go-xdr/@v/v0.0.0-20161123171359-e6a2ba005892.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/davecgh/go-xdr@v0.0.0-20161123171359-e6a2ba005892", + "Sum": "h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o=", + "GoModSum": "h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE=" +} +{ + "Path": "github.com/google/uuid", + "Version": "v1.2.0", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/google/uuid/@v/v1.2.0.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/google/uuid/@v/v1.2.0.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/google/uuid/@v/v1.2.0.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/google/uuid@v1.2.0", + "Sum": "h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=", + "GoModSum": "h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=" +} +{ + "Path": "github.com/kr/pretty", + "Version": "v0.1.0", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pretty/@v/v0.1.0.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pretty/@v/v0.1.0.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pretty/@v/v0.1.0.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/kr/pretty@v0.1.0", + "Sum": "h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=", + "GoModSum": "h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=" +} +{ + "Path": "github.com/kr/pty", + "Version": "v1.1.1", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pty/@v/v1.1.1.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pty/@v/v1.1.1.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pty/@v/v1.1.1.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/kr/pty@v1.1.1", + "Sum": "h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=", + "GoModSum": "h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=" +} +{ + "Path": "github.com/kr/text", + "Version": "v0.1.0", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/text/@v/v0.1.0.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/text/@v/v0.1.0.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/text/@v/v0.1.0.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/kr/text@v0.1.0", + "Sum": "h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=", + "GoModSum": "h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=" +} +{ + "Path": "github.com/vmware/govmomi", + "Version": "v0.27.0", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/govmomi/@v/v0.27.0.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/govmomi/@v/v0.27.0.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/govmomi/@v/v0.27.0.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/vmware/govmomi@v0.27.0", + "Sum": "h1:KoQ8IsLAa7V78s5d7dgpZA8d039GBM83cVxgAq9uWuw=", + "GoModSum": "h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o=" +} +{ + "Path": "github.com/vmware/vmw-guestinfo", + "Version": "v0.0.0-20170707015358-25eff159a728", + "Info": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/vmw-guestinfo/@v/v0.0.0-20170707015358-25eff159a728.info", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/vmw-guestinfo/@v/v0.0.0-20170707015358-25eff159a728.mod", + "Zip": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/vmw-guestinfo/@v/v0.0.0-20170707015358-25eff159a728.zip", + "Dir": "/Users/hhalil/go/pkg/mod/github.com/vmware/vmw-guestinfo@v0.0.0-20170707015358-25eff159a728", + "Sum": "h1:sH9mEk+flyDxiUa5BuPiuhDETMbzrt9A20I2wktMvRQ=", + "GoModSum": "h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk=" +} +`), nil + }, + stubGoListModules: func(dir string) ([]byte, error) { + return []byte(` +{ + "Path": "project1", + "Main": true, + "Dir": "/Users/hhalil/Documents/VMware/mirrors_github_bazel-gazelle/hakan/project1", + "GoMod": "/Users/hhalil/Documents/VMware/mirrors_github_bazel-gazelle/hakan/project1/go.mod", + "GoVersion": "1.18" +} +{ + "Path": "project2", + "Main": true, + "Dir": "/Users/hhalil/Documents/VMware/mirrors_github_bazel-gazelle/hakan/project2", + "GoMod": "/Users/hhalil/Documents/VMware/mirrors_github_bazel-gazelle/hakan/project2/go.mod", + "GoVersion": "1.18" +} +{ + "Path": "project3", + "Main": true, + "Dir": "/Users/hhalil/Documents/VMware/mirrors_github_bazel-gazelle/hakan/project3", + "GoMod": "/Users/hhalil/Documents/VMware/mirrors_github_bazel-gazelle/hakan/project3/go.mod", + "GoVersion": "1.18" +} +{ + "Path": "github.com/a8m/tree", + "Version": "v0.0.0-20210115125333-10a5fd5b637d", + "Time": "2021-01-15T12:53:33Z", + "Indirect": true, + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/a8m/tree/@v/v0.0.0-20210115125333-10a5fd5b637d.mod" +} +{ + "Path": "github.com/davecgh/go-xdr", + "Version": "v0.0.0-20161123171359-e6a2ba005892", + "Time": "2016-11-23T17:13:59Z", + "Indirect": true, + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/davecgh/go-xdr/@v/v0.0.0-20161123171359-e6a2ba005892.mod" +} +{ + "Path": "github.com/google/uuid", + "Version": "v1.2.0", + "Time": "2021-01-22T18:20:15Z", + "Indirect": true, + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/google/uuid/@v/v1.2.0.mod" +} +{ + "Path": "github.com/kr/pretty", + "Version": "v0.1.0", + "Time": "2018-05-06T08:33:45Z", + "Indirect": true, + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pretty/@v/v0.1.0.mod" +} +{ + "Path": "github.com/kr/pty", + "Version": "v1.1.1", + "Time": "2018-01-13T18:08:13Z", + "Indirect": true, + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/pty/@v/v1.1.1.mod" +} +{ + "Path": "github.com/kr/text", + "Version": "v0.1.0", + "Time": "2018-05-06T08:24:08Z", + "Indirect": true, + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/kr/text/@v/v0.1.0.mod" +} +{ + "Path": "github.com/vmware/govmomi", + "Version": "v0.27.0", + "Time": "2021-10-14T20:30:09Z", + "Indirect": true, + "Dir": "/Users/hhalil/go/pkg/mod/github.com/vmware/govmomi@v0.27.0", + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/govmomi/@v/v0.27.0.mod", + "GoVersion": "1.14" +} +{ + "Path": "github.com/vmware/vmw-guestinfo", + "Version": "v0.0.0-20170707015358-25eff159a728", + "Time": "2017-07-07T01:53:58Z", + "Indirect": true, + "GoMod": "/Users/hhalil/go/pkg/mod/cache/download/github.com/vmware/vmw-guestinfo/@v/v0.0.0-20170707015358-25eff159a728.mod" +} +`), nil + }, + }, } { t.Run(tc.desc, func(t *testing.T) { if tc.stubGoModDownload != nil { diff --git a/language/go/work.go b/language/go/work.go new file mode 100644 index 000000000..53e8f03b7 --- /dev/null +++ b/language/go/work.go @@ -0,0 +1,142 @@ +/* Copyright 2022 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package golang + +import ( + "bytes" + "encoding/json" + "fmt" + "go/build" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + + "github.com/bazelbuild/bazel-gazelle/label" + "github.com/bazelbuild/bazel-gazelle/language" + "github.com/bazelbuild/bazel-gazelle/rule" +) + +func importReposFromWork(args language.ImportReposArgs) language.ImportReposResult { + // dir where go.work is located + dir := filepath.Dir(args.Path) + + // List all modules except for the main module, including implicit indirect + // dependencies. + type module struct { + Path, Version, Sum, Error string + Main bool + Replace *struct { + Path, Version string + } + } + // path@version can be used as a unique identifier for looking up sums + pathToModule := map[string]*module{} + data, err := goListModules(dir) + if err != nil { + return language.ImportReposResult{Error: err} + } + dec := json.NewDecoder(bytes.NewReader(data)) + for dec.More() { + mod := new(module) + if err := dec.Decode(mod); err != nil { + return language.ImportReposResult{Error: err} + } + if mod.Main { + continue + } + if mod.Replace != nil { + if filepath.IsAbs(mod.Replace.Path) || build.IsLocalImport(mod.Replace.Path) { + log.Printf("go_repository does not support file path replacements for %s -> %s", mod.Path, + mod.Replace.Path) + continue + } + pathToModule[mod.Replace.Path+"@"+mod.Replace.Version] = mod + } else { + pathToModule[mod.Path+"@"+mod.Version] = mod + } + } + + // Sums are missing by default for go.work. + // That's why we run 'go mod download' to get them. + // Once https://github.com/golang/go/issues/52792 is resolved, we can use go list. + // This must be done in a temporary directory because 'go mod download' + // may modify go.mod and go.sum. It does not support -mod=readonly. + var missingSumArgs []string + for pathVer, mod := range pathToModule { + if mod.Sum == "" { + missingSumArgs = append(missingSumArgs, pathVer) + } + } + + if len(missingSumArgs) > 0 { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + return language.ImportReposResult{Error: fmt.Errorf("finding module sums: %v", err)} + } + defer os.RemoveAll(tmpDir) + data, err := goModDownload(tmpDir, missingSumArgs) + dec = json.NewDecoder(bytes.NewReader(data)) + if err != nil { + // Best-effort try to adorn specific error details from the JSON output. + for dec.More() { + var dl module + if err := dec.Decode(&dl); err != nil { + // If we couldn't parse a possible error description, just ignore this part of the output. + continue + } + if dl.Error != "" { + err = fmt.Errorf("%v\nError downloading %v: %v", err, dl.Path, dl.Error) + } + } + + return language.ImportReposResult{Error: err} + } + for dec.More() { + var dl module + if err := dec.Decode(&dl); err != nil { + return language.ImportReposResult{Error: err} + } + if mod, ok := pathToModule[dl.Path+"@"+dl.Version]; ok { + mod.Sum = dl.Sum + } + } + } + + // Translate to repository rules. + gen := make([]*rule.Rule, 0, len(pathToModule)) + for pathVer, mod := range pathToModule { + if mod.Sum == "" { + log.Printf("could not determine sum for module %s", pathVer) + continue + } + r := rule.NewRule("go_repository", label.ImportPathToBazelRepoName(mod.Path)) + r.SetAttr("importpath", mod.Path) + r.SetAttr("sum", mod.Sum) + if mod.Replace == nil { + r.SetAttr("version", mod.Version) + } else { + r.SetAttr("replace", mod.Replace.Path) + r.SetAttr("version", mod.Replace.Version) + } + gen = append(gen, r) + } + sort.Slice(gen, func(i, j int) bool { + return gen[i].Name() < gen[j].Name() + }) + return language.ImportReposResult{Gen: gen} +} From 100ee0de8086ce3b1e96c710c1126e8fbf3ec92a Mon Sep 17 00:00:00 2001 From: HakanSunay Date: Fri, 13 May 2022 16:44:51 +0300 Subject: [PATCH 2/7] Add forgotten change in BUILD.bazel for the go package --- language/go/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/language/go/BUILD.bazel b/language/go/BUILD.bazel index 0fc7bcbaa..1320972fc 100644 --- a/language/go/BUILD.bazel +++ b/language/go/BUILD.bazel @@ -30,6 +30,7 @@ go_library( "resolve.go", "std_package_list.go", "update.go", + "work.go", ], importpath = "github.com/bazelbuild/bazel-gazelle/language/go", visibility = ["//visibility:public"], From aed34af23031307584949a48511d333f8d0bf581 Mon Sep 17 00:00:00 2001 From: HakanSunay Date: Fri, 13 May 2022 16:49:39 +0300 Subject: [PATCH 3/7] Regenerate go_repository_tools_srcs.bzl to reflect work.go --- internal/go_repository_tools_srcs.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/go_repository_tools_srcs.bzl b/internal/go_repository_tools_srcs.bzl index 48de4e3a2..4f5380281 100644 --- a/internal/go_repository_tools_srcs.bzl +++ b/internal/go_repository_tools_srcs.bzl @@ -62,6 +62,7 @@ GO_REPOSITORY_TOOLS_SRCS = [ "@bazel_gazelle//language/go:resolve.go", "@bazel_gazelle//language/go:std_package_list.go", "@bazel_gazelle//language/go:update.go", + "@bazel_gazelle//language/go:work.go", "@bazel_gazelle//language:lang.go", "@bazel_gazelle//language/proto:BUILD.bazel", "@bazel_gazelle//language/proto:config.go", From 5b12c58eb70fd7d50d66ad2e86e7f6b7581506bf Mon Sep 17 00:00:00 2001 From: HakanSunay Date: Fri, 13 May 2022 17:07:13 +0300 Subject: [PATCH 4/7] Hopefully the last file to update before getting a successful run --- language/go/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/language/go/BUILD.bazel b/language/go/BUILD.bazel index 1320972fc..fc77d9e7f 100644 --- a/language/go/BUILD.bazel +++ b/language/go/BUILD.bazel @@ -113,6 +113,7 @@ filegroup( "stubs_test.go", "update.go", "update_import_test.go", + "work.go", "//language/go/gen_std_package_list:all_files", ], visibility = ["//visibility:public"], From f17add54c70504eab9662238d5203bf8afa88233 Mon Sep 17 00:00:00 2001 From: HakanSunay Date: Fri, 13 May 2022 19:42:20 +0300 Subject: [PATCH 5/7] Extract common functionality between and into a new location to prevent code repetition --- internal/go_repository_tools_srcs.bzl | 1 + language/go/BUILD.bazel | 2 + language/go/modules.go | 214 +--------------------- language/go/update_import_test.go | 4 +- language/go/utils.go | 251 ++++++++++++++++++++++++++ language/go/work.go | 117 +----------- 6 files changed, 276 insertions(+), 313 deletions(-) create mode 100644 language/go/utils.go diff --git a/internal/go_repository_tools_srcs.bzl b/internal/go_repository_tools_srcs.bzl index 4f5380281..6fd16d2fc 100644 --- a/internal/go_repository_tools_srcs.bzl +++ b/internal/go_repository_tools_srcs.bzl @@ -62,6 +62,7 @@ GO_REPOSITORY_TOOLS_SRCS = [ "@bazel_gazelle//language/go:resolve.go", "@bazel_gazelle//language/go:std_package_list.go", "@bazel_gazelle//language/go:update.go", + "@bazel_gazelle//language/go:utils.go", "@bazel_gazelle//language/go:work.go", "@bazel_gazelle//language:lang.go", "@bazel_gazelle//language/proto:BUILD.bazel", diff --git a/language/go/BUILD.bazel b/language/go/BUILD.bazel index fc77d9e7f..0d0ed71b6 100644 --- a/language/go/BUILD.bazel +++ b/language/go/BUILD.bazel @@ -30,6 +30,7 @@ go_library( "resolve.go", "std_package_list.go", "update.go", + "utils.go", "work.go", ], importpath = "github.com/bazelbuild/bazel-gazelle/language/go", @@ -113,6 +114,7 @@ filegroup( "stubs_test.go", "update.go", "update_import_test.go", + "utils.go", "work.go", "//language/go/gen_std_package_list:all_files", ], diff --git a/language/go/modules.go b/language/go/modules.go index 5e92d5b2b..bee1bb361 100644 --- a/language/go/modules.go +++ b/language/go/modules.go @@ -17,77 +17,25 @@ package golang import ( "bytes" - "encoding/json" - "errors" "fmt" - "go/build" "io/ioutil" - "log" - "os" - "os/exec" "path/filepath" - "runtime" - "sort" "strings" - "github.com/bazelbuild/bazel-gazelle/label" "github.com/bazelbuild/bazel-gazelle/language" - "github.com/bazelbuild/bazel-gazelle/rule" ) func importReposFromModules(args language.ImportReposArgs) language.ImportReposResult { - dir := filepath.Dir(args.Path) - - // List all modules except for the main module, including implicit indirect - // dependencies. - // Schema is documented at https://go.dev/ref/mod#go-list-m - type moduleFromList struct { - Path, Version, Sum, Error string - Main bool - Replace *struct { - Path, Version string - } - } - // path@version can be used as a unique identifier for looking up sums - pathToModule := map[string]*moduleFromList{} - data, err := goListModules(dir) - dec := json.NewDecoder(bytes.NewReader(data)) + // run go list in the dir where go.mod is located + data, err := goListModules(filepath.Dir(args.Path)) if err != nil { - // Best-effort try to adorn specific error details from the JSON output. - for dec.More() { - var dl moduleFromList - if decodeErr := dec.Decode(&dl); decodeErr != nil { - // If we couldn't parse a possible error description, just return the raw error. - err = fmt.Errorf("%w\nError parsing module for more error information: %v", err, decodeErr) - break - } - if dl.Error != "" { - err = fmt.Errorf("%w\nError listing %v: %v", err, dl.Path, dl.Error) - } - } - err = fmt.Errorf("error from go list: %w", err) + return language.ImportReposResult{Error: processGoListError(err, data)} + } + pathToModule, err := extractModules(data) + if err != nil { return language.ImportReposResult{Error: err} } - for dec.More() { - mod := new(moduleFromList) - if err := dec.Decode(mod); err != nil { - return language.ImportReposResult{Error: err} - } - if mod.Main { - continue - } - if mod.Replace != nil { - if filepath.IsAbs(mod.Replace.Path) || build.IsLocalImport(mod.Replace.Path) { - log.Printf("go_repository does not support file path replacements for %s -> %s", mod.Path, - mod.Replace.Path) - continue - } - pathToModule[mod.Replace.Path+"@"+mod.Replace.Version] = mod - } else { - pathToModule[mod.Path+"@"+mod.Version] = mod - } - } // Load sums from go.sum. Ideally, they're all there. goSumPath := filepath.Join(filepath.Dir(args.Path), "go.sum") @@ -108,154 +56,12 @@ func importReposFromModules(args language.ImportReposArgs) language.ImportReposR } } - // If sums are missing, run 'go mod download' to get them. - // This must be done in a temporary directory because 'go mod download' - // may modify go.mod and go.sum. It does not support -mod=readonly. - var missingSumArgs []string - for pathVer, mod := range pathToModule { - if mod.Sum == "" { - missingSumArgs = append(missingSumArgs, pathVer) - } - } - - type downloadError struct { - Err string - } - - // Schema is documented at https://go.dev/ref/mod#go-mod-download - type moduleFromDownload struct { - Path, Version, Sum string - Main bool - Replace *struct { - Path, Version string - } - Error *downloadError - } - - if len(missingSumArgs) > 0 { - tmpDir, err := ioutil.TempDir("", "") - if err != nil { - return language.ImportReposResult{Error: fmt.Errorf("finding module sums: %v", err)} - } - defer os.RemoveAll(tmpDir) - data, err := goModDownload(tmpDir, missingSumArgs) - dec = json.NewDecoder(bytes.NewReader(data)) - if err != nil { - // Best-effort try to adorn specific error details from the JSON output. - for dec.More() { - var dl moduleFromDownload - if decodeErr := dec.Decode(&dl); decodeErr != nil { - // If we couldn't parse a possible error description, just return the raw error. - err = fmt.Errorf("%w\nError parsing module for more error information: %v", err, decodeErr) - break - } - if dl.Error != nil { - err = fmt.Errorf("%w\nError downloading %v: %v", err, dl.Path, dl.Error.Err) - } - } - err = fmt.Errorf("error from go mod download: %w", err) - - return language.ImportReposResult{Error: err} - } - for dec.More() { - var dl moduleFromDownload - if err := dec.Decode(&dl); err != nil { - return language.ImportReposResult{Error: err} - } - if mod, ok := pathToModule[dl.Path+"@"+dl.Version]; ok { - mod.Sum = dl.Sum - } - } + pathToModule, err = fillMissingSums(pathToModule) + if err != nil { + return language.ImportReposResult{Error: fmt.Errorf("finding module sums: %v", err)} } // Translate to repository rules. - gen := make([]*rule.Rule, 0, len(pathToModule)) - for pathVer, mod := range pathToModule { - if mod.Sum == "" { - log.Printf("could not determine sum for module %s", pathVer) - continue - } - r := rule.NewRule("go_repository", label.ImportPathToBazelRepoName(mod.Path)) - r.SetAttr("importpath", mod.Path) - r.SetAttr("sum", mod.Sum) - if mod.Replace == nil { - r.SetAttr("version", mod.Version) - } else { - r.SetAttr("replace", mod.Replace.Path) - r.SetAttr("version", mod.Replace.Version) - } - gen = append(gen, r) - } - sort.Slice(gen, func(i, j int) bool { - return gen[i].Name() < gen[j].Name() - }) + gen := toRepositoryRules(pathToModule) return language.ImportReposResult{Gen: gen} } - -// goListModules invokes "go list" in a directory containing a go.mod file. -var goListModules = func(dir string) ([]byte, error) { - return runGoCommandForOutput(dir, "list", "-mod=readonly", "-e", "-m", "-json", "all") -} - -// goModDownload invokes "go mod download" in a directory containing a -// go.mod file. -var goModDownload = func(dir string, args []string) ([]byte, error) { - dlArgs := []string{"mod", "download", "-json"} - dlArgs = append(dlArgs, args...) - return runGoCommandForOutput(dir, dlArgs...) -} - -// findGoTool attempts to locate the go executable. If GOROOT is set, we'll -// prefer the one in there; otherwise, we'll rely on PATH. If the wrapper -// script generated by the gazelle rule is invoked by Bazel, it will set -// GOROOT to the configured SDK. We don't want to rely on the host SDK in -// that situation. -func findGoTool() string { - path := "go" // rely on PATH by default - if goroot, ok := os.LookupEnv("GOROOT"); ok { - path = filepath.Join(goroot, "bin", "go") - } - if runtime.GOOS == "windows" { - path += ".exe" - } - return path -} - -func runGoCommandForOutput(dir string, args ...string) ([]byte, error) { - goTool := findGoTool() - env := os.Environ() - env = append(env, "GO111MODULE=on") - if os.Getenv("GOCACHE") == "" && os.Getenv("HOME") == "" { - gocache, err := ioutil.TempDir("", "") - if err != nil { - return nil, err - } - env = append(env, "GOCACHE="+gocache) - defer os.RemoveAll(gocache) - } - if os.Getenv("GOPATH") == "" && os.Getenv("HOME") == "" { - gopath, err := ioutil.TempDir("", "") - if err != nil { - return nil, err - } - env = append(env, "GOPATH="+gopath) - defer os.RemoveAll(gopath) - } - cmd := exec.Command(goTool, args...) - stderr := &bytes.Buffer{} - cmd.Stderr = stderr - cmd.Dir = dir - cmd.Env = env - out, err := cmd.Output() - if err != nil { - var errStr string - var xerr *exec.ExitError - if errors.As(err, &xerr) { - errStr = strings.TrimSpace(stderr.String()) - } else { - errStr = err.Error() - } - return out, fmt.Errorf("running '%s %s': %s", cmd.Path, strings.Join(cmd.Args, " "), errStr) - } - return out, nil -} diff --git a/language/go/update_import_test.go b/language/go/update_import_test.go index 3d644a365..fe8d859f5 100644 --- a/language/go/update_import_test.go +++ b/language/go/update_import_test.go @@ -248,7 +248,7 @@ definitely.doesnotexist/ever v0.1.0/go.mod h1:HI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqw }, }, want: "", - wantErr: "error from go mod download: failed to download\nError downloading definitely.doesnotexist/ever: Did not exist", + wantErr: "finding module sums: error from go mod download: failed to download\nError downloading definitely.doesnotexist/ever: Did not exist", stubGoModDownload: func(dir string, args []string) ([]byte, error) { return []byte(`{ "Path": "definitely.doesnotexist/ever", @@ -278,7 +278,7 @@ definitely.doesnotexist/ever v0.1.0/go.mod h1:HI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqw }, }, want: "", - wantErr: "error from go mod download: failed to download\nError parsing module for more error information: invalid character 'o' in literal null (expecting 'u')", + wantErr: "finding module sums: error from go mod download: failed to download\nError parsing module for more error information: invalid character 'o' in literal null (expecting 'u')", stubGoModDownload: func(dir string, args []string) ([]byte, error) { return []byte(`{ "Path": "definitely.doesnotexist/ever", diff --git a/language/go/utils.go b/language/go/utils.go new file mode 100644 index 000000000..1b2b8cf68 --- /dev/null +++ b/language/go/utils.go @@ -0,0 +1,251 @@ +/* Copyright 2022 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package golang + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go/build" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/bazelbuild/bazel-gazelle/label" + "github.com/bazelbuild/bazel-gazelle/rule" +) + +// goListModules invokes "go list" in a directory containing a go.mod file. +var goListModules = func(dir string) ([]byte, error) { + return runGoCommandForOutput(dir, "list", "-mod=readonly", "-e", "-m", "-json", "all") +} + +// goModDownload invokes "go mod download" in a directory containing a +// go.mod file. +var goModDownload = func(dir string, args []string) ([]byte, error) { + dlArgs := []string{"mod", "download", "-json"} + dlArgs = append(dlArgs, args...) + return runGoCommandForOutput(dir, dlArgs...) +} + +// modulesFromList is an abstraction to preserve the output of `go list`. +// The output schema is documented at https://go.dev/ref/mod#go-list-m +type moduleFromList struct { + Path, Version, Sum, Error string + Main bool + Replace *struct { + Path, Version string + } +} + +type downloadError struct { + Err string +} + +// moduleFromDownload is an abstraction to preserve the output of `go mod download`. +// The output schema is documented at https://go.dev/ref/mod#go-mod-download +type moduleFromDownload struct { + Path, Version, Sum string + Main bool + Replace *struct { + Path, Version string + } + Error *downloadError +} + +// extractModules lists all modules except for the main module, +// including implicit indirect dependencies. +func extractModules(data []byte) (map[string]*moduleFromList, error) { + // path@version can be used as a unique identifier for looking up sums + pathToModule := map[string]*moduleFromList{} + dec := json.NewDecoder(bytes.NewReader(data)) + for dec.More() { + mod := new(moduleFromList) + if err := dec.Decode(mod); err != nil { + return nil, err + } + if mod.Main { + continue + } + if mod.Replace != nil { + if filepath.IsAbs(mod.Replace.Path) || build.IsLocalImport(mod.Replace.Path) { + log.Printf("go_repository does not support file path replacements for %s -> %s", mod.Path, + mod.Replace.Path) + continue + } + pathToModule[mod.Replace.Path+"@"+mod.Replace.Version] = mod + } else { + pathToModule[mod.Path+"@"+mod.Version] = mod + } + } + return pathToModule, nil +} + +// fillMissingSums runs `go mod download` to get missing sums. +// This must be done in a temporary directory because 'go mod download' +// may modify go.mod and go.sum. It does not support -mod=readonly. +func fillMissingSums(pathToModule map[string]*moduleFromList) (map[string]*moduleFromList, error) { + var missingSumArgs []string + for pathVer, mod := range pathToModule { + if mod.Sum == "" { + missingSumArgs = append(missingSumArgs, pathVer) + } + } + + if len(missingSumArgs) > 0 { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) + data, err := goModDownload(tmpDir, missingSumArgs) + dec := json.NewDecoder(bytes.NewReader(data)) + if err != nil { + // Best-effort try to adorn specific error details from the JSON output. + for dec.More() { + var dl moduleFromDownload + if decodeErr := dec.Decode(&dl); decodeErr != nil { + // If we couldn't parse a possible error description, just return the raw error. + err = fmt.Errorf("%w\nError parsing module for more error information: %v", err, decodeErr) + break + } + if dl.Error != nil { + err = fmt.Errorf("%w\nError downloading %v: %v", err, dl.Path, dl.Error.Err) + } + } + err = fmt.Errorf("error from go mod download: %w", err) + + return nil, err + } + for dec.More() { + var dl moduleFromDownload + if err := dec.Decode(&dl); err != nil { + return nil, err + } + if mod, ok := pathToModule[dl.Path+"@"+dl.Version]; ok { + mod.Sum = dl.Sum + } + } + } + + return pathToModule, nil +} + +// toRepositoryRules transforms the input map into repository rules. +func toRepositoryRules(pathToModule map[string]*moduleFromList) []*rule.Rule { + gen := make([]*rule.Rule, 0, len(pathToModule)) + for pathVer, mod := range pathToModule { + if mod.Sum == "" { + log.Printf("could not determine sum for module %s", pathVer) + continue + } + r := rule.NewRule("go_repository", label.ImportPathToBazelRepoName(mod.Path)) + r.SetAttr("importpath", mod.Path) + r.SetAttr("sum", mod.Sum) + if mod.Replace == nil { + r.SetAttr("version", mod.Version) + } else { + r.SetAttr("replace", mod.Replace.Path) + r.SetAttr("version", mod.Replace.Version) + } + gen = append(gen, r) + } + sort.Slice(gen, func(i, j int) bool { + return gen[i].Name() < gen[j].Name() + }) + + return gen +} + +// processGoListError attempts a best-effort try to adorn specific error details from the JSON output of `go list`. +func processGoListError(err error, data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + for dec.More() { + var dl moduleFromList + if decodeErr := dec.Decode(&dl); decodeErr != nil { + // If we couldn't parse a possible error description, just return the raw error. + err = fmt.Errorf("%w\nError parsing module for more error information: %v", err, decodeErr) + break + } + if dl.Error != "" { + err = fmt.Errorf("%w\nError listing %v: %v", err, dl.Path, dl.Error) + } + } + err = fmt.Errorf("error from go list: %w", err) + + return err +} + +// findGoTool attempts to locate the go executable. If GOROOT is set, we'll +// prefer the one in there; otherwise, we'll rely on PATH. If the wrapper +// script generated by the gazelle rule is invoked by Bazel, it will set +// GOROOT to the configured SDK. We don't want to rely on the host SDK in +// that situation. +func findGoTool() string { + path := "go" // rely on PATH by default + if goroot, ok := os.LookupEnv("GOROOT"); ok { + path = filepath.Join(goroot, "bin", "go") + } + if runtime.GOOS == "windows" { + path += ".exe" + } + return path +} + +func runGoCommandForOutput(dir string, args ...string) ([]byte, error) { + goTool := findGoTool() + env := os.Environ() + env = append(env, "GO111MODULE=on") + if os.Getenv("GOCACHE") == "" && os.Getenv("HOME") == "" { + gocache, err := ioutil.TempDir("", "") + if err != nil { + return nil, err + } + env = append(env, "GOCACHE="+gocache) + defer os.RemoveAll(gocache) + } + if os.Getenv("GOPATH") == "" && os.Getenv("HOME") == "" { + gopath, err := ioutil.TempDir("", "") + if err != nil { + return nil, err + } + env = append(env, "GOPATH="+gopath) + defer os.RemoveAll(gopath) + } + cmd := exec.Command(goTool, args...) + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + cmd.Dir = dir + cmd.Env = env + out, err := cmd.Output() + if err != nil { + var errStr string + var xerr *exec.ExitError + if errors.As(err, &xerr) { + errStr = strings.TrimSpace(stderr.String()) + } else { + errStr = err.Error() + } + return out, fmt.Errorf("running '%s %s': %s", cmd.Path, strings.Join(cmd.Args, " "), errStr) + } + return out, nil +} diff --git a/language/go/work.go b/language/go/work.go index 53e8f03b7..757265e1a 100644 --- a/language/go/work.go +++ b/language/go/work.go @@ -16,127 +16,30 @@ limitations under the License. package golang import ( - "bytes" - "encoding/json" "fmt" - "go/build" - "io/ioutil" - "log" - "os" "path/filepath" - "sort" - "github.com/bazelbuild/bazel-gazelle/label" "github.com/bazelbuild/bazel-gazelle/language" - "github.com/bazelbuild/bazel-gazelle/rule" ) func importReposFromWork(args language.ImportReposArgs) language.ImportReposResult { - // dir where go.work is located - dir := filepath.Dir(args.Path) - - // List all modules except for the main module, including implicit indirect - // dependencies. - type module struct { - Path, Version, Sum, Error string - Main bool - Replace *struct { - Path, Version string - } - } - // path@version can be used as a unique identifier for looking up sums - pathToModule := map[string]*module{} - data, err := goListModules(dir) + // run go list in the dir where go.work is located + data, err := goListModules(filepath.Dir(args.Path)) if err != nil { - return language.ImportReposResult{Error: err} - } - dec := json.NewDecoder(bytes.NewReader(data)) - for dec.More() { - mod := new(module) - if err := dec.Decode(mod); err != nil { - return language.ImportReposResult{Error: err} - } - if mod.Main { - continue - } - if mod.Replace != nil { - if filepath.IsAbs(mod.Replace.Path) || build.IsLocalImport(mod.Replace.Path) { - log.Printf("go_repository does not support file path replacements for %s -> %s", mod.Path, - mod.Replace.Path) - continue - } - pathToModule[mod.Replace.Path+"@"+mod.Replace.Version] = mod - } else { - pathToModule[mod.Path+"@"+mod.Version] = mod - } + return language.ImportReposResult{Error: processGoListError(nil, data)} } - // Sums are missing by default for go.work. - // That's why we run 'go mod download' to get them. - // Once https://github.com/golang/go/issues/52792 is resolved, we can use go list. - // This must be done in a temporary directory because 'go mod download' - // may modify go.mod and go.sum. It does not support -mod=readonly. - var missingSumArgs []string - for pathVer, mod := range pathToModule { - if mod.Sum == "" { - missingSumArgs = append(missingSumArgs, pathVer) - } + pathToModule, err := extractModules(data) + if err != nil { + return language.ImportReposResult{Error: err} } - if len(missingSumArgs) > 0 { - tmpDir, err := ioutil.TempDir("", "") - if err != nil { - return language.ImportReposResult{Error: fmt.Errorf("finding module sums: %v", err)} - } - defer os.RemoveAll(tmpDir) - data, err := goModDownload(tmpDir, missingSumArgs) - dec = json.NewDecoder(bytes.NewReader(data)) - if err != nil { - // Best-effort try to adorn specific error details from the JSON output. - for dec.More() { - var dl module - if err := dec.Decode(&dl); err != nil { - // If we couldn't parse a possible error description, just ignore this part of the output. - continue - } - if dl.Error != "" { - err = fmt.Errorf("%v\nError downloading %v: %v", err, dl.Path, dl.Error) - } - } - - return language.ImportReposResult{Error: err} - } - for dec.More() { - var dl module - if err := dec.Decode(&dl); err != nil { - return language.ImportReposResult{Error: err} - } - if mod, ok := pathToModule[dl.Path+"@"+dl.Version]; ok { - mod.Sum = dl.Sum - } - } + pathToModule, err = fillMissingSums(pathToModule) + if err != nil { + return language.ImportReposResult{Error: fmt.Errorf("finding module sums: %v", err)} } // Translate to repository rules. - gen := make([]*rule.Rule, 0, len(pathToModule)) - for pathVer, mod := range pathToModule { - if mod.Sum == "" { - log.Printf("could not determine sum for module %s", pathVer) - continue - } - r := rule.NewRule("go_repository", label.ImportPathToBazelRepoName(mod.Path)) - r.SetAttr("importpath", mod.Path) - r.SetAttr("sum", mod.Sum) - if mod.Replace == nil { - r.SetAttr("version", mod.Version) - } else { - r.SetAttr("replace", mod.Replace.Path) - r.SetAttr("version", mod.Replace.Version) - } - gen = append(gen, r) - } - sort.Slice(gen, func(i, j int) bool { - return gen[i].Name() < gen[j].Name() - }) + gen := toRepositoryRules(pathToModule) return language.ImportReposResult{Gen: gen} } From d8c18ce4f718d68dccd2e1854189e8732c1b8480 Mon Sep 17 00:00:00 2001 From: HakanSunay Date: Fri, 13 May 2022 20:38:49 +0300 Subject: [PATCH 6/7] Simpliy return expressions --- language/go/modules.go | 4 +--- language/go/work.go | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/language/go/modules.go b/language/go/modules.go index bee1bb361..2440c186a 100644 --- a/language/go/modules.go +++ b/language/go/modules.go @@ -61,7 +61,5 @@ func importReposFromModules(args language.ImportReposArgs) language.ImportReposR return language.ImportReposResult{Error: fmt.Errorf("finding module sums: %v", err)} } - // Translate to repository rules. - gen := toRepositoryRules(pathToModule) - return language.ImportReposResult{Gen: gen} + return language.ImportReposResult{Gen: toRepositoryRules(pathToModule)} } diff --git a/language/go/work.go b/language/go/work.go index 757265e1a..83a4dc4d5 100644 --- a/language/go/work.go +++ b/language/go/work.go @@ -39,7 +39,5 @@ func importReposFromWork(args language.ImportReposArgs) language.ImportReposResu return language.ImportReposResult{Error: fmt.Errorf("finding module sums: %v", err)} } - // Translate to repository rules. - gen := toRepositoryRules(pathToModule) - return language.ImportReposResult{Gen: gen} + return language.ImportReposResult{Gen: toRepositoryRules(pathToModule)} } From ebe0e538a6634d4c8eb172bfb2826306c67b6be6 Mon Sep 17 00:00:00 2001 From: HakanSunay Date: Mon, 16 May 2022 20:04:51 +0300 Subject: [PATCH 7/7] Update general README with information about --- README.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6797bb63c..19330f6aa 100644 --- a/README.rst +++ b/README.rst @@ -540,7 +540,7 @@ The following flags are accepted: The ``update-repos`` command updates repository rules. It can write the rules to either the WORKSPACE (by default) or a .bzl file macro function. It can be used to add new repository rules or update existing rules to the specified -version. It can also import repository rules from a ``go.mod`` file or a +version. It can also import repository rules from a ``go.mod``, ``go.work`` or a ``Gopkg.lock`` file. .. code:: bash @@ -554,9 +554,15 @@ version. It can also import repository rules from a ``go.mod`` file or a # Import repositories from go.mod $ gazelle update-repos -from_file=go.mod + # Import repositories from go.work + $ gazelle update-repos -from_file=go.work + # Import repositories from go.mod and update macro $ gazelle update-repos -from_file=go.mod -to_macro=repositories.bzl%go_repositories + # Import repositories from go.work and update macro + $ gazelle update-repos -from_file=go.work -to_macro=repositories.bzl%go_repositories + The following flags are accepted: +----------------------------------------------------------------------------------------------------------+----------------------------------------------+ @@ -566,7 +572,7 @@ The following flags are accepted: +----------------------------------------------------------------------------------------------------------+----------------------------------------------+ | Import repositories from a file as `go_repository`_ rules. These rules will be added to the bottom of the WORKSPACE file or merged with existing rules. | | | -| The lock file format is inferred from the file name. ``go.mod`` and, ``Gopkg.lock`` (the dep lock format) are both supported. | +| The lock file format is inferred from the file name. ``go.mod``, ``go.work` and, ``Gopkg.lock`` (the dep lock format) are all supported. | +----------------------------------------------------------------------------------------------------------+----------------------------------------------+ | :flag:`-repo_root dir` | | +----------------------------------------------------------------------------------------------------------+----------------------------------------------+