From e0a3261fa21bd7afbd728c26883d2148b3a42c71 Mon Sep 17 00:00:00 2001 From: Tony Worm Date: Thu, 9 Mar 2023 03:09:53 -0500 Subject: [PATCH] hof/mod: add validate, introspection, and psuedoversions --- .hof/shadow/cli/cmd/hof/cmd/mod.go | 1 + .hof/shadow/cli/cmd/hof/cmd/mod/verify.go | 88 ++++++++++ cmd/hof/cmd/mod.go | 1 + cmd/hof/cmd/mod/verify.go | 88 ++++++++++ cmd/hof/verinfo/verinfo.go | 2 +- design/cmds/mod.cue | 9 + docs/code/cmd-help/mod | 1 + flow/tasks/csp/schema.cue | 4 +- lib/create/create.go | 11 +- lib/mod/find.go | 170 +++++++++++++++++++ lib/mod/get.go | 23 +-- lib/mod/modder/README.md | 4 - lib/mod/modder/modder.go | 113 ------------- lib/mod/modder/modder_check.go | 181 -------------------- lib/mod/modder/modder_deps.go | 133 --------------- lib/mod/modder/modder_errors.go | 53 ------ lib/mod/modder/modder_init.go | 130 --------------- lib/mod/modder/modder_load.go | 163 ------------------ lib/mod/modder/modder_vendor.go | 195 ---------------------- lib/mod/modder/modder_verify.go | 83 --------- lib/mod/modder/modder_write.go | 169 ------------------- lib/mod/modder/module.go | 62 ------- lib/mod/modder/module_load.go | 151 ----------------- lib/mod/mvs.go | 31 +++- lib/mod/testdata/get__branch_dep.txt | 7 + lib/mod/testdata/get__commit_dep.txt | 8 + lib/mod/testdata/get__newer_dep.txt | 4 +- lib/mod/testdata/vendor__with_deps.txt | 2 +- lib/mod/testdata/vendor__with_psuedo.txt | 16 ++ lib/mod/tidy.go | 32 +++- lib/mod/verify.go | 87 ++++++++++ lib/repos/cache/fetch.go | 148 ++-------------- lib/repos/cache/info.go | 118 +++++++++++++ lib/repos/cache/load.go | 7 +- lib/repos/cache/write.go | 42 ++++- lib/repos/git/fetch.go | 9 +- script/runtime/cmd_fs.go | 2 +- 37 files changed, 721 insertions(+), 1627 deletions(-) create mode 100644 .hof/shadow/cli/cmd/hof/cmd/mod/verify.go create mode 100644 cmd/hof/cmd/mod/verify.go create mode 100644 lib/mod/find.go delete mode 100644 lib/mod/modder/README.md delete mode 100644 lib/mod/modder/modder.go delete mode 100644 lib/mod/modder/modder_check.go delete mode 100644 lib/mod/modder/modder_deps.go delete mode 100644 lib/mod/modder/modder_errors.go delete mode 100644 lib/mod/modder/modder_init.go delete mode 100644 lib/mod/modder/modder_load.go delete mode 100644 lib/mod/modder/modder_vendor.go delete mode 100644 lib/mod/modder/modder_verify.go delete mode 100644 lib/mod/modder/modder_write.go delete mode 100644 lib/mod/modder/module.go delete mode 100644 lib/mod/modder/module_load.go create mode 100644 lib/mod/testdata/get__branch_dep.txt create mode 100644 lib/mod/testdata/get__commit_dep.txt create mode 100644 lib/mod/testdata/vendor__with_psuedo.txt create mode 100644 lib/repos/cache/info.go diff --git a/.hof/shadow/cli/cmd/hof/cmd/mod.go b/.hof/shadow/cli/cmd/hof/cmd/mod.go index 3df3e8958..074c79494 100644 --- a/.hof/shadow/cli/cmd/hof/cmd/mod.go +++ b/.hof/shadow/cli/cmd/hof/cmd/mod.go @@ -143,6 +143,7 @@ func init() { ModCmd.AddCommand(cmdmod.InitCmd) ModCmd.AddCommand(cmdmod.GetCmd) + ModCmd.AddCommand(cmdmod.VerifyCmd) ModCmd.AddCommand(cmdmod.TidyCmd) ModCmd.AddCommand(cmdmod.LinkCmd) ModCmd.AddCommand(cmdmod.VendorCmd) diff --git a/.hof/shadow/cli/cmd/hof/cmd/mod/verify.go b/.hof/shadow/cli/cmd/hof/cmd/mod/verify.go new file mode 100644 index 000000000..a22e77554 --- /dev/null +++ b/.hof/shadow/cli/cmd/hof/cmd/mod/verify.go @@ -0,0 +1,88 @@ +package cmdmod + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/hofstadter-io/hof/lib/mod" + + "github.com/hofstadter-io/hof/cmd/hof/flags" + + "github.com/hofstadter-io/hof/cmd/hof/ga" +) + +var verifyLong = `verify integrity of dependencies` + +func VerifyRun(args []string) (err error) { + + err = mod.Verify(flags.RootPflags) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + return err +} + +var VerifyCmd = &cobra.Command{ + + Use: "verify", + + Short: "verify integrity of dependencies", + + Long: verifyLong, + + PreRun: func(cmd *cobra.Command, args []string) { + + ga.SendCommandPath(cmd.CommandPath()) + + }, + + Run: func(cmd *cobra.Command, args []string) { + var err error + + // Argument Parsing + + err = VerifyRun(args) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + }, +} + +func init() { + extra := func(cmd *cobra.Command) bool { + + return false + } + + ohelp := VerifyCmd.HelpFunc() + ousage := VerifyCmd.UsageFunc() + help := func(cmd *cobra.Command, args []string) { + if extra(cmd) { + return + } + ohelp(cmd, args) + } + usage := func(cmd *cobra.Command) error { + if extra(cmd) { + return nil + } + return ousage(cmd) + } + + thelp := func(cmd *cobra.Command, args []string) { + ga.SendCommandPath(cmd.CommandPath() + " help") + help(cmd, args) + } + tusage := func(cmd *cobra.Command) error { + ga.SendCommandPath(cmd.CommandPath() + " usage") + return usage(cmd) + } + VerifyCmd.SetHelpFunc(thelp) + VerifyCmd.SetUsageFunc(tusage) + +} diff --git a/cmd/hof/cmd/mod.go b/cmd/hof/cmd/mod.go index 3df3e8958..074c79494 100644 --- a/cmd/hof/cmd/mod.go +++ b/cmd/hof/cmd/mod.go @@ -143,6 +143,7 @@ func init() { ModCmd.AddCommand(cmdmod.InitCmd) ModCmd.AddCommand(cmdmod.GetCmd) + ModCmd.AddCommand(cmdmod.VerifyCmd) ModCmd.AddCommand(cmdmod.TidyCmd) ModCmd.AddCommand(cmdmod.LinkCmd) ModCmd.AddCommand(cmdmod.VendorCmd) diff --git a/cmd/hof/cmd/mod/verify.go b/cmd/hof/cmd/mod/verify.go new file mode 100644 index 000000000..a22e77554 --- /dev/null +++ b/cmd/hof/cmd/mod/verify.go @@ -0,0 +1,88 @@ +package cmdmod + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/hofstadter-io/hof/lib/mod" + + "github.com/hofstadter-io/hof/cmd/hof/flags" + + "github.com/hofstadter-io/hof/cmd/hof/ga" +) + +var verifyLong = `verify integrity of dependencies` + +func VerifyRun(args []string) (err error) { + + err = mod.Verify(flags.RootPflags) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + return err +} + +var VerifyCmd = &cobra.Command{ + + Use: "verify", + + Short: "verify integrity of dependencies", + + Long: verifyLong, + + PreRun: func(cmd *cobra.Command, args []string) { + + ga.SendCommandPath(cmd.CommandPath()) + + }, + + Run: func(cmd *cobra.Command, args []string) { + var err error + + // Argument Parsing + + err = VerifyRun(args) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + }, +} + +func init() { + extra := func(cmd *cobra.Command) bool { + + return false + } + + ohelp := VerifyCmd.HelpFunc() + ousage := VerifyCmd.UsageFunc() + help := func(cmd *cobra.Command, args []string) { + if extra(cmd) { + return + } + ohelp(cmd, args) + } + usage := func(cmd *cobra.Command) error { + if extra(cmd) { + return nil + } + return ousage(cmd) + } + + thelp := func(cmd *cobra.Command, args []string) { + ga.SendCommandPath(cmd.CommandPath() + " help") + help(cmd, args) + } + tusage := func(cmd *cobra.Command) error { + ga.SendCommandPath(cmd.CommandPath() + " usage") + return usage(cmd) + } + VerifyCmd.SetHelpFunc(thelp) + VerifyCmd.SetUsageFunc(tusage) + +} diff --git a/cmd/hof/verinfo/verinfo.go b/cmd/hof/verinfo/verinfo.go index fea37d1d0..3f10d88df 100644 --- a/cmd/hof/verinfo/verinfo.go +++ b/cmd/hof/verinfo/verinfo.go @@ -20,7 +20,7 @@ var ( // the value gets injected into templates in various places // the default here is set to something useful for dev // the release version is the same as the cli running it - HofVersion = "v0.6.8-create.1" + HofVersion = "v0.6.8-beta.9" ) func init() { diff --git a/design/cmds/mod.cue b/design/cmds/mod.cue index eed02b7a7..70465dda5 100644 --- a/design/cmds/mod.cue +++ b/design/cmds/mod.cue @@ -87,6 +87,15 @@ import ( os.Exit(1) } """ + }, { + Name: "verify" + Usage: "verify" + Short: "verify integrity of dependencies" + Long: Short + + Imports: #ModCmdImports + + Body: (#body & { func: "Verify" }).content }, { Name: "tidy" Usage: "tidy" diff --git a/docs/code/cmd-help/mod b/docs/code/cmd-help/mod index 315bb7100..c27804d27 100644 --- a/docs/code/cmd-help/mod +++ b/docs/code/cmd-help/mod @@ -90,6 +90,7 @@ Available Commands: link symlink dependencies to cue.mod/pkg tidy recalculate dependencies and update mod files vendor copy dependencies to cue.mod/pkg + verify verify integrity of dependencies Flags: -h, --help help for mod diff --git a/flow/tasks/csp/schema.cue b/flow/tasks/csp/schema.cue index 64d2df0ca..99f2c445c 100644 --- a/flow/tasks/csp/schema.cue +++ b/flow/tasks/csp/schema.cue @@ -20,7 +20,7 @@ Chan: { // Send a message to a mailbox Send: { @task(csp.Send) - $task(csp.Send) + $task: "csp.Send" // the name of the channel mailbox: string @@ -35,7 +35,7 @@ Send: { // Recv is a coroutine which runs indefinitely Recv: { @task(csp.Recv) - $task(csp.Recv) + $task: "csp.Recv" // the name of the channel mailbox: string diff --git a/lib/create/create.go b/lib/create/create.go index 39ef030d5..89c7cad75 100644 --- a/lib/create/create.go +++ b/lib/create/create.go @@ -60,20 +60,19 @@ func Create(module string, extra []string, rootflags flags.RootPflagpole, cmdfla } ref := ver - - if looksLikeRepo(url) && ver == "latest" { - // ensure we have the most up-to-date code - _, err = cache.FetchRepoSource(url, "") + // ensure we have the most up-to-date code + if looksLikeRepo(url) { + _, err = cache.FetchRepoSource(url, ver) if err != nil { return err } - - ref, err = cache.GetLatestTag(url, false) + ref, err = cache.UpgradePsuedoVersion(url, ver) if err != nil { return err } } + fmt.Println("setting up...") tmpdir, subdir, err = setupTmpdir(url, ref) if err != nil { diff --git a/lib/mod/find.go b/lib/mod/find.go new file mode 100644 index 000000000..2d59f41be --- /dev/null +++ b/lib/mod/find.go @@ -0,0 +1,170 @@ +package mod + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "cuelang.org/go/cue/parser" + + "github.com/hofstadter-io/hof/lib/repos/utils" + "github.com/hofstadter-io/hof/lib/yagu" +) + +var wellKnownDirs = []string { + ".hof/", + ".workdir/", + "cue.mod/", + "docs/", + "testdata/", +} + +func (cm *CueMod) findDepsFromImports() (err error) { + + // dirs, err := cm.findDirsWithCueFiles() + files, err := cm.findCueFiles() + if err != nil { + return err + } + + // fmt.Println("finding missing:") + depMap := make(map[string]bool) + // for _, dir := range dirs { + // deps, _ := cm.dir2deps(dir) + for _, file := range files { + // fmt.Println("dir:", dir) + deps, _ := cm.file2deps(file) + for _, dep := range deps { + // check to see if we know about dep already + if cm.checkIfDepAlreadyKnown(dep) { + continue + } + + // memo if we haven't already + if _, ok := depMap[dep]; !ok { + depMap[dep] = true + } + } + } + + if len(depMap) == 0 { + return nil + } + + // fmt.Println("adding deps: ") + for dep, _ := range depMap { + // fmt.Println(" d:", dep) + // need to walk up to find + m, err := cm.dep2module(dep) + if err != nil { + fmt.Println(err) + continue + } + // fmt.Println(" ", m) + cm.Require[m] = "latest" + } + + return nil +} + +func (cm *CueMod) checkIfDepAlreadyKnown(dep string) (found bool) { + // from this module + if dep == cm.Module || strings.HasPrefix(dep, cm.Module + "/") { + return true + } + + for path, _ := range cm.Replace { + if dep == path || strings.HasPrefix(dep, path + "/") { + return true + } + } + + for path, _ := range cm.Require { + if dep == path || strings.HasPrefix(dep, path + "/") { + return true + } + } + + for path, _ := range cm.Indirect { + if dep == path || strings.HasPrefix(dep, path + "/") { + return true + } + } + + return false +} + +func (cm *CueMod) findCueFiles() (files []string, err error) { + // fmt.Println("findDirs:", cm.Basedir) + + files, err = yagu.FilesFromGlobs([]string{filepath.Join(cm.Basedir, "**/*.cue")}) + if err != nil { + return nil, err + } + + final := make([]string, 0, len(files)) + + for _, file := range files { + // skip wellknown dirs + match := false + for _, wkd := range wellKnownDirs { + if strings.Contains(file, wkd) { + match = true + break + } + } + if match { + continue + } + + final = append(final, file) + } + + return final, nil +} + +func (cm *CueMod) file2deps(file string) ([]string, []error) { + content, err := os.ReadFile(file) + if err != nil { + return nil, []error{err} + } + + fn, err := filepath.Rel(cm.Basedir, file) + if err != nil { + return nil, []error{err} + } + + f, err := parser.ParseFile(fn, content) + if err != nil { + return nil, []error{err} + } + + paths := []string{} + + for _, imp := range f.Imports { + // this is the import path string value + path := imp.Path.Value + + // remove the quotes + path = strings.Replace(path, "\"", "", -1) + + // filter builtin packages + if !strings.Contains(path, ".") { + continue + } + + // include in our deps + paths = append(paths, path) + } + + return paths, nil +} + +func (cm *CueMod) dep2module(dep string) (string, error) { + remote, owner, repo := utils.ParseModURL(dep) + + m := path.Join(remote, owner, repo) + return m, nil +} diff --git a/lib/mod/get.go b/lib/mod/get.go index df8a9b465..950c5e2fa 100644 --- a/lib/mod/get.go +++ b/lib/mod/get.go @@ -39,6 +39,7 @@ func Get(module string, rflags flags.RootPflagpole, gflags flags.Mod__GetFlagpol } fns := []func () error { + cm.UpgradePseudoVersions, func () error { return cm.SolveMVS(updateMvs) }, cm.CleanDeps, cm.CleanSums, @@ -73,13 +74,8 @@ func updateOne(cm *CueMod, path, ver string, rflags flags.RootPflagpole, gflags } // if latest, update - if ver == "latest" { - // make sure we have the latest - _, err = cache.FetchRepoSource(path, "") - if err != nil { - return err - } - ver, err = cache.GetLatestTag(path, gflags.Prerelease) + if ver == "latest" || ver == "next" { + ver, err = cache.GetLatestTag(path, gflags.Prerelease || ver == "next") if err != nil { return err } @@ -104,17 +100,11 @@ func updateOne(cm *CueMod, path, ver string, rflags flags.RootPflagpole, gflags func updateAll(cm *CueMod, rflags flags.RootPflagpole, gflags flags.Mod__GetFlagpole) (err error) { for path, ver := range cm.Require { - // make sure we have the latest - _, err = cache.FetchRepoSource(path, "") - if err != nil { - return err - } nver, err := cache.GetLatestTag(path, false) if err != nil { return err } - // fmt.Println(" ur:", path, ver, nver) // only update if newer, incase we have specific prereleases if semver.Compare(nver, ver) > 1 { cm.Require[path] = nver @@ -122,18 +112,13 @@ func updateAll(cm *CueMod, rflags flags.RootPflagpole, gflags flags.Mod__GetFlag } for path, ver := range cm.Indirect { - // make sure we have the latest - _, err = cache.FetchRepoSource(path, "") - if err != nil { - return err - } // explicitly not doing prerelease here, should we allow updating all to pre-releases? nver, err := cache.GetLatestTag(path, false) if err != nil { return err } - // fmt.Println(" ui:", path, ver) + // only update if newer, incase we have specific prereleases if semver.Compare(nver, ver) > 1 { cm.Indirect[path] = ver } diff --git a/lib/mod/modder/README.md b/lib/mod/modder/README.md deleted file mode 100644 index 0d421d42b..000000000 --- a/lib/mod/modder/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This folder is deprecated and will be removed in the future. - -The logic now lives up a directory and has been significantly reduced. - diff --git a/lib/mod/modder/modder.go b/lib/mod/modder/modder.go deleted file mode 100644 index dd13b515e..000000000 --- a/lib/mod/modder/modder.go +++ /dev/null @@ -1,113 +0,0 @@ -package modder - -import ( - "fmt" - "os" - - "github.com/go-git/go-billy/v5" - - "cuelang.org/go/cue" - - "github.com/hofstadter-io/hof/lib/yagu" -) - -// This modder is for more complex, yet configurable module processing. -// You can have system wide and local custom configurations. -// The fields in this struct are alpha and are likely to change -type Modder struct { - // MetaConfiguration - Name string `yaml:"Name"` - Version string `yaml:"Version",omitempty` - - // Module information - ModFile string `yaml:"ModFile",omitempty` - SumFile string `yaml:"SumFile",omitempty` - ModsDir string `yaml:"ModsDir",omitempty` - MappingFile string `yaml:"MappingFile",omitempty` - PrivateEnvVar string `yaml:"PrivateEnvVar",omitempty` - - // Commands override default, configuragble processing - // for things like golang - NoLoad bool `yaml:"NoLoad",omitempty` - CommandInit [][]string `yaml:"CommandInit",omitempty` - CommandGraph [][]string `yaml:"CommandGraph",omitempty` - CommandTidy [][]string `yaml:"CommandTidy",omitempty` - CommandVendor [][]string `yaml:"CommandVendor",omitempty` - CommandVerify [][]string `yaml:"CommandVerify",omitempty` - CommandStatus [][]string `yaml:"CommandStatus",omitempty` - - // Init related fields - // we need to create things like directories and files beyond the - InitTemplates map[string]string `yaml:"InitTemplates",omitempty` - InitPreCommands [][]string `yaml:"InitPreCommands",omitempty` - InitPostCommands [][]string `yaml:"InitPostCommands",omitempty` - - // Vendor related fields - // filesystem globs for discovering files we should copy over - VendorIncludeGlobs []string `yaml:"VendorIncludeGlobs",omitempty` - VendorExcludeGlobs []string `yaml:"VendorExcludeGlobs",omitempty` - // Any files we need to generate - VendorTemplates map[string]string `yaml:"VendorTemplates",omitempty` - VendorPreCommands [][]string `yaml:"VendorPreCommands",omitempty` - VendorPostCommands [][]string `yaml:"VendorPostCommands",omitempty` - - // Some more vendor controls - ManageFileOnly bool `yaml:"ManageFileOnly",omitempty` - SymlinkLocalReplaces bool `yaml:"SymlinkLocalReplaces",omitempty` - - // Introspection Configuration(s) - // filesystem globs for discovering files we should introspect - // regexs for extracting package information - IntrospectIncludeGlobs []string `yaml:"IntrospectIncludeGlobs",omitempty` - IntrospectExcludeGlobs []string `yaml:"IntrospectExcludeGlobs",omitempty` - IntrospectExtractRegex []string `yaml:"IntrospectExtractRegex",omitempty` - - PackageManagerDefaultPrefix string `yaml:"PackageManagerDefaultPrefix",omitempty` - - // filesystem - FS billy.Filesystem `yaml:"-"` - - // root module - workdir string `yaml:"-"` - module *Module `yaml:"-"` - errors []error `yaml:"-"` - - // dependency modules (requires/replace) - // dependencies should respect any .mvsconfig it finds along side the module files - // module writers can then have local control over how their module is handeled during vendoring - depsMap map[string]*Module `yaml:"-"` - - // compiled cue, used for merging - CueInstance *cue.Instance `yaml:"-"` -} - -func NewFromFile(lang, filepath string, FS billy.Filesystem) (*Modder, error) { - - bytes, err := yagu.BillyReadAll(filepath, FS) - if err != nil { - if _, ok := err.(*os.PathError); !ok && err.Error() != "file does not exist" && err.Error() != "no such file or directory" { - return nil, err - } - // The user has not setup a global $HOME/.mvs/mvsconfig file - return nil, nil - } - - var mdrMap map[string]*Modder - - rt := cue.Runtime{} - i, err := rt.Compile(filepath, string(bytes)) - if err != nil { - return nil, err - } - err = i.Value().Decode(&mdrMap) - if err != nil { - return nil, err - } - - mdr, ok := mdrMap[lang] - if !ok { - return nil, fmt.Errorf("lang %q not found in %s", lang, filepath) - } - - return mdr, nil -} diff --git a/lib/mod/modder/modder_check.go b/lib/mod/modder/modder_check.go deleted file mode 100644 index 781c60374..000000000 --- a/lib/mod/modder/modder_check.go +++ /dev/null @@ -1,181 +0,0 @@ -package modder - -import ( - "fmt" - "path" - "strings" - - "github.com/go-git/go-billy/v5/osfs" - - "github.com/hofstadter-io/hof/lib/mod/parse/sumfile" - "github.com/hofstadter-io/hof/lib/yagu" -) - -func (mdr *Modder) PartitionSumEntries() ([]string, []string, []string, error) { - present := []string{} - missing := []string{} - local := []string{} - - mod := mdr.module - sf := mod.SumFile - if sf == nil { - return nil, nil, nil, fmt.Errorf("No sum file %q for %s, run 'mvs vendor [%s]' to fix.", mdr.SumFile, mdr.Name, mdr.Name) - } - - for path, R := range mod.SelfDeps { - // local replace? - if strings.HasPrefix(R.NewPath, ".") { - local = append(local, path) - continue - } - - ver := sumfile.Version{ - Path: path, - Version: R.NewVersion, - } - - _, ok := sf.Mods[ver] - if ok { - present = append(present, path) - } else { - missing = append(missing, path) - } - } - - return present, missing, local, nil -} - -func (mdr *Modder) CompareSumEntryToVendor(R Replace) error { - sf := mdr.module.SumFile - - // sumfile ver from Replace - driver := sumfile.Version{ - Path: R.NewPath, - Version: R.NewVersion, - } - modver := sumfile.Version{ - Path: R.NewPath, - Version: path.Join(driver.Version, mdr.ModFile), - } - - // sumfile dirhash - d, ok := sf.Mods[driver] - if !ok { - merr := fmt.Errorf("Missing module dirhash in sumfile for '%v' from modfile entry '%v'", driver, R) - // mdr.errors = append(mdr.errors, merr) - return merr - } - sumDirhash := d[0] - - // sumfile modhash - m, ok := sf.Mods[modver] - if !ok { - merr := fmt.Errorf("Missing module modhash in sumfile for '%v' from modfile entry '%v'", modver, R) - // mdr.errors = append(mdr.errors, merr) - return merr - } - sumModhash := m[0] - - // fmt.Println("SUMHASH", sumDirhash, sumModhash) - - // load vendor copy, oldpath because that is how it will be imported - rpath := R.OldPath - if R.OldPath == "" { - rpath = R.NewPath - } - vpath := path.Join(mdr.ModsDir, rpath) - // fmt.Println("VPATH", vpath) - FS := osfs.New(vpath) - - // Calc hashes for vendor from billy - vdrDirhash, err := yagu.BillyCalcHash(FS) - if err != nil { - merr := fmt.Errorf("While calculating vendor dirhash for '%v' from '%v'\n%w\n", driver, R, err) - // mdr.errors = append(mdr.errors, merr) - return merr - } - - vdrModhash, err := yagu.BillyCalcFileHash(mdr.ModFile, FS) - if err != nil { - merr := fmt.Errorf("While calculating vendor modhash for '%v' from '%v'\n%w\n", modver, R, err) - // mdr.errors = append(mdr.errors, merr) - return merr - } - // fmt.Println("VDRHASH", vdrDirhash, vdrModhash) - - mismatch := false - if sumDirhash != vdrDirhash { - // merr := fmt.Errorf("Mismatched dir hashes in sumfile for '%v' from modfile entry '%v'", driver, R) - // mdr.errors = append(mdr.errors, merr) - mismatch = true - } - if sumModhash != vdrModhash { - // merr := fmt.Errorf("Mismatched modfile hashes in sumfile for '%v' from modfile entry '%v'", modver, R) - // mdr.errors = append(mdr.errors, merr) - mismatch = true - } - - if mismatch { - return fmt.Errorf("Errors with vendor integrity") - } - - return nil -} - -func (mdr *Modder) CompareLocalReplaceToVendor(R Replace) error { - - // load both into billy - LFS := osfs.New(R.NewPath) - VFS := osfs.New(path.Join(mdr.ModsDir, R.OldPath)) - - // Calc hashes for replace from billy - localDirhash, err := yagu.BillyGlobCalcHash(LFS, mdr.VendorIncludeGlobs, mdr.VendorExcludeGlobs) - if err != nil { - merr := fmt.Errorf("While calculating local dirhash for '%#+v'\n%w\n", R, err) - mdr.errors = append(mdr.errors, merr) - return merr - } - - localModhash, err := yagu.BillyCalcFileHash(mdr.ModFile, LFS) - if err != nil { - merr := fmt.Errorf("While calculating local modhash for '%#+v'\n%w\n", R, err) - mdr.errors = append(mdr.errors, merr) - return merr - } - // fmt.Println("LCLHASH", localDirhash, localModhash) - - // Calc hashes for vendor from billy - vdrDirhash, err := yagu.BillyGlobCalcHash(VFS, mdr.VendorIncludeGlobs, mdr.VendorExcludeGlobs) - if err != nil { - merr := fmt.Errorf("While calculating vendor dirhash for '%#+v'\n%w\n", R, err) - mdr.errors = append(mdr.errors, merr) - return merr - } - - vdrModhash, err := yagu.BillyCalcFileHash(mdr.ModFile, VFS) - if err != nil { - merr := fmt.Errorf("While calculating vendor modhash for '%#+v'\n%w\n", R, err) - mdr.errors = append(mdr.errors, merr) - return merr - } - // fmt.Println("VDRHASH", vdrDirhash, vdrModhash) - - // Do the check - mismatch := false - if localDirhash != vdrDirhash { - merr := fmt.Errorf("Mismatched dirhash for '%#+v'\n%w\n", R, err) - mdr.errors = append(mdr.errors, merr) - mismatch = true - } - if localModhash != vdrModhash { - merr := fmt.Errorf("Mismatched modhash for '%#+v'\n%w\n", R, err) - mdr.errors = append(mdr.errors, merr) - mismatch = true - } - - if mismatch { - return fmt.Errorf("Errors with vendor integrity") - } - - return nil -} diff --git a/lib/mod/modder/modder_deps.go b/lib/mod/modder/modder_deps.go deleted file mode 100644 index 9b9d88888..000000000 --- a/lib/mod/modder/modder_deps.go +++ /dev/null @@ -1,133 +0,0 @@ -package modder - -import ( - "fmt" - "strings" - - "golang.org/x/mod/semver" - - "github.com/hofstadter-io/hof/lib/repos/git" -) - -func (mdr *Modder) PrintRootDeps() error { - fmt.Println("Root module self deps for", mdr.module.Module) - err := mdr.module.PrintSelfDeps() - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) LoadRootDeps() error { - fmt.Println("Loading self deps for", mdr.module.Module) - - err := mdr.module.LoadSelfDeps() - if err != nil { - return err - } - return nil -} - -// This sets or overwrites the module -func (mdr *Modder) ReplaceDependency(m *Module) error { - // Don't add the root module to the dependencies - if mdr.module.Module == m.Module { - return nil - } - // save module to depsMap, that's it? (yes) - mdr.depsMap[m.Module] = m - - return nil -} - -// If not set, just adds. If set, takes the one with the greater version. -func (mdr *Modder) MvsMergeDependency(m *Module) error { - // Don't add the root module to the dependencies - if mdr.module.Module == m.Module { - return nil - } - - // check for existing module - e, ok := mdr.depsMap[m.Module] - if !ok { - // just add - mdr.depsMap[m.Module] = m - - } else { - // check local replace - if strings.HasPrefix(e.ReplaceModule, ".") { - // do nothing - return nil - } - - // check remote replace - if m.ReplaceModule != "" { - if e.ReplaceModule == m.ReplaceModule { - // check version, is what we have a newer version? - if semver.Compare(e.ReplaceVersion, m.ReplaceVersion) >= 0 { - // do nothing, only 1/4 cases - return nil - } - } - // all other cases, want to update module - } else { - // check version, is what we have a newer version? - if semver.Compare(e.Version, m.Version) >= 0 { - // do nothing - return nil - } - } - - mdr.depsMap[m.Module] = m - - } - - // fmt.Printf("Merge %-48s => %s\n", m.Module + "@" + m.Version, m.ReplaceModule + "@" + m.ReplaceVersion) - - // NOTE This is what basically makes us BFS - for _, R := range m.SelfDeps { - err := mdr.VendorDep(R) - if err != nil { - mdr.errors = append(mdr.errors, err) - } - } - - return nil -} - -// TODO, break this function apart -func (mdr *Modder) addDependency(m *Module) error { - // Don't add the root module to the dependencies - if mdr.module.Module == m.Module { - return nil - } - // save module to depsMap - mdr.depsMap[m.Module] = m - - // TODO ADD par.Work here - clone and ilook for sum..., then do checks and actions - - // Should only happen with local replace right now - if m.Ref == nil { - clone, err := git.CloneLocalRepo(m.ReplaceModule) - m.Clone = clone - if err != nil { - return err - } - return nil - } - - // clone the module and load - clone, err := git.CloneRepoRef(m.Module, m.Ref) - m.Clone = clone - if err != nil { - return err - } - - // Pushi into parallel worker here? - // load dep module - // dm, err := mdr.LoadModule("...") - // if err != nil { return err } - - return nil -} diff --git a/lib/mod/modder/modder_errors.go b/lib/mod/modder/modder_errors.go deleted file mode 100644 index d20b0f0e7..000000000 --- a/lib/mod/modder/modder_errors.go +++ /dev/null @@ -1,53 +0,0 @@ -package modder - -import ( - "fmt" -) - -func (mdr *Modder) CheckForErrors() error { - if len(mdr.errors) > 0 { - return fmt.Errorf("Exiting due to errors in modder %s", mdr.Name) - } - if len(mdr.module.Errors) > 0 { - return fmt.Errorf("Exiting due to errors in module %s", mdr.module.Module) - } - - for _, dep := range mdr.depsMap { - if len(dep.Errors) > 0 { - return fmt.Errorf("Exiting due to errors in module %s", dep.Module) - } - } - - return nil -} - -func (mdr *Modder) PrintErrors() error { - var wasError error - - if len(mdr.errors) > 0 { - wasError = fmt.Errorf("Exiting due to errors.") - for _, err := range mdr.errors { - fmt.Println(err) - } - } - - if len(mdr.module.Errors) > 0 { - wasError = fmt.Errorf("Exiting due to errors.") - for _, err := range mdr.module.Errors { - fmt.Println(err) - } - } - - for _, dep := range mdr.depsMap { - if len(dep.Errors) > 0 { - if wasError != nil { - wasError = fmt.Errorf("Exiting due to errors.") - } - for _, err := range dep.Errors { - fmt.Println(err) - } - } - } - - return wasError -} diff --git a/lib/mod/modder/modder_init.go b/lib/mod/modder/modder_init.go deleted file mode 100644 index ea6f1037b..000000000 --- a/lib/mod/modder/modder_init.go +++ /dev/null @@ -1,130 +0,0 @@ -package modder - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "text/template" - - gomod "golang.org/x/mod/module" - - "github.com/hofstadter-io/hof/lib/mod/parse/modfile" - "github.com/hofstadter-io/hof/lib/yagu" -) - -/* TODO -- more configuration for initialization -*/ - -func (mdr *Modder) Init(module string) error { - // exec commands override configurable behavior - if len(mdr.CommandInit) > 0 { - for _, cmd := range mdr.CommandGraph { - out, err := yagu.Exec(cmd) - fmt.Println(out) - if err != nil { - return err - } - } - } else { - // or load from file - err := mdr.initModFile(module) - if err != nil { - return err - } - } - - // Run init templates regardless of the init main method - err := mdr.writeInitTemplates(module) - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) initModFile(module string) error { - lang := mdr.Name - filename := mdr.ModFile - - err := gomod.CheckPath(module) - if err != nil { - return fmt.Errorf("bad module format %q, should be 'domain.com/repo/proj'", module) - } - - // make sure file does not exist - _, err = ioutil.ReadFile(filename) - // we read the file and it exists - if err == nil { - return fmt.Errorf("%s already exists", filename) - } - // error was not path error, so return - if _, ok := err.(*os.PathError); !ok && err.Error() != "file does not exist" && err.Error() != "no such file or directory" { - return err - } - - // Create empty modfile - f, err := modfile.Parse(filename, nil, nil) - if err != nil { - return err - } - - err = f.AddModuleStmt(module) - if err != nil { - return err - } - - err = f.AddLanguageStmt(lang, mdr.Version) - if err != nil { - return err - } - - bytes, err := f.Format() - if err != nil { - return err - } - - err = ioutil.WriteFile(filename, bytes, 0644) - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) writeInitTemplates(module string) error { - - for filename, templateStr := range mdr.InitTemplates { - - tmpl, err := template.New(filename).Parse(templateStr) - if err != nil { - return err - } - - data := map[string]interface{}{ - "Language": mdr.Name, - "Module": module, - "Modder": mdr, - } - - err = os.MkdirAll(path.Dir(filename), 0755) - if err != nil { - return err - } - - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - - err = tmpl.Execute(file, data) - if err != nil { - return err - } - - } - - return nil -} diff --git a/lib/mod/modder/modder_load.go b/lib/mod/modder/modder_load.go deleted file mode 100644 index 5bbde1364..000000000 --- a/lib/mod/modder/modder_load.go +++ /dev/null @@ -1,163 +0,0 @@ -package modder - -import ( - "fmt" - - "github.com/go-git/go-billy/v5/osfs" -) - -/* Reads the module files relative to the supplied dir from local FS -- ModFile -- SumFile -- MappingFile -*/ - -/* -func (mdr *Modder) LoadRootFromFS() error { - // Load the root module - err := mdr.LoadMetaFromFS(".") - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) LoadModderFromFS(dir string) error { - // Load the root module - err := mdr.LoadMetaFromFS(dir) - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) LoadIndexDepsFromFS(dir string) error { - // Load the root module - err := mdr.LoadMetaFromFS(".") - if err != nil { - return err - } - - // Load the root module's deps - err = mdr.LoadRootDeps() - if err != nil { - return err - } - // Recurse - - return nil -} -*/ - -func (mdr *Modder) LoadMetaFromBilly() error { - // Load the root module - err := mdr.LoadMetaFromFS(".") - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) LoadMetaFromZip() error { - // Load the root module - err := mdr.LoadMetaFromFS(".") - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) LoadMetaFromFS(dir string) error { - // Shortcut for no load modules, forget the reason for no load... - if mdr.NoLoad { - return nil - } - - // Initialize filesystem - mdr.FS = osfs.New(dir) - - // Initialize Module related fields - mdr.module = &Module{ - FS: mdr.FS, - } - mdr.depsMap = map[string]*Module{} - - // Load module files - var err error - err = mdr.LoadModFile() - if err != nil { - return err - } - - err = mdr.LoadSumFile() - if err != nil { - mdr.errors = append(mdr.errors, err) - return nil - } - - err = mdr.LoadMappingsFile() - if err != nil { - mdr.errors = append(mdr.errors, err) - return nil - } - - return nil -} - -// Loads the root modules mod file -func (mdr *Modder) LoadModFile() error { - fn := mdr.ModFile - m := mdr.module - - err := m.LoadModFile(fn, false /* Do load replace directives! */) - if err != nil { - return err - } - - // no file - if m.ModFile == nil { - return fmt.Errorf("no %q file found, directory not initialized for %s", mdr.ModFile, mdr.Name) - } - - // empty file checks (based on parsed out values) - if m.ModFile.Module == nil { - return fmt.Errorf("your %q file appears to be empty", mdr.ModFile) - } - if m.ModFile.Module.Mod.Path == "" { - return fmt.Errorf("your %q file appears to be missing the module clause", mdr.ModFile) - } - - m.Module = m.ModFile.Module.Mod.Path - - return nil -} - -// Loads the root modules sum file -func (mdr *Modder) LoadSumFile() error { - fn := mdr.SumFile - m := mdr.module - - err := m.LoadSumFile(fn) - if err != nil { - return err - } - - return nil -} - -// Loads the root modules mapping file -func (mdr *Modder) LoadMappingsFile() error { - fn := mdr.MappingFile - m := mdr.module - - err := m.LoadMappingFile(fn) - if err != nil { - return err - } - - return nil -} diff --git a/lib/mod/modder/modder_vendor.go b/lib/mod/modder/modder_vendor.go deleted file mode 100644 index 399ce14ed..000000000 --- a/lib/mod/modder/modder_vendor.go +++ /dev/null @@ -1,195 +0,0 @@ -package modder - -import ( - "fmt" - "strings" - - "github.com/go-git/go-billy/v5/osfs" - - "github.com/hofstadter-io/hof/lib/repos/cache" - "github.com/hofstadter-io/hof/lib/yagu" -) - -/* Vendor reads in a module, determines dependencies, and writes out the vendor folder. - -Will there be infinite recursion, or maybe just two levels? - - -This module & deps - 1) load this module - a) check sum <-> mod files, need to determine if any updates here, for now w/o sum file - 2) process its sum/mod files - 3) for each of this mods deps dependency - a) fetch all refs - b) find minimum - c) add to depMap, when / if ... ? guessing how right now - - replaces are processed first - - requires are processed second, so only add if not there, we shouldn't have duplicates in owr own mods files - - - d) if added, clond the desired ref to memory - - 4) Now loop over depMap to pull in secondary dependencies - - probably want to create a "newDeps" map here if we need to support wider recursion - - basically follow the last block, but load independently and merge after - - do we need a separate modder when we process each dep? - - probably if we are going to enable each module to optionally specify local behavior - - so first file we should read is the .mvsconfig, that maps to whatever - - F) Finally, write out the vendor directory - a) check /modules.txt and checksums - b) write out if necessary -*/ -func (mdr *Modder) Vendor() error { - // TODO, run pre vendor commands here - - // Vendor Command Override - if len(mdr.CommandVendor) > 0 { - for _, cmd := range mdr.CommandVendor { - out, err := yagu.Exec(cmd) - fmt.Println(out) - if err != nil { - return err - } - } - } else { - // Otherwise, MVS venodiring - err := mdr.VendorMVS() - if err != nil { - mdr.PrintErrors() - return err - } - } - - // TODO, run post vendor commands here - - return nil -} - -// The entrypoint to the MVS internal vendoring process -func (mdr *Modder) VendorMVS() error { - var err error - - // Load minimal root module - err = mdr.LoadMetaFromFS(".") - if err != nil { - // fmt.Println(err) - return err - } - for _, R := range mdr.module.SelfDeps { - err := mdr.VendorDep(R) - if err != nil { - mdr.errors = append(mdr.errors, err) - } - } - - if err := mdr.CheckForErrors(); err != nil { - return err - } - - // Finally, write out anything that needs to be - err = mdr.WriteVendor() - if err != nil { - return err - } - return nil -} - -func (mdr *Modder) VendorDep(R Replace) error { - // fmt.Printf("VendorDep %#+v\n", R) - - // Fetch and Load module - if strings.HasPrefix(R.NewPath, "/") || strings.HasPrefix(R.NewPath, "./") || strings.HasPrefix(R.NewPath, "../") { - // fmt.Println("local replace:", R) - err := mdr.LoadLocalReplace(R) - if err != nil { - mdr.errors = append(mdr.errors, err) - return err - } - return nil - } else { - err := mdr.LoadRemoteModule(R) - if err != nil { - mdr.errors = append(mdr.errors, err) - return err - } - return nil - } - - return nil -} - -func (mdr *Modder) LoadRemoteModule(R Replace) (err error) { - // fmt.Printf("LoadRemoteReplace %#+v\n", R) - - // If sumfile, check integrity and possibly shortcut - if mdr.module.SumFile != nil { - mdr.CompareSumEntryToVendor(R) - } - - // TODO, check if valid and just add m.deps to processing - // We can do this in a BFS manner - - m := &Module{ - Module: R.OldPath, - Version: R.OldVersion, - ReplaceModule: R.NewPath, - ReplaceVersion: R.NewVersion, - } - - if m.Module == "" { - m.Module = R.NewPath - m.Version = R.NewVersion - } - - m.FS, err = cache.Load(R.NewPath, R.NewVersion) - if err != nil { - return err - } - - err = m.LoadMetaFiles(mdr.ModFile, mdr.SumFile, mdr.MappingFile, true /* ignoreReplace directives */) - if err != nil { - return err - } - - // TODO, check if valid and just add m.deps to processing - - // XXX Within this call is what basically makes us BFS - // It's where we load the dependencies - err = mdr.MvsMergeDependency(m) - if err != nil { - return err - } - - return nil -} - -func (mdr *Modder) LoadLocalReplace(R Replace) error { - // fmt.Printf("LoadLocalReplace %#+v\n", R) - var err error - - m := &Module{ - Module: R.OldPath, - Version: R.OldVersion, - ReplaceModule: R.NewPath, - ReplaceVersion: R.NewVersion, - } - // fmt.Printf("LoadLocalReplace %#+v\n", m) - - m.FS = osfs.New(m.ReplaceModule) - - err = m.LoadMetaFiles(mdr.ModFile, mdr.SumFile, mdr.MappingFile, true /* ignoreReplace directives */) - if err != nil { - return err - } - - // TODO, check if valid and just add m.deps to processing - - // XXX Within this call is what basically makes us BFS - // It's where we load the dependencies - err = mdr.MvsMergeDependency(m) - if err != nil { - return err - } - - return nil -} diff --git a/lib/mod/modder/modder_verify.go b/lib/mod/modder/modder_verify.go deleted file mode 100644 index 219473df6..000000000 --- a/lib/mod/modder/modder_verify.go +++ /dev/null @@ -1,83 +0,0 @@ -package modder - -import ( - "fmt" - - "github.com/hofstadter-io/hof/lib/yagu" -) - -func (mdr *Modder) Verify() error { - - // Verify Command Override - if len(mdr.CommandVerify) > 0 { - for _, cmd := range mdr.CommandVerify { - out, err := yagu.Exec(cmd) - fmt.Println(out) - if err != nil { - return err - } - } - } else { - // Otherwise, MVS venodiring - err := mdr.VerifyMVS() - if err != nil { - mdr.PrintErrors() - return err - } - } - - return nil -} - -// The entrypoint to the MVS internal verify process -func (mdr *Modder) VerifyMVS() error { - - valid := true - - // Load minimal root module - err := mdr.LoadMetaFromFS(".") - if err != nil { - return err - } - - // Load the root module's deps - present, missing, local, err := mdr.PartitionSumEntries() - if err != nil { - return err - } - - // Invalid if there are missing deps - for _, m := range missing { - valid = false - R := mdr.module.SelfDeps[m] - err := fmt.Errorf("Sumfile missing: %s@%s", R.NewPath, R.NewVersion) - mdr.errors = append(mdr.errors, err) - } - - for _, p := range present { - R := mdr.module.SelfDeps[p] - err := mdr.CompareSumEntryToVendor(R) - // Something is wrong with the vendored copy or the hash - if err != nil { - valid = false - mdr.errors = append(mdr.errors, err) - } - } - - for _, p := range local { - R := mdr.module.SelfDeps[p] - err := mdr.CompareLocalReplaceToVendor(R) - // Something is wrong with the vendored copy or the hash - if err != nil { - valid = false - mdr.errors = append(mdr.errors, err) - } - } - - if !valid { - return fmt.Errorf("Vendoring is in an inconsistent state, please run 'mvs vendor %s' ", mdr.Name) - } - - // We are OK! - return nil -} diff --git a/lib/mod/modder/modder_write.go b/lib/mod/modder/modder_write.go deleted file mode 100644 index a6bf23cfc..000000000 --- a/lib/mod/modder/modder_write.go +++ /dev/null @@ -1,169 +0,0 @@ -package modder - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "strings" - - "github.com/hofstadter-io/hof/lib/mod/parse/sumfile" - "github.com/hofstadter-io/hof/lib/yagu" -) - -var ( - // Common files to copy from modules, also includes the .md version of the filename - definiteVendors = []string{ - "README", - "SECURITY", - "AUTHORS", - "CONTRIBUTORS", - "COPYLEFT", - "COPYING", - "COPYRIGHT", - "LEGAL", - "LICENSE", - "NOTICE", - "PATENTS", - } -) - -func (mdr *Modder) WriteVendor() error { - // fmt.Println("Writing Vendor from scratch") - os.RemoveAll(mdr.ModsDir) - - // make vendor dir if not present - err := yagu.Mkdir(mdr.ModsDir) - if err != nil { - return err - } - - // write out each dep - for _, m := range mdr.depsMap { - // fmt.Printf(" writing: %#+v\n", m) - // XXX, this only (?) happens with local replaces with no matching require entry - if m.Version == "" { - m.Version = "v0.0.0" - } - - dirhash, err := yagu.BillyCalcHash(m.FS) - if err != nil { - mdr.errors = append(mdr.errors, err) - return fmt.Errorf("While calculating billy dir hash for %q\n%w\n", mdr.ModsDir, err) - } - - modhash, err := yagu.BillyCalcFileHash(mdr.ModFile, m.FS) - if err != nil { - mdr.errors = append(mdr.errors, err) - return fmt.Errorf("While calculating billy mod hash\n%w\n", err) - } - - dver := sumfile.Version{ - Path: strings.Join([]string{m.Module}, "/"), - Version: m.Version, - } - if mdr.module.SumFile == nil { - mdr.module.SumFile = &sumfile.Sum{} - } - mdr.module.SumFile.Add(dver, dirhash) - - mver := sumfile.Version{ - Path: strings.Join([]string{m.Module}, "/"), - Version: strings.Join([]string{m.Version, mdr.ModFile}, "/"), - } - mdr.module.SumFile.Add(mver, modhash) - - baseDir := path.Join(mdr.ModsDir, m.Module) - - // fmt.Printf("Writing %-48s => %s\n", m.ReplaceModule + "@" + m.ReplaceVersion, baseDir) - - // Should we make a symlink for a local replace? - hasLocalPrefix := strings.HasPrefix(m.ReplaceModule, "../") || strings.HasPrefix(m.ReplaceModule, "./") || strings.HasPrefix(m.ReplaceModule, "/") - // fmt.Println("symlinking?", mdr.SymlinkLocalReplaces, hasLocalPrefix, m.ReplaceModule, strings.HasPrefix(m.ReplaceModule, "/")) - // fmt.Printf("m: %#v\n", m) - if mdr.SymlinkLocalReplaces && hasLocalPrefix { - // count and create backPaths string - backPaths := strings.Repeat("../", strings.Count(baseDir, "/")) - // create final symlink string - originalBaseDir := path.Join(backPaths, m.ReplaceModule) - - // strip last dir (module core name) - targetBaseDir := baseDir[:strings.LastIndex(baseDir, "/")] - targetSymlink := baseDir // for readability - - // make the base dir - err := yagu.Mkdir(targetBaseDir) - if err != nil { - mdr.errors = append(mdr.errors, err) - return fmt.Errorf("While creating baseDir for local replace\n%w\n", err) - } - - fmt.Printf("local replace: symlinking %s to %s\n", m.Module, m.ReplaceModule) - - // create the actual symlink - err = os.Symlink(originalBaseDir, targetSymlink) - if err != nil { - mdr.errors = append(mdr.errors, err) - return fmt.Errorf("While writing symlink for local replace\n%w\n", err) - } - - // next dep as we don't need to write things - continue - } - - // copy definite files always - files, err := m.FS.ReadDir("/") - if err != nil { - return err - } - for _, file := range files { - for _, fn := range definiteVendors { - // Found one! - if strings.HasPrefix(strings.ToUpper(file.Name()), fn) { - // TODO, these functions should just take 2 billy FS - err = yagu.BillyWriteFileToOS(baseDir, "/"+file.Name(), m.FS) - if err != nil { - return err - } - } - - } - } - - // fmt.Println("writing local vendor") - - if len(mdr.VendorIncludeGlobs) > 0 || len(mdr.VendorExcludeGlobs) > 0 { - // Just copy everything - // TODO, these functions should just take 2 billy FS - err = yagu.BillyGlobWriteDirToOS(baseDir, "/", m.FS, mdr.VendorIncludeGlobs, mdr.VendorExcludeGlobs) - if err != nil { - return err - } - - } else { - // Just copy everything - // TODO, these functions should just take 2 billy FS - err = yagu.BillyWriteDirToOS(baseDir, "/", m.FS) - if err != nil { - return err - } - - } - - } // end loop over deps - - // possibly no deps, so lets write an empty sumfile - if mdr.module.SumFile == nil { - mdr.module.SumFile = &sumfile.Sum{} - } - - // Write sumfile - out, err := mdr.module.SumFile.Write() - if err != nil { - return err - } - - ioutil.WriteFile(mdr.SumFile, []byte(out), 0644) - - return nil -} diff --git a/lib/mod/modder/module.go b/lib/mod/modder/module.go deleted file mode 100644 index afac83838..000000000 --- a/lib/mod/modder/module.go +++ /dev/null @@ -1,62 +0,0 @@ -package modder - -import ( - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-git/v5/plumbing" - - "github.com/hofstadter-io/hof/lib/mod/parse/mappingfile" - "github.com/hofstadter-io/hof/lib/mod/parse/modfile" - "github.com/hofstadter-io/hof/lib/mod/parse/sumfile" - "github.com/hofstadter-io/hof/lib/repos/git" -) - -type Module struct { - // From mod/sum files - Language string - LangVer string - Module string - Version string - Require []Require - Replace []Replace - - // Merged version of local require / replace - // Requires as replaces will not have the old fields set - SelfDeps map[string]Replace - - // If this module gets replaced - ReplaceModule string - ReplaceVersion string - - // Module files in memory - ModFile *modfile.File - SumFile *sumfile.Sum - Mappings *mappingfile.Mappings - // TODO modules.txt for mapping imports to vendors - // TODO also a checksum? - - // TODO, is this modder a good idea for our nested - // .mvsconfig processing and vendoring - Mdr *Modder - - Errors []error - Ref *plumbing.Reference - Refs []*plumbing.Reference - Clone *git.GitRepo - FS billy.Filesystem -} - -type Require struct { - Path string - Version string -} - -type Replace struct { - OldPath string - OldVersion string - NewPath string - NewVersion string -} - -// If no lang.sum, calc sum, degenerate of next -// if both, look for differences, calc sumc -// if diff, fetch and do normal thing diff --git a/lib/mod/modder/module_load.go b/lib/mod/modder/module_load.go deleted file mode 100644 index 4e1a190d5..000000000 --- a/lib/mod/modder/module_load.go +++ /dev/null @@ -1,151 +0,0 @@ -package modder - -import ( - "fmt" - "os" - - "github.com/hofstadter-io/hof/lib/mod/parse/mappingfile" - "github.com/hofstadter-io/hof/lib/mod/parse/modfile" - "github.com/hofstadter-io/hof/lib/mod/parse/sumfile" - "github.com/hofstadter-io/hof/lib/yagu" -) - -func (m *Module) LoadModFile(fn string, ignoreReplace bool) error { - - modBytes, err := yagu.BillyReadAll(fn, m.FS) - if err != nil { - if _, ok := err.(*os.PathError); !ok && err.Error() != "file does not exist" && err.Error() != "no such file or directory" { - return err - } - } else { - f, err := modfile.Parse(fn, modBytes, nil) - if err != nil { - return err - } - m.ModFile = f - m.Language = f.Language.Name - m.LangVer = f.Language.Version - // m.Module = f.Module.Mod.Path - // m.Version = f.Module.Mod.Version - - for _, req := range f.Require { - r := Require{Path: req.Mod.Path, Version: req.Mod.Version} - m.Require = append(m.Require, r) - } - - // Let's just not load them for now to be sure - if !ignoreReplace { - for _, rep := range f.Replace { - r := Replace{OldPath: rep.Old.Path, OldVersion: rep.Old.Version, NewPath: rep.New.Path, NewVersion: rep.New.Version} - m.Replace = append(m.Replace, r) - } - } - - err = m.MergeSelfDeps(ignoreReplace) - if err != nil { - return err - } - - } - - return nil -} - -func (m *Module) MergeSelfDeps(ignoreReplace bool) error { - // Now merge self deps - m.SelfDeps = map[string]Replace{} - for _, req := range m.Require { - if _, ok := m.SelfDeps[req.Path]; ok { - return fmt.Errorf("Dependency %q required twice in %q", req.Path, m.Module) - } - m.SelfDeps[req.Path] = Replace{ - NewPath: req.Path, - NewVersion: req.Version, - } - } - - // we typically ignore replaces from when not the root module - if !ignoreReplace { - dblReplace := map[string]Replace{} - for _, rep := range m.Replace { - // Check if replaced twice - if _, ok := dblReplace[rep.OldPath]; ok { - return fmt.Errorf("Dependency %q replaced twice in %q", rep.OldPath, m.Module) - } - dblReplace[rep.OldPath] = rep - - // Pull in require info if not in replace - if req, ok := m.SelfDeps[rep.OldPath]; ok { - if rep.OldVersion == "" { - rep.OldVersion = req.NewVersion - } - } - m.SelfDeps[rep.OldPath] = rep - } - } - - return nil -} - -func (m *Module) LoadSumFile(fn string) error { - - sumBytes, err := yagu.BillyReadAll(fn, m.FS) - if err != nil { - if _, ok := err.(*os.PathError); !ok && err.Error() != "file does not exist" && err.Error() != "no such file or directory" { - return err - } - } else { - sumMod, err := sumfile.ParseSum(sumBytes, fn) - if err != nil { - return err - } - m.SumFile = &sumMod - } - - return nil -} - -func (m *Module) LoadMappingFile(fn string) error { - - mapBytes, err := yagu.BillyReadAll(fn, m.FS) - if err != nil { - if _, ok := err.(*os.PathError); !ok && err.Error() != "file does not exist" && err.Error() != "no such file or directory" { - return err - } - } else { - mapMod, err := mappingfile.ParseMapping(mapBytes, fn) - if err != nil { - return err - } - m.Mappings = &mapMod - } - - return nil -} - -func (m *Module) LoadMetaFiles(modname, sumname, mapname string, ignoreReplace bool) error { - var err error - - // TODO load the modules .mvsconfig if present - - err = m.LoadModFile(modname, ignoreReplace) - if err != nil { - return err - } - - // Suppressing the next to errors here - // because we can handle them not being around else where - err = m.LoadSumFile(sumname) - if err != nil { - // fmt.Println(err) - // return err - } - - err = m.LoadMappingFile(mapname) - if err != nil { - // fmt.Println(err) - // return err - } - - return nil -} diff --git a/lib/mod/mvs.go b/lib/mod/mvs.go index 5110eee82..e4555d9e0 100644 --- a/lib/mod/mvs.go +++ b/lib/mod/mvs.go @@ -3,6 +3,7 @@ package mod import ( "fmt" "path/filepath" + "strings" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" @@ -20,8 +21,8 @@ func (cm *CueMod) SolveMVS(latest bool) error { targets := []module.Version{} for _, dep := range cm.Replace { - dep := module.Version{ Path: dep.Path, Version: dep.Version } - targets = append(targets, dep) + d := module.Version{ Path: dep.Path, Version: dep.Version } + targets = append(targets, d) } for path, ver := range cm.Require { // skip any replaced modules @@ -60,16 +61,17 @@ func cmpVersion(v1, v2 string) int { if v1 == "" { return 1 } + return semver.Compare(v1, v2) } func (rr *RequirementResolver) Max(v1, v2 string) string { - if cmpVersion(v1, v2) < 0 { + x := cmpVersion(v1, v2) + // fmt.Println("MAX", v1, v2, x) + if x < 0 { return v2 } return v1 - - return v1 } func (rr *RequirementResolver) Required(m module.Version) ([]module.Version, error) { @@ -98,16 +100,13 @@ func (rr *RequirementResolver) Required(m module.Version) ([]module.Version, err deps := []module.Version{} for path, ver := range M.Require { - // filter a replaced module or same module? - if _, ok := rr.rootMod.Replace[path]; ok || path == rr.rootMod.Module { - continue - } if rr.latest { // possibly extra calls here...? _, err = cache.FetchRepoSource(path, "") if err != nil { return nil, err } + // intentionally not supporting @next or prereleases when updating via all@latest nver, err := cache.GetLatestTag(path, false) if err != nil { return nil, err @@ -118,7 +117,21 @@ func (rr *RequirementResolver) Required(m module.Version) ([]module.Version, err } } + // filter a replaced module or same module? + _, ok1 := rr.rootMod.Replace[path] + if ok1 || path == rr.rootMod.Module { + // fmt.Println("mvs-skip:", path, ver) + continue + } + rver, ok2 := rr.rootMod.Require[path] + // ok2 = ok2 && path == rr.rootMod.Module + if ok2 && strings.HasPrefix(rver, "v0.0.0-") { + // fmt.Println("mvs-skip2:", path, ver, rver) + ver = rver + } + dep := module.Version{ Path: path, Version: ver } + // fmt.Println("mvs-add:", dep) deps = append(deps, dep) } diff --git a/lib/mod/testdata/get__branch_dep.txt b/lib/mod/testdata/get__branch_dep.txt new file mode 100644 index 000000000..3e16042ad --- /dev/null +++ b/lib/mod/testdata/get__branch_dep.txt @@ -0,0 +1,7 @@ +# this works, but the result drifts over time +# so we probably need to think about a better check + +exec hof mod init hof.io/test +exec hof mod get github.com/hofstadter-io/hof@_dev +exec hof mod vendor +exists cue.mod/pkg/github.com/hofstadter-io/hof/cue.mod/module.cue diff --git a/lib/mod/testdata/get__commit_dep.txt b/lib/mod/testdata/get__commit_dep.txt new file mode 100644 index 000000000..119405de3 --- /dev/null +++ b/lib/mod/testdata/get__commit_dep.txt @@ -0,0 +1,8 @@ +# this works, but the result drifts over time +# so we probably need to think about a better check + +exec hof mod init hof.io/test +exec hof mod get github.com/hofstadter-io/hof@a7d892fc4363e2b9d8ee41183fd12c32f71abf06 +exec hof mod vendor +exists cue.mod/pkg/github.com/hofstadter-io/hof/cue.mod/module.cue + diff --git a/lib/mod/testdata/get__newer_dep.txt b/lib/mod/testdata/get__newer_dep.txt index 3647a3317..8bfb199bf 100644 --- a/lib/mod/testdata/get__newer_dep.txt +++ b/lib/mod/testdata/get__newer_dep.txt @@ -1,6 +1,6 @@ exec hof mod vendor -exec hof mod get github.com/hofstadter-io/hof@v0.6.8-create.1 +exec hof mod get github.com/hofstadter-io/hof@v0.6.8-beta.9 cmp cue.mod/module.cue match/cue.mod/module-get.cue -- cue.mod/module.cue -- @@ -15,7 +15,7 @@ module: "hof.io/test" cue: "0.5.0" require: { - "github.com/hofstadter-io/hof": "v0.6.8-create.1" + "github.com/hofstadter-io/hof": "v0.6.8-beta.9" } indirect: { diff --git a/lib/mod/testdata/vendor__with_deps.txt b/lib/mod/testdata/vendor__with_deps.txt index d648005fe..e5f2424e8 100644 --- a/lib/mod/testdata/vendor__with_deps.txt +++ b/lib/mod/testdata/vendor__with_deps.txt @@ -7,5 +7,5 @@ module: "github.com/test/withsolo" cue: "0.5.0" require: { - "github.com/hofstadter-io/hof": "v0.6.8-create.1" + "github.com/hofstadter-io/hof": "v0.6.8-beta.9" } diff --git a/lib/mod/testdata/vendor__with_psuedo.txt b/lib/mod/testdata/vendor__with_psuedo.txt new file mode 100644 index 000000000..9250ce8bf --- /dev/null +++ b/lib/mod/testdata/vendor__with_psuedo.txt @@ -0,0 +1,16 @@ +# hof mod vendor - with deps +exec hof mod vendor +exists cue.mod/pkg/github.com/hofstadter-io/hof/cue.mod/module.cue + +-- cue.mod/module.cue -- +module: "github.com/test/withsolo" +cue: "0.5.0" + +require: { + "github.com/hofstadter-io/hof": "v0.0.0-20230309083347-a7d892fc4363e2b9d8ee41183fd12c32f71abf06" +} + +indirect: { + "github.com/hofstadter-io/ghacue": "v0.2.0" + "github.com/hofstadter-io/hofmod-cli": "v0.8.1" +} diff --git a/lib/mod/tidy.go b/lib/mod/tidy.go index a83c7120a..ff435f5ac 100644 --- a/lib/mod/tidy.go +++ b/lib/mod/tidy.go @@ -16,14 +16,19 @@ func Tidy(rflags flags.RootPflagpole) (error) { return err } + // introspect and add any missing dependencies + // remove any unused? + cm.findDepsFromImports() + + // if no deps, we can bail early if len(cm.Require) == 0 { fmt.Println("no requirements found") return nil } - // fmt.Println(cm.Module, cm.Require) - + // define & run a sequence of functions fns := []func () error { + cm.UpgradePseudoVersions, func () error { return cm.SolveMVS(false) }, cm.CleanDeps, cm.CleanSums, @@ -37,12 +42,28 @@ func Tidy(rflags flags.RootPflagpole) (error) { } } - // fmt.Println("tidy:", cm.Require, cm.Indirect) - - // TODO, figure out link / vendor style + // finally, write updated {module,sums}.cue return cm.Vendor("", rflags.Verbosity) } +func (cm *CueMod) UpgradePseudoVersions() (err error) { + for path, dep := range cm.Replace { + ver, _ := cache.UpgradePsuedoVersion(dep.Path, dep.Version) + dep.Version = ver + cm.Replace[path] = dep + } + + for path, ver := range cm.Require { + if _, ok := cm.Replace[path]; ok { + continue + } + ver, _ = cache.UpgradePsuedoVersion(path, ver) + cm.Require[path] = ver + } + + return nil +} + func (cm *CueMod) CleanDeps() error { // fmt.Println("clean deps:", cm.Module) @@ -141,3 +162,4 @@ func (cm *CueMod) CleanSums() error { return nil } + diff --git a/lib/mod/verify.go b/lib/mod/verify.go index 0c751fae1..cb2183a92 100644 --- a/lib/mod/verify.go +++ b/lib/mod/verify.go @@ -1 +1,88 @@ package mod + +import ( + "fmt" + + "github.com/hofstadter-io/hof/cmd/hof/flags" + "github.com/hofstadter-io/hof/lib/repos/cache" +) + +func Verify(rflags flags.RootPflagpole) (error) { + upgradeHofMods() + + cm, err := loadRootMod() + if err != nil { + return err + } + + if len(cm.Require) == 0 { + fmt.Println("no requirements found") + return nil + } + + err = cm.ensureCached() + if err != nil { + return err + } + + return cm.Verify() +} + +func (cm *CueMod) Verify() (err error) { + + for path, ver := range cm.Require { + if _, ok := cm.Replace[path]; !ok { + err := cm.verifyModule(path, ver) + if err != nil { + return err + } + } + } + + for path, ver := range cm.Indirect { + if _, ok := cm.Replace[path]; !ok { + err := cm.verifyModule(path, ver) + if err != nil { + return err + } + } + } + + return nil +} + + +func (cm *CueMod) verifyModule(path, ver string) (error) { + + // get hash from cache + hc, err := cache.Checksum(path, ver) + if err != nil { + return err + } + + // get has from sums + hs, ok := cm.Sums[Dep{path,ver}] + if !ok { + return fmt.Errorf("%s@%s missing from sum file", path, ver) + } + + // search + match := false + for _, h := range hs { + if h == hc { + match = true + break + } + } + + if !match { + return fmt.Errorf(mismatchMsg, path, ver, []string{hc}, hs) + } + + return nil +} + +var mismatchMsg = `unable to verify %s@%s incorrect or missing hashes + cache: %v + sums.cue: %v +` diff --git a/lib/repos/cache/fetch.go b/lib/repos/cache/fetch.go index ae783fc51..870b5ae7d 100644 --- a/lib/repos/cache/fetch.go +++ b/lib/repos/cache/fetch.go @@ -2,20 +2,16 @@ package cache import ( "fmt" - "strings" + "path" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "golang.org/x/mod/semver" "github.com/hofstadter-io/hof/lib/repos/git" "github.com/hofstadter-io/hof/lib/repos/utils" ) -const hofPrivateVar = "HOF_PRIVATE" - func OpenRepoSource(path string) (*gogit.Repository, error) { if debug { fmt.Println("cache.OpenRepoSource:", path) @@ -26,144 +22,28 @@ func OpenRepoSource(path string) (*gogit.Repository, error) { return gogit.PlainOpen(dir) } -func FetchRepoSource(path, ver string) (billy.Filesystem, error) { +func FetchRepoSource(rpath, ver string) (billy.Filesystem, error) { if debug { - fmt.Println("cache.FetchRepoSource:", path) + fmt.Println("cache.FetchRepoSource:", rpath) } - remote, owner, repo := utils.ParseModURL(path) + remote, owner, repo := utils.ParseModURL(rpath) dir := SourceOutdir(remote, owner, repo) + url := path.Join(remote, owner, repo) - err := git.SyncSource(dir, remote, owner, repo, ver) - if err != nil { - return nil, err - } - - FS := osfs.New(dir) - - return FS, nil -} - -func GetLatestTag(path string, pre bool) (string, error) { - R, err := OpenRepoSource(path) - if err != nil { - return "", err - } + // only fetch if we haven't already this run + _, ok := syncedRepos.Load(url) + if !ok { - refs, err := R.Tags() - if err != nil { - return "", err - } - - best := "" - refs.ForEach(func (ref *plumbing.Reference) error { - n := ref.Name().Short() - - // skip any tags which do not start with v - if !strings.HasPrefix(n, "v") { - return nil - } - - // maybe filter prereleases - if !pre && semver.Prerelease(n) != "" { - return nil + err := git.SyncSource(dir, remote, owner, repo, ver) + if err != nil { + return nil, err } - // update best? - if best == "" || semver.Compare(n, best) > 0 { - best = n - } - return nil - }) - - return best, nil -} - -/* -func FindBestRef(url, ver string) (_,_,_ string, err error) { - // fmt.Println("finding latest version for", url) - if ver == "" { - ver = "latest" + syncedRepos.Store(url, true) } - repo, err := git.NewRemote(url) - if err != nil { - return "", "", "", err - } - - refs, err := repo.RemoteRefs() - if err != nil { - return "", "", "", err - } - - refVal := "" - refType := "" - refCommit := "" - - for _, ref := range refs { - parts := strings.Fields(ref.String()) - if len(parts) == 2 { - commit, refStr := parts[0], parts[1] - - rs := strings.Split(refStr, "/") - // HEAD & PRs, other? - if len(rs) != 3 { - continue - } - // ensure it is a ref - if rs[0] != "refs" { - continue - } - - rType := rs[1] - rVal := rs[2] - - // latest requires semver tags (optional leading 'v') - if ver == "latest" { - if rType != "tags" { - continue - } - - // add the optional 'v' if not present - if rVal[0:1] != "v" { - rVal = "v" + rVal - } - - // continue if prerelease semver - if semver.Prerelease(semver.Canonical(rVal)) != "" { - continue - } - - // update if empty - if refVal == "" { - refVal = rVal - refCommit = commit - refType = "tags" - } - - if semver.IsValid(rVal) { - if semver.Compare(rVal, refVal) > 0 { - refVal = rVal - } - } - - // is this tag greater than a best so far? (semver wise) - - } else { - // check if match - // break if found - if rVal == ver { - refVal = ver - refCommit = commit - refType = rType - break - } - } - } - } - - // fmt.Println(refVal, refType, refCommit) + FS := osfs.New(dir) - return refVal, refType, refCommit, nil + return FS, nil } -*/ diff --git a/lib/repos/cache/info.go b/lib/repos/cache/info.go new file mode 100644 index 000000000..b7aa1aa66 --- /dev/null +++ b/lib/repos/cache/info.go @@ -0,0 +1,118 @@ +package cache + +import ( + "fmt" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "golang.org/x/mod/semver" +) + + +func GetLatestTag(path string, pre bool) (string, error) { + _, err := FetchRepoSource(path, "") + if err != nil { + return "", err + } + + R, err := OpenRepoSource(path) + if err != nil { + return "", err + } + + refs, err := R.Tags() + if err != nil { + return "", err + } + + best := "" + refs.ForEach(func (ref *plumbing.Reference) error { + n := ref.Name().Short() + + // skip any tags which do not start with v + if !strings.HasPrefix(n, "v") { + return nil + } + + // maybe filter prereleases + if !pre && semver.Prerelease(n) != "" { + return nil + } + + // update best? + if best == "" || semver.Compare(n, best) > 0 { + best = n + } + return nil + }) + + return best, nil +} + +func GetLatestBranch(path, branch string) (string, error) { + // sync + _, err := FetchRepoSource(path, "") + if err != nil { + return branch, err + } + // open repo + R, err := OpenRepoSource(path) + if err != nil { + return branch, fmt.Errorf("(glb) open source error: %w for %s@%s", err, path, branch) + } + + // get working tree + wt, err := R.Worktree() + if err != nil { + return branch, fmt.Errorf("(glb) worktree error: %w for %s@%s", err, path, branch) + } + + // try to checkout branch + err = wt.Checkout(&gogit.CheckoutOptions{ + Branch: plumbing.NewRemoteReferenceName("origin", branch), + Force: true, + }) + if err == nil { + h, err := R.Head() + if err != nil { + return branch, err + } + return h.Hash().String(), nil + } + + return branch, nil +} + +func UpgradePsuedoVersion(path, ver string) (s string, err error) { + // semver tag? + if semver.IsValid(ver) { + return ver, nil + } + + if ver == "latest" || ver == "next" { + ver, err = GetLatestTag(path, ver == "next") + if err != nil { + return ver, err + } + } + + // branch? need to find commit + nver, err := GetLatestBranch(path, ver) + if err != nil { + return ver, err + } + if nver != "" { + ver = nver + } + + if !strings.HasPrefix(ver, "v") { + now := time.Now().UTC().Format("20060102150405") + ver = fmt.Sprintf("v0.0.0-%s-%s", now, ver) + } + + + return ver, nil +} + diff --git a/lib/repos/cache/load.go b/lib/repos/cache/load.go index da2ca5525..ee4ee782d 100644 --- a/lib/repos/cache/load.go +++ b/lib/repos/cache/load.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "sync" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" @@ -19,11 +20,15 @@ var cacheBaseDir string var modBaseDir string var srcBaseDir string +// hacky, but we only want to sync once per repo per process +var syncedRepos *sync.Map + // CacheDir/{mod,src} var debug = false func init() { + syncedRepos = new(sync.Map) nc := os.Getenv(NoCacheEnvVar) if nc != "" { noCache = true @@ -132,7 +137,7 @@ func CacheModule(url, ver string) (billy.Filesystem, error) { } // fmt.Println("making:", url, ver) - err = CopyRepoTag(url, ver) + ver, err = CopyRepoTag(url, ver) if err != nil { return nil, err } diff --git a/lib/repos/cache/write.go b/lib/repos/cache/write.go index 0b6d8c59e..d8d59ea68 100644 --- a/lib/repos/cache/write.go +++ b/lib/repos/cache/write.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" @@ -63,7 +64,7 @@ func Write(remote, owner, repo, tag string, FS billy.Filesystem) error { return nil } -func CopyRepoTag(path, ver string) error { +func CopyRepoTag(path, ver string) (string, error) { if debug { fmt.Println("cache.CopyRepoTag:", path, ver) } @@ -73,31 +74,58 @@ func CopyRepoTag(path, ver string) error { // open git source R, err := OpenRepoSource(path) if err != nil { - return fmt.Errorf("(crt) open source error: %w for %s@%s", err, path, ver) + return ver, fmt.Errorf("(crt) open source error: %w for %s@%s", err, path, ver) } // get working tree wt, err := R.Worktree() if err != nil { - return fmt.Errorf("(crt) worktree error: %w for %s@%s", err, path, ver) + return ver, fmt.Errorf("(crt) worktree error: %w for %s@%s", err, path, ver) + } + + lver := ver + parts := strings.Split(lver, "-") + if strings.HasPrefix(lver, "v0.0.0-") { + lver = strings.Join(parts[2:], "-") } // checkout tag err = wt.Checkout(&gogit.CheckoutOptions{ - Branch: plumbing.ReferenceName("refs/tags/" + ver), + Branch: plumbing.NewTagReferenceName(lver), Force: true, }) if err != nil { - return fmt.Errorf("(crt) checkout error: %w for %s@%s", err, path, ver) + // err = fmt.Errorf("(crt) checkout error: %w for %s@%s", err, path, ver) + // try branch + err = wt.Checkout(&gogit.CheckoutOptions{ + Branch: plumbing.NewRemoteReferenceName("origin", lver), + Force: true, + }) + + if err != nil { + err = wt.Checkout(&gogit.CheckoutOptions{ + Hash: plumbing.NewHash(lver), + Force: true, + }) + if err != nil { + return ver, fmt.Errorf("(crt) checkout error: unable to find version %q for module %q", ver, path) + } + + } else { + h, err := R.Head() + lver = strings.Join(append(parts[:2], h.Hash().String()), "-") + fmt.Println("Checking out branch:", path, ver, lver, err) + ver = lver + } } // copy FS := osfs.New(dir) err = Write(remote, owner, repo, ver, FS) if err != nil { - return fmt.Errorf("(crt) writing error: %w for %s@%s", err, path, ver) + return ver, fmt.Errorf("(crt) writing error: %w for %s@%s", err, path, ver) } - return nil + return ver, nil } diff --git a/lib/repos/git/fetch.go b/lib/repos/git/fetch.go index e8cfebaf1..4af57ecd8 100644 --- a/lib/repos/git/fetch.go +++ b/lib/repos/git/fetch.go @@ -19,9 +19,9 @@ import ( var debug = false func SyncSource(dir, remote, owner, repo, ver string) error { - url := path.Join(repo,owner,repo) + url := path.Join(remote,owner,repo) if debug { - fmt.Println("git.SyncSource:", dir, url) + fmt.Println("git.SyncSource:", dir, remote, owner, repo, ver, url) } _, err := os.Lstat(dir) // does not exist @@ -41,6 +41,11 @@ func SyncSource(dir, remote, owner, repo, ver string) error { // jenky... foundTag := false if ver != "" { + if strings.HasPrefix(ver, "v0.0.0-") { + parts := strings.Split(ver, "-") + ver = strings.Join(parts[2:], "-") + } + ref, err := R.Tag(ver) if err == nil && ref != nil { foundTag = true diff --git a/script/runtime/cmd_fs.go b/script/runtime/cmd_fs.go index 5805e31fd..302f37e24 100644 --- a/script/runtime/cmd_fs.go +++ b/script/runtime/cmd_fs.go @@ -132,7 +132,7 @@ func (ts *Script) CmdExists(neg int, args []string) { ts.Fatalf("%s %s unexpectedly exists", what, file) } if err != nil && neg == 0 { - ts.Fatalf("%s does not exist", file) + ts.Fatalf("%s does not exist: %s", file, err) } if err == nil && neg == 0 && readonly && info.Mode()&0222 != 0 { ts.Fatalf("%s exists but is writable", file)