diff --git a/cmd/gazelle/fix-update.go b/cmd/gazelle/fix-update.go index 644203f51..1b47753fa 100644 --- a/cmd/gazelle/fix-update.go +++ b/cmd/gazelle/fix-update.go @@ -55,6 +55,9 @@ type updateConfig struct { patchBuffer bytes.Buffer print0 bool profile profiler + + // EXPERIMENTAL: caching of the rule index across runs + ruleIndexFile string } type emitFunc func(c *config.Config, f *rule.File) error @@ -94,6 +97,7 @@ func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *conf fs.StringVar(&ucr.memProfile, "memprofile", "", "write memory profile to `file`") fs.Var(&gzflag.MultiFlag{Values: &ucr.knownImports}, "known_import", "import path for which external resolution is skipped (can specify multiple times)") fs.StringVar(&ucr.repoConfigPath, "repo_config", "", "file where Gazelle should load repository configuration. Defaults to WORKSPACE.") + fs.StringVar(&uc.ruleIndexFile, "indexdb", "", "EXPERIMENTAL: file to cache the rule index") } func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error { @@ -110,6 +114,9 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro if uc.patchPath != "" && !filepath.IsAbs(uc.patchPath) { uc.patchPath = filepath.Join(c.WorkDir, uc.patchPath) } + if uc.ruleIndexFile != "" && !filepath.IsAbs(uc.ruleIndexFile) { + uc.ruleIndexFile = filepath.Join(c.WorkDir, uc.ruleIndexFile) + } p, err := newProfiler(ucr.cpuProfile, ucr.memProfile) if err != nil { return err @@ -311,15 +318,38 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { } }() + walkMode := uc.walkMode + + updateRels := walk.NewUpdateFilter(c.RepoRoot, uc.dirs, uc.walkMode) + preindexed := false + + // Load the rule index file if it exists. + if c.IndexLibraries && uc.ruleIndexFile != "" { + // Do not load index entries from directories that are being updated. + indexLoaded, err := ruleIndex.LoadIndex(uc.ruleIndexFile, updateRels.ShouldReIndex) + if err != nil { + log.Printf("Failed to load index file %s: %v", uc.ruleIndexFile, err) + } else if indexLoaded { + // Drop "visit all" since indexing has been loaded from disk. + if walkMode == walk.VisitAllUpdateSubdirsMode { + walkMode = walk.UpdateSubdirsMode + } else if walkMode == walk.VisitAllUpdateDirsMode { + walkMode = walk.UpdateDirsMode + } + + preindexed = true + } + } + var errorsFromWalk []error - walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) { + walk.Walk(c, cexts, uc.dirs, walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) { // If this file is ignored or if Gazelle was not asked to update this // directory, just index the build file and move on. if !update { for _, repl := range c.KindMap { mrslv.MappedKind(rel, repl) } - if c.IndexLibraries && f != nil { + if c.IndexLibraries && f != nil && (!preindexed || updateRels.ShouldReIndex(rel)) { for _, r := range f.Rules { ruleIndex.AddRule(c, r, f) } @@ -446,7 +476,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { }) // Add library rules to the dependency resolution table. - if c.IndexLibraries { + if c.IndexLibraries && (!preindexed || updateRels.ShouldReIndex(rel)) { for _, r := range f.Rules { ruleIndex.AddRule(c, r, f) } @@ -475,6 +505,14 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) { // Finish building the index for dependency resolution. ruleIndex.Finish() + // Persist the index for future runs. + if c.IndexLibraries && uc.ruleIndexFile != "" { + err := ruleIndex.SaveIndex(uc.ruleIndexFile) + if err != nil { + fmt.Printf("Failed to save index file %s: %v", uc.ruleIndexFile, err) + } + } + // Resolve dependencies. rc, cleanupRc := repo.NewRemoteCache(uc.repos) defer func() { diff --git a/resolve/index.go b/resolve/index.go index 109fa6557..ac9e471bd 100644 --- a/resolve/index.go +++ b/resolve/index.go @@ -16,7 +16,10 @@ limitations under the License. package resolve import ( + "encoding/json" + "io" "log" + "os" "github.com/bazelbuild/bazel-gazelle/config" "github.com/bazelbuild/bazel-gazelle/label" @@ -145,6 +148,66 @@ func NewRuleIndex(mrslv func(ruleKind, pkgRel string) Resolver, exts ...interfac } } +func (ix *RuleIndex) LoadIndex(indexDbPath string, isPkgExcluded func(string) bool) (bool, error) { + if ix.indexed { + log.Fatal("LoadIndex called after Finish") + } + + indexDbFile, err := os.Open(indexDbPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + defer indexDbFile.Close() + + indexDbContent, err := io.ReadAll(indexDbFile) + if err != nil { + return false, err + } + + // TODO: read & verify index version + + var rules []*ruleRecord + + err = json.Unmarshal(indexDbContent, &rules) + if err != nil { + return false, err + } + + ix.rules = make([]*ruleRecord, 0, len(rules)) + for _, r := range rules { + if !isPkgExcluded(r.Label.Pkg) { + ix.rules = append(ix.rules, r) + } + } + + return true, nil +} + +func (ix *RuleIndex) SaveIndex(indexDbPath string) error { + // TODO: write index version + + indexDbContent, err := json.Marshal(ix.rules) + if err != nil { + return err + } + + indexDbFile, err := os.OpenFile(indexDbPath, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + return err + } + defer indexDbFile.Close() + + _, err = indexDbFile.Write(indexDbContent) + if err != nil { + return err + } + + return nil +} + // AddRule adds a rule r to the index. The rule will only be indexed if there // is a known resolver for the rule's kind and Resolver.Imports returns a // non-nil slice. diff --git a/walk/walk.go b/walk/walk.go index 7128d4684..0152d1980 100644 --- a/walk/walk.go +++ b/walk/walk.go @@ -115,7 +115,7 @@ func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode, } } - updateRels := buildUpdateRelMap(c.RepoRoot, dirs) + updateRels := NewUpdateFilter(c.RepoRoot, dirs, mode) var visit func(*config.Config, string, string, bool) visit = func(c *config.Config, dir, rel string, updateParent bool) { @@ -162,15 +162,15 @@ func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode, } } - shouldUpdate := shouldUpdate(rel, mode, updateParent, updateRels) + shouldUpdate := updateRels.shouldUpdate(rel, updateParent) for _, sub := range subdirs { - if subRel := path.Join(rel, sub); shouldVisit(subRel, mode, shouldUpdate, updateRels) { + if subRel := path.Join(rel, sub); updateRels.shouldVisit(subRel, shouldUpdate) { visit(c, filepath.Join(dir, sub), subRel, shouldUpdate) } } update := !haveError && !wc.ignore && shouldUpdate - if shouldCall(rel, mode, updateParent, updateRels) { + if updateRels.shouldCall(rel, updateParent) { genFiles := findGenFiles(wc, f) wf(dir, rel, c, update, f, subdirs, regularFiles, genFiles) } @@ -178,16 +178,22 @@ func Walk(c *config.Config, cexts []config.Configurer, dirs []string, mode Mode, visit(c, c.RepoRoot, "", false) } -// buildUpdateRelMap builds a table of prefixes, used to determine which +// An UpdateFilter tracks which directories need to be updated +type UpdateFilter struct { + mode Mode + + // map from slash-separated paths relative to the + // root directory ("" for the root itself) to a boolean indicating whether + // the directory should be updated. + updateRels map[string]bool +} + +// NewUpdateFilter builds a table of prefixes, used to determine which // directories to update and visit. // // root and dirs must be absolute, canonical file paths. Each entry in dirs // must be a subdirectory of root. The caller is responsible for checking this. -// -// buildUpdateRelMap returns a map from slash-separated paths relative to the -// root directory ("" for the root itself) to a boolean indicating whether -// the directory should be updated. -func buildUpdateRelMap(root string, dirs []string) map[string]bool { +func NewUpdateFilter(root string, dirs []string, mode Mode) *UpdateFilter { relMap := make(map[string]bool) for _, dir := range dirs { rel, _ := filepath.Rel(root, dir) @@ -210,42 +216,58 @@ func buildUpdateRelMap(root string, dirs []string) map[string]bool { i = next + 1 } } - return relMap + return &UpdateFilter{mode, relMap} +} + +func (u *UpdateFilter) ShouldReIndex(rel string) bool { + if rel == "." { + rel = "" + } + + if should, found := u.updateRels[rel]; found { + return should + } + + if rel != "" && (u.mode == UpdateSubdirsMode || u.mode == VisitAllUpdateSubdirsMode) { + return u.ShouldReIndex(path.Dir(rel)) + } + + return false } // shouldCall returns true if Walk should call the callback in the // directory rel. -func shouldCall(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool { - switch mode { +func (u *UpdateFilter) shouldCall(rel string, updateParent bool) bool { + switch u.mode { case VisitAllUpdateSubdirsMode, VisitAllUpdateDirsMode: return true case UpdateSubdirsMode: - return updateParent || updateRels[rel] + return updateParent || u.updateRels[rel] default: // UpdateDirsMode - return updateRels[rel] + return u.updateRels[rel] } } // shouldUpdate returns true if Walk should pass true to the callback's update // parameter in the directory rel. This indicates the build file should be // updated. -func shouldUpdate(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool { - if (mode == VisitAllUpdateSubdirsMode || mode == UpdateSubdirsMode) && updateParent { +func (u *UpdateFilter) shouldUpdate(rel string, updateParent bool) bool { + if (u.mode == VisitAllUpdateSubdirsMode || u.mode == UpdateSubdirsMode) && updateParent { return true } - return updateRels[rel] + return u.updateRels[rel] } // shouldVisit returns true if Walk should visit the subdirectory rel. -func shouldVisit(rel string, mode Mode, updateParent bool, updateRels map[string]bool) bool { - switch mode { +func (u *UpdateFilter) shouldVisit(rel string, updateParent bool) bool { + switch u.mode { case VisitAllUpdateSubdirsMode, VisitAllUpdateDirsMode: return true case UpdateSubdirsMode: - _, ok := updateRels[rel] + _, ok := u.updateRels[rel] return ok || updateParent default: // UpdateDirsMode - _, ok := updateRels[rel] + _, ok := u.updateRels[rel] return ok } }