Skip to content
This repository has been archived by the owner on Jun 26, 2023. It is now read-only.

Commit

Permalink
Sync pinner on every pin operation by default
Browse files Browse the repository at this point in the history
This PR builds on PR #13 to sync the pinner on every pin operation. This PR does the following that #13 does not:

- Sync's dag service separately from pin data
- Does not release and immediately reacquire lock to sync, syncs while still holding pinner lock.
- Syncs only pin data for most operations

In addition to sync of pin data, this PR also revises how indexes are rebuilt.  Instead of reading through all pins and loading all previous index data, now only a single pass through the pins is needed to rebuild missing indexes resulting from incomplete add or delete operations. This is operationally much simpler, but also does not require storing entire pin sets or index sets in memory as did the previous solution.
  • Loading branch information
gammazero committed Jul 16, 2021
1 parent 3565d71 commit 92dbc3e
Show file tree
Hide file tree
Showing 6 changed files with 402 additions and 173 deletions.
32 changes: 11 additions & 21 deletions dsindex/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,40 +112,30 @@ func (x *indexer) ForEach(ctx context.Context, key string, fn func(key, value st
if err != nil {
return err
}
defer results.Close()

for {
r, ok := results.NextSync()
if !ok {
break
for r := range results.Next() {
if ctx.Err() != nil {
return ctx.Err()
}
if r.Error != nil {
err = r.Error
break
}
if ctx.Err() != nil {
err = ctx.Err()
break
return fmt.Errorf("cannot read index: %v", r.Error)
}
ent := r.Entry
var decIdx string
decIdx, err = decode(path.Base(path.Dir(ent.Key)))
decIdx, err := decode(path.Base(path.Dir(ent.Key)))
if err != nil {
err = fmt.Errorf("cannot decode index: %v", err)
break
return fmt.Errorf("cannot decode index: %v", err)
}
var decKey string
decKey, err = decode(path.Base(ent.Key))
decKey, err := decode(path.Base(ent.Key))
if err != nil {
err = fmt.Errorf("cannot decode key: %v", err)
break
return fmt.Errorf("cannot decode key: %v", err)
}
if !fn(decIdx, decKey) {
break
return nil
}
}
results.Close()

return err
return nil
}

func (x *indexer) HasValue(ctx context.Context, key, value string) (bool, error) {
Expand Down
Loading

0 comments on commit 92dbc3e

Please sign in to comment.