diff --git a/conn/node_test.go b/conn/node_test.go index 1e094e6ba9e..1d569c14fa6 100644 --- a/conn/node_test.go +++ b/conn/node_test.go @@ -34,14 +34,6 @@ import ( "golang.org/x/net/context" ) -func openBadger(dir string) (*badger.DB, error) { - opt := badger.DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - - return badger.Open(opt) -} - func (n *Node) run(wg *sync.WaitGroup) { ticker := time.NewTicker(20 * time.Millisecond) defer ticker.Stop() @@ -73,7 +65,7 @@ func TestProposal(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) store := raftwal.Init(db, 0, 0) diff --git a/dgraph/cmd/bulk/reduce.go b/dgraph/cmd/bulk/reduce.go index e822865b1d9..0177d5e7aa8 100644 --- a/dgraph/cmd/bulk/reduce.go +++ b/dgraph/cmd/bulk/reduce.go @@ -86,13 +86,9 @@ func (r *reducer) run() error { } func (r *reducer) createBadger(i int) *badger.DB { - opt := badger.DefaultOptions - opt.SyncWrites = false - opt.TableLoadingMode = bo.MemoryMap - opt.ValueThreshold = 1 << 10 // 1 KB. - opt.Dir = r.opt.shardOutputDirs[i] - opt.ValueDir = opt.Dir - opt.Logger = nil + opt := badger.DefaultOptions(r.opt.shardOutputDirs[i]).WithSyncWrites(false). + WithTableLoadingMode(bo.MemoryMap).WithValueThreshold(1 << 10 /* 1 KB */). + WithLogger(nil) db, err := badger.OpenManaged(opt) x.Check(err) r.dbs = append(r.dbs, db) diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index 27457591824..fd7370bfe1f 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -743,11 +743,9 @@ func run() { dir = opt.wdir isWal = true } - bopts := badger.DefaultOptions - bopts.Dir = dir - bopts.ValueDir = dir - bopts.TableLoadingMode = options.MemoryMap - bopts.ReadOnly = opt.readOnly + bopts := badger.DefaultOptions(dir). + WithTableLoadingMode(options.MemoryMap). + WithReadOnly(opt.readOnly) x.AssertTruef(len(bopts.Dir) > 0, "No posting or wal dir specified.") fmt.Printf("Opening DB: %s\n", bopts.Dir) diff --git a/dgraph/cmd/live/run.go b/dgraph/cmd/live/run.go index d124d120a4a..83bf76f436d 100644 --- a/dgraph/cmd/live/run.go +++ b/dgraph/cmd/live/run.go @@ -244,14 +244,11 @@ func setup(opts batchMutationOptions, dc *dgo.Dgraph) *loader { var db *badger.DB if len(opt.clientDir) > 0 { x.Check(os.MkdirAll(opt.clientDir, 0700)) - o := badger.DefaultOptions - o.Dir = opt.clientDir - o.ValueDir = opt.clientDir - o.TableLoadingMode = bopt.MemoryMap - o.SyncWrites = false var err error - db, err = badger.Open(o) + db, err = badger.Open(badger.DefaultOptions(opt.clientDir). + WithTableLoadingMode(bopt.MemoryMap). + WithSyncWrites(false)) x.Checkf(err, "Error while creating badger KV posting store") } diff --git a/dgraph/cmd/zero/run.go b/dgraph/cmd/zero/run.go index d2b0259abdd..0c6f27a6c25 100644 --- a/dgraph/cmd/zero/run.go +++ b/dgraph/cmd/zero/run.go @@ -203,12 +203,8 @@ func run() { // Open raft write-ahead log and initialize raft node. x.Checkf(os.MkdirAll(opts.w, 0700), "Error while creating WAL dir.") - kvOpt := badger.LSMOnlyOptions - kvOpt.SyncWrites = false - kvOpt.Truncate = true - kvOpt.Dir = opts.w - kvOpt.ValueDir = opts.w - kvOpt.ValueLogFileSize = 64 << 20 + kvOpt := badger.LSMOnlyOptions(opts.w).WithSyncWrites(false).WithTruncate(true). + WithValueLogFileSize(64 << 20) kv, err := badger.Open(kvOpt) x.Checkf(err, "Error while opening WAL store") defer kv.Close() diff --git a/edgraph/server.go b/edgraph/server.go index c3ee310aa8d..823c738e6f8 100644 --- a/edgraph/server.go +++ b/edgraph/server.go @@ -119,12 +119,8 @@ func (s *ServerState) runVlogGC(store *badger.DB) { } } -func setBadgerOptions(opt badger.Options, dir string) badger.Options { - opt.SyncWrites = false - opt.Truncate = true - opt.Dir = dir - opt.ValueDir = dir - opt.Logger = &x.ToGlog{} +func setBadgerOptions(opt badger.Options) badger.Options { + opt = opt.WithSyncWrites(false).WithTruncate(true).WithLogger(&x.ToGlog{}) glog.Infof("Setting Badger table load option: %s", Config.BadgerTables) switch Config.BadgerTables { @@ -155,8 +151,8 @@ func (s *ServerState) initStorage() { { // Write Ahead Log directory x.Checkf(os.MkdirAll(Config.WALDir, 0700), "Error while creating WAL dir.") - opt := badger.LSMOnlyOptions - opt = setBadgerOptions(opt, Config.WALDir) + opt := badger.LSMOnlyOptions(Config.WALDir) + opt = setBadgerOptions(opt) opt.ValueLogMaxEntries = 10000 // Allow for easy space reclamation. // We should always force load LSM tables to memory, disregarding user settings, because @@ -175,10 +171,9 @@ func (s *ServerState) initStorage() { // All the writes to posting store should be synchronous. We use batched writers // for posting lists, so the cost of sync writes is amortized. x.Check(os.MkdirAll(Config.PostingDir, 0700)) - opt := badger.DefaultOptions - opt.ValueThreshold = 1 << 10 // 1KB - opt.NumVersionsToKeep = math.MaxInt32 - opt = setBadgerOptions(opt, Config.PostingDir) + opt := badger.DefaultOptions(Config.PostingDir).WithValueThreshold(1 << 10 /* 1KB */). + WithNumVersionsToKeep(math.MaxInt32) + opt = setBadgerOptions(opt) glog.Infof("Opening postings BadgerDB with options: %+v\n", opt) s.Pstore, err = badger.OpenManaged(opt) diff --git a/ee/backup/run.go b/ee/backup/run.go index 66d8998a7c7..3e2fc3daf66 100644 --- a/ee/backup/run.go +++ b/ee/backup/run.go @@ -215,26 +215,22 @@ func runRestoreCmd() error { // RunRestore calls badger.Load and tries to load data into a new DB. func RunRestore(pdir, location, backupId string) (uint64, error) { - bo := badger.DefaultOptions - bo.SyncWrites = true - bo.TableLoadingMode = options.MemoryMap - bo.ValueThreshold = 1 << 10 - bo.NumVersionsToKeep = math.MaxInt32 - // Scan location for backup files and load them. Each file represents a node group, // and we create a new p dir for each. return Load(location, backupId, func(r io.Reader, groupId int) error { - bo := bo - bo.Dir = filepath.Join(pdir, fmt.Sprintf("p%d", groupId)) - bo.ValueDir = bo.Dir - db, err := badger.OpenManaged(bo) + dir := filepath.Join(pdir, fmt.Sprintf("p%d", groupId)) + db, err := badger.OpenManaged(badger.DefaultOptions(dir). + WithSyncWrites(true). + WithTableLoadingMode(options.MemoryMap). + WithValueThreshold(1 << 10). + WithNumVersionsToKeep(math.MaxInt32)) if err != nil { return err } defer db.Close() fmt.Printf("Restoring groupId: %d\n", groupId) - if !pathExist(bo.Dir) { - fmt.Println("Creating new db:", bo.Dir) + if !pathExist(dir) { + fmt.Println("Creating new db:", dir) } gzReader, err := gzip.NewReader(r) if err != nil { diff --git a/posting/list_test.go b/posting/list_test.go index de0c7da0d2d..ed4ea02c3e8 100644 --- a/posting/list_test.go +++ b/posting/list_test.go @@ -1247,10 +1247,7 @@ func TestMain(m *testing.M) { dir, err := ioutil.TempDir("", "storetest_") x.Check(err) - opt := badger.DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - ps, err = badger.OpenManaged(opt) + ps, err = badger.OpenManaged(badger.DefaultOptions(dir)) x.Check(err) Init(ps) schema.Init(ps) diff --git a/raftwal/storage_test.go b/raftwal/storage_test.go index a9e8a832c82..46eb96659a0 100644 --- a/raftwal/storage_test.go +++ b/raftwal/storage_test.go @@ -45,20 +45,12 @@ import ( pb "go.etcd.io/etcd/raft/raftpb" ) -func openBadger(dir string) (*badger.DB, error) { - opt := badger.DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - - return badger.Open(opt) -} - func TestStorageTerm(t *testing.T) { dir, err := ioutil.TempDir("", "badger") require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) ds := Init(db, 0, 0) @@ -108,7 +100,7 @@ func TestStorageEntries(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) ds := Init(db, 0, 0) @@ -153,7 +145,7 @@ func TestStorageLastIndex(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) ds := Init(db, 0, 0) @@ -183,7 +175,7 @@ func TestStorageFirstIndex(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) ds := Init(db, 0, 0) @@ -216,7 +208,7 @@ func TestStorageCompact(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) ds := Init(db, 0, 0) @@ -265,7 +257,7 @@ func TestStorageCreateSnapshot(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) ds := Init(db, 0, 0) @@ -302,7 +294,7 @@ func TestStorageAppend(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - db, err := openBadger(dir) + db, err := badger.Open(badger.DefaultOptions(dir)) require.NoError(t, err) ds := Init(db, 0, 0) diff --git a/schema/parse_test.go b/schema/parse_test.go index 9d86905de51..11ad5c22ab5 100644 --- a/schema/parse_test.go +++ b/schema/parse_test.go @@ -802,9 +802,7 @@ func TestMain(m *testing.M) { dir, err := ioutil.TempDir("", "storetest_") x.Check(err) - kvOpt := badger.DefaultOptions - kvOpt.Dir = dir - kvOpt.ValueDir = dir + kvOpt := badger.DefaultOptions(dir) ps, err = badger.OpenManaged(kvOpt) x.Check(err) Init(ps) diff --git a/vendor/github.com/dgraph-io/badger/README.md b/vendor/github.com/dgraph-io/badger/README.md index 873316025c7..5779cb6bd90 100644 --- a/vendor/github.com/dgraph-io/badger/README.md +++ b/vendor/github.com/dgraph-io/badger/README.md @@ -52,7 +52,7 @@ few months. The [Changelog] is kept fairly up-to-date. ## Getting Started ### Installing -To start using Badger, install Go 1.8 or above and run `go get`: +To start using Badger, install Go 1.11 or above and run `go get`: ```sh $ go get github.com/dgraph-io/badger/... @@ -76,7 +76,7 @@ package main import ( "log" - "github.com/dgraph-io/badger" + badger "github.com/dgraph-io/badger" ) func main() { diff --git a/vendor/github.com/dgraph-io/badger/appveyor.yml b/vendor/github.com/dgraph-io/badger/appveyor.yml index 79dac338e76..afa54ca0ad0 100644 --- a/vendor/github.com/dgraph-io/badger/appveyor.yml +++ b/vendor/github.com/dgraph-io/badger/appveyor.yml @@ -13,6 +13,7 @@ clone_folder: c:\gopath\src\github.com\dgraph-io\badger environment: GOVERSION: 1.8.3 GOPATH: c:\gopath + GO111MODULE: on # scripts that run after cloning repository install: diff --git a/vendor/github.com/dgraph-io/badger/backup.go b/vendor/github.com/dgraph-io/badger/backup.go index 0eddb346996..fe62aa61114 100644 --- a/vendor/github.com/dgraph-io/badger/backup.go +++ b/vendor/github.com/dgraph-io/badger/backup.go @@ -127,20 +127,20 @@ func writeTo(list *pb.KVList, w io.Writer) error { return err } -type Loader struct { +type loader struct { db *DB throttle *y.Throttle entries []*Entry } -func (db *DB) NewLoader(maxPendingWrites int) *Loader { - return &Loader{ +func (db *DB) newLoader(maxPendingWrites int) *loader { + return &loader{ db: db, throttle: y.NewThrottle(maxPendingWrites), } } -func (l *Loader) Set(kv *pb.KV) error { +func (l *loader) set(kv *pb.KV) error { var userMeta, meta byte if len(kv.UserMeta) > 0 { userMeta = kv.UserMeta[0] @@ -162,19 +162,21 @@ func (l *Loader) Set(kv *pb.KV) error { return nil } -func (l *Loader) send() error { +func (l *loader) send() error { if err := l.throttle.Do(); err != nil { return err } - l.db.batchSetAsync(l.entries, func(err error) { + if err := l.db.batchSetAsync(l.entries, func(err error) { l.throttle.Done(err) - }) + }); err != nil { + return err + } l.entries = make([]*Entry, 0, 1000) return nil } -func (l *Loader) Finish() error { +func (l *loader) finish() error { if len(l.entries) > 0 { if err := l.send(); err != nil { return err @@ -193,7 +195,7 @@ func (db *DB) Load(r io.Reader, maxPendingWrites int) error { br := bufio.NewReaderSize(r, 16<<10) unmarshalBuf := make([]byte, 1<<10) - loader := db.NewLoader(maxPendingWrites) + ldr := db.newLoader(maxPendingWrites) for { var sz uint64 err := binary.Read(br, binary.LittleEndian, &sz) @@ -217,7 +219,7 @@ func (db *DB) Load(r io.Reader, maxPendingWrites int) error { } for _, kv := range list.Kv { - if err := loader.Set(kv); err != nil { + if err := ldr.set(kv); err != nil { return err } @@ -229,7 +231,7 @@ func (db *DB) Load(r io.Reader, maxPendingWrites int) error { } } - if err := loader.Finish(); err != nil { + if err := ldr.finish(); err != nil { return err } db.orc.txnMark.Done(db.orc.nextTxnTs - 1) diff --git a/vendor/github.com/dgraph-io/badger/db.go b/vendor/github.com/dgraph-io/badger/db.go index c0fce4e5b18..21bb22d6f08 100644 --- a/vendor/github.com/dgraph-io/badger/db.go +++ b/vendor/github.com/dgraph-io/badger/db.go @@ -192,7 +192,7 @@ func Open(opt Options) (db *DB, err error) { opt.maxBatchSize = (15 * opt.MaxTableSize) / 100 opt.maxBatchCount = opt.maxBatchSize / int64(skl.MaxNodeSize) - if opt.ValueThreshold > math.MaxUint16-16 { + if opt.ValueThreshold > ValueThresholdLimit { return nil, ErrValueThreshold } @@ -294,7 +294,9 @@ func Open(opt Options) (db *DB, err error) { db.lc.startCompact(db.closers.compactors) db.closers.memtable = y.NewCloser(1) - go db.flushMemtable(db.closers.memtable) // Need levels controller to be up. + go func() { + _ = db.flushMemtable(db.closers.memtable) // Need levels controller to be up. + }() } headKey := y.KeyWithTs(head, math.MaxUint64) @@ -353,6 +355,11 @@ func (db *DB) Close() error { func (db *DB) close() (err error) { db.elog.Printf("Closing database") + + if err := db.vlog.flushDiscardStats(); err != nil { + return errors.Wrap(err, "failed to flush discard stats") + } + atomic.StoreInt32(&db.blockWrites, 1) // Stop value GC first. @@ -462,22 +469,6 @@ func (db *DB) Sync() error { return db.vlog.sync(math.MaxUint32) } -// When you create or delete a file, you have to ensure the directory entry for the file is synced -// in order to guarantee the file is visible (if the system crashes). (See the man page for fsync, -// or see https://github.com/coreos/etcd/issues/6368 for an example.) -func syncDir(dir string) error { - f, err := openDir(dir) - if err != nil { - return errors.Wrapf(err, "While opening directory: %s.", dir) - } - err = f.Sync() - closeErr := f.Close() - if err != nil { - return errors.Wrapf(err, "While syncing directory: %s.", dir) - } - return errors.Wrapf(closeErr, "While closing directory: %s.", dir) -} - // getMemtables returns the current memtables and get references. func (db *DB) getMemTables() ([]*skl.Skiplist, func()) { db.RLock() @@ -880,10 +871,6 @@ func (db *DB) handleFlushTask(ft flushTask) error { headTs := y.KeyWithTs(head, db.orc.nextTs()) ft.mt.Put(headTs, y.ValueStruct{Value: offset}) - // Also store lfDiscardStats before flushing memtables - discardStatsKey := y.KeyWithTs(lfDiscardStatsKey, 1) - ft.mt.Put(discardStatsKey, y.ValueStruct{Value: db.vlog.encodedDiscardStats()}) - fileID := db.lc.reserveFileID() fd, err := y.CreateSyncedFile(table.NewFilename(fileID, db.opt.Dir), true) if err != nil { @@ -913,7 +900,7 @@ func (db *DB) handleFlushTask(ft flushTask) error { } // We own a ref on tbl. err = db.lc.addLevel0Table(tbl) // This will incrRef (if we don't error, sure) - tbl.DecrRef() // Releases our ref. + _ = tbl.DecrRef() // Releases our ref. return err } @@ -1231,7 +1218,9 @@ func (db *DB) startCompactions() { if db.closers.memtable != nil { db.flushChan = make(chan flushTask, db.opt.NumMemtables) db.closers.memtable = y.NewCloser(1) - go db.flushMemtable(db.closers.memtable) + go func() { + _ = db.flushMemtable(db.closers.memtable) + }() } } diff --git a/vendor/github.com/dgraph-io/badger/dir_unix.go b/vendor/github.com/dgraph-io/badger/dir_unix.go index 146713a3601..d56e6e821a7 100644 --- a/vendor/github.com/dgraph-io/badger/dir_unix.go +++ b/vendor/github.com/dgraph-io/badger/dir_unix.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" + "github.com/dgraph-io/badger/y" "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -99,3 +100,19 @@ func (guard *directoryLockGuard) release() error { // openDir opens a directory for syncing. func openDir(path string) (*os.File, error) { return os.Open(path) } + +// When you create or delete a file, you have to ensure the directory entry for the file is synced +// in order to guarantee the file is visible (if the system crashes). (See the man page for fsync, +// or see https://github.com/coreos/etcd/issues/6368 for an example.) +func syncDir(dir string) error { + f, err := openDir(dir) + if err != nil { + return errors.Wrapf(err, "While opening directory: %s.", dir) + } + err = y.FileSync(f) + closeErr := f.Close() + if err != nil { + return errors.Wrapf(err, "While syncing directory: %s.", dir) + } + return errors.Wrapf(closeErr, "While closing directory: %s.", dir) +} diff --git a/vendor/github.com/dgraph-io/badger/dir_windows.go b/vendor/github.com/dgraph-io/badger/dir_windows.go index 28ccb7aaa31..60f982e2c5c 100644 --- a/vendor/github.com/dgraph-io/badger/dir_windows.go +++ b/vendor/github.com/dgraph-io/badger/dir_windows.go @@ -104,3 +104,7 @@ func (g *directoryLockGuard) release() error { g.path = "" return syscall.CloseHandle(g.h) } + +// Windows doesn't support syncing directories to the file system. See +// https://github.com/dgraph-io/badger/issues/699#issuecomment-504133587 for more details. +func syncDir(dir string) error { return nil } diff --git a/vendor/github.com/dgraph-io/badger/errors.go b/vendor/github.com/dgraph-io/badger/errors.go index b8c0d9c2942..8d2df6833a4 100644 --- a/vendor/github.com/dgraph-io/badger/errors.go +++ b/vendor/github.com/dgraph-io/badger/errors.go @@ -17,9 +17,16 @@ package badger import ( + "math" + "github.com/pkg/errors" ) +const ( + // ValueThresholdLimit is the maximum permissible value of opt.ValueThreshold. + ValueThresholdLimit = math.MaxUint16 - 16 + 1 +) + var ( // ErrValueLogSize is returned when opt.ValueLogFileSize option is not within the valid // range. @@ -27,7 +34,8 @@ var ( // ErrValueThreshold is returned when ValueThreshold is set to a value close to or greater than // uint16. - ErrValueThreshold = errors.New("Invalid ValueThreshold, must be lower than uint16") + ErrValueThreshold = errors.Errorf( + "Invalid ValueThreshold, must be less than %d", ValueThresholdLimit) // ErrKeyNotFound is returned when key isn't found on a txn.Get. ErrKeyNotFound = errors.New("Key not found") diff --git a/vendor/github.com/dgraph-io/badger/iterator.go b/vendor/github.com/dgraph-io/badger/iterator.go index d1fa05794b5..f4af4058d29 100644 --- a/vendor/github.com/dgraph-io/badger/iterator.go +++ b/vendor/github.com/dgraph-io/badger/iterator.go @@ -334,7 +334,7 @@ type IteratorOptions struct { Prefix []byte // Only iterate over this given prefix. prefixIsKey bool // If set, use the prefix for bloom filter lookup. - internalAccess bool // Used to allow internal access to badger keys. + InternalAccess bool // Used to allow internal access to badger keys. } func (opt *IteratorOptions) pickTable(t table.TableInterface) bool { @@ -539,7 +539,7 @@ func (it *Iterator) parseItem() bool { } // Skip badger keys. - if !it.opt.internalAccess && bytes.HasPrefix(key, badgerPrefix) { + if !it.opt.InternalAccess && bytes.HasPrefix(key, badgerPrefix) { mi.Next() return false } diff --git a/vendor/github.com/dgraph-io/badger/levels.go b/vendor/github.com/dgraph-io/badger/levels.go index bfd7e62feba..a4efd6624fe 100644 --- a/vendor/github.com/dgraph-io/badger/levels.go +++ b/vendor/github.com/dgraph-io/badger/levels.go @@ -610,7 +610,7 @@ func (s *levelsController) compactBuildTables( // -- we're the only holders of a ref). for j := 0; j < numBuilds; j++ { if newTables[j] != nil { - newTables[j].DecrRef() + _ = newTables[j].DecrRef() } } errorReturn := errors.Wrapf(firstErr, "While running compaction for: %+v", cd) @@ -620,7 +620,9 @@ func (s *levelsController) compactBuildTables( sort.Slice(newTables, func(i, j int) bool { return y.CompareKeys(newTables[i].Biggest(), newTables[j].Biggest()) < 0 }) - s.kv.vlog.updateDiscardStats(discardStats) + if err := s.kv.vlog.updateDiscardStats(discardStats); err != nil { + return nil, nil, errors.Wrap(err, "failed to update discard stats") + } s.kv.opt.Debugf("Discard stats: %v", discardStats) return newTables, func() error { return decrRefs(newTables) }, nil } diff --git a/vendor/github.com/dgraph-io/badger/manifest.go b/vendor/github.com/dgraph-io/badger/manifest.go index 4d2d83fa47e..a5818829473 100644 --- a/vendor/github.com/dgraph-io/badger/manifest.go +++ b/vendor/github.com/dgraph-io/badger/manifest.go @@ -216,7 +216,7 @@ func (mf *manifestFile) addChanges(changesParam []*pb.ManifestChange) error { } mf.appendLock.Unlock() - return mf.fp.Sync() + return y.FileSync(mf.fp) } // Has to be 4 bytes. The value can never change, ever, anyway. @@ -255,7 +255,7 @@ func helpRewrite(dir string, m *Manifest) (*os.File, int, error) { fp.Close() return nil, 0, err } - if err := fp.Sync(); err != nil { + if err := y.FileSync(fp); err != nil { fp.Close() return nil, 0, err } @@ -321,7 +321,8 @@ func (r *countingReader) ReadByte() (b byte, err error) { } var ( - errBadMagic = errors.New("manifest has bad magic") + errBadMagic = errors.New("manifest has bad magic") + errBadChecksum = errors.New("manifest has checksum mismatch") ) // ReplayManifestFile reads the manifest file and constructs two manifest objects. (We need one @@ -366,7 +367,7 @@ func ReplayManifestFile(fp *os.File) (ret Manifest, truncOffset int64, err error return Manifest{}, 0, err } if crc32.Checksum(buf, y.CastagnoliCrcTable) != binary.BigEndian.Uint32(lenCrcBuf[4:8]) { - break + return Manifest{}, 0, errBadChecksum } var changeSet pb.ManifestChangeSet diff --git a/vendor/github.com/dgraph-io/badger/options.go b/vendor/github.com/dgraph-io/badger/options.go index 560b65b243c..b91fdc5e300 100644 --- a/vendor/github.com/dgraph-io/badger/options.go +++ b/vendor/github.com/dgraph-io/badger/options.go @@ -20,92 +20,48 @@ import ( "github.com/dgraph-io/badger/options" ) -// NOTE: Keep the comments in the following to 75 chars width, so they -// format nicely in godoc. +// Note: If you add a new option X make sure you also add a WithX method on Options. // Options are params for creating DB object. // // This package provides DefaultOptions which contains options that should // work for most applications. Consider using that as a starting point before // customizing it for your own needs. +// +// Each option X is documented on the WithX method. type Options struct { - // 1. Mandatory flags - // ------------------- - // Directory to store the data in. If it doesn't exist, Badger will - // try to create it for you. - Dir string - // Directory to store the value log in. Can be the same as Dir. If it - // doesn't exist, Badger will try to create it for you. - ValueDir string + // Required options. - // 2. Frequently modified flags - // ----------------------------- - // Sync all writes to disk. Setting this to false would achieve better - // performance, but may cause data to be lost. - SyncWrites bool + Dir string + ValueDir string - // How should LSM tree be accessed. - TableLoadingMode options.FileLoadingMode + // Usually modified options. - // How should value log be accessed. + SyncWrites bool + TableLoadingMode options.FileLoadingMode ValueLogLoadingMode options.FileLoadingMode + NumVersionsToKeep int + ReadOnly bool + Truncate bool + Logger Logger - // How many versions to keep per key. - NumVersionsToKeep int - - // Open the DB as read-only. With this set, multiple processes can - // open the same Badger DB. Note: if the DB being opened had crashed - // before and has vlog data to be replayed, ReadOnly will cause Open - // to fail with an appropriate message. - ReadOnly bool - - // Truncate value log to delete corrupt data, if any. Would not truncate if ReadOnly is set. - Truncate bool - - // DB-specific logger which will override the global logger. - Logger Logger - - // 3. Flags that user might want to review - // ---------------------------------------- - // The following affect all levels of LSM tree. - MaxTableSize int64 // Each table (or file) is at most this size. - LevelSizeMultiplier int // Equals SizeOf(Li+1)/SizeOf(Li). - MaxLevels int // Maximum number of levels of compaction. - // If value size >= this threshold, only store value offsets in tree. - ValueThreshold int - // Maximum number of tables to keep in memory, before stalling. - NumMemtables int - // The following affect how we handle LSM tree L0. - // Maximum number of Level 0 tables before we start compacting. - NumLevelZeroTables int - - // If we hit this number of Level 0 tables, we will stall until L0 is - // compacted away. - NumLevelZeroTablesStall int + // Fine tuning options. - // Maximum total size for L1. - LevelOneSize int64 + MaxTableSize int64 + LevelSizeMultiplier int + MaxLevels int + ValueThreshold int + NumMemtables int - // Size of single value log file. - ValueLogFileSize int64 + NumLevelZeroTables int + NumLevelZeroTablesStall int - // Max number of entries a value log file can hold (approximately). A value log file would be - // determined by the smaller of its file size and max entries. + LevelOneSize int64 + ValueLogFileSize int64 ValueLogMaxEntries uint32 - // Number of compaction workers to run concurrently. Setting this to zero would stop compactions - // to happen within LSM tree. If set to zero, writes could block forever. - NumCompactors int - - // When closing the DB, force compact Level 0. This ensures that both reads and writes are - // efficient when the DB is opened later. - CompactL0OnClose bool - - // After this many number of value log file rotates, there would be a force flushing of memtable - // to disk. This is useful in write loads with fewer keys and larger values. This work load - // would fill up the value logs quickly, while not filling up the Memtables. Thus, on a crash - // and restart, the value log head could cause the replay of a good number of value log files - // which can slow things on start. + NumCompactors int + CompactL0OnClose bool LogRotatesToFlush int32 // Transaction start and commit timestamps are managed by end-user. @@ -121,46 +77,46 @@ type Options struct { } // DefaultOptions sets a list of recommended options for good performance. -// Feel free to modify these to suit your needs. -var DefaultOptions = Options{ - LevelOneSize: 256 << 20, - LevelSizeMultiplier: 10, - TableLoadingMode: options.MemoryMap, - ValueLogLoadingMode: options.MemoryMap, - // table.MemoryMap to mmap() the tables. - // table.Nothing to not preload the tables. - MaxLevels: 7, - MaxTableSize: 64 << 20, - NumCompactors: 2, // Compactions can be expensive. Only run 2. - NumLevelZeroTables: 5, - NumLevelZeroTablesStall: 10, - NumMemtables: 5, - SyncWrites: true, - NumVersionsToKeep: 1, - CompactL0OnClose: true, - // Nothing to read/write value log using standard File I/O - // MemoryMap to mmap() the value log files - // (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32. - // -1 so 2*ValueLogFileSize won't overflow on 32-bit systems. - ValueLogFileSize: 1<<30 - 1, - - ValueLogMaxEntries: 1000000, - ValueThreshold: 32, - Truncate: false, - Logger: defaultLogger, - LogRotatesToFlush: 2, +// Feel free to modify these to suit your needs with the WithX methods. +func DefaultOptions(path string) Options { + return Options{ + Dir: path, + ValueDir: path, + LevelOneSize: 256 << 20, + LevelSizeMultiplier: 10, + TableLoadingMode: options.MemoryMap, + ValueLogLoadingMode: options.MemoryMap, + // table.MemoryMap to mmap() the tables. + // table.Nothing to not preload the tables. + MaxLevels: 7, + MaxTableSize: 64 << 20, + NumCompactors: 2, // Compactions can be expensive. Only run 2. + NumLevelZeroTables: 5, + NumLevelZeroTablesStall: 10, + NumMemtables: 5, + SyncWrites: true, + NumVersionsToKeep: 1, + CompactL0OnClose: true, + // Nothing to read/write value log using standard File I/O + // MemoryMap to mmap() the value log files + // (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32. + // -1 so 2*ValueLogFileSize won't overflow on 32-bit systems. + ValueLogFileSize: 1<<30 - 1, + + ValueLogMaxEntries: 1000000, + ValueThreshold: 32, + Truncate: false, + Logger: defaultLogger, + LogRotatesToFlush: 2, + } } // LSMOnlyOptions follows from DefaultOptions, but sets a higher ValueThreshold // so values would be colocated with the LSM tree, with value log largely acting // as a write-ahead log only. These options would reduce the disk usage of value // log, and make Badger act more like a typical LSM tree. -var LSMOnlyOptions = Options{} - -func init() { - LSMOnlyOptions = DefaultOptions - - LSMOnlyOptions.ValueThreshold = 65500 // Max value length which fits in uint16. +func LSMOnlyOptions(path string) Options { + // Max value length which fits in uint16. // Let's not set any other options, because they can cause issues with the // size of key-value a user can pass to Badger. For e.g., if we set // ValueLogFileSize to 64MB, a user can't pass a value more than that. @@ -171,4 +127,248 @@ func init() { // NOTE: If a user does not want to set 64KB as the ValueThreshold because // of performance reasons, 1KB would be a good option too, allowing // values smaller than 1KB to be colocated with the keys in the LSM tree. + return DefaultOptions(path).WithValueThreshold(65500) +} + +// WithDir returns a new Options value with Dir set to the given value. +// +// Dir is the path of the directory where key data will be stored in. +// If it doesn't exist, Badger will try to create it for you. +// This is set automatically to be the path given to `DefaultOptions`. +func (opt Options) WithDir(val string) Options { + opt.Dir = val + return opt +} + +// WithValueDir returns a new Options value with ValueDir set to the given value. +// +// ValueDir is the path of the directory where value data will be stored in. +// If it doesn't exist, Badger will try to create it for you. +// This is set automatically to be the path given to `DefaultOptions`. +func (opt Options) WithValueDir(val string) Options { + opt.ValueDir = val + return opt +} + +// WithSyncWrites returns a new Options value with SyncWrites set to the given value. +// +// When SyncWrites is true all writes are synced to disk. Setting this to false would achieve better +// performance, but may cause data loss in case of crash. +// +// The default value of SyncWrites is true. +func (opt Options) WithSyncWrites(val bool) Options { + opt.SyncWrites = val + return opt +} + +// WithTableLoadingMode returns a new Options value with TableLoadingMode set to the given value. +// +// TableLoadingMode indicates which file loading mode should be used for the LSM tree data files. +// +// The default value of TableLoadingMode is options.MemoryMap. +func (opt Options) WithTableLoadingMode(val options.FileLoadingMode) Options { + opt.TableLoadingMode = val + return opt +} + +// WithValueLogLoadingMode returns a new Options value with ValueLogLoadingMode set to the given +// value. +// +// ValueLogLoadingMode indicates which file loading mode should be used for the value log data +// files. +// +// The default value of ValueLogLoadingMode is options.MemoryMap. +func (opt Options) WithValueLogLoadingMode(val options.FileLoadingMode) Options { + opt.ValueLogLoadingMode = val + return opt +} + +// WithNumVersionsToKeep returns a new Options value with NumVersionsToKeep set to the given value. +// +// NumVersionsToKeep sets how many versions to keep per key at most. +// +// The default value of NumVersionsToKeep is 1. +func (opt Options) WithNumVersionsToKeep(val int) Options { + opt.NumVersionsToKeep = val + return opt +} + +// WithReadOnly returns a new Options value with ReadOnly set to the given value. +// +// When ReadOnly is true the DB will be opened on read-only mode. +// Multiple processes can open the same Badger DB. +// Note: if the DB being opened had crashed before and has vlog data to be replayed, +// ReadOnly will cause Open to fail with an appropriate message. +// +// The default value of ReadOnly is false. +func (opt Options) WithReadOnly(val bool) Options { + opt.ReadOnly = val + return opt +} + +// WithTruncate returns a new Options value with Truncate set to the given value. +// +// Truncate indicates whether value log files should be truncated to delete corrupt data, if any. +// This option is ignored when ReadOnly is true. +// +// The default value of Truncate is false. +func (opt Options) WithTruncate(val bool) Options { + opt.Truncate = val + return opt +} + +// WithLogger returns a new Options value with Logger set to the given value. +// +// Logger provides a way to configure what logger each value of badger.DB uses. +// +// The default value of Logger writes to stderr using the log package from the Go standard library. +func (opt Options) WithLogger(val Logger) Options { + opt.Logger = val + return opt +} + +// WithMaxTableSize returns a new Options value with MaxTableSize set to the given value. +// +// MaxTableSize sets the maximum size in bytes for each LSM table or file. +// +// The default value of MaxTableSize is 64MB. +func (opt Options) WithMaxTableSize(val int64) Options { + opt.MaxTableSize = val + return opt +} + +// WithLevelSizeMultiplier returns a new Options value with LevelSizeMultiplier set to the given +// value. +// +// LevelSizeMultiplier sets the ratio between the maximum sizes of contiguous levels in the LSM. +// Once a level grows to be larger than this ratio allowed, the compaction process will be +// triggered. +// +// The default value of LevelSizeMultiplier is 10. +func (opt Options) WithLevelSizeMultiplier(val int) Options { + opt.LevelSizeMultiplier = val + return opt +} + +// WithMaxLevels returns a new Options value with MaxLevels set to the given value. +// +// Maximum number of levels of compaction allowed in the LSM. +// +// The default value of MaxLevels is 7. +func (opt Options) WithMaxLevels(val int) Options { + opt.MaxLevels = val + return opt +} + +// WithValueThreshold returns a new Options value with ValueThreshold set to the given value. +// +// ValueThreshold sets the threshold used to decide whether a value is stored directly in the LSM +// tree or separatedly in the log value files. +// +// The default value of ValueThreshold is 32, but LSMOnlyOptions sets it to 65500. +func (opt Options) WithValueThreshold(val int) Options { + opt.ValueThreshold = val + return opt +} + +// WithNumMemtables returns a new Options value with NumMemtables set to the given value. +// +// NumMemtables sets the maximum number of tables to keep in memory before stalling. +// +// The default value of NumMemtables is 5. +func (opt Options) WithNumMemtables(val int) Options { + opt.NumMemtables = val + return opt +} + +// WithNumLevelZeroTables returns a new Options value with NumLevelZeroTables set to the given +// value. +// +// NumLevelZeroTables sets the maximum number of Level 0 tables before compaction starts. +// +// The default value of NumLevelZeroTables is 5. +func (opt Options) WithNumLevelZeroTables(val int) Options { + opt.NumLevelZeroTables = val + return opt +} + +// WithNumLevelZeroTablesStall returns a new Options value with NumLevelZeroTablesStall set to the +// given value. +// +// NumLevelZeroTablesStall sets the number of Level 0 tables that once reached causes the DB to +// stall until compaction succeeds. +// +// The default value of NumLevelZeroTablesStall is 10. +func (opt Options) WithNumLevelZeroTablesStall(val int) Options { + opt.NumLevelZeroTablesStall = val + return opt +} + +// WithLevelOneSize returns a new Options value with LevelOneSize set to the given value. +// +// LevelOneSize sets the maximum total size for Level 1. +// +// The default value of LevelOneSize is 20MB. +func (opt Options) WithLevelOneSize(val int64) Options { + opt.LevelOneSize = val + return opt +} + +// WithValueLogFileSize returns a new Options value with ValueLogFileSize set to the given value. +// +// ValueLogFileSize sets the maximum size of a single value log file. +// +// The default value of ValueLogFileSize is 1GB. +func (opt Options) WithValueLogFileSize(val int64) Options { + opt.ValueLogFileSize = val + return opt +} + +// WithValueLogMaxEntries returns a new Options value with ValueLogMaxEntries set to the given +// value. +// +// ValueLogMaxEntries sets the maximum number of entries a value log file can hold approximately. +// A actual size limit of a value log file is the minimum of ValueLogFileSize and +// ValueLogMaxEntries. +// +// The default value of ValueLogMaxEntries is one million (1000000). +func (opt Options) WithValueLogMaxEntries(val uint32) Options { + opt.ValueLogMaxEntries = val + return opt +} + +// WithNumCompactors returns a new Options value with NumCompactors set to the given value. +// +// NumCompactors sets the number of compaction workers to run concurrently. +// Setting this to zero stops compactions, which could eventually cause writes to block forever. +// +// The default value of NumCompactors is 2. +func (opt Options) WithNumCompactors(val int) Options { + opt.NumCompactors = val + return opt +} + +// WithCompactL0OnClose returns a new Options value with CompactL0OnClose set to the given value. +// +// CompactL0OnClose determines whether Level 0 should be compacted before closing the DB. +// This ensures that both reads and writes are efficient when the DB is opened later. +// +// The default value of CompactL0OnClose is true. +func (opt Options) WithCompactL0OnClose(val bool) Options { + opt.CompactL0OnClose = val + return opt +} + +// WithLogRotatesToFlush returns a new Options value with LogRotatesToFlush set to the given value. +// +// LogRotatesToFlush sets the number of value log file rotates after which the Memtables are +// flushed to disk. This is useful in write loads with fewer keys and larger values. This work load +// would fill up the value logs quickly, while not filling up the Memtables. Thus, on a crash +// and restart, the value log head could cause the replay of a good number of value log files +// which can slow things on start. +// +// The default value of LogRotatesToFlush is 2. +func (opt Options) WithLogRotatesToFlush(val int32) Options { + opt.LogRotatesToFlush = val + return opt } diff --git a/vendor/github.com/dgraph-io/badger/publisher.go b/vendor/github.com/dgraph-io/badger/publisher.go index 068cece60ba..24588f5c68d 100644 --- a/vendor/github.com/dgraph-io/badger/publisher.go +++ b/vendor/github.com/dgraph-io/badger/publisher.go @@ -96,7 +96,7 @@ func (p *publisher) publishUpdates(reqs requests) { kv := &pb.KV{ Key: y.ParseKey(k), Value: y.SafeCopy(nil, e.Value), - Meta: []byte{e.UserMeta}, + UserMeta: []byte{e.UserMeta}, ExpiresAt: e.ExpiresAt, Version: y.ParseTs(k), } diff --git a/vendor/github.com/dgraph-io/badger/skl/README.md b/vendor/github.com/dgraph-io/badger/skl/README.md index 92fa68bb539..e22e4590bbf 100644 --- a/vendor/github.com/dgraph-io/badger/skl/README.md +++ b/vendor/github.com/dgraph-io/badger/skl/README.md @@ -110,4 +110,4 @@ Showing top 10 nodes out of 41 (cum >= 185.62MB) 0 signals received 46434 voluntary context switches 597049 involuntary context switches -``` \ No newline at end of file +``` diff --git a/vendor/github.com/dgraph-io/badger/skl/skl.go b/vendor/github.com/dgraph-io/badger/skl/skl.go index b465b09ecc5..fc2eff982ba 100644 --- a/vendor/github.com/dgraph-io/badger/skl/skl.go +++ b/vendor/github.com/dgraph-io/badger/skl/skl.go @@ -97,10 +97,11 @@ func (s *Skiplist) DecrRef() { // Indicate we are closed. Good for testing. Also, lets GC reclaim memory. Race condition // here would suggest we are accessing skiplist when we are supposed to have no reference! s.arena = nil + // Since the head references the arena's buf, as long as the head is kept around + // GC can't release the buf. + s.head = nil } -func (s *Skiplist) valid() bool { return s.arena != nil } - func newNode(arena *Arena, key []byte, v y.ValueStruct, height int) *node { // The base level is already allocated in the node struct. offset := arena.putNode(height) diff --git a/vendor/github.com/dgraph-io/badger/stream_writer.go b/vendor/github.com/dgraph-io/badger/stream_writer.go index b134f87cd2a..3d2a7992efd 100644 --- a/vendor/github.com/dgraph-io/badger/stream_writer.go +++ b/vendor/github.com/dgraph-io/badger/stream_writer.go @@ -157,6 +157,9 @@ func (sw *StreamWriter) Flush() error { } if !sw.db.opt.managedTxns { + if sw.db.orc != nil { + sw.db.orc.Stop() + } sw.db.orc = newOracle(sw.db.opt) sw.db.orc.nextTxnTs = sw.maxVersion sw.db.orc.txnMark.Done(sw.maxVersion) diff --git a/vendor/github.com/dgraph-io/badger/structs.go b/vendor/github.com/dgraph-io/badger/structs.go index ffab4ad82e4..51d16cdb2de 100644 --- a/vendor/github.com/dgraph-io/badger/structs.go +++ b/vendor/github.com/dgraph-io/badger/structs.go @@ -114,13 +114,19 @@ func encodeEntry(e *Entry, buf *bytes.Buffer) (int, error) { hash := crc32.New(y.CastagnoliCrcTable) buf.Write(headerEnc[:]) - hash.Write(headerEnc[:]) + if _, err := hash.Write(headerEnc[:]); err != nil { + return 0, err + } buf.Write(e.Key) - hash.Write(e.Key) + if _, err := hash.Write(e.Key); err != nil { + return 0, err + } buf.Write(e.Value) - hash.Write(e.Value) + if _, err := hash.Write(e.Value); err != nil { + return 0, err + } var crcBuf [crc32.Size]byte binary.BigEndian.PutUint32(crcBuf[:], hash.Sum32()) diff --git a/vendor/github.com/dgraph-io/badger/table/README.md b/vendor/github.com/dgraph-io/badger/table/README.md index 5d33e96ab57..a784f12680a 100644 --- a/vendor/github.com/dgraph-io/badger/table/README.md +++ b/vendor/github.com/dgraph-io/badger/table/README.md @@ -1,51 +1,69 @@ -# BenchmarkRead +Size of table is 122,173,606 bytes for all benchmarks. +# BenchmarkRead ``` -$ go test -bench Read$ -count 3 - -Size of table: 105843444 -BenchmarkRead-8 3 343846914 ns/op -BenchmarkRead-8 3 351790907 ns/op -BenchmarkRead-8 3 351762823 ns/op +$ go test -bench ^BenchmarkRead$ -run ^$ -count 3 +goos: linux +goarch: amd64 +pkg: github.com/dgraph-io/badger/table +BenchmarkRead-16 10 153281932 ns/op +BenchmarkRead-16 10 153454443 ns/op +BenchmarkRead-16 10 155349696 ns/op +PASS +ok github.com/dgraph-io/badger/table 23.549s ``` -Size of table is 105,843,444 bytes, which is ~101M. - -The rate is ~287M/s which matches our read speed. This is using mmap. +Size of table is 122,173,606 bytes, which is ~117MB. -To read a 64M table, this would take ~0.22s, which is negligible. +The rate is ~750MB/s using LoadToRAM (when table is in RAM). -``` -$ go test -bench BenchmarkReadAndBuild -count 3 +To read a 64MB table, this would take ~0.0853s, which is negligible. -BenchmarkReadAndBuild-8 1 2341034225 ns/op -BenchmarkReadAndBuild-8 1 2346349671 ns/op -BenchmarkReadAndBuild-8 1 2364064576 ns/op +# BenchmarkReadAndBuild +```go +$ go test -bench BenchmarkReadAndBuild -run ^$ -count 3 +goos: linux +goarch: amd64 +pkg: github.com/dgraph-io/badger/table +BenchmarkReadAndBuild-16 2 945041628 ns/op +BenchmarkReadAndBuild-16 2 947120893 ns/op +BenchmarkReadAndBuild-16 2 954909506 ns/op +PASS +ok github.com/dgraph-io/badger/table 26.856s ``` -The rate is ~43M/s. To build a ~64M table, this would take ~1.5s. Note that this +The rate is ~122MB/s. To build a 64MB table, this would take ~0.52s. Note that this does NOT include the flushing of the table to disk. All we are doing above is -to read one table (mmaped) and write one table in memory. +reading one table (which is in RAM) and write one table in memory. + +The table building takes 0.52-0.0853s ~ 0.4347s. -The table building takes 1.5-0.22 ~ 1.3s. +# BenchmarkReadMerged +Below, we merge 5 tables. The total size remains unchanged at ~122M. -If we are writing out up to 10 tables, this would take 1.5*10 ~ 15s, and ~13s -is spent building the tables. +```go +$ go test -bench ReadMerged -run ^$ -count 3 +BenchmarkReadMerged-16 2 954475788 ns/op +BenchmarkReadMerged-16 2 955252462 ns/op +BenchmarkReadMerged-16 2 956857353 ns/op +PASS +ok github.com/dgraph-io/badger/table 33.327s +``` -When running populate, building one table in memory tends to take ~1.5s to ~2.5s -on my system. Where does this overhead come from? Let's investigate the merging. +The rate is ~122MB/s. To read a 64MB table using merge iterator, this would take ~0.52s. -Below, we merge 5 tables. The total size remains unchanged at ~101M. +# BenchmarkRandomRead -``` -$ go test -bench ReadMerged -count 3 -BenchmarkReadMerged-8 1 1321190264 ns/op -BenchmarkReadMerged-8 1 1296958737 ns/op -BenchmarkReadMerged-8 1 1314381178 ns/op +```go +go test -bench BenchmarkRandomRead$ -run ^$ -count 3 +goos: linux +goarch: amd64 +pkg: github.com/dgraph-io/badger/table +BenchmarkRandomRead-16 300000 3596 ns/op +BenchmarkRandomRead-16 300000 3621 ns/op +BenchmarkRandomRead-16 300000 3596 ns/op +PASS +ok github.com/dgraph-io/badger/table 44.727s ``` -The rate is ~76M/s. To build a 64M table, this would take ~0.84s. The writing -takes ~1.3s as we saw above. So in total, we expect around 0.84+1.3 ~ 2.1s. -This roughly matches what we observe when running populate. There might be -some additional overhead due to the concurrent writes going on, in flushing the -table to disk. Also, the tables tend to be slightly bigger than 64M/s. \ No newline at end of file +For random read benchmarking, we are randomly reading a key and verifying its value. diff --git a/vendor/github.com/dgraph-io/badger/table/table.go b/vendor/github.com/dgraph-io/badger/table/table.go index 147b167b7b6..0a1f42d464f 100644 --- a/vendor/github.com/dgraph-io/badger/table/table.go +++ b/vendor/github.com/dgraph-io/badger/table/table.go @@ -87,7 +87,9 @@ func (t *Table) DecrRef() error { // It's necessary to delete windows files if t.loadingMode == options.MemoryMap { - y.Munmap(t.mmap) + if err := y.Munmap(t.mmap); err != nil { + return err + } } if err := t.fd.Truncate(0); err != nil { // This is very important to let the FS know that the file is deleted. @@ -191,7 +193,9 @@ func OpenTable(fd *os.File, mode options.FileLoadingMode, cksum []byte) (*Table, // Close closes the open table. (Releases resources back to the OS.) func (t *Table) Close() error { if t.loadingMode == options.MemoryMap { - y.Munmap(t.mmap) + if err := y.Munmap(t.mmap); err != nil { + return err + } } return t.fd.Close() diff --git a/vendor/github.com/dgraph-io/badger/test.sh b/vendor/github.com/dgraph-io/badger/test.sh index e2df230eed5..5b14bfd8f59 100755 --- a/vendor/github.com/dgraph-io/badger/test.sh +++ b/vendor/github.com/dgraph-io/badger/test.sh @@ -12,12 +12,12 @@ go test -v --manual=true -run='TestBigKeyValuePairs$' go test -v --manual=true -run='TestPushValueLogLimit' # Run the special Truncate test. -rm -R p || true +rm -rf p go test -v --manual=true -run='TestTruncateVlogNoClose$' . truncate --size=4096 p/000000.vlog go test -v --manual=true -run='TestTruncateVlogNoClose2$' . go test -v --manual=true -run='TestTruncateVlogNoClose3$' . -rm -R p || true +rm -rf p # Then the normal tests. echo diff --git a/vendor/github.com/dgraph-io/badger/util.go b/vendor/github.com/dgraph-io/badger/util.go index 02952a80c5f..c5173e26cc9 100644 --- a/vendor/github.com/dgraph-io/badger/util.go +++ b/vendor/github.com/dgraph-io/badger/util.go @@ -28,31 +28,6 @@ import ( "github.com/pkg/errors" ) -// summary is produced when DB is closed. Currently it is used only for testing. -type summary struct { - fileIDs map[uint64]bool -} - -func (s *levelsController) getSummary() *summary { - out := &summary{ - fileIDs: make(map[uint64]bool), - } - for _, l := range s.levels { - l.getSummary(out) - } - return out -} - -func (s *levelHandler) getSummary(sum *summary) { - s.RLock() - defer s.RUnlock() - for _, t := range s.tables { - sum.fileIDs[t.ID()] = true - } -} - -func (s *DB) validate() error { return s.lc.validate() } - func (s *levelsController) validate() error { for _, l := range s.levels { if err := l.validate(); err != nil { diff --git a/vendor/github.com/dgraph-io/badger/value.go b/vendor/github.com/dgraph-io/badger/value.go index fe2474c4cd5..f57f1b3ba84 100644 --- a/vendor/github.com/dgraph-io/badger/value.go +++ b/vendor/github.com/dgraph-io/badger/value.go @@ -54,6 +54,9 @@ const ( bitFinTxn byte = 1 << 7 // Set if the entry is to indicate end of txn in value log. mi int64 = 1 << 20 + + // The number of updates after which discard map should be flushed into badger. + discardStatsFlushThreshold = 100 ) type logFile struct { @@ -147,7 +150,7 @@ func (lf *logFile) read(p valuePointer, s *y.Slice) (buf []byte, err error) { func (lf *logFile) doneWriting(offset uint32) error { // Sync before acquiring lock. (We call this from write() and thus know we have shared access // to the fd.) - if err := lf.fd.Sync(); err != nil { + if err := y.FileSync(lf.fd); err != nil { return errors.Wrapf(err, "Unable to sync value log: %q", lf.path) } // Close and reopen the file read-only. Acquire lock because fd will become invalid for a bit. @@ -176,7 +179,7 @@ func (lf *logFile) doneWriting(offset uint32) error { // You must hold lf.lock to sync() func (lf *logFile) sync() error { - return lf.fd.Sync() + return y.FileSync(lf.fd) } var errStop = errors.New("Stop iteration") @@ -463,7 +466,9 @@ func (vlog *valueLog) rewrite(f *logFile, tr trace.Trace) error { } if deleteFileNow { - vlog.deleteLogFile(f) + if err := vlog.deleteLogFile(f); err != nil { + return err + } } return nil @@ -476,7 +481,7 @@ func (vlog *valueLog) deleteMoveKeysFor(fid uint32, tr trace.Trace) error { tr.LazyPrintf("Iterating over move keys to find invalids for fid: %d", fid) err := db.View(func(txn *Txn) error { opt := DefaultIteratorOptions - opt.internalAccess = true + opt.InternalAccess = true opt.PrefetchValues = false itr := txn.NewIterator(opt) defer itr.Close() @@ -604,7 +609,8 @@ func (vlog *valueLog) dropAll() (int, error) { // a given logfile. type lfDiscardStats struct { sync.Mutex - m map[uint32]int64 + m map[uint32]int64 + updatesSinceFlush int } type valueLog struct { @@ -762,11 +768,7 @@ func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry) error { vlog.db = db vlog.elog = trace.NewEventLog("Badger", "Valuelog") vlog.garbageCh = make(chan struct{}, 1) // Only allow one GC at a time. - - if err := vlog.populateDiscardStats(); err != nil { - return err - } - + vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)} if err := vlog.populateFilesMap(); err != nil { return err } @@ -850,6 +852,9 @@ func (vlog *valueLog) open(db *DB, ptr valuePointer, replayFn logEntry) error { if err = last.mmap(2 * opt.ValueLogFileSize); err != nil { return errFile(err, last.path, "Map log file") } + if err := vlog.populateDiscardStats(); err != nil { + return err + } return nil } @@ -1370,12 +1375,36 @@ func (vlog *valueLog) runGC(discardRatio float64, head valuePointer) error { } } -func (vlog *valueLog) updateDiscardStats(stats map[uint32]int64) { +func (vlog *valueLog) updateDiscardStats(stats map[uint32]int64) error { vlog.lfDiscardStats.Lock() for fid, sz := range stats { vlog.lfDiscardStats.m[fid] += sz + vlog.lfDiscardStats.updatesSinceFlush++ } vlog.lfDiscardStats.Unlock() + if vlog.lfDiscardStats.updatesSinceFlush > discardStatsFlushThreshold { + if err := vlog.flushDiscardStats(); err != nil { + return err + } + vlog.lfDiscardStats.updatesSinceFlush = 0 + } + return nil +} + +// flushDiscardStats inserts discard stats into badger. Returns error on failure. +func (vlog *valueLog) flushDiscardStats() error { + if len(vlog.lfDiscardStats.m) == 0 { + return nil + } + entries := []*Entry{{ + Key: y.KeyWithTs(lfDiscardStatsKey, 1), + Value: vlog.encodedDiscardStats(), + }} + req, err := vlog.db.sendToWriteCh(entries) + if err != nil { + return errors.Wrapf(err, "failed to push discard stats to write channel") + } + return req.Wait() } // encodedDiscardStats returns []byte representation of lfDiscardStats @@ -1399,13 +1428,26 @@ func (vlog *valueLog) populateDiscardStats() error { // check if value is Empty if vs.Value == nil || len(vs.Value) == 0 { - vlog.lfDiscardStats = &lfDiscardStats{m: make(map[uint32]int64)} return nil } var statsMap map[uint32]int64 - if err := json.Unmarshal(vs.Value, &statsMap); err != nil { - return err + // discard map is stored in the vlog file. + if vs.Meta&bitValuePointer > 0 { + var vp valuePointer + vp.Decode(vs.Value) + result, cb, err := vlog.Read(vp, new(y.Slice)) + if err != nil { + return errors.Wrapf(err, "failed to read value pointer from vlog file: %+v", vp) + } + defer runCallback(cb) + if err := json.Unmarshal(result, &statsMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal discard stats") + } + } else { + if err := json.Unmarshal(vs.Value, &statsMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal discard stats") + } } vlog.opt.Debugf("Value Log Discard stats: %v", statsMap) vlog.lfDiscardStats = &lfDiscardStats{m: statsMap} diff --git a/vendor/github.com/dgraph-io/badger/y/file_sync.go b/vendor/github.com/dgraph-io/badger/y/file_sync.go new file mode 100644 index 00000000000..19016ef6984 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/y/file_sync.go @@ -0,0 +1,28 @@ +// +build !darwin go1.12 + +/* + * Copyright 2019 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package y + +import "os" + +// FileSync calls os.File.Sync with the right parameters. +// This function can be removed once we stop supporting Go 1.11 +// on MacOS. +// +// More info: https://golang.org/issue/26650. +func FileSync(f *os.File) error { return f.Sync() } diff --git a/vendor/github.com/dgraph-io/badger/y/file_sync_darwin.go b/vendor/github.com/dgraph-io/badger/y/file_sync_darwin.go new file mode 100644 index 00000000000..01c79f23013 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/y/file_sync_darwin.go @@ -0,0 +1,37 @@ +// +build darwin,!go1.12 + +/* + * Copyright 2019 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package y + +import ( + "os" + "syscall" +) + +// FileSync calls os.File.Sync with the right parameters. +// This function can be removed once we stop supporting Go 1.11 +// on MacOS. +// +// More info: https://golang.org/issue/26650. +func FileSync(f *os.File) error { + _, _, err := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), syscall.F_FULLFSYNC, 0) + if err == 0 { + return nil + } + return err +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 61c5df3eafc..b741e1b26d5 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -459,22 +459,22 @@ "revisionTime": "2016-09-07T16:21:46Z" }, { - "checksumSHA1": "LmxXlda5frwORaWcfAZW0OWyERI=", + "checksumSHA1": "xNw25IDral2xen+ekQicZluFJfw=", "path": "github.com/dgraph-io/badger", - "revision": "3f5a4cb1b0b2f1c11d28943247216f8b87866a4c", - "revisionTime": "2019-06-07T23:13:53Z" + "revision": "b116882676f2894a1d901ed06e474ac369e37999", + "revisionTime": "2019-06-26T23:27:49Z" }, { "checksumSHA1": "oOuT7ebEiZ1ViHLKdFxKFOvobAQ=", "path": "github.com/dgraph-io/badger/options", - "revision": "3f5a4cb1b0b2f1c11d28943247216f8b87866a4c", - "revisionTime": "2019-06-07T23:13:53Z" + "revision": "b116882676f2894a1d901ed06e474ac369e37999", + "revisionTime": "2019-06-26T23:27:49Z" }, { "checksumSHA1": "SV7o4+eEK7/XNWC7H7Z5vWCoHP0=", "path": "github.com/dgraph-io/badger/pb", - "revision": "3f5a4cb1b0b2f1c11d28943247216f8b87866a4c", - "revisionTime": "2019-06-07T23:13:53Z" + "revision": "b116882676f2894a1d901ed06e474ac369e37999", + "revisionTime": "2019-06-26T23:27:49Z" }, { "checksumSHA1": "d8wE18ae6lOhmJqh0jwwhmQCkII=", @@ -483,22 +483,22 @@ "revisionTime": "2018-11-26T21:07:12Z" }, { - "checksumSHA1": "00T6XbLV4d95J7hm6kTXDReaQHM=", + "checksumSHA1": "7dxXjygrynDxvPE1UBHFDxVn7kE=", "path": "github.com/dgraph-io/badger/skl", - "revision": "3f5a4cb1b0b2f1c11d28943247216f8b87866a4c", - "revisionTime": "2019-06-07T23:13:53Z" + "revision": "b116882676f2894a1d901ed06e474ac369e37999", + "revisionTime": "2019-06-26T23:27:49Z" }, { - "checksumSHA1": "6b+M6HOosiA2HlPwO0DGN8jDn4w=", + "checksumSHA1": "0nZcXky/WdTWm3io4j6z2XbH4oM=", "path": "github.com/dgraph-io/badger/table", - "revision": "3f5a4cb1b0b2f1c11d28943247216f8b87866a4c", - "revisionTime": "2019-06-07T23:13:53Z" + "revision": "b116882676f2894a1d901ed06e474ac369e37999", + "revisionTime": "2019-06-26T23:27:49Z" }, { - "checksumSHA1": "ozi+AzOCsxkBoTXDQiwS4a7M0mU=", + "checksumSHA1": "6Ya7Kr114SVcXRqa3UaglreBrY8=", "path": "github.com/dgraph-io/badger/y", - "revision": "3f5a4cb1b0b2f1c11d28943247216f8b87866a4c", - "revisionTime": "2019-06-07T23:13:53Z" + "revision": "b116882676f2894a1d901ed06e474ac369e37999", + "revisionTime": "2019-06-26T23:27:49Z" }, { "checksumSHA1": "g06KOc9faGeu0J6262AQfpL1x/o=", diff --git a/worker/draft_test.go b/worker/draft_test.go index b313c7e08ca..26b66bc6a89 100644 --- a/worker/draft_test.go +++ b/worker/draft_test.go @@ -31,10 +31,7 @@ import ( ) func openBadger(dir string) (*badger.DB, error) { - opt := badger.DefaultOptions - opt.Dir = dir - opt.ValueDir = dir - + opt := badger.DefaultOptions(dir) return badger.Open(opt) } diff --git a/worker/worker_test.go b/worker/worker_test.go index ad2d96bc8e7..c0f313be400 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -379,9 +379,7 @@ func TestMain(m *testing.M) { x.Check(err) defer os.RemoveAll(dir) - opt := badger.DefaultOptions - opt.Dir = dir - opt.ValueDir = dir + opt := badger.DefaultOptions(dir) ps, err := badger.OpenManaged(opt) x.Check(err) pstore = ps diff --git a/xidmap/xidmap_test.go b/xidmap/xidmap_test.go index f70694a1885..eb3ec8d9a3e 100644 --- a/xidmap/xidmap_test.go +++ b/xidmap/xidmap_test.go @@ -22,10 +22,7 @@ func withDB(t *testing.T, test func(db *badger.DB)) { require.NoError(t, err) defer os.RemoveAll(dir) - opt := badger.LSMOnlyOptions - opt.Dir = dir - opt.ValueDir = dir - + opt := badger.LSMOnlyOptions(dir) db, err := badger.Open(opt) require.NoError(t, err) defer db.Close() diff --git a/z/backup.go b/z/backup.go index 42c70d208f5..deae6ea1a27 100644 --- a/z/backup.go +++ b/z/backup.go @@ -37,11 +37,8 @@ import ( // attribute in a map. // TODO(martinmr): See if this method can be simplified (e.g not use stream framework). func GetPValues(pdir, attr string, readTs uint64) (map[string]string, error) { - opt := badger.DefaultOptions - opt.Dir = pdir - opt.ValueDir = pdir - opt.TableLoadingMode = options.MemoryMap - opt.ReadOnly = true + opt := badger.DefaultOptions(pdir).WithTableLoadingMode(options.MemoryMap). + WithReadOnly(true) db, err := badger.OpenManaged(opt) if err != nil { return nil, err