From cab58681b07a21837f145b63a458d7d3c336fca3 Mon Sep 17 00:00:00 2001 From: rht Date: Thu, 12 Nov 2015 22:28:04 +0700 Subject: [PATCH] Add config option for flatfs no-sync License: MIT Signed-off-by: rht --- Godeps/Godeps.json | 2 +- .../jbenet/go-datastore/coalesce/coalesce.go | 6 +-- .../jbenet/go-datastore/elastigo/datastore.go | 3 +- .../jbenet/go-datastore/flatfs/flatfs.go | 47 ++++++++++++------- .../jbenet/go-datastore/flatfs/flatfs_test.go | 35 +++++++------- .../jbenet/go-datastore/lru/datastore_test.go | 3 +- .../go-datastore/timecache/timecache.go | 6 +-- repo/config/datastore.go | 1 + repo/fsrepo/defaultds.go | 3 +- 9 files changed, 63 insertions(+), 43 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 0ae2ff3f373..a56ee039f96 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -166,7 +166,7 @@ }, { "ImportPath": "github.com/jbenet/go-datastore", - "Rev": "c835c30f206c1e97172e428f052e225adab9abde" + "Rev": "bec407bccea1cfaf56ee946e947642e3ac5a9258" }, { "ImportPath": "github.com/jbenet/go-detect-race", diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce.go index e85a4b49132..976ae4dbf7c 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce.go @@ -8,10 +8,10 @@ import ( dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" ) +// parent keys var ( - putKey = "put" - getKey = // parent keys - "get" + putKey = "put" + getKey = "get" hasKey = "has" deleteKey = "delete" ) diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go index 8058d19a853..e77bf755423 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go @@ -6,9 +6,10 @@ import ( "net/url" "strings" - "github.com/codahale/blake2" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" query "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" + + "github.com/codahale/blake2" "github.com/mattbaird/elastigo/api" "github.com/mattbaird/elastigo/core" ) diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go index 07502114e20..f85ad05ddb4 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go @@ -15,6 +15,7 @@ import ( "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-os-rename" + logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log" ) @@ -33,11 +34,14 @@ type Datastore struct { path string // length of the dir splay prefix, in bytes of hex digits hexPrefixLen int + + // sychronize all writes and directory changes for added safety + sync bool } var _ datastore.Datastore = (*Datastore)(nil) -func New(path string, prefixLen int) (*Datastore, error) { +func New(path string, prefixLen int, sync bool) (*Datastore, error) { if prefixLen <= 0 || prefixLen > maxPrefixLen { return nil, ErrBadPrefixLen } @@ -45,6 +49,7 @@ func New(path string, prefixLen int) (*Datastore, error) { path: path, // convert from binary bytes to bytes of hex encoding hexPrefixLen: prefixLen * hex.EncodedLen(1), + sync: sync, } return fs, nil } @@ -80,8 +85,10 @@ func (fs *Datastore) makePrefixDir(dir string) error { // it, the creation of the prefix dir itself might not be // durable yet. Sync the root dir after a successful mkdir of // a prefix dir, just to be paranoid. - if err := syncDir(fs.path); err != nil { - return err + if fs.sync { + if err := syncDir(fs.path); err != nil { + return err + } } return nil } @@ -148,8 +155,10 @@ func (fs *Datastore) doPut(key datastore.Key, val []byte) error { if _, err := tmp.Write(val); err != nil { return err } - if err := tmp.Sync(); err != nil { - return err + if fs.sync { + if err := tmp.Sync(); err != nil { + return err + } } if err := tmp.Close(); err != nil { return err @@ -162,8 +171,10 @@ func (fs *Datastore) doPut(key datastore.Key, val []byte) error { } removed = true - if err := syncDir(dir); err != nil { - return err + if fs.sync { + if err := syncDir(dir); err != nil { + return err + } } return nil } @@ -213,8 +224,10 @@ func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error { // Now we sync everything // sync and close files for fi, _ := range files { - if err := fi.Sync(); err != nil { - return err + if fs.sync { + if err := fi.Sync(); err != nil { + return err + } } if err := fi.Close(); err != nil { @@ -236,15 +249,17 @@ func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error { } // now sync the dirs for those files - for _, dir := range dirsToSync { - if err := syncDir(dir); err != nil { - return err + if fs.sync { + for _, dir := range dirsToSync { + if err := syncDir(dir); err != nil { + return err + } } - } - // sync top flatfs dir - if err := syncDir(fs.path); err != nil { - return err + // sync top flatfs dir + if err := syncDir(fs.path); err != nil { + return err + } } return nil diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs_test.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs_test.go index cd36d684e2b..f63b74bf763 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs_test.go @@ -8,11 +8,12 @@ import ( "runtime" "testing" - rand "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/randbo" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" dstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/test" + + rand "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/randbo" ) func tempdir(t testing.TB) (path string, cleanup func()) { @@ -34,7 +35,7 @@ func TestBadPrefixLen(t *testing.T) { defer cleanup() for i := 0; i > -3; i-- { - _, err := flatfs.New(temp, 0) + _, err := flatfs.New(temp, i, false) if g, e := err, flatfs.ErrBadPrefixLen; g != e { t.Errorf("expected ErrBadPrefixLen, got: %v", g) } @@ -45,7 +46,7 @@ func TestPutBadValueType(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -60,7 +61,7 @@ func TestPut(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -75,7 +76,7 @@ func TestGet(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -103,7 +104,7 @@ func TestPutOverwrite(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -135,7 +136,7 @@ func TestGetNotFoundError(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -153,7 +154,7 @@ func TestStorage(t *testing.T) { const prefixLen = 2 const prefix = "7175" const target = prefix + string(os.PathSeparator) + "71757578.data" - fs, err := flatfs.New(temp, prefixLen) + fs, err := flatfs.New(temp, prefixLen, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -208,7 +209,7 @@ func TestHasNotFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -226,7 +227,7 @@ func TestHasFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -248,7 +249,7 @@ func TestDeleteNotFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -263,7 +264,7 @@ func TestDeleteFound(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -288,7 +289,7 @@ func TestQuerySimple(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -324,7 +325,7 @@ func TestBatchPut(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -336,7 +337,7 @@ func TestBatchDelete(t *testing.T) { temp, cleanup := tempdir(t) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { t.Fatalf("New fail: %v\n", err) } @@ -359,7 +360,7 @@ func BenchmarkConsecutivePut(b *testing.B) { temp, cleanup := tempdir(b) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { b.Fatalf("New fail: %v\n", err) } @@ -389,7 +390,7 @@ func BenchmarkBatchedPut(b *testing.B) { temp, cleanup := tempdir(b) defer cleanup() - fs, err := flatfs.New(temp, 2) + fs, err := flatfs.New(temp, 2, false) if err != nil { b.Fatalf("New fail: %v\n", err) } diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore_test.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore_test.go index b1822471d8a..dc31b19a16e 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore_test.go @@ -5,10 +5,11 @@ import ( "testing" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" - lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru" // Hook up gocheck into the "go test" runner. + lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru" . "gopkg.in/check.v1" ) +// Hook up gocheck into the "go test" runner. func Test(t *testing.T) { TestingT(t) } type DSSuite struct{} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache.go index 1da1ef02c2d..5ac675d598c 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache.go @@ -9,10 +9,10 @@ import ( dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" ) +// op keys var ( - putKey = "put" - getKey = // op keys - "get" + putKey = "put" + getKey = "get" hasKey = "has" deleteKey = "delete" ) diff --git a/repo/config/datastore.go b/repo/config/datastore.go index 89ded36f1a2..52582bd5cb5 100644 --- a/repo/config/datastore.go +++ b/repo/config/datastore.go @@ -16,6 +16,7 @@ type Datastore struct { GCPeriod string // in ns, us, ms, s, m, h Params *json.RawMessage + NoSync bool } func (d *Datastore) ParamData() []byte { diff --git a/repo/fsrepo/defaultds.go b/repo/fsrepo/defaultds.go index 6ac20261f10..4bca3107188 100644 --- a/repo/fsrepo/defaultds.go +++ b/repo/fsrepo/defaultds.go @@ -39,7 +39,8 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) { // including "/" from datastore.Key and 2 bytes from multihash. To // reach a uniform 256-way split, we need approximately 4 bytes of // prefix. - blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4) + syncfs := !r.config.Datastore.NoSync + blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4, syncfs) if err != nil { return nil, fmt.Errorf("unable to open flatfs datastore: %v", err) }