From 4bf43047d8a342b0e3c34aa736831f7677676b0f Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 3 Feb 2017 16:20:51 -0500 Subject: [PATCH 01/11] filestore util: basic filestore commands. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/filestore.go | 132 +++++++++++++++++++++++++++++++++++ core/commands/root.go | 2 + filestore/util.go | 138 +++++++++++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 core/commands/filestore.go create mode 100644 filestore/util.go diff --git a/core/commands/filestore.go b/core/commands/filestore.go new file mode 100644 index 00000000000..5624cff7929 --- /dev/null +++ b/core/commands/filestore.go @@ -0,0 +1,132 @@ +package commands + +import ( + "context" + "fmt" + + cmds "github.com/ipfs/go-ipfs/commands" + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/filestore" + u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util" +) + +var FileStoreCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Interact with filestore objects.", + }, + Subcommands: map[string]*cmds.Command{ + "ls": lsFileStore, + "verify": verifyFileStore, + }, +} + +var lsFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List objects in filestore.", + }, + Run: func(req cmds.Request, res cmds.Response) { + _, fs, err := getFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + next, err := filestore.ListAll(fs) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + out := listResAsChan(next, req.Context()) + res.SetOutput(out) + }, + PostRun: func(req cmds.Request, res cmds.Response) { + if res.Error() != nil { + return + } + outChan, ok := res.Output().(<-chan interface{}) + if !ok { + res.SetError(u.ErrCast(), cmds.ErrNormal) + return + } + res.SetOutput(nil) + errors := false + for r0 := range outChan { + r := r0.(*filestore.ListRes) + if r.ErrorMsg != "" { + fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg) + } else { + fmt.Fprintf(res.Stdout(), "%s\n", r.FormatLong()) + } + } + if errors { + res.SetError(fmt.Errorf("errors while displaying some entries"), cmds.ErrNormal) + } + }, + Type: filestore.ListRes{}, +} + +var verifyFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Verify objects in filestore.", + }, + Run: func(req cmds.Request, res cmds.Response) { + _, fs, err := getFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + next, err := filestore.VerifyAll(fs) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + out := listResAsChan(next, req.Context()) + res.SetOutput(out) + }, + PostRun: func(req cmds.Request, res cmds.Response) { + if res.Error() != nil { + return + } + outChan, ok := res.Output().(<-chan interface{}) + if !ok { + res.SetError(u.ErrCast(), cmds.ErrNormal) + return + } + res.SetOutput(nil) + for r0 := range outChan { + r := r0.(*filestore.ListRes) + fmt.Fprintf(res.Stdout(), "%s %s\n", r.Status.Format(), r.FormatLong()) + } + }, + Type: filestore.ListRes{}, +} + +func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) { + n, err := req.InvocContext().GetNode() + if err != nil { + return nil, nil, err + } + fs := n.Filestore + if fs == nil { + return n, nil, fmt.Errorf("filestore not enabled") + } + return n, fs, err +} + +func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} { + out := make(chan interface{}, 128) + go func() { + defer close(out) + for { + r := next() + if r == nil { + return + } + select { + case out <- r: + case <-ctx.Done(): + return + } + } + }() + return out +} diff --git a/core/commands/root.go b/core/commands/root.go index b0bbde403ac..031917e2866 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -47,6 +47,7 @@ ADVANCED COMMANDS pin Pin objects to local storage repo Manipulate the IPFS repository stats Various operational stats + filestore Manage the filestore (experimental) NETWORK COMMANDS id Show info about IPFS peers @@ -124,6 +125,7 @@ var rootSubcommands = map[string]*cmds.Command{ "update": ExternalBinary(), "version": VersionCmd, "bitswap": BitswapCmd, + "filestore": FileStoreCmd, } // RootRO is the readonly version of Root diff --git a/filestore/util.go b/filestore/util.go new file mode 100644 index 00000000000..ac6909bc6d7 --- /dev/null +++ b/filestore/util.go @@ -0,0 +1,138 @@ +package filestore + +import ( + "fmt" + + pb "github.com/ipfs/go-ipfs/filestore/pb" + dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + + ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore" + dsq "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore/query" + proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto" + cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid" +) + +type Status int32 + +const ( + StatusOk Status = 0 + StatusFileError Status = 10 // Backing File Error + //StatusFileNotFound Status = 11 // Backing File Not Found + //StatusFileChanged Status = 12 // Contents of the file changed + StatusOtherError Status = 20 // Internal Error, likely corrupt entry +) + +func (s Status) String() string { + switch s { + case StatusOk: + return "ok" + case StatusFileError: + return "error" + case StatusOtherError: + return "ERROR" + default: + return "???" + } +} + +func (s Status) Format() string { + return fmt.Sprintf("%-5s", s.String()) +} + +type ListRes struct { + Status Status + ErrorMsg string + Key *cid.Cid + FilePath string + Offset uint64 + Size uint64 +} + +func (r *ListRes) FormatLong() string { + switch { + case r.Key == nil: + return "?????????????????????????????????????????????????" + default: + return fmt.Sprintf("%-50s %6d %s %d", r.Key, r.Size, r.FilePath, r.Offset) + } +} + +func ListAll(fs *Filestore) (func() *ListRes, error) { + return listAll(fs, false) +} + +func VerifyAll(fs *Filestore) (func() *ListRes, error) { + return listAll(fs, true) +} + +func listAll(fs *Filestore, verify bool) (func() *ListRes, error) { + q := dsq.Query{} + qr, err := fs.fm.ds.Query(q) + if err != nil { + return nil, err + } + + return func() *ListRes { + cid, dobj, err := next(qr) + if dobj == nil && err == nil { + return nil + } else if err == nil && verify { + _, err = fs.fm.readDataObj(cid, dobj) + } + return mkListRes(cid, dobj, err) + }, nil +} + +func next(qr dsq.Results) (*cid.Cid, *pb.DataObj, error) { + v, ok := qr.NextSync() + if !ok { + return nil, nil, nil + } + + k := ds.RawKey(v.Key) + c, err := dshelp.DsKeyToCid(k) + if err != nil { + return nil, nil, fmt.Errorf("decoding cid from filestore: %s", err) + } + + data, ok := v.Value.([]byte) + if !ok { + return c, nil, fmt.Errorf("stored filestore dataobj was not a []byte") + } + + var dobj pb.DataObj + if err := proto.Unmarshal(data, &dobj); err != nil { + return c, nil, err + } + + return c, &dobj, nil +} + +func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes { + status := StatusOk + errorMsg := "" + if err != nil { + if _, ok := err.(*CorruptReferenceError); ok { + status = StatusFileError + } else { + status = StatusOtherError + } + errorMsg = err.Error() + } + if d == nil { + return &ListRes{ + Status: status, + ErrorMsg: errorMsg, + Key: c, + } + } else { + return &ListRes{ + Status: status, + ErrorMsg: errorMsg, + Key: c, + FilePath: *d.FilePath, + Size: *d.Size_, + Offset: *d.Offset, + } + } +} From 13f617d139687a1f6ffa7cb9c0caba45693bd685 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 7 Mar 2017 00:48:53 -0500 Subject: [PATCH 02/11] filestore util: allow listing/verifying of individual blocks. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/filestore.go | 77 +++++++++++++++++++++++++++++++------- filestore/util.go | 33 ++++++++++++++-- 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 5624cff7929..4b6daa709e2 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/filestore" u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util" + cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid" ) var FileStoreCmd = &cmds.Command{ @@ -24,19 +25,30 @@ var lsFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects in filestore.", }, + Arguments: []cmds.Argument{ + cmds.StringArg("obj", false, true, "Cid of objects to list."), + }, Run: func(req cmds.Request, res cmds.Response) { _, fs, err := getFilestore(req) if err != nil { res.SetError(err, cmds.ErrNormal) return } - next, err := filestore.ListAll(fs) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return + args := req.Arguments() + if len(args) > 0 { + out := perKeyActionToChan(args, func(c *cid.Cid) *filestore.ListRes { + return filestore.List(fs, c) + }, req.Context()) + res.SetOutput(out) + } else { + next, err := filestore.ListAll(fs) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + out := listResToChan(next, req.Context()) + res.SetOutput(out) } - out := listResAsChan(next, req.Context()) - res.SetOutput(out) }, PostRun: func(req cmds.Request, res cmds.Response) { if res.Error() != nil { @@ -52,6 +64,7 @@ var lsFileStore = &cmds.Command{ for r0 := range outChan { r := r0.(*filestore.ListRes) if r.ErrorMsg != "" { + errors = true fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg) } else { fmt.Fprintf(res.Stdout(), "%s\n", r.FormatLong()) @@ -68,19 +81,30 @@ var verifyFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Verify objects in filestore.", }, + Arguments: []cmds.Argument{ + cmds.StringArg("obj", false, true, "Cid of objects to verify."), + }, Run: func(req cmds.Request, res cmds.Response) { _, fs, err := getFilestore(req) if err != nil { res.SetError(err, cmds.ErrNormal) return } - next, err := filestore.VerifyAll(fs) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return + args := req.Arguments() + if len(args) > 0 { + out := perKeyActionToChan(args, func(c *cid.Cid) *filestore.ListRes { + return filestore.Verify(fs, c) + }, req.Context()) + res.SetOutput(out) + } else { + next, err := filestore.VerifyAll(fs) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + out := listResToChan(next, req.Context()) + res.SetOutput(out) } - out := listResAsChan(next, req.Context()) - res.SetOutput(out) }, PostRun: func(req cmds.Request, res cmds.Response) { if res.Error() != nil { @@ -94,6 +118,9 @@ var verifyFileStore = &cmds.Command{ res.SetOutput(nil) for r0 := range outChan { r := r0.(*filestore.ListRes) + if r.Status == filestore.StatusOtherError { + fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg) + } fmt.Fprintf(res.Stdout(), "%s %s\n", r.Status.Format(), r.FormatLong()) } }, @@ -112,7 +139,7 @@ func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error return n, fs, err } -func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} { +func listResToChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} { out := make(chan interface{}, 128) go func() { defer close(out) @@ -130,3 +157,27 @@ func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan i }() return out } + +func perKeyActionToChan(args []string, action func(*cid.Cid) *filestore.ListRes, ctx context.Context) <-chan interface{} { + out := make(chan interface{}, 128) + go func() { + defer close(out) + for _, arg := range args { + c, err := cid.Decode(arg) + if err != nil { + out <- &filestore.ListRes{ + Status: filestore.StatusOtherError, + ErrorMsg: fmt.Sprintf("%s: %v", arg, err), + } + continue + } + r := action(c) + select { + case out <- r: + case <-ctx.Done(): + return + } + } + }() + return out +} diff --git a/filestore/util.go b/filestore/util.go index ac6909bc6d7..75bb76bc1bf 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -3,6 +3,7 @@ package filestore import ( "fmt" + "github.com/ipfs/go-ipfs/blocks/blockstore" pb "github.com/ipfs/go-ipfs/filestore/pb" dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" @@ -19,7 +20,8 @@ const ( StatusFileError Status = 10 // Backing File Error //StatusFileNotFound Status = 11 // Backing File Not Found //StatusFileChanged Status = 12 // Contents of the file changed - StatusOtherError Status = 20 // Internal Error, likely corrupt entry + StatusOtherError Status = 20 // Internal Error, likely corrupt entry + StatusKeyNotFound Status = 30 ) func (s Status) String() string { @@ -30,13 +32,15 @@ func (s Status) String() string { return "error" case StatusOtherError: return "ERROR" + case StatusKeyNotFound: + return "missing" default: return "???" } } func (s Status) Format() string { - return fmt.Sprintf("%-5s", s.String()) + return fmt.Sprintf("%-7s", s.String()) } type ListRes struct { @@ -52,19 +56,40 @@ func (r *ListRes) FormatLong() string { switch { case r.Key == nil: return "?????????????????????????????????????????????????" + case r.FilePath == "": + return r.Key.String() default: return fmt.Sprintf("%-50s %6d %s %d", r.Key, r.Size, r.FilePath, r.Offset) } } +func List(fs *Filestore, key *cid.Cid) *ListRes { + return list(fs, false, key) +} + func ListAll(fs *Filestore) (func() *ListRes, error) { return listAll(fs, false) } +func Verify(fs *Filestore, key *cid.Cid) *ListRes { + return list(fs, true, key) +} + func VerifyAll(fs *Filestore) (func() *ListRes, error) { return listAll(fs, true) } +func list(fs *Filestore, verify bool, key *cid.Cid) *ListRes { + dobj, err := fs.fm.getDataObj(key) + if err != nil { + return mkListRes(key, nil, err) + } + if verify { + _, err = fs.fm.readDataObj(key, dobj) + } + return mkListRes(key, dobj, err) +} + func listAll(fs *Filestore, verify bool) (func() *ListRes, error) { q := dsq.Query{} qr, err := fs.fm.ds.Query(q) @@ -112,7 +137,9 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes { status := StatusOk errorMsg := "" if err != nil { - if _, ok := err.(*CorruptReferenceError); ok { + if err == ds.ErrNotFound || err == blockstore.ErrNotFound { + status = StatusKeyNotFound + } else if _, ok := err.(*CorruptReferenceError); ok { status = StatusFileError } else { status = StatusOtherError From 886a6fe166c7144e01da66e8a3822137f6cd9838 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 3 Feb 2017 18:34:08 -0500 Subject: [PATCH 03/11] filestore: be more specific when there is a problem reading the backing file. License: MIT Signed-off-by: Kevin Atkinson --- filestore/fsrefstore.go | 29 ++++++++++++++++++++++------- filestore/util.go | 29 +++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index f333a845d6b..5af21d4196a 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -27,8 +27,18 @@ type FileManager struct { root string } +type CorruptReferenceCode int + +const ( + OtherErr CorruptReferenceCode = 0 + FileError CorruptReferenceCode = 1 + FileMissing CorruptReferenceCode = 2 + FileChanged CorruptReferenceCode = 3 +) + type CorruptReferenceError struct { - Err error + Code CorruptReferenceCode + Err error } func (c CorruptReferenceError) Error() string { @@ -127,20 +137,24 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { abspath := filepath.Join(f.root, p) fi, err := os.Open(abspath) - if err != nil { - return nil, &CorruptReferenceError{err} + if os.IsNotExist(err) { + return nil, &CorruptReferenceError{FileMissing, err} + } else if err != nil { + return nil, &CorruptReferenceError{FileError, err} } defer fi.Close() _, err = fi.Seek(int64(d.GetOffset()), os.SEEK_SET) if err != nil { - return nil, &CorruptReferenceError{err} + return nil, &CorruptReferenceError{FileError, err} } outbuf := make([]byte, d.GetSize_()) _, err = io.ReadFull(fi, outbuf) - if err != nil { - return nil, &CorruptReferenceError{err} + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil, &CorruptReferenceError{FileChanged, err} + } else if err != nil { + return nil, &CorruptReferenceError{FileError, err} } outcid, err := c.Prefix().Sum(outbuf) @@ -149,7 +163,8 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { } if !c.Equals(outcid) { - return nil, &CorruptReferenceError{fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} + return nil, &CorruptReferenceError{FileChanged, + fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} } return outbuf, nil diff --git a/filestore/util.go b/filestore/util.go index 75bb76bc1bf..8049f7ec145 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -16,12 +16,12 @@ import ( type Status int32 const ( - StatusOk Status = 0 - StatusFileError Status = 10 // Backing File Error - //StatusFileNotFound Status = 11 // Backing File Not Found - //StatusFileChanged Status = 12 // Contents of the file changed - StatusOtherError Status = 20 // Internal Error, likely corrupt entry - StatusKeyNotFound Status = 30 + StatusOk Status = 0 + StatusFileError Status = 10 // Backing File Error + StatusFileNotFound Status = 11 // Backing File Not Found + StatusFileChanged Status = 12 // Contents of the file changed + StatusOtherError Status = 20 // Internal Error, likely corrupt entry + StatusKeyNotFound Status = 30 ) func (s Status) String() string { @@ -30,6 +30,10 @@ func (s Status) String() string { return "ok" case StatusFileError: return "error" + case StatusFileNotFound: + return "no-file" + case StatusFileChanged: + return "changed" case StatusOtherError: return "ERROR" case StatusKeyNotFound: @@ -139,8 +143,17 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes { if err != nil { if err == ds.ErrNotFound || err == blockstore.ErrNotFound { status = StatusKeyNotFound - } else if _, ok := err.(*CorruptReferenceError); ok { - status = StatusFileError + } else if err, ok := err.(*CorruptReferenceError); ok { + switch err.Code { + case FileError: + status = StatusFileError + case FileMissing: + status = StatusFileNotFound + case FileChanged: + status = StatusFileChanged + default: + status = StatusOtherError + } } else { status = StatusOtherError } From 064bb538843765392a7f4f1b7cb5a0bc8f4a5144 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 3 Feb 2017 20:46:35 -0500 Subject: [PATCH 04/11] filestore util: add documentation for 'filestore ls' and 'verify' commands. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/filestore.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 4b6daa709e2..9bb2994f21e 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -24,6 +24,16 @@ var FileStoreCmd = &cmds.Command{ var lsFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects in filestore.", + LongDescription: ` +List objects in the filestore. + +If one or more is specified only list those specific objects, +otherwise list all objects. + +The output is: + + +`, }, Arguments: []cmds.Argument{ cmds.StringArg("obj", false, true, "Cid of objects to list."), @@ -80,6 +90,26 @@ var lsFileStore = &cmds.Command{ var verifyFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Verify objects in filestore.", + LongDescription: ` +Verify objects in the filestore. + +If one or more is specified only verify those specific objects, +otherwise verify all objects. + +The output is: + + + +Where is one of: +ok: the block can be reconstructed +changed: the contents of the backing file have changed +no-file: the backing file could not be found +error: there was some other problem reading the file +missing: could not be found in the filestore +ERROR: internal error, most likely due to a corrupt database + +For ERROR entries the error will also be printed to stderr. +`, }, Arguments: []cmds.Argument{ cmds.StringArg("obj", false, true, "Cid of objects to verify."), From 033c442d32c48f3e36eb3e4cb740e43ca67cdd9f Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 6 Feb 2017 15:50:35 -0500 Subject: [PATCH 05/11] filestore util: add tests for verifying and listing filestore blocks. License: MIT Signed-off-by: Kevin Atkinson --- test/sharness/t0271-filestore-verify.sh | 153 ++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100755 test/sharness/t0271-filestore-verify.sh diff --git a/test/sharness/t0271-filestore-verify.sh b/test/sharness/t0271-filestore-verify.sh new file mode 100755 index 00000000000..5d38115e693 --- /dev/null +++ b/test/sharness/t0271-filestore-verify.sh @@ -0,0 +1,153 @@ +#!/bin/sh +# +# Copyright (c) 2017 Jeromy Johnson +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test out the filestore nocopy functionality" + +. lib/test-lib.sh + + +test_init_dataset() { + test_expect_success "create a dataset" ' + rm -r somedir + mkdir somedir && + random 1000 1 > somedir/file1 && + random 10000 2 > somedir/file2 && + random 1000000 3 > somedir/file3 + ' +} + +EXPHASH="QmRueCuPMYYvdxWz1vWncF7wzCScEx4qasZXo5aVBb1R4V" + +cat < ls_expect +zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo 10000 somedir/file2 0 +zb2rhav4wcdvNXtaKDTWHYAqtUHMEpygT1cxqMsfK7QrDuHxH 262144 somedir/file3 524288 +zb2rhbcZ3aUXYcrbhhDH1JyrpDcpdw1KFJ5Xs5covjnvMpxDR 1000 somedir/file1 0 +zb2rhe28UqCDm7TFib7PRyQYEkvuq8iahcXA2AbgaxCLvNhfk 262144 somedir/file3 0 +zb2rhebtyTTuHKyTbJPnkDUSruU5Uma4DN8t2EkvYZ6fP36mm 262144 somedir/file3 262144 +zb2rhm9VTrX2mfatggYUk8mHLz78XBxVUTTzLvM2N3d6frdAU 213568 somedir/file3 786432 +EOF + +FILE1_HASH=zb2rhbcZ3aUXYcrbhhDH1JyrpDcpdw1KFJ5Xs5covjnvMpxDR +FILE2_HASH=zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo +FILE3_HASH=QmfE4SDQazxTD7u8VTYs9AJqQL8rrJPUAorLeJXKSZrVf9 + +cat < verify_expect +ok zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo 10000 somedir/file2 0 +ok zb2rhav4wcdvNXtaKDTWHYAqtUHMEpygT1cxqMsfK7QrDuHxH 262144 somedir/file3 524288 +ok zb2rhbcZ3aUXYcrbhhDH1JyrpDcpdw1KFJ5Xs5covjnvMpxDR 1000 somedir/file1 0 +ok zb2rhe28UqCDm7TFib7PRyQYEkvuq8iahcXA2AbgaxCLvNhfk 262144 somedir/file3 0 +ok zb2rhebtyTTuHKyTbJPnkDUSruU5Uma4DN8t2EkvYZ6fP36mm 262144 somedir/file3 262144 +ok zb2rhm9VTrX2mfatggYUk8mHLz78XBxVUTTzLvM2N3d6frdAU 213568 somedir/file3 786432 +EOF + +test_filestore_adds() { + test_expect_success "nocopy add succeeds" ' + HASH=$(ipfs add --raw-leaves --nocopy -r -q somedir | tail -n1) + ' + + test_expect_success "nocopy add has right hash" ' + test "$HASH" = "$EXPHASH" + ' + + test_expect_success "'ipfs filestore ls' output looks good'" ' + ipfs filestore ls | sort > ls_actual && + test_cmp ls_expect ls_actual + ' + + test_expect_success "'ipfs filestore ls HASH' works" ' + ipfs filestore ls $FILE1_HASH > ls_actual && + grep -q somedir/file1 ls_actual + ' + + test_expect_success "can retrieve multi-block file" ' + ipfs cat $FILE3_HASH > file3.data && + test_cmp somedir/file3 file3.data + ' +} + +test_filestore_verify() { + test_expect_success "ipfs filestore verify' output looks good'" ' + ipfs filestore verify | LC_ALL=C sort > verify_actual + test_cmp verify_expect verify_actual + ' + + test_expect_success "'ipfs filestore verify HASH' works" ' + ipfs filestore verify $FILE1_HASH > verify_actual && + grep -q somedir/file1 verify_actual + ' + + test_expect_success "rename a file" ' + mv somedir/file1 somedir/file1.bk + ' + + test_expect_success "can not retrieve block after backing file moved" ' + test_must_fail ipfs cat $FILE1_HASH + ' + + test_expect_success "'ipfs filestore verify' shows file as missing" ' + ipfs filestore verify > verify_actual && + grep no-file verify_actual | grep -q somedir/file1 + ' + + test_expect_success "move file back" ' + mv somedir/file1.bk somedir/file1 + ' + + test_expect_success "block okay now" ' + ipfs cat $FILE1_HASH > file1.data && + test_cmp somedir/file1 file1.data + ' + + test_expect_success "change first bit of file" ' + dd if=/dev/zero of=somedir/file3 bs=1024 count=1 + ' + + test_expect_success "can not retrieve block after backing file changed" ' + test_must_fail ipfs cat $FILE3_HASH + ' + + test_expect_success "'ipfs filestore verify' shows file as changed" ' + ipfs filestore verify > verify_actual && + grep changed verify_actual | grep -q somedir/file3 + ' +} + +init_ipfs_filestore() { + test_expect_success "clean up old node" ' + rm -rf "$IPFS_PATH" mountdir ipfs ipns + ' + + test_init_ipfs + + test_expect_success "enable filestore config setting" ' + ipfs config --json Experimental.FilestoreEnabled true + ' +} + +test_init_dataset + +init_ipfs_filestore + +test_filestore_adds + +test_filestore_verify + +echo "WORKING DIR" +echo "IPFS PATH = " $IPFS_PATH +pwd + + +test_init_dataset + +init_ipfs_filestore + +test_launch_ipfs_daemon + +test_filestore_adds + +test_kill_ipfs_daemon + +test_done From 1f2e174e6a989e57b6fe5f58c5487048f95249e3 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 6 Feb 2017 15:54:36 -0500 Subject: [PATCH 06/11] filestore: use the same codes in ListRes and CorruptReferenceError. License: MIT Signed-off-by: Kevin Atkinson --- filestore/fsrefstore.go | 23 +++++++---------------- filestore/util.go | 11 +---------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 5af21d4196a..f493139e5c5 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -27,17 +27,8 @@ type FileManager struct { root string } -type CorruptReferenceCode int - -const ( - OtherErr CorruptReferenceCode = 0 - FileError CorruptReferenceCode = 1 - FileMissing CorruptReferenceCode = 2 - FileChanged CorruptReferenceCode = 3 -) - type CorruptReferenceError struct { - Code CorruptReferenceCode + Code Status Err error } @@ -138,23 +129,23 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { fi, err := os.Open(abspath) if os.IsNotExist(err) { - return nil, &CorruptReferenceError{FileMissing, err} + return nil, &CorruptReferenceError{StatusFileNotFound, err} } else if err != nil { - return nil, &CorruptReferenceError{FileError, err} + return nil, &CorruptReferenceError{StatusFileError, err} } defer fi.Close() _, err = fi.Seek(int64(d.GetOffset()), os.SEEK_SET) if err != nil { - return nil, &CorruptReferenceError{FileError, err} + return nil, &CorruptReferenceError{StatusFileError, err} } outbuf := make([]byte, d.GetSize_()) _, err = io.ReadFull(fi, outbuf) if err == io.EOF || err == io.ErrUnexpectedEOF { - return nil, &CorruptReferenceError{FileChanged, err} + return nil, &CorruptReferenceError{StatusFileChanged, err} } else if err != nil { - return nil, &CorruptReferenceError{FileError, err} + return nil, &CorruptReferenceError{StatusFileError, err} } outcid, err := c.Prefix().Sum(outbuf) @@ -163,7 +154,7 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { } if !c.Equals(outcid) { - return nil, &CorruptReferenceError{FileChanged, + return nil, &CorruptReferenceError{StatusFileChanged, fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} } diff --git a/filestore/util.go b/filestore/util.go index 8049f7ec145..652dfa22915 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -144,16 +144,7 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes { if err == ds.ErrNotFound || err == blockstore.ErrNotFound { status = StatusKeyNotFound } else if err, ok := err.(*CorruptReferenceError); ok { - switch err.Code { - case FileError: - status = StatusFileError - case FileMissing: - status = StatusFileNotFound - case FileChanged: - status = StatusFileChanged - default: - status = StatusOtherError - } + status = err.Code } else { status = StatusOtherError } From f7efb34e68c60a844a21221de8b741f83954cd4a Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 6 Feb 2017 21:53:51 -0500 Subject: [PATCH 07/11] filestore util: Add 'filestore dups' command. Enhance tests. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/filestore.go | 40 +++++++++++++++++++ filestore/filestore.go | 8 ++++ ...ore-verify.sh => t0271-filestore-utils.sh} | 22 +++++++++- 3 files changed, 69 insertions(+), 1 deletion(-) rename test/sharness/{t0271-filestore-verify.sh => t0271-filestore-utils.sh} (90%) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 9bb2994f21e..529ec4f0c1d 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -18,6 +18,7 @@ var FileStoreCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "ls": lsFileStore, "verify": verifyFileStore, + "dups": dupsFileStore, }, } @@ -157,6 +158,44 @@ For ERROR entries the error will also be printed to stderr. Type: filestore.ListRes{}, } +var dupsFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Print block both in filestore and non-filestore.", + }, + Run: func(req cmds.Request, res cmds.Response) { + _, fs, err := getFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + ch, err := fs.FileManager().AllKeysChan(req.Context()) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + out := make(chan interface{}, 128) + res.SetOutput((<-chan interface{})(out)) + + go func() { + defer close(out) + for cid := range ch { + have, err := fs.MainBlockstore().Has(cid) + if err != nil { + out <- &RefWrapper{Err: err.Error()} + return + } + if have { + out <- &RefWrapper{Ref: cid.String()} + } + } + }() + }, + Marshalers: refsMarshallerMap, + Type: RefWrapper{}, +} + + func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) { n, err := req.InvocContext().GetNode() if err != nil { @@ -211,3 +250,4 @@ func perKeyActionToChan(args []string, action func(*cid.Cid) *filestore.ListRes, }() return out } + diff --git a/filestore/filestore.go b/filestore/filestore.go index eefd925e358..da3dc17c6dc 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -19,6 +19,14 @@ type Filestore struct { bs blockstore.Blockstore } +func (f *Filestore) FileManager() *FileManager { + return f.fm +} + +func (f *Filestore) MainBlockstore() blockstore.Blockstore { + return f.bs +} + func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore { return &Filestore{fm, bs} } diff --git a/test/sharness/t0271-filestore-verify.sh b/test/sharness/t0271-filestore-utils.sh similarity index 90% rename from test/sharness/t0271-filestore-verify.sh rename to test/sharness/t0271-filestore-utils.sh index 5d38115e693..38a7aac1dab 100755 --- a/test/sharness/t0271-filestore-verify.sh +++ b/test/sharness/t0271-filestore-utils.sh @@ -115,6 +115,18 @@ test_filestore_verify() { ' } +cat < dups_expect +$FILE1_HASH +EOF + +test_filestore_dups() { + test_expect_success "'ipfs filestore dups'" ' + ipfs add --raw-leaves somedir/file1 && + ipfs filestore dups > dups_actual && + test_cmp dups_expect dups_actual +' +} + init_ipfs_filestore() { test_expect_success "clean up old node" ' rm -rf "$IPFS_PATH" mountdir ipfs ipns @@ -135,6 +147,8 @@ test_filestore_adds test_filestore_verify +test_filestore_dups + echo "WORKING DIR" echo "IPFS PATH = " $IPFS_PATH pwd @@ -144,10 +158,16 @@ test_init_dataset init_ipfs_filestore -test_launch_ipfs_daemon +# must be in offline mode so tests of retrieving non-exist blocks +# don't hang +test_launch_ipfs_daemon --offline test_filestore_adds +test_filestore_verify + +test_filestore_dups + test_kill_ipfs_daemon test_done From 81af0347e9ac61a190d730cb9b6cd54fceb6d9a3 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Thu, 9 Feb 2017 11:59:16 -0500 Subject: [PATCH 08/11] filestore util: change "???..." to "" License: MIT Signed-off-by: Kevin Atkinson --- filestore/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filestore/util.go b/filestore/util.go index 652dfa22915..ef77b417ab5 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -59,7 +59,7 @@ type ListRes struct { func (r *ListRes) FormatLong() string { switch { case r.Key == nil: - return "?????????????????????????????????????????????????" + return "" case r.FilePath == "": return r.Key.String() default: From 96269e025662f8c6ee8bd8704fc113e5c870f548 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Thu, 9 Feb 2017 12:16:36 -0500 Subject: [PATCH 09/11] filestore: Refactor. License: MIT Signed-off-by: Kevin Atkinson --- filestore/fsrefstore.go | 4 ++++ filestore/util.go | 12 +++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index f493139e5c5..3e3750cb9df 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -109,6 +109,10 @@ func (f *FileManager) getDataObj(c *cid.Cid) (*pb.DataObj, error) { // } + return unmarshalDataObj(o) +} + +func unmarshalDataObj(o interface{}) (*pb.DataObj, error) { data, ok := o.([]byte) if !ok { return nil, fmt.Errorf("stored filestore dataobj was not a []byte") diff --git a/filestore/util.go b/filestore/util.go index ef77b417ab5..f098d2e177c 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -9,7 +9,6 @@ import ( ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore" dsq "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore/query" - proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto" cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid" ) @@ -124,17 +123,12 @@ func next(qr dsq.Results) (*cid.Cid, *pb.DataObj, error) { return nil, nil, fmt.Errorf("decoding cid from filestore: %s", err) } - data, ok := v.Value.([]byte) - if !ok { - return c, nil, fmt.Errorf("stored filestore dataobj was not a []byte") - } - - var dobj pb.DataObj - if err := proto.Unmarshal(data, &dobj); err != nil { + dobj, err := unmarshalDataObj(v.Value) + if err != nil { return c, nil, err } - return c, &dobj, nil + return c, dobj, nil } func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes { From 620b52b2aa3e26c1d9d4336fed06098c6c65ce78 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 13 Feb 2017 15:56:35 -0500 Subject: [PATCH 10/11] filestore util: refactor and clean up tests License: MIT Signed-off-by: Kevin Atkinson --- test/sharness/t0271-filestore-utils.sh | 67 +++++++++++++++----------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/test/sharness/t0271-filestore-utils.sh b/test/sharness/t0271-filestore-utils.sh index 38a7aac1dab..371bbbd7b4c 100755 --- a/test/sharness/t0271-filestore-utils.sh +++ b/test/sharness/t0271-filestore-utils.sh @@ -8,6 +8,17 @@ test_description="Test out the filestore nocopy functionality" . lib/test-lib.sh +test_init_filestore() { + test_expect_success "clean up old node" ' + rm -rf "$IPFS_PATH" mountdir ipfs ipns + ' + + test_init_ipfs + + test_expect_success "enable filestore config setting" ' + ipfs config --json Experimental.FilestoreEnabled true + ' +} test_init_dataset() { test_expect_success "create a dataset" ' @@ -19,6 +30,11 @@ test_init_dataset() { ' } +test_init() { + test_init_filestore + test_init_dataset +} + EXPHASH="QmRueCuPMYYvdxWz1vWncF7wzCScEx4qasZXo5aVBb1R4V" cat < ls_expect @@ -68,11 +84,16 @@ test_filestore_adds() { ' } -test_filestore_verify() { +# check that the filestore is in a clean state +test_filestore_state() { test_expect_success "ipfs filestore verify' output looks good'" ' ipfs filestore verify | LC_ALL=C sort > verify_actual test_cmp verify_expect verify_actual ' +} + +test_filestore_verify() { + test_filestore_state test_expect_success "'ipfs filestore verify HASH' works" ' ipfs filestore verify $FILE1_HASH > verify_actual && @@ -113,35 +134,28 @@ test_filestore_verify() { ipfs filestore verify > verify_actual && grep changed verify_actual | grep -q somedir/file3 ' -} -cat < dups_expect -$FILE1_HASH -EOF + # reset the state for the next test + test_init_dataset +} test_filestore_dups() { + # make sure the filestore is in a clean state + test_filestore_state + test_expect_success "'ipfs filestore dups'" ' ipfs add --raw-leaves somedir/file1 && ipfs filestore dups > dups_actual && + echo "$FILE1_HASH" > dups_expect test_cmp dups_expect dups_actual -' -} - -init_ipfs_filestore() { - test_expect_success "clean up old node" ' - rm -rf "$IPFS_PATH" mountdir ipfs ipns - ' - - test_init_ipfs - - test_expect_success "enable filestore config setting" ' - ipfs config --json Experimental.FilestoreEnabled true ' } -test_init_dataset +# +# No daemon +# -init_ipfs_filestore +test_init test_filestore_adds @@ -149,17 +163,14 @@ test_filestore_verify test_filestore_dups -echo "WORKING DIR" -echo "IPFS PATH = " $IPFS_PATH -pwd - - -test_init_dataset +# +# With daemon +# -init_ipfs_filestore +test_init -# must be in offline mode so tests of retrieving non-exist blocks -# don't hang +# must be in offline mode so tests that retrieve non-existent blocks +# doesn't hang test_launch_ipfs_daemon --offline test_filestore_adds From 1eddb60f73a072e23800e5eafae57f47e1190068 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Wed, 22 Mar 2017 20:56:35 -0400 Subject: [PATCH 11/11] filestore util: doc improvement License: MIT Signed-off-by: Kevin Atkinson --- core/commands/filestore.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 529ec4f0c1d..9ce103b5075 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -7,8 +7,8 @@ import ( cmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/filestore" - u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util" cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid" + u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util" ) var FileStoreCmd = &cmds.Command{ @@ -160,7 +160,7 @@ For ERROR entries the error will also be printed to stderr. var dupsFileStore = &cmds.Command{ Helptext: cmds.HelpText{ - Tagline: "Print block both in filestore and non-filestore.", + Tagline: "List blocks that are both in the filestore and standard block storage.", }, Run: func(req cmds.Request, res cmds.Response) { _, fs, err := getFilestore(req) @@ -195,7 +195,6 @@ var dupsFileStore = &cmds.Command{ Type: RefWrapper{}, } - func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) { n, err := req.InvocContext().GetNode() if err != nil { @@ -250,4 +249,3 @@ func perKeyActionToChan(args []string, action func(*cid.Cid) *filestore.ListRes, }() return out } -