Add 'filestore dups' command. Enhance tests.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

core/commands/filestore.go

@@ -3,6 +3,7 @@ package commands
 import (
 	"context"
 	"fmt"
+	"io"
 
 	//ds "github.com/ipfs/go-datastore"
 	//bs "github.com/ipfs/go-ipfs/blocks/blockstore"
@@ -24,6 +25,7 @@ var FileStoreCmd = &cmds.Command{
 	Subcommands: map[string]*cmds.Command{
 		"ls":     lsFileStore,
 		"verify": verifyFileStore,
+		"dups":   dupsFileStore,
 	},
 }
 
@@ -163,6 +165,45 @@ For ERROR entries the error will also be printed to stderr.
 	Type: filestore.ListRes{},
 }
 
+var dupsFileStore = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Print block both in filestore and non-filestore.",
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		_, fs, err := getFilestore(req)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		ch, err := fs.FileManager().AllKeysChan(req.Context())
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		reader, writer := io.Pipe()
+		go func() {
+			defer writer.Close()
+			for cid := range ch {
+				have, err := fs.MainBlockstore().Has(cid)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+				if have {
+					fmt.Fprintf(writer, "%v\n", cid)
+				}
+			}
+		}()
+		res.SetOutput(reader)
+	},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			return res.(io.Reader), nil
+		},
+	},
+}
+
+
 func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) {
 	n, err := req.InvocContext().GetNode()
 	if err != nil {
@@ -217,3 +258,4 @@ func perKeyActionToChan(args []string, action func(*cid.Cid) *filestore.ListRes,
 	}()
 	return out
 }
+

filestore/filestore.go

@@ -19,6 +19,14 @@ type Filestore struct {
 	bs blockstore.Blockstore
 }
 
+func (f *Filestore) FileManager() *FileManager {
+	return f.fm
+}
+
+func (f *Filestore) MainBlockstore() blockstore.Blockstore {
+	return f.bs
+}
+
 func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore {
 	return &Filestore{fm, bs}
 }

test/sharness/t0271-filestore-verify.sh → test/sharness/t0271-filestore-utils.sh

@@ -114,6 +114,18 @@ test_filestore_verify() {
 '
 }
 
+cat <<EOF > dups_expect
+$FILE1_HASH
+EOF
+
+test_filestore_dups() {
+	test_expect_success "'ipfs filestore dups'" '
+		ipfs add --raw-leaves somedir/file1 &&
+		ipfs filestore dups > dups_actual &&
+		test_cmp dups_expect dups_actual
+'
+}
+
 init_ipfs_filestore() {
 	test_expect_success "clean up old node" '
 		rm -rf "$IPFS_PATH" mountdir ipfs ipns
@@ -134,6 +146,8 @@ test_filestore_adds
 
 test_filestore_verify
 
+test_filestore_dups
+
 echo "WORKING DIR"
 echo "IPFS PATH = " $IPFS_PATH
 pwd
@@ -143,10 +157,16 @@ test_init_dataset
 
 init_ipfs_filestore
 
-test_launch_ipfs_daemon
+# must be in offline mode so tests of retrieving non-exist blocks
+# don't hang
+test_launch_ipfs_daemon --offline
 
 test_filestore_adds
 
+test_filestore_verify
+
+test_filestore_dups
+
 test_kill_ipfs_daemon
 
 test_done