From 89e1d9fd493522678a5347b45ed48040939454c8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 16 Jul 2018 12:29:29 -0700 Subject: [PATCH 1/4] switch urlstore to the new commands lib License: MIT Signed-off-by: Steven Allen --- core/commands/root.go | 2 +- core/commands/urlstore.go | 24 ++++++++++-------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/core/commands/root.go b/core/commands/root.go index 648a2a88610..d5e884d3901 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -136,7 +136,7 @@ var rootSubcommands = map[string]*cmds.Command{ "tar": lgc.NewCommand(TarCmd), "file": lgc.NewCommand(unixfs.UnixFSCmd), "update": lgc.NewCommand(ExternalBinary()), - "urlstore": lgc.NewCommand(urlStoreCmd), + "urlstore": urlStoreCmd, "version": lgc.NewCommand(VersionCmd), "shutdown": lgc.NewCommand(daemonShutdownCmd), } diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index b903f95ed8f..98606f7d58a 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -4,13 +4,12 @@ import ( "fmt" "io" "net/http" - "strings" - cmds "github.com/ipfs/go-ipfs/commands" filestore "github.com/ipfs/go-ipfs/filestore" balanced "github.com/ipfs/go-ipfs/importer/balanced" ihelper "github.com/ipfs/go-ipfs/importer/helpers" + cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" chunk "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" @@ -18,7 +17,6 @@ import ( ) var urlStoreCmd = &cmds.Command{ - Subcommands: map[string]*cmds.Command{ "add": urlAdd, }, @@ -49,9 +47,9 @@ time. }, Type: BlockStat{}, - Run: func(req cmds.Request, res cmds.Response) { - url := req.Arguments()[0] - n, err := req.InvocContext().GetNode() + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) { + url := req.Arguments[0] + n, err := GetNode(env) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -106,17 +104,15 @@ time. return } - res.SetOutput(BlockStat{ + cmds.EmitOnce(res, BlockStat{ Key: blc.Cid().String(), Size: int(hres.ContentLength), }) }, - Marshalers: cmds.MarshalerMap{ - cmds.Text: func(res cmds.Response) (io.Reader, error) { - ch := res.Output().(<-chan interface{}) - bs0 := <-ch - bs := bs0.(*BlockStat) - return strings.NewReader(bs.Key + "\n"), nil - }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, bs *BlockStat) error { + _, err := fmt.Fprintln(w, bs.Key) + return err + }), }, } From 19caad2301e5cccb53627b1ba550c1530dd085b1 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 16 Jul 2018 12:33:33 -0700 Subject: [PATCH 2/4] add trickle-dag support to the urlstore fixes #5241 License: MIT Signed-off-by: Steven Allen --- core/commands/urlstore.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index 98606f7d58a..5f415f0ab92 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -8,6 +8,7 @@ import ( filestore "github.com/ipfs/go-ipfs/filestore" balanced "github.com/ipfs/go-ipfs/importer/balanced" ihelper "github.com/ipfs/go-ipfs/importer/helpers" + trickle "github.com/ipfs/go-ipfs/importer/trickle" cmds "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" @@ -42,6 +43,9 @@ found. It may disappear or the semantics can change at any time. `, }, + Options: []cmdkit.Option{ + cmdkit.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), + }, Arguments: []cmdkit.Argument{ cmdkit.StringArg("url", true, false, "URL to add to IPFS"), }, @@ -71,6 +75,8 @@ time. return } + useTrickledag, _ := req.Options[trickleOptionName].(bool) + hreq, err := http.NewRequest("GET", url, nil) if err != nil { res.SetError(err, cmdkit.ErrNormal) @@ -98,14 +104,18 @@ time. URL: url, } - blc, err := balanced.Layout(dbp.New(chk)) + layout := balanced.Layout + if useTrickledag { + layout = trickle.Layout + } + root, err := layout(dbp.New(chk)) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } cmds.EmitOnce(res, BlockStat{ - Key: blc.Cid().String(), + Key: root.Cid().String(), Size: int(hres.ContentLength), }) }, From 1358b28204d2c8f077d78d61fdd32c9abf9294e4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 16 Jul 2018 15:03:10 -0700 Subject: [PATCH 3/4] add sharness for trickle importer License: MIT Signed-off-by: Steven Allen --- test/sharness/t0272-urlstore.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/sharness/t0272-urlstore.sh b/test/sharness/t0272-urlstore.sh index 312247f2f07..346c06da523 100755 --- a/test/sharness/t0272-urlstore.sh +++ b/test/sharness/t0272-urlstore.sh @@ -124,6 +124,12 @@ test_expect_success "get large file via urlstore" ' test_cmp file3 file3.actual ' +test_expect_success "check that the trickle option works" ' + HASHat=$(ipfs add -q --cid-version=1 --raw-leaves=true -n --trickle file3) && + HASHut=$(ipfs urlstore add --trickle http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a) && + test $HASHat = $HASHut +' + test_kill_ipfs_daemon test_expect_success "files can not be retrieved via the urlstore" ' From 4d4f343df776e3bb202472967ffe7f3b16643232 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 16 Jul 2018 15:03:25 -0700 Subject: [PATCH 4/4] check hashes in urlstore sharness tests License: MIT Signed-off-by: Steven Allen --- test/sharness/t0272-urlstore.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/sharness/t0272-urlstore.sh b/test/sharness/t0272-urlstore.sh index 346c06da523..bc33efbd730 100755 --- a/test/sharness/t0272-urlstore.sh +++ b/test/sharness/t0272-urlstore.sh @@ -138,4 +138,13 @@ test_expect_success "files can not be retrieved via the urlstore" ' test_must_fail ipfs cat $HASH3 > /dev/null ' +test_expect_success "check that the hashes were correct" ' + HASH1e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file1) && + HASH2e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file2) && + HASH3e=$(ipfs add -q -n --cid-version=1 --raw-leaves=true file3) && + test $HASH1e = $HASH1 && + test $HASH2e = $HASH2 && + test $HASH3e = $HASH3 +' + test_done