From d3422a2e649506cd15f593b7454478e7e3e64e51 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Wed, 25 Jul 2018 01:38:49 -0400 Subject: [PATCH] Add support for inlining via the id-hash to the add command. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/add.go | 10 ++++++++++ core/commands/inliner.go | 26 ++++++++++++++++++++++++++ test/sharness/t0046-id-hash.sh | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 core/commands/inliner.go diff --git a/core/commands/add.go b/core/commands/add.go index 42b0b3548cf..649a3f93574 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -44,6 +44,8 @@ const ( fstoreCacheOptionName = "fscache" cidVersionOptionName = "cid-version" hashOptionName = "hash" + inlineOptionName = "inline" + idHashLimitOptionName = "id-hash-limit" ) const adderOutChanSize = 8 @@ -120,6 +122,8 @@ You can now check what blocks have been created by: cmdkit.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), cmdkit.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)"), cmdkit.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"), + cmdkit.BoolOption(inlineOptionName, "Inline small objects using identity hash. (experimental)"), + cmdkit.IntOption(idHashLimitOptionName, "Identity hash maxium size. (experimental)").WithDefault(64), }, PreRun: func(req *cmds.Request, env cmds.Environment) error { quiet, _ := req.Options[quietOptionName].(bool) @@ -173,6 +177,8 @@ You can now check what blocks have been created by: fscache, _ := req.Options[fstoreCacheOptionName].(bool) cidVer, cidVerSet := req.Options[cidVersionOptionName].(int) hashFunStr, _ := req.Options[hashOptionName].(string) + inline, _ := req.Options[inlineOptionName].(bool) + idHashLimit, _ := req.Options[idHashLimitOptionName].(int) // The arguments are subject to the following constraints. // @@ -281,6 +287,10 @@ You can now check what blocks have been created by: fileAdder.NoCopy = nocopy fileAdder.Prefix = &prefix + if inline { + fileAdder.Prefix = Inliner{fileAdder.Prefix, idHashLimit} + } + if hash { md := dagtest.Mock() emptyDirNode := ft.EmptyDirNode() diff --git a/core/commands/inliner.go b/core/commands/inliner.go new file mode 100644 index 00000000000..ba78832a18d --- /dev/null +++ b/core/commands/inliner.go @@ -0,0 +1,26 @@ +package commands + +import ( + cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" + mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" +) + +type Inliner struct { + base cid.Format + limit int +} + +func (p Inliner) GetCodec() uint64 { + return p.base.GetCodec() +} + +func (p Inliner) WithCodec(c uint64) cid.Format { + return Inliner{p.base.WithCodec(c), p.limit} +} + +func (p Inliner) Sum(data []byte) (*cid.Cid, error) { + if len(data) > p.limit { + return p.base.Sum(data) + } + return cid.FormatV1{Codec: p.base.GetCodec(), HashFun: mh.ID}.Sum(data) +} diff --git a/test/sharness/t0046-id-hash.sh b/test/sharness/t0046-id-hash.sh index b265200ca11..de9187b6c7c 100755 --- a/test/sharness/t0046-id-hash.sh +++ b/test/sharness/t0046-id-hash.sh @@ -43,6 +43,40 @@ test_expect_success "can still fetch it" ' test_cmp junk.txt actual ' +test_expect_success "ipfs add --inline works as expected" ' + echo $ID_HASH0_CONTENTS > afile && + HASH=$(ipfs add -q --inline afile) +' + +test_expect_success "ipfs add --inline outputs the correct hash" ' + echo zrjkGSFoQBjsuadykdx7acCCDAmcLX4HmLUFCs = "$HASH" && + test zrjkGSFoQBjsuadykdx7acCCDAmcLX4HmLUFCs = "$HASH" +' + +test_expect_success "ipfs add --inline --raw-leaves works as expected" ' + echo $ID_HASH0_CONTENTS > afile && + HASH=$(ipfs add -q --inline --raw-leaves afile) +' + +test_expect_success "ipfs add --inline --raw-leaves outputs the correct hash" ' + echo "$ID_HASH0" = "$HASH" && + test "$ID_HASH0" = "$HASH" +' + +test_expect_success "create 1000 bytes file and get its hash" ' + random 1000 2 > 1000bytes && + HASH0=$(ipfs add -q --raw-leaves --only-hash 1000bytes) +' + +test_expect_success "ipfs add --inline --raw-leaves works as expected on large file" ' + HASH=$(ipfs add -q --inline --raw-leaves 1000bytes) +' + +test_expect_success "ipfs add --inline --raw-leaves outputs the correct hash on large file" ' + echo "$HASH0" = "$HASH" && + test "$HASH0" = "$HASH" +' + test_expect_success "enable filestore" ' ipfs config --json Experimental.FilestoreEnabled true '