From 485a5e59e9ad8b48d362fa36f52d99739065ed04 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 13 Feb 2018 17:15:51 -0500 Subject: [PATCH 1/2] Create a raw node instead of a file node when there is no content. This fixes things so when raw-leaves are enabled a zero size file creates a zero size raw leaf. When raw-leaves are not enabled the hash created changes from QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH to Qmdsf68UUYTSSx3i4GtDJfxzpAEZt7Mp23m3qa36LYMSiW, since the type field changed from TFile to TRaw. License: MIT Signed-off-by: Kevin Atkinson --- core/coreapi/unixfs_test.go | 2 +- importer/balanced/builder.go | 7 +++- importer/helpers/dagbuilder.go | 53 +++++++++++++----------- test/sharness/t0040-add-and-cat.sh | 36 ++++++++++++++++ test/sharness/t0111-gateway-writeable.sh | 2 +- 5 files changed, 73 insertions(+), 27 deletions(-) diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 3ccaa1e4c5b..912013f60ea 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -36,7 +36,7 @@ var helloStr = "hello, world!" var emptyDir = coreapi.ResolvedPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn", nil, nil) // `echo -n | ipfs add` -var emptyFile = coreapi.ResolvedPath("/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH", nil, nil) +var emptyFile = coreapi.ResolvedPath("/ipfs/Qmdsf68UUYTSSx3i4GtDJfxzpAEZt7Mp23m3qa36LYMSiW", nil, nil) func makeAPIIdent(ctx context.Context, fullIdentity bool) (*core.IpfsNode, coreiface.CoreAPI, error) { var ident config.Identity diff --git a/importer/balanced/builder.go b/importer/balanced/builder.go index c15c56c62de..4ea84df8fbe 100644 --- a/importer/balanced/builder.go +++ b/importer/balanced/builder.go @@ -51,7 +51,12 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { } if root == nil { - root = db.NewUnixfsNode() + // this should only happen with an empty node, so return a leaf + var err error + root, err = db.NewLeaf(nil) + if err != nil { + return nil, err + } } out, err := db.Add(root) diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index d0532aa227c..90b160d9f1c 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -122,6 +122,34 @@ func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { return n } +// NewLeaf creates a leaf node filled with data +func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) { + if len(data) > BlockSizeLimit { + return nil, ErrSizeLimitExceeded + } + + if db.rawLeaves { + if db.prefix == nil { + return &UnixfsNode{ + rawnode: dag.NewRawNode(data), + raw: true, + }, nil + } + rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix) + if err != nil { + return nil, err + } + return &UnixfsNode{ + rawnode: rawnode, + raw: true, + }, nil + } + + blk := db.newUnixfsBlock() + blk.SetData(data) + return blk, nil +} + // newUnixfsBlock creates a new Unixfs node to represent a raw data block func (db *DagBuilderHelper) newUnixfsBlock() *UnixfsNode { n := &UnixfsNode{ @@ -164,30 +192,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { return nil, nil } - if len(data) > BlockSizeLimit { - return nil, ErrSizeLimitExceeded - } - - if db.rawLeaves { - if db.prefix == nil { - return &UnixfsNode{ - rawnode: dag.NewRawNode(data), - raw: true, - }, nil - } - rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix) - if err != nil { - return nil, err - } - return &UnixfsNode{ - rawnode: rawnode, - raw: true, - }, nil - } - - blk := db.newUnixfsBlock() - blk.SetData(data) - return blk, nil + return db.NewLeaf(data) } // SetPosInfo sets the offset information of a node using the fullpath and stat diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index b9c633b2973..aa21ed9cf70 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -166,6 +166,24 @@ test_add_cat_file() { echo "added $HASH .hello.txt" >expected && test_cmp expected actual ' + + test_expect_success "add zero length file" ' + touch zero-length-file && + ZEROHASH=$(ipfs add -q zero-length-file) && + echo $ZEROHASH + ' + + test_expect_success "zero length file has correct hash" ' + test "$ZEROHASH" = Qmdsf68UUYTSSx3i4GtDJfxzpAEZt7Mp23m3qa36LYMSiW + ' + + test_expect_success "cat zero length file" ' + ipfs cat $ZEROHASH > zero-length-file_out + ' + + test_expect_success "make sure it looks good" ' + test_cmp zero-length-file zero-length-file_out + ' } test_add_cat_5MB() { @@ -221,6 +239,24 @@ test_add_cat_raw() { test_expect_success "make sure it looks good" ' test_cmp afile afile_out ' + + test_expect_success "add zero length file with raw-leaves" ' + touch zero-length-file && + ZEROHASH=$(ipfs add -q --raw-leaves zero-length-file) && + echo $ZEROHASH + ' + + test_expect_success "zero length file has correct hash" ' + test "$ZEROHASH" = zb2rhmy65F3REf8SZp7De11gxtECBGgUKaLdiDj7MCGCHxbDW + ' + + test_expect_success "cat zero length file" ' + ipfs cat $ZEROHASH > zero-length-file_out + ' + + test_expect_success "make sure it looks good" ' + test_cmp zero-length-file zero-length-file_out + ' } test_add_cat_expensive() { diff --git a/test/sharness/t0111-gateway-writeable.sh b/test/sharness/t0111-gateway-writeable.sh index 71775d5d1e4..bb7064a4abe 100755 --- a/test/sharness/t0111-gateway-writeable.sh +++ b/test/sharness/t0111-gateway-writeable.sh @@ -14,7 +14,7 @@ test_launch_ipfs_daemon --writable test_expect_success "ipfs daemon --writable overrides config" ' curl -v -X POST http://$GWAY_ADDR/ipfs/ 2> outfile && grep "HTTP/1.1 201 Created" outfile && - grep "Location: /ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" outfile + grep "Location: /ipfs/Qmdsf68UUYTSSx3i4GtDJfxzpAEZt7Mp23m3qa36LYMSiW" outfile ' test_kill_ipfs_daemon From 6ab1e714731c833a4c18efdf3ff550ad2e156210 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Thu, 31 May 2018 04:05:17 -0400 Subject: [PATCH 2/2] Avoid changing hash for empty non-raw leaves. License: MIT Signed-off-by: Kevin Atkinson --- core/coreapi/unixfs_test.go | 2 +- importer/helpers/dagbuilder.go | 9 ++++++++- test/sharness/t0040-add-and-cat.sh | 2 +- test/sharness/t0111-gateway-writeable.sh | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 912013f60ea..3ccaa1e4c5b 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -36,7 +36,7 @@ var helloStr = "hello, world!" var emptyDir = coreapi.ResolvedPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn", nil, nil) // `echo -n | ipfs add` -var emptyFile = coreapi.ResolvedPath("/ipfs/Qmdsf68UUYTSSx3i4GtDJfxzpAEZt7Mp23m3qa36LYMSiW", nil, nil) +var emptyFile = coreapi.ResolvedPath("/ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH", nil, nil) func makeAPIIdent(ctx context.Context, fullIdentity bool) (*core.IpfsNode, coreiface.CoreAPI, error) { var ident config.Identity diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 90b160d9f1c..2b481007c62 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -122,7 +122,10 @@ func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { return n } -// NewLeaf creates a leaf node filled with data +// NewLeaf creates a leaf node filled with data. If rawLeaves is +// defined than a raw leaf will be returned. Otherwise, if data is +// nil the type field will be TRaw (for backwards compatibility), if +// data is defined (but possibly empty) the type field will be TRaw. func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) { if len(data) > BlockSizeLimit { return nil, ErrSizeLimitExceeded @@ -145,6 +148,10 @@ func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) { }, nil } + if data == nil { + return db.NewUnixfsNode(), nil + } + blk := db.newUnixfsBlock() blk.SetData(data) return blk, nil diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index aa21ed9cf70..82c194e9e03 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -174,7 +174,7 @@ test_add_cat_file() { ' test_expect_success "zero length file has correct hash" ' - test "$ZEROHASH" = Qmdsf68UUYTSSx3i4GtDJfxzpAEZt7Mp23m3qa36LYMSiW + test "$ZEROHASH" = QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH ' test_expect_success "cat zero length file" ' diff --git a/test/sharness/t0111-gateway-writeable.sh b/test/sharness/t0111-gateway-writeable.sh index bb7064a4abe..71775d5d1e4 100755 --- a/test/sharness/t0111-gateway-writeable.sh +++ b/test/sharness/t0111-gateway-writeable.sh @@ -14,7 +14,7 @@ test_launch_ipfs_daemon --writable test_expect_success "ipfs daemon --writable overrides config" ' curl -v -X POST http://$GWAY_ADDR/ipfs/ 2> outfile && grep "HTTP/1.1 201 Created" outfile && - grep "Location: /ipfs/Qmdsf68UUYTSSx3i4GtDJfxzpAEZt7Mp23m3qa36LYMSiW" outfile + grep "Location: /ipfs/QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" outfile ' test_kill_ipfs_daemon