diff --git a/core/coreapi/interface/interface.go b/core/coreapi/interface/interface.go index d521e713d0b..ce45430d2cb 100644 --- a/core/coreapi/interface/interface.go +++ b/core/coreapi/interface/interface.go @@ -206,6 +206,14 @@ type ObjectAPI interface { // * "json" WithInputEnc(e string) options.ObjectPutOption + // WithDataType specifies the encoding of data field when using Josn or XML + // input encoding. + // + // Supported types: + // * "text" (default) + // * "base64" + WithDataType(t string) options.ObjectPutOption + // Get returns the node for the path Get(context.Context, Path) (Node, error) @@ -221,20 +229,20 @@ type ObjectAPI interface { // AddLink adds a link under the specified path. child path can point to a // subdirectory within the patent which must be present (can be overridden // with WithCreate option). - AddLink(ctx context.Context, base Path, name string, child Path, opts ...options.ObjectAddLinkOption) (Node, error) + AddLink(ctx context.Context, base Path, name string, child Path, opts ...options.ObjectAddLinkOption) (Path, error) // WithCreate is an option for AddLink which specifies whether create required // directories for the child WithCreate(create bool) options.ObjectAddLinkOption // RmLink removes a link from the node - RmLink(ctx context.Context, base Path, link string) (Node, error) + RmLink(ctx context.Context, base Path, link string) (Path, error) // AppendData appends data to the node - AppendData(context.Context, Path, io.Reader) (Node, error) + AppendData(context.Context, Path, io.Reader) (Path, error) // SetData sets the data contained in the node - SetData(context.Context, Path, io.Reader) (Node, error) + SetData(context.Context, Path, io.Reader) (Path, error) } // ObjectStat provides information about dag nodes @@ -254,7 +262,7 @@ type ObjectStat struct { // DataSize is the size of data block section DataSize int - // CumulativeSize is size of node + // CumulativeSize is size of the tree (BlockSize + link sizes) CumulativeSize int } diff --git a/core/coreapi/interface/options/object.go b/core/coreapi/interface/options/object.go index fe86a1cdede..9c8c9a9ddd1 100644 --- a/core/coreapi/interface/options/object.go +++ b/core/coreapi/interface/options/object.go @@ -6,6 +6,7 @@ type ObjectNewSettings struct { type ObjectPutSettings struct { InputEnc string + DataType string } type ObjectAddLinkSettings struct { @@ -33,6 +34,7 @@ func ObjectNewOptions(opts ...ObjectNewOption) (*ObjectNewSettings, error) { func ObjectPutOptions(opts ...ObjectPutOption) (*ObjectPutSettings, error) { options := &ObjectPutSettings{ InputEnc: "json", + DataType: "text", } for _, opt := range opts { @@ -74,6 +76,13 @@ func (api *ObjectOptions) WithInputEnc(e string) ObjectPutOption { } } +func (api *ObjectOptions) WithDataType(t string) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.DataType = t + return nil + } +} + func (api *ObjectOptions) WithCreate(create bool) ObjectAddLinkOption { return func(settings *ObjectAddLinkSettings) error { settings.Create = create diff --git a/core/coreapi/object.go b/core/coreapi/object.go index 915c5ed8c2c..772f90bf291 100644 --- a/core/coreapi/object.go +++ b/core/coreapi/object.go @@ -13,15 +13,32 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" + "encoding/base64" + "encoding/json" + "encoding/xml" + "errors" + "fmt" node "gx/ipfs/QmNwUEK7QbwSqyKBu3mMtToo8SUc6wQJ7gdZq4gGGJqfnf/go-ipld-format" cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" ) +const inputLimit = 2 << 20 + type ObjectAPI struct { *CoreAPI *caopts.ObjectOptions } +type Link struct { + Name, Hash string + Size uint64 +} + +type Node struct { + Links []Link + Data string +} + func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (coreiface.Node, error) { options, err := caopts.ObjectNewOptions(opts...) if err != nil { @@ -49,8 +66,66 @@ func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Obj return nil, err } - dagApi := api.Dag() - return dagApi.Put(ctx, src, dagApi.WithInputEnc(options.InputEnc), dagApi.WithCodec(cid.DagProtobuf)) + data, err := ioutil.ReadAll(io.LimitReader(src, inputLimit+10)) + if err != nil { + return nil, err + } + + var dagnode *dag.ProtoNode + switch options.InputEnc { + case "json": + node := new(Node) + err = json.Unmarshal(data, node) + if err != nil { + return nil, err + } + + // check that we have data in the Node to add + // otherwise we will add the empty object without raising an error + if nodeEmpty(node) { + return nil, errors.New("no data or links in this node") + } + + dagnode, err = deserializeNode(node, options.DataType) + if err != nil { + return nil, err + } + + case "protobuf": + dagnode, err = dag.DecodeProtobuf(data) + + case "xml": + node := new(Node) + err = xml.Unmarshal(data, node) + if err != nil { + return nil, err + } + + // check that we have data in the Node to add + // otherwise we will add the empty object without raising an error + if nodeEmpty(node) { + return nil, errors.New("no data or links in this node") + } + + dagnode, err = deserializeNode(node, options.DataType) + if err != nil { + return nil, err + } + + default: + return nil, errors.New("unknown object encoding") + } + + if err != nil { + return nil, err + } + + _, err = api.node.DAG.Add(dagnode) + if err != nil { + return nil, err + } + + return ParseCid(dagnode.Cid()), nil } func (api *ObjectAPI) Get(ctx context.Context, path coreiface.Path) (coreiface.Node, error) { @@ -109,7 +184,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, path coreiface.Path) (*coreiface return out, nil } -func (api *ObjectAPI) AddLink(ctx context.Context, base coreiface.Path, name string, child coreiface.Path, opts ...caopts.ObjectAddLinkOption) (coreiface.Node, error) { +func (api *ObjectAPI) AddLink(ctx context.Context, base coreiface.Path, name string, child coreiface.Path, opts ...caopts.ObjectAddLinkOption) (coreiface.Path, error) { options, err := caopts.ObjectAddLinkOptions(opts...) if err != nil { return nil, err @@ -147,10 +222,10 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base coreiface.Path, name str return nil, err } - return nnode, nil + return ParseCid(nnode.Cid()), nil } -func (api *ObjectAPI) RmLink(ctx context.Context, base coreiface.Path, link string) (coreiface.Node, error) { +func (api *ObjectAPI) RmLink(ctx context.Context, base coreiface.Path, link string) (coreiface.Path, error) { baseNd, err := api.core().ResolveNode(ctx, base) if err != nil { return nil, err @@ -173,18 +248,18 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base coreiface.Path, link stri return nil, err } - return nnode, nil + return ParseCid(nnode.Cid()), nil } -func (api *ObjectAPI) AppendData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Node, error) { +func (api *ObjectAPI) AppendData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Path, error) { return api.patchData(ctx, path, r, true) } -func (api *ObjectAPI) SetData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Node, error) { +func (api *ObjectAPI) SetData(ctx context.Context, path coreiface.Path, r io.Reader) (coreiface.Path, error) { return api.patchData(ctx, path, r, false) } -func (api *ObjectAPI) patchData(ctx context.Context, path coreiface.Path, r io.Reader, appendData bool) (coreiface.Node, error) { +func (api *ObjectAPI) patchData(ctx context.Context, path coreiface.Path, r io.Reader, appendData bool) (coreiface.Path, error) { nd, err := api.core().ResolveNode(ctx, path) if err != nil { return nil, err @@ -210,9 +285,41 @@ func (api *ObjectAPI) patchData(ctx context.Context, path coreiface.Path, r io.R return nil, err } - return pbnd, nil + return ParseCid(pbnd.Cid()), nil } func (api *ObjectAPI) core() coreiface.CoreAPI { return api.CoreAPI } + +func deserializeNode(nd *Node, dataFieldEncoding string) (*dag.ProtoNode, error) { + dagnode := new(dag.ProtoNode) + switch dataFieldEncoding { + case "text": + dagnode.SetData([]byte(nd.Data)) + case "base64": + data, _ := base64.StdEncoding.DecodeString(nd.Data) + dagnode.SetData(data) + default: + return nil, fmt.Errorf("Unkown data field encoding") + } + + dagnode.SetLinks(make([]*node.Link, len(nd.Links))) + for i, link := range nd.Links { + c, err := cid.Decode(link.Hash) + if err != nil { + return nil, err + } + dagnode.Links()[i] = &node.Link{ + Name: link.Name, + Size: link.Size, + Cid: c, + } + } + + return dagnode, nil +} + +func nodeEmpty(node *Node) bool { + return node.Data == "" && len(node.Links) == 0 +} diff --git a/core/coreapi/object_test.go b/core/coreapi/object_test.go new file mode 100644 index 00000000000..1ff90d32de6 --- /dev/null +++ b/core/coreapi/object_test.go @@ -0,0 +1,385 @@ +package coreapi_test + +import ( + "bytes" + "context" + "encoding/hex" + "io/ioutil" + "strings" + "testing" +) + +func TestNew(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + emptyNode, err := api.Object().New(ctx) + if err != nil { + t.Fatal(err) + } + + dirNode, err := api.Object().New(ctx, api.Object().WithType("unixfs-dir")) + if err != nil { + t.Fatal(err) + } + + if emptyNode.String() != "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" { + t.Errorf("Unexpected emptyNode path: %s", emptyNode.String()) + } + + if dirNode.String() != "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" { + t.Errorf("Unexpected dirNode path: %s", dirNode.String()) + } +} + +func TestObjectPut(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"YmFy"}`), api.Object().WithDataType("base64")) //bar + if err != nil { + t.Fatal(err) + } + + pbBytes, err := hex.DecodeString("0a0362617a") + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().Put(ctx, bytes.NewReader(pbBytes), api.Object().WithInputEnc("protobuf")) + if err != nil { + t.Fatal(err) + } + + if p1.String() != "/ipfs/QmQeGyS87nyijii7kFt1zbe4n2PsXTFimzsdxyE9qh9TST" { + t.Errorf("unexpected path: %s", p1.String()) + } + + if p2.String() != "/ipfs/QmNeYRbCibmaMMK6Du6ChfServcLqFvLJF76PzzF76SPrZ" { + t.Errorf("unexpected path: %s", p2.String()) + } + + if p3.String() != "/ipfs/QmZreR7M2t7bFXAdb1V5FtQhjk4t36GnrvueLJowJbQM9m" { + t.Errorf("unexpected path: %s", p3.String()) + } +} + +func TestObjectGet(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + nd, err := api.Object().Get(ctx, p1) + if err != nil { + t.Fatal(err) + } + + if string(nd.RawData()[len(nd.RawData())-3:]) != "foo" { + t.Fatal("got non-matching data") + } +} + +func TestObjectData(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p1) + if err != nil { + t.Fatal(err) + } + + data, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(data) != "foo" { + t.Fatal("got non-matching data") + } +} + +func TestObjectLinks(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`"}]}`)) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p2) + if err != nil { + t.Fatal(err) + } + + if len(links) != 1 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Cid.String() != p1.Cid().String() { + t.Fatal("cids didn't batch") + } + + if links[0].Name != "bar" { + t.Fatal("unexpected link name") + } +} + +func TestObjectStat(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + stat, err := api.Object().Stat(ctx, p2) + if err != nil { + t.Fatal(err) + } + + if stat.Cid.String() != p2.Cid().String() { + t.Error("unexpected stat.Cid") + } + + if stat.NumLinks != 1 { + t.Errorf("unexpected stat.NumLinks") + } + + if stat.BlockSize != 51 { + t.Error("unexpected stat.BlockSize") + } + + if stat.LinksSize != 47 { + t.Errorf("unexpected stat.LinksSize: %d", stat.LinksSize) + } + + if stat.DataSize != 4 { + t.Error("unexpected stat.DataSize") + } + + if stat.CumulativeSize != 54 { + t.Error("unexpected stat.DataSize") + } +} + +func TestObjectAddLink(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().AddLink(ctx, p2, "abc", p2) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 2 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Name != "abc" { + t.Errorf("unexpected link 0 name: %s", links[0].Name) + } + + if links[1].Name != "bar" { + t.Errorf("unexpected link 1 name: %s", links[1].Name) + } +} + +func TestObjectAddLinkCreate(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().AddLink(ctx, p2, "abc/d", p2) + if err == nil { + t.Fatal("expected an error") + } + if err.Error() != "no link by that name" { + t.Fatal("unexpected error: %s", err.Error()) + } + + p3, err = api.Object().AddLink(ctx, p2, "abc/d", p2, api.Object().WithCreate(true)) + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 2 { + t.Errorf("unexpected number of links: %d", len(links)) + } + + if links[0].Name != "abc" { + t.Errorf("unexpected link 0 name: %s", links[0].Name) + } + + if links[1].Name != "bar" { + t.Errorf("unexpected link 1 name: %s", links[1].Name) + } +} + +func TestObjectRmLink(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + if err != nil { + t.Fatal(err) + } + + p3, err := api.Object().RmLink(ctx, p2, "bar") + if err != nil { + t.Fatal(err) + } + + links, err := api.Object().Links(ctx, p3) + if err != nil { + t.Fatal(err) + } + + if len(links) != 0 { + t.Errorf("unexpected number of links: %d", len(links)) + } +} + +func TestObjectAddData(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().AppendData(ctx, p1, strings.NewReader("bar")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p2) + if err != nil { + t.Fatal(err) + } + + data, err := ioutil.ReadAll(r) + + if string(data) != "foobar" { + t.Error("unexpected data") + } +} + +func TestObjectSetData(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + p1, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"foo"}`)) + if err != nil { + t.Fatal(err) + } + + p2, err := api.Object().SetData(ctx, p1, strings.NewReader("bar")) + if err != nil { + t.Fatal(err) + } + + r, err := api.Object().Data(ctx, p2) + if err != nil { + t.Fatal(err) + } + + data, err := ioutil.ReadAll(r) + + if string(data) != "bar" { + t.Error("unexpected data") + } +}