feat: dag API (OLD PR)

package.json

@@ -50,7 +50,7 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs#readme",
   "devDependencies": {
-    "aegir": "^9.2.1",
+    "aegir": "^9.3.0",
     "buffer-loader": "0.0.1",
     "chai": "^3.5.0",
     "detect-node": "^2.0.3",
@@ -65,7 +65,7 @@
     "lodash": "^4.17.2",
     "ncp": "^2.0.0",
     "nexpect": "^0.5.0",
-    "pre-commit": "^1.2.1",
+    "pre-commit": "^1.2.2",
     "rimraf": "^2.5.4",
     "stream-to-promise": "^2.2.0",
     "transform-loader": "^0.2.3"
@@ -74,25 +74,26 @@
     "async": "^2.1.4",
     "bl": "^1.1.2",
     "boom": "^4.2.0",
-    "debug": "^2.3.3",
+    "cids": "^0.3.5",
+    "debug": "^2.4.5",
     "fs-pull-blob-store": "^0.3.0",
     "glob": "^7.1.1",
-    "hapi": "^16.0.1",
+    "hapi": "^16.0.2",
     "hapi-set-header": "^1.0.2",
     "idb-pull-blob-store": "^0.5.1",
     "ipfs-api": "^12.1.0",
-    "ipfs-bitswap": "^0.8.2",
+    "ipfs-bitswap": "^0.8.3",
     "ipfs-block": "^0.5.3",
-    "ipfs-block-service": "^0.7.1",
+    "ipfs-block-service": "^0.7.2",
     "ipfs-multipart": "^0.1.0",
-    "ipfs-repo": "^0.11.1",
-    "ipfs-unixfs": "^0.1.8",
-    "ipfs-unixfs-engine": "^0.14.1",
-    "ipld-resolver": "^0.4.0",
+    "ipfs-repo": "^0.11.2",
+    "ipfs-unixfs": "^0.1.9",
+    "ipfs-unixfs-engine": "^0.14.2",
+    "ipld-resolver": "^0.4.1",
     "isstream": "^0.1.2",
-    "joi": "^10.0.5",
-    "libp2p-ipfs-nodejs": "^0.17.0",
-    "libp2p-ipfs-browser": "^0.17.2",
+    "joi": "^10.0.6",
+    "libp2p-ipfs-nodejs": "^0.17.1",
+    "libp2p-ipfs-browser": "^0.17.3",
     "lodash.flatmap": "^4.5.0",
     "lodash.get": "^4.4.2",
     "lodash.has": "^4.5.2",
@@ -101,10 +102,11 @@
     "lodash.values": "^4.3.0",
     "mafmt": "^2.1.2",
     "multiaddr": "^2.1.1",
-    "multihashes": "^0.3.0",
+    "multihashes": "^0.3.1",
+    "multihashing-async": "^0.3.0",
     "path-exists": "^3.0.0",
     "peer-book": "^0.3.0",
-    "peer-id": "^0.8.0",
+    "peer-id": "^0.8.1",
     "peer-info": "^0.8.1",
     "promisify-es6": "^1.0.2",
     "pull-file": "^1.0.0",

src/cli/commands/block/get.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const utils = require('../../utils')
-const mh = require('multihashes')
+const CID = require('cids')
 const debug = require('debug')
 const log = debug('cli:block')
 log.error = debug('cli:block:error')
@@ -19,17 +19,17 @@ module.exports = {
         throw err
       }
 
-      const hash = utils.isDaemonOn()
-        ? argv.key
-        : mh.fromB58String(argv.key)
+      const cid = new CID(argv.key)
 
-      ipfs.block.get(hash, (err, block) => {
+      ipfs.block.get(cid, (err, block) => {
         if (err) {
           throw err
         }
 
         if (block.data) {
-          console.log(block.data.toString())
+          // writing the buffer to stdout seems to be the only way
+          // to send out binary data correctly
+          process.stdout.write(block.data)
           return
         }
 

src/cli/commands/block/put.js

@@ -1,52 +1,93 @@
 'use strict'
 
 const utils = require('../../utils')
-const mh = require('multihashes')
 const bl = require('bl')
 const fs = require('fs')
 const Block = require('ipfs-block')
+const CID = require('cids')
+const multihashing = require('multihashing-async')
 const waterfall = require('async/waterfall')
 const debug = require('debug')
 const log = debug('cli:block')
 log.error = debug('cli:block:error')
 
-function addBlock (buf) {
+function addBlock (buf, opts) {
+  let block = new Block(buf)
+  let cid
+
   utils.getIPFS((err, ipfs) => {
     if (err) {
       throw err
     }
 
     waterfall([
-      (cb) => ipfs.block.put(new Block(buf), cb),
-      (block, cb) => block.key(cb)
-    ], (err, key) => {
+      (cb) => generateHash(block, opts, cb),
+      (mhash, cb) => generateCid(mhash, block, opts, cb),
+      (cb) => ipfs.block.put(block, cid, cb)
+    ], (err) => {
       if (err) {
         throw err
       }
 
-      console.log(mh.toB58String(key))
+      console.log(cid.toBaseEncodedString())
     })
   })
+
+  function generateHash (block, opts, cb) {
+    if (opts.mhlen === undefined) {
+      multihashing(buf, opts.mhtype, cb)
+    } else {
+      multihashing(buf, opts.mhtype, opts.mhlen, cb)
+    }
+  }
+
+  function generateCid (mhash, block, opts, cb) {
+    cid = new CID(opts.verison, opts.format, mhash)
+    cb()
+  }
 }
 
 module.exports = {
   command: 'put [data]',
 
   describe: 'Stores input as an IPFS block',
 
-  builder: {},
+  builder: {
+    format: {
+      alias: 'f',
+      describe: 'cid format for blocks to be created with.',
+      default: 'v0'
+    },
+    mhtype: {
+      describe: 'multihash hash function',
+      default: 'sha2-256'
+    },
+    mhlen: {
+      describe: 'multihash hash length',
+      default: undefined
+    }
+  },
 
   handler (argv) {
+    // parse options
+    if (argv.format === 'v0') {
+      argv.verison = 0
+      argv.format = 'dag-pb'
+      argv.mhtype = 'sha2-256'
+    } else {
+      argv.verison = 1
+    }
+
     if (argv.data) {
-      return addBlock(fs.readFileSync(argv.data))
+      return addBlock(fs.readFileSync(argv.data), argv)
     }
 
     process.stdin.pipe(bl((err, input) => {
       if (err) {
         throw err
       }
 
-      addBlock(input)
+      addBlock(input, argv)
     }))
   }
 }

src/cli/commands/block/stat.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const utils = require('../../utils')
+const CID = require('cids')
 const debug = require('debug')
 const log = debug('cli:block')
 log.error = debug('cli:block:error')
@@ -13,12 +14,14 @@ module.exports = {
   builder: {},
 
   handler (argv) {
+    const cid = new CID(argv.key)
+
     utils.getIPFS((err, ipfs) => {
       if (err) {
         throw err
       }
 
-      ipfs.block.stat(argv.key, (err, stats) => {
+      ipfs.block.stat(cid, (err, stats) => {
         if (err) {
           throw err
         }

src/core/components/dag.js

@@ -0,0 +1,39 @@
+'use strict'
+
+const promisify = require('promisify-es6')
+
+const dagPB = require('ipld-dag-pb')
+const dagCBOR = require('ipld-dag-cbor')
+
+module.exports = function dag (self) {
+  return {
+    put: promisify((dagNode, multicodec, hashAlg, callback) => {
+      switch (multicodec) {
+        case 'dag-pb': dagPB.util.cid(dagNode, gotCid); break
+        case 'dag-cbor': dagCBOR.util.cid(dagNode, gotCid); break
+        default: callback(new Error('IPLD Format not supported'))
+      }
+
+      function gotCid (err, cid) {
+        if (err) {
+          return callback(err)
+        }
+        self._ipldResolver.put({
+          node: dagNode,
+          cid: cid
+        }, callback)
+      }
+    }),
+    get: promisify((cid, callback) => {
+      self._ipldResolver.get(cid, callback)
+    }),
+    rm: promisify((cid, callback) => {
+      // TODO once pinning is complete, this remove operation has to first
+      // verify that some pinning chain is not broken with the operation
+      self._ipldResolver.remove(cid, callback)
+    }),
+    resolve: promisify((cid, path, callback) => {
+      self._ipldResolver.resolve(cid, path, callback)
+    })
+  }
+}

src/core/components/go-online.js

@@ -14,7 +14,6 @@ module.exports = function goOnline (self) {
       }
 
       self._bitswap = new Bitswap(
-        self._libp2pNode.peerInfo,
         self._libp2pNode,
         self._repo.blockstore,
         self._libp2pNode.peerBook

src/core/index.js

@@ -18,6 +18,7 @@ const bootstrap = require('./components/bootstrap')
 const config = require('./components/config')
 const block = require('./components/block')
 const object = require('./components/object')
+const dag = require('./components/dag')
 const libp2p = require('./components/libp2p')
 const swarm = require('./components/swarm')
 const ping = require('./components/ping')
@@ -62,6 +63,7 @@ function IPFS (repoInstance) {
   this.config = config(this)
   this.block = block(this)
   this.object = object(this)
+  this.dag = dag(this)
   this.libp2p = libp2p(this)
   this.swarm = swarm(this)
   this.files = files(this)

test/cli/test-block.js

@@ -16,9 +16,17 @@ describe('block', () => {
       })
     })
 
+    it('put with flags, format and mhtype', () => {
+      return ipfs('block put --format eth-block --mhtype keccak-256 test/test-data/eth-block').then((out) => {
+        expect(out).to.be.eql(
+          'z43AaGF23fmvRnDP56Ub9WcJCfzSfqtmzNCCvmz5eudT8dtdCDS'
+        )
+      })
+    })
+
     it('get', () => {
       return ipfs('block get QmZjTnYw2TFhn9Nn7tjmPSoTBoY7YRkwPzwSrSbabY24Kp').then((out) => {
-        expect(out).to.be.eql('hello world\n')
+        expect(out).to.be.eql('hello world')
       })
     })
 

test/core/both/index.js

@@ -2,12 +2,13 @@
 'use strict'
 
 describe('--both', () => {
-  require('./test-bitswap')
   require('./test-block')
   require('./test-bootstrap')
   require('./test-config')
   require('./test-files')
   require('./test-generic')
   require('./test-init')
   require('./test-object')
+  require('./test-dag')
+  require('./test-bitswap')
 })

test/core/both/test-dag.js

@@ -0,0 +1,20 @@
+/* eslint-env mocha */
+
+'use strict'
+
+const test = require('interface-ipfs-core')
+const IPFSFactory = require('../../utils/factory-core')
+
+let factory
+
+const common = {
+  setup: function (cb) {
+    factory = new IPFSFactory()
+    cb(null, factory)
+  },
+  teardown: function (cb) {
+    factory.dismantle(cb)
+  }
+}
+
+test.dag(common)