Files add progress

package.json

@@ -92,9 +92,12 @@
     "async": "^2.5.0",
     "bl": "^1.2.1",
     "boom": "^5.2.0",
+    "byteman": "^1.3.5",
     "cids": "^0.5.1",
     "debug": "^3.0.0",
+    "duplexify": "^3.5.1",
     "fsm-event": "^2.1.0",
+    "get-folder-size": "^1.0.0",
     "glob": "^7.1.2",
     "hapi": "^16.5.2",
     "hapi-set-header": "^1.0.2",
@@ -130,11 +133,14 @@
     "mkdirp": "^0.5.1",
     "multiaddr": "^2.3.0",
     "multihashes": "~0.4.5",
+    "multiplex": "^6.7.0",
+    "nan": "^2.7.0",
     "once": "^1.4.0",
     "path-exists": "^3.0.0",
     "peer-book": "^0.5.0",
     "peer-id": "^0.9.0",
     "peer-info": "^0.10.0",
+    "progress": "^2.0.0",
     "promisify-es6": "^1.0.3",
     "pull-file": "^1.0.0",
     "pull-paramap": "^1.2.2",
@@ -151,6 +157,7 @@
     "temp": "^0.8.3",
     "through2": "^2.0.3",
     "update-notifier": "^2.2.0",
+    "webrtcsupport": "^2.2.0",
     "yargs": "8.0.2"
   },
   "contributors": [
@@ -204,4 +211,4 @@
     "tcme <hi@this-connect.me>",
     "ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <victorbjelkholm@gmail.com>"
   ]
-}
+}

src/cli/bin.js

@@ -15,12 +15,6 @@ updateNotifier({
 }).notify()
 
 const cli = yargs
-  .option('q', {
-    alias: 'quiet',
-    desc: 'suppress output',
-    type: 'boolean',
-    coerce: (quiet) => { if (quiet) { utils.disablePrinting() } }
-  })
   .commandDir('commands')
   .demandCommand(1)
   .fail((msg, err, yargs) => {

src/cli/commands/files/add.js

@@ -2,12 +2,16 @@
 
 const fs = require('fs')
 const path = require('path')
+const async = require('async')
 const glob = require('glob')
 const sortBy = require('lodash.sortby')
 const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 const zip = require('pull-zip')
 const toPull = require('stream-to-pull-stream')
+const Progress = require('progress')
+const getFolderSize = require('get-folder-size')
+const byteman = require('byteman')
 const utils = require('../../utils')
 const print = require('../../utils').print
 
@@ -37,6 +41,27 @@ function checkPath (inPath, recursive) {
   return inPath
 }
 
+function getTotalBytes (path, recursive, cb) {
+  if (recursive) {
+    getFolderSize(path, cb)
+  } else {
+    fs.stat(path, (err, stat) => cb(err, stat.size))
+  }
+}
+
+function createProgressBar (totalBytes) {
+  const total = byteman(totalBytes, 2, 'MB')
+  const barFormat = `:progress / ${total} [:bar] :percent :etas`
+
+  // 16 MB / 34 MB [===========             ] 48% 5.8s //
+  return new Progress(barFormat, {
+    incomplete: ' ',
+    clear: true,
+    stream: process.stdout,
+    total: totalBytes
+  })
+}
+
 function addPipeline (index, addStream, list, wrapWithDirectory) {
   pull(
     zip(
@@ -89,6 +114,12 @@ module.exports = {
   describe: 'Add a file to IPFS using the UnixFS data format',
 
   builder: {
+    progress: {
+      alias: 'p',
+      type: 'boolean',
+      default: true,
+      describe: 'Stream progress data'
+    },
     recursive: {
       alias: 'r',
       type: 'boolean',
@@ -116,46 +147,69 @@ module.exports = {
   },
 
   handler (argv) {
-    const inPath = checkPath(argv.file, argv.recursive)
+    const {
+      file,
+      recursive,
+      progress,
+      wrapWithDirectory,
+      trickle,
+      enableShardingExperiment,
+      shardSplitThreshold
+    } = argv
+
+    const inPath = checkPath(file, recursive)
     const index = inPath.lastIndexOf('/') + 1
+
     const options = {
-      strategy: argv.trickle ? 'trickle' : 'balanced',
-      shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity
+      strategy: trickle ? 'trickle' : 'balanced',
+      shardSplitThreshold: enableShardingExperiment ? shardSplitThreshold : Infinity
     }
 
-    if (argv.enableShardingExperiment && utils.isDaemonOn()) {
+    if (enableShardingExperiment && utils.isDaemonOn()) {
       throw new Error('Error: Enabling the sharding experiment should be done on the daemon')
     }
-    const ipfs = argv.ipfs
 
-    // TODO: revist when interface-ipfs-core exposes pull-streams
-    let createAddStream = (cb) => {
-      ipfs.files.createAddStream(options, (err, stream) => {
-        cb(err, err ? null : toPull.transform(stream))
-      })
-    }
+    const ipfs = argv.ipfs
 
-    if (typeof ipfs.files.createAddPullStream === 'function') {
-      createAddStream = (cb) => {
-        cb(null, ipfs.files.createAddPullStream(options))
-      }
-    }
+    let list = []
+    let currentBytes = 0
+
+    async.waterfall([
+      (next) => glob(path.join(inPath, '/**/*'), next),
+      (globResult, next) => {
+        list = globResult.length === 0 ? [inPath] : globResult
+
+        getTotalBytes(inPath, recursive, next)
+      },
+      (totalBytes, next) => {
+        if (progress) {
+          const bar = createProgressBar(totalBytes)
+          options.progress = function (byteLength) {
+            currentBytes += byteLength
+            bar.tick(byteLength, {progress: byteman(currentBytes, 2, 'MB')})
+          }
+        }
 
-    createAddStream((err, addStream) => {
-      if (err) {
-        throw err
-      }
+        // TODO: revist when interface-ipfs-core exposes pull-streams
 
-      glob(path.join(inPath, '/**/*'), (err, list) => {
-        if (err) {
-          throw err
+        let createAddStream = (cb) => {
+          ipfs.files.createAddStream(options, (err, stream) => {
+            cb(err, err ? null : toPull.transform(stream))
+          })
         }
-        if (list.length === 0) {
-          list = [inPath]
+
+        if (typeof ipfs.files.createAddPullStream === 'function') {
+          createAddStream = (cb) => {
+            cb(null, ipfs.files.createAddPullStream(options))
+          }
         }
 
-        addPipeline(index, addStream, list, argv.wrapWithDirectory)
-      })
+        createAddStream(next)
+      }
+    ], (err, addStream) => {
+      if (err) throw err
+
+      addPipeline(index, addStream, list, wrapWithDirectory)
     })
   }
 }

src/core/components/files.js

@@ -19,7 +19,6 @@ module.exports = function files (self) {
     const opts = Object.assign({}, {
       shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity
     }, options)
-
     return pull(
       pull.map(normalizeContent),
       pull.flatten(),

test/cli/files.js

@@ -8,7 +8,7 @@ const compareDir = require('dir-compare').compareSync
 const rimraf = require('rimraf').sync
 const runOnAndOff = require('../utils/on-and-off')
 
-describe('files', () => runOnAndOff((thing) => {
+describe.only('files', () => runOnAndOff((thing) => {
   let ipfs
   const readme = fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme'))
                    .toString('utf-8')
@@ -128,12 +128,13 @@ describe('files', () => runOnAndOff((thing) => {
   })
 
   it('add and wrap with a directory', () => {
-    return ipfs('add -w src/init-files/init-docs/readme').then((out) => {
-      expect(out).to.be.eql([
-        'added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme',
-        'added QmapdaVPjHXdcswef82FnGQUauMNpk9xYFkLDZKgAxhMwq'
-      ].join('\n') + '\n')
-    })
+    return ipfs('add -w src/init-files/init-docs/readme')
+      .then((out) => {
+        expect(out).to.be.eql([
+          'added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme',
+          'added QmapdaVPjHXdcswef82FnGQUauMNpk9xYFkLDZKgAxhMwq'
+        ].join('\n') + '\n')
+      })
   })
 
   it('cat', () => {