From 2f8f90c070a53584e292401b3ea611edd1f4902f Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Wed, 30 Aug 2017 16:42:49 +0100 Subject: [PATCH 1/4] Adds a progress bar --- src/cli/commands/files/add.js | 25 +++++++++++++++++++++++-- test/cli/files.js | 13 +++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 9e7d192106..cc2f03ca3a 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,6 +8,7 @@ const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') +const Progress = require('progress') const utils = require('../../utils') const print = require('../../utils').print @@ -37,7 +38,12 @@ function checkPath (inPath, recursive) { return inPath } -function addPipeline (index, addStream, list, wrapWithDirectory) { +function addPipeline (index, addStream, list, bar, argv) { + const { + wrapWithDirectory, + progress + } = argv + pull( zip( pull.values(list), @@ -60,6 +66,9 @@ function addPipeline (index, addStream, list, wrapWithDirectory) { content: fs.createReadStream(file.originalPath) })), addStream, + pull.through((file) => { + if (progress) bar.tick(file.size) + }), pull.map((file) => ({ hash: file.hash, path: wrapWithDirectory ? file.path.substring(WRAPPER.length) : file.path @@ -89,6 +98,12 @@ module.exports = { describe: 'Add a file to IPFS using the UnixFS data format', builder: { + progress: { + alias: 'p', + type: 'boolean', + default: true, + describe: 'Stream progress data' + }, recursive: { alias: 'r', type: 'boolean', @@ -118,6 +133,12 @@ module.exports = { handler (argv) { const inPath = checkPath(argv.file, argv.recursive) const index = inPath.lastIndexOf('/') + 1 + const bar = new Progress('[:bar]', { + complete: '=', + incomplete: ' ', + width: 20, + total: fs.statSync(argv.file).size + }) const options = { strategy: argv.trickle ? 'trickle' : 'balanced', shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity @@ -154,7 +175,7 @@ module.exports = { list = [inPath] } - addPipeline(index, addStream, list, argv.wrapWithDirectory) + addPipeline(index, addStream, list, bar, argv) }) }) } diff --git a/test/cli/files.js b/test/cli/files.js index 7dc01599c6..063641b8cf 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -128,12 +128,13 @@ describe('files', () => runOnAndOff((thing) => { }) it('add and wrap with a directory', () => { - return ipfs('add -w src/init-files/init-docs/readme').then((out) => { - expect(out).to.be.eql([ - 'added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme', - 'added QmapdaVPjHXdcswef82FnGQUauMNpk9xYFkLDZKgAxhMwq' - ].join('\n') + '\n') - }) + return ipfs('add -w src/init-files/init-docs/readme') + .then((out) => { + expect(out).to.be.eql([ + 'added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme', + 'added QmapdaVPjHXdcswef82FnGQUauMNpk9xYFkLDZKgAxhMwq' + ].join('\n') + '\n') + }) }) it('cat', () => { From 6a09e29916e8d0a96a0ae5816bf752323e3acb74 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 1 Sep 2017 16:14:44 +0100 Subject: [PATCH 2/4] updates bar format to match go-ipfs --- package.json | 5 +++- src/cli/commands/files/add.js | 47 ++++++++++++++++++++++++----------- src/core/components/files.js | 1 - test/cli/files.js | 2 +- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index be5c0cfef3..7859f97f78 100644 --- a/package.json +++ b/package.json @@ -92,9 +92,11 @@ "async": "^2.5.0", "bl": "^1.2.1", "boom": "^5.2.0", + "byteman": "^1.3.5", "cids": "^0.5.1", "debug": "^3.0.0", "fsm-event": "^2.1.0", + "get-folder-size": "^1.0.0", "glob": "^7.1.2", "hapi": "^16.5.2", "hapi-set-header": "^1.0.2", @@ -135,6 +137,7 @@ "peer-book": "^0.5.0", "peer-id": "^0.9.0", "peer-info": "^0.10.0", + "progress": "^2.0.0", "promisify-es6": "^1.0.3", "pull-file": "^1.0.0", "pull-paramap": "^1.2.2", @@ -204,4 +207,4 @@ "tcme ", "ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ " ] -} \ No newline at end of file +} diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index cc2f03ca3a..a8ec5e0782 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -9,6 +9,8 @@ const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') const Progress = require('progress') +const getFolderSize = require('get-folder-size') +const byteman = require('byteman') const utils = require('../../utils') const print = require('../../utils').print @@ -38,12 +40,27 @@ function checkPath (inPath, recursive) { return inPath } -function addPipeline (index, addStream, list, bar, argv) { - const { - wrapWithDirectory, - progress - } = argv +function getTotalBytes (path, recursive) { + return recursive ? getFolderSize(path).size : fs.statSync(path).size +} + +function createProgressBar (totalBytes) { + const totalDisplay = byteman(totalBytes) + const barFormat = `:current KB / ${totalDisplay} [:bar] :percent :etas ` + + // 2340 KB / 34MB [======== ] 48% 5.8s // + return new Progress(barFormat, { + complete: '=', + incomplete: ' ', + clear: true, + width: 40, + stream: process.stdout, + total: totalBytes + }) +} + +function addPipeline (index, addStream, list, wrapWithDirectory) { pull( zip( pull.values(list), @@ -66,9 +83,6 @@ function addPipeline (index, addStream, list, bar, argv) { content: fs.createReadStream(file.originalPath) })), addStream, - pull.through((file) => { - if (progress) bar.tick(file.size) - }), pull.map((file) => ({ hash: file.hash, path: wrapWithDirectory ? file.path.substring(WRAPPER.length) : file.path @@ -133,12 +147,7 @@ module.exports = { handler (argv) { const inPath = checkPath(argv.file, argv.recursive) const index = inPath.lastIndexOf('/') + 1 - const bar = new Progress('[:bar]', { - complete: '=', - incomplete: ' ', - width: 20, - total: fs.statSync(argv.file).size - }) + const options = { strategy: argv.trickle ? 'trickle' : 'balanced', shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity @@ -147,9 +156,17 @@ module.exports = { if (argv.enableShardingExperiment && utils.isDaemonOn()) { throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } + const ipfs = argv.ipfs + if (argv.progress) { + const totalBytes = getTotalBytes(argv.file, argv.recursive) + const bar = createProgressBar(totalBytes) + options.progress = bar.tick.bind(bar) + } + // TODO: revist when interface-ipfs-core exposes pull-streams + let createAddStream = (cb) => { ipfs.files.createAddStream(options, (err, stream) => { cb(err, err ? null : toPull.transform(stream)) @@ -175,7 +192,7 @@ module.exports = { list = [inPath] } - addPipeline(index, addStream, list, bar, argv) + addPipeline(index, addStream, list, argv.wrapWithDirectory) }) }) } diff --git a/src/core/components/files.js b/src/core/components/files.js index 3d650e74bc..df64c6c47f 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -19,7 +19,6 @@ module.exports = function files (self) { const opts = Object.assign({}, { shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity }, options) - return pull( pull.map(normalizeContent), pull.flatten(), diff --git a/test/cli/files.js b/test/cli/files.js index 063641b8cf..70792cec50 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -8,7 +8,7 @@ const compareDir = require('dir-compare').compareSync const rimraf = require('rimraf').sync const runOnAndOff = require('../utils/on-and-off') -describe('files', () => runOnAndOff((thing) => { +describe.only('files', () => runOnAndOff((thing) => { let ipfs const readme = fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme')) .toString('utf-8') From 4b8295067000cb41b37e59051d61377c8c284ebf Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Sat, 2 Sep 2017 18:33:42 +0100 Subject: [PATCH 3/4] progress bar initially formated ok and working with folders --- package.json | 4 ++ src/cli/bin.js | 6 --- src/cli/commands/files/add.js | 95 +++++++++++++++++++++-------------- 3 files changed, 60 insertions(+), 45 deletions(-) diff --git a/package.json b/package.json index 7859f97f78..4048f7e3b7 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "byteman": "^1.3.5", "cids": "^0.5.1", "debug": "^3.0.0", + "duplexify": "^3.5.1", "fsm-event": "^2.1.0", "get-folder-size": "^1.0.0", "glob": "^7.1.2", @@ -132,6 +133,8 @@ "mkdirp": "^0.5.1", "multiaddr": "^2.3.0", "multihashes": "~0.4.5", + "multiplex": "^6.7.0", + "nan": "^2.7.0", "once": "^1.4.0", "path-exists": "^3.0.0", "peer-book": "^0.5.0", @@ -154,6 +157,7 @@ "temp": "^0.8.3", "through2": "^2.0.3", "update-notifier": "^2.2.0", + "webrtcsupport": "^2.2.0", "yargs": "8.0.2" }, "contributors": [ diff --git a/src/cli/bin.js b/src/cli/bin.js index 8da1473cdd..fc3858ccf6 100755 --- a/src/cli/bin.js +++ b/src/cli/bin.js @@ -15,12 +15,6 @@ updateNotifier({ }).notify() const cli = yargs - .option('q', { - alias: 'quiet', - desc: 'suppress output', - type: 'boolean', - coerce: (quiet) => { if (quiet) { utils.disablePrinting() } } - }) .commandDir('commands') .demandCommand(1) .fail((msg, err, yargs) => { diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index a8ec5e0782..aad0c3f446 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -2,6 +2,7 @@ const fs = require('fs') const path = require('path') +const async = require('async') const glob = require('glob') const sortBy = require('lodash.sortby') const pull = require('pull-stream') @@ -40,21 +41,24 @@ function checkPath (inPath, recursive) { return inPath } -function getTotalBytes (path, recursive) { - return recursive ? getFolderSize(path).size : fs.statSync(path).size +function getTotalBytes (path, recursive, cb) { + if (recursive) { + getFolderSize(path, cb) + } else { + fs.stat(path, (err, stat) => cb(err, stat.size)) + } } function createProgressBar (totalBytes) { - const totalDisplay = byteman(totalBytes) - const barFormat = `:current KB / ${totalDisplay} [:bar] :percent :etas ` - - // 2340 KB / 34MB [======== ] 48% 5.8s // + const totalDisplay = byteman(totalBytes, 1, 'MB') + const barFormat = `:size MB / ${totalDisplay} [:bar] :percent :etas` + // 2340 MB / 34 MB [======== ] 48% 5.8s // return new Progress(barFormat, { - complete: '=', + complete : '=', incomplete: ' ', clear: true, - width: 40, + width: 60, stream: process.stdout, total: totalBytes }) @@ -145,55 +149,68 @@ module.exports = { }, handler (argv) { - const inPath = checkPath(argv.file, argv.recursive) + const { + file, + recursive, + progress, + wrapWithDirectory, + trickle, + enableShardingExperiment, + shardSplitThreshold + } = argv + + const inPath = checkPath(file, recursive) const index = inPath.lastIndexOf('/') + 1 const options = { - strategy: argv.trickle ? 'trickle' : 'balanced', - shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity + strategy: trickle ? 'trickle' : 'balanced', + shardSplitThreshold: enableShardingExperiment ? shardSplitThreshold : Infinity } - if (argv.enableShardingExperiment && utils.isDaemonOn()) { + if (enableShardingExperiment && utils.isDaemonOn()) { throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } const ipfs = argv.ipfs - if (argv.progress) { - const totalBytes = getTotalBytes(argv.file, argv.recursive) - const bar = createProgressBar(totalBytes) - options.progress = bar.tick.bind(bar) - } + let list = [] - // TODO: revist when interface-ipfs-core exposes pull-streams + async.waterfall([ + (next) => glob(path.join(inPath, '/**/*'), next), + (globResult, next) => { + list = globResult.length === 0 ? [inPath] : globResult - let createAddStream = (cb) => { - ipfs.files.createAddStream(options, (err, stream) => { - cb(err, err ? null : toPull.transform(stream)) - }) - } + getTotalBytes(inPath, recursive, next) + }, + (totalBytes, next) => { + if (progress) { + // create a new instance of a progress bar and + // pass the tick function into the `addStream` as an option - if (typeof ipfs.files.createAddPullStream === 'function') { - createAddStream = (cb) => { - cb(null, ipfs.files.createAddPullStream(options)) - } - } + const bar = createProgressBar(totalBytes) + options.progress = bar.tick.bind(bar) + } - createAddStream((err, addStream) => { - if (err) { - throw err - } + // TODO: revist when interface-ipfs-core exposes pull-streams - glob(path.join(inPath, '/**/*'), (err, list) => { - if (err) { - throw err + let createAddStream = (cb) => { + ipfs.files.createAddStream(options, (err, stream) => { + cb(err, err ? null : toPull.transform(stream)) + }) } - if (list.length === 0) { - list = [inPath] + + if (typeof ipfs.files.createAddPullStream === 'function') { + createAddStream = (cb) => { + cb(null, ipfs.files.createAddPullStream(options)) + } } - addPipeline(index, addStream, list, argv.wrapWithDirectory) - }) + createAddStream(next) + } + ], (err, addStream) => { + if (err) throw err + + addPipeline(index, addStream, list, wrapWithDirectory) }) } } From e9af1bf31a7d0fec394b69b582fe0e17e9ac84dc Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Sun, 3 Sep 2017 07:06:36 +0100 Subject: [PATCH 4/4] progress bar format matching go implementation --- src/cli/commands/files/add.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index aad0c3f446..0f130cba60 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -50,15 +50,13 @@ function getTotalBytes (path, recursive, cb) { } function createProgressBar (totalBytes) { - const totalDisplay = byteman(totalBytes, 1, 'MB') - const barFormat = `:size MB / ${totalDisplay} [:bar] :percent :etas` + const total = byteman(totalBytes, 2, 'MB') + const barFormat = `:progress / ${total} [:bar] :percent :etas` - // 2340 MB / 34 MB [======== ] 48% 5.8s // + // 16 MB / 34 MB [=========== ] 48% 5.8s // return new Progress(barFormat, { - complete : '=', incomplete: ' ', clear: true, - width: 60, stream: process.stdout, total: totalBytes }) @@ -174,6 +172,7 @@ module.exports = { const ipfs = argv.ipfs let list = [] + let currentBytes = 0 async.waterfall([ (next) => glob(path.join(inPath, '/**/*'), next), @@ -184,11 +183,11 @@ module.exports = { }, (totalBytes, next) => { if (progress) { - // create a new instance of a progress bar and - // pass the tick function into the `addStream` as an option - const bar = createProgressBar(totalBytes) - options.progress = bar.tick.bind(bar) + options.progress = function (byteLength) { + currentBytes += byteLength + bar.tick(byteLength, {progress: byteman(currentBytes, 2, 'MB')}) + } } // TODO: revist when interface-ipfs-core exposes pull-streams