diff --git a/package.json b/package.json index bcafc88fdb..1e0b66701a 100644 --- a/package.json +++ b/package.json @@ -37,46 +37,47 @@ }, "homepage": "https://github.com/ipfs/js-ipfs#readme", "devDependencies": { - "aegir": "^3.0.0", + "aegir": "^3.0.1", "async": "^2.0.0-rc.3", "buffer-loader": "0.0.1", "chai": "^3.5.0", "expose-loader": "^0.7.1", "form-data": "^1.0.0-rc3", "idb-plus-blob-store": "^1.1.2", - "lodash": "^4.11.1", - "mocha": "^2.3.4", + "lodash": "^4.11.2", + "mocha": "^2.4.5", "ncp": "^2.0.0", "nexpect": "^0.5.0", "pre-commit": "^1.1.2", - "rimraf": "^2.4.4", + "rimraf": "^2.5.2", "stream-to-promise": "^1.1.0", "transform-loader": "^0.2.3" }, "dependencies": { "babel-runtime": "^6.6.1", "bl": "^1.1.2", - "boom": "^3.1.1", + "boom": "^3.1.2", "bs58": "^3.0.0", "debug": "^2.2.0", "fs-blob-store": "^5.2.1", "glob": "^7.0.3", "hapi": "^13.3.0", - "ipfs-api": "^3.0.1", + "ipfs-api": "^3.0.2", "ipfs-block": "^0.3.0", "ipfs-block-service": "^0.3.0", - "ipfs-data-importing": "^0.3.3", "ipfs-merkle-dag": "^0.5.0", "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.8.0", - "joi": "^8.0.2", - "libp2p-ipfs": "^0.3.3", + "ipfs-unixfs-engine": "^0.6.1", + "joi": "^8.0.5", + "libp2p-ipfs": "^0.3.5", "lodash.get": "^4.2.1", - "lodash.set": "^4.0.0", - "multiaddr": "^1.3.0", + "lodash.set": "^4.1.0", + "multiaddr": "^1.4.1", "peer-book": "0.1.0", "peer-id": "^0.6.6", "peer-info": "^0.6.2", + "readable-stream": "^1.1.13", "ronin": "^0.3.11", "temp": "^0.8.3" }, diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 5a0fd79138..298f8fc9e9 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -1,11 +1,16 @@ 'use strict' const Command = require('ronin').Command -const IPFS = require('../../../core') +const utils = require('../../utils') const debug = require('debug') const log = debug('cli:version') log.error = debug('cli:version:error') const bs58 = require('bs58') +const Readable = require('stream').Readable +const fs = require('fs') +const async = require('async') +const pathj = require('path') +const glob = require('glob') module.exports = Command.extend({ desc: 'Add a file to IPFS using the UnixFS data format', @@ -19,15 +24,71 @@ module.exports = Command.extend({ }, run: (recursive, path) => { - var node = new IPFS() - path = process.cwd() + '/' + path - node.files.add(path, { - recursive: recursive - }, (err, stats) => { + let rs + + if (!path) { + throw new Error('Error: Argument \'path\' is required') + } + + var s = fs.statSync(path) + + if (s.isDirectory() && recursive === false) { + throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') + } + if (path === '.' && recursive === true) { + path = process.cwd() + s = fs.statSync(process.cwd()) + } else if (path === '.' && recursive === false) { + s = fs.statSync(process.cwd()) + if (s.isDirectory()) { + throw new Error('Error: ' + process.cwd() + ' is a directory, use the \'-r\' flag to specify directories') + } + } + + glob(pathj.join(path, '/**/*'), (err, res) => { if (err) { - return console.log(err) + throw err } - console.log('added', bs58.encode(stats.Hash).toString(), stats.Name) + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + const i = ipfs.files.add() + i.on('data', (file) => { + console.log('added', bs58.encode(file.multihash).toString(), file.path) + }) + if (res.length !== 0) { + const index = path.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + rs = new Readable() + const addPath = element.substring(index + 1, element.length) + if (fs.statSync(element).isDirectory()) { + callback() + } else { + const buffered = fs.readFileSync(element) + rs.push(buffered) + rs.push(null) + const filePair = {path: addPath, stream: rs} + i.write(filePair) + callback() + } + }, (err) => { + if (err) { + throw err + } + i.end() + }) + } else { + rs = new Readable() + const buffered = fs.readFileSync(path) + path = path.substring(path.lastIndexOf('/') + 1, path.length) + rs.push(buffered) + rs.push(null) + const filePair = {path: path, stream: rs} + i.write(filePair) + i.end() + } + }) }) } }) diff --git a/src/cli/commands/files/cat.js b/src/cli/commands/files/cat.js new file mode 100644 index 0000000000..fe61d61316 --- /dev/null +++ b/src/cli/commands/files/cat.js @@ -0,0 +1,37 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + options: {}, + + run: (path, options) => { + if (!path) { + throw new Error("Argument 'path' is required") + } + if (!options) { + options = {} + } + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + ipfs.files.cat(path, (err, res) => { + if (err) { + throw (err) + } + if (res) { + res.on('file', (data) => { + data.stream.pipe(process.stdout) + }) + } + }) + }) + } +}) diff --git a/src/cli/commands/files/get.js b/src/cli/commands/files/get.js new file mode 100644 index 0000000000..b33e57541e --- /dev/null +++ b/src/cli/commands/files/get.js @@ -0,0 +1,69 @@ +'use strict' + +const Command = require('ronin').Command +const debug = require('debug') +const utils = require('../../utils') +const log = debug('cli:files') +log.error = debug('cli:files:error') +var fs = require('fs') +const pathj = require('path') + +module.exports = Command.extend({ + desc: 'Download IPFS objects', + + options: {}, + + run: (path, options) => { + let dir + let filepath + let ws + + if (!path) { + throw new Error("Argument 'path' is required") + } + if (!options) { + options = {} + dir = process.cwd() + } else { + if (options.slice(-1) !== '/') { + options += '/' + } + dir = options + } + + utils.getIPFS((err, ipfs) => { + if (err) { + throw err + } + ipfs.files.get(path, (err, data) => { + if (err) { + throw err + } + data.on('file', (data) => { + if (data.path.lastIndexOf('/') === -1) { + filepath = data.path + if (data.dir === false) { + ws = fs.createWriteStream(pathj.join(dir, data.path)) + data.stream.pipe(ws) + } else { + try { + fs.mkdirSync(pathj.join(dir, data.path)) + } catch (err) { + throw err + } + } + } else { + filepath = data.path.substring(0, data.path.lastIndexOf('/') + 1) + try { + fs.mkdirSync(pathj.join(dir, filepath)) + } catch (err) { + throw err + } + ws = fs.createWriteStream(pathj.join(dir, data.path)) + data.stream.pipe(ws) + } + }) + }) + }) + } +}) diff --git a/src/core/index.js b/src/core/index.js index 6b0c7f4e21..0290fa7ec8 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -1,416 +1,52 @@ 'use strict' const BlockService = require('ipfs-block-service') -const Block = require('ipfs-block') const mDAG = require('ipfs-merkle-dag') -const DAGNode = mDAG.DAGNode const DAGService = mDAG.DAGService -const peerId = require('peer-id') -const PeerInfo = require('peer-info') -const multiaddr = require('multiaddr') -const importer = require('ipfs-data-importing').import -const libp2p = require('libp2p-ipfs') const IPFSRepo = require('ipfs-repo') const PeerBook = require('peer-book') -const init = require('./init') const defaultRepo = require('./default-repo') -const utils = require('./utils') + +const load = require('./ipfs/load') +const version = require('./ipfs/version') +const id = require('./ipfs/id') +const repo = require('./ipfs/repo') +const init = require('./ipfs/init') +const bootstrap = require('./ipfs/bootstrap') +const config = require('./ipfs/config') +const block = require('./ipfs/block') +const object = require('./ipfs/object') +const libp2p = require('./ipfs/libp2p') +const files = require('./ipfs/files') exports = module.exports = IPFS -function IPFS (repo) { +function IPFS (repoInstance) { if (!(this instanceof IPFS)) { throw new Error('Must be instantiated with new') } - if (!(repo instanceof IPFSRepo)) { - repo = defaultRepo(repo) - } - - const blockS = new BlockService(repo) - const dagS = new DAGService(blockS) - var peerInfo - var libp2pNode - const peerInfoBook = new PeerBook() - - this.load = (callback) => { - utils.ifRepoExists(repo, (err) => { - if (err) { - throw err - } - - repo.config.get((err, config) => { - if (err) { - throw err - } - const pid = peerId.createFromPrivKey(config.Identity.PrivKey) - peerInfo = new PeerInfo(pid) - config.Addresses.Swarm.forEach((addr) => { - peerInfo.multiaddr.add(multiaddr(addr)) - }) - callback() - }) - }) - } - - this.version = (opts, callback) => { - if (typeof opts === 'function') { - callback = opts - opts = {} - } - - utils.ifRepoExists(repo, (err) => { - if (err) { - return callback(err) - } - - repo.config.get((err, config) => { - if (err) { - return callback(err) - } - - callback(null, config.Version.Current) - }) - }) - } - - this.id = (opts, callback) => { - if (typeof opts === 'function') { - callback = opts - opts = {} - } - if (!peerInfo) { // because of split second warmup - setTimeout(ready, 100) - } else { - ready() - } - function ready () { - callback(null, { - ID: peerInfo.id.toB58String(), - PublicKey: peerInfo.id.pubKey.toString('base64'), - Addresses: peerInfo.multiaddrs.map((ma) => { return ma.toString() }), - AgentVersion: 'js-ipfs', - ProtocolVersion: '9000' - }) - } - } - - this.repo = { - init: (bits, empty, callback) => { - // 1. check if repo already exists - }, - - version: (opts, callback) => { - if (typeof opts === 'function') { - callback = opts - opts = {} - } - - utils.ifRepoExists(repo, (err, res) => { - if (err) { - return callback(err) - } - - repo.version.get(callback) - }) - }, - - gc: function () {}, - - path: () => repo.path - } - - this.init = (opts, callback) => { init(repo, opts, callback) } - - this.bootstrap = { - list: (callback) => { - repo.config.get((err, config) => { - if (err) { return callback(err) } - callback(null, config.Bootstrap) - }) - }, - add: (multiaddr, callback) => { - repo.config.get((err, config) => { - if (err) { return callback(err) } - config.Bootstrap.push(multiaddr) - repo.config.set(config, (err) => { - if (err) { return callback(err) } - - callback() - }) - }) - }, - rm: (multiaddr, callback) => { - repo.config.get((err, config) => { - if (err) { return callback(err) } - config.Bootstrap = config.Bootstrap.filter((mh) => { - if (mh === multiaddr) { - return false - } else { return true } - }) - repo.config.set(config, (err) => { - if (err) { return callback(err) } - callback() - }) - }) - } - } - - this.config = { - // cli only feature built with show and replace - // edit: (callback) => {}, - replace: (config, callback) => { - repo.config.set(config, callback) - }, - show: (callback) => { - repo.config.get((err, config) => { - if (err) { return callback(err) } - callback(null, config) - }) - } - } - - this.block = { - get: (multihash, callback) => { - blockS.getBlock(multihash, callback) - }, - put: (block, callback) => { - blockS.addBlock(block, callback) - }, - del: (multihash, callback) => { - blockS.deleteBlock(multihash, callback) - }, - stat: (multihash, callback) => { - blockS.getBlock(multihash, (err, block) => { - if (err) { - return callback(err) - } - callback(null, { - Key: multihash, - Size: block.data.length - }) - }) - } - } - - this.object = { - new: (template, callback) => { - if (!callback) { - callback = template - } - var node = new DAGNode() - var block = new Block(node.marshal()) - blockS.addBlock(block, function (err) { - if (err) { - return callback(err) - } - callback(null, { - Hash: block.key, - Size: node.size(), - Name: '' - }) - }) - }, - patch: { - appendData: (multihash, data, callback) => { - this.object.get(multihash, (err, obj) => { - if (err) { - return callback(err) - } - obj.data = Buffer.concat([obj.data, data]) - dagS.add(obj, (err) => { - if (err) { - return callback(err) - } - callback(null, obj) - }) - }) - }, - addLink: (multihash, link, callback) => { - this.object.get(multihash, (err, obj) => { - if (err) { - return callback(err) - } - obj.addRawLink(link) - dagS.add(obj, (err) => { - if (err) { - return callback(err) - } - callback(null, obj) - }) - }) - }, - rmLink: (multihash, linkRef, callback) => { - this.object.get(multihash, (err, obj) => { - if (err) { - return callback(err) - } - obj.links = obj.links.filter((link) => { - // filter by name when linkRef is a string, or by hash otherwise - if (typeof linkRef === 'string') { - return link.name !== linkRef - } - return !link.hash.equals(linkRef) - }) - dagS.add(obj, (err) => { - if (err) { - return callback(err) - } - callback(null, obj) - }) - }) - }, - setData: (multihash, data, callback) => { - this.object.get(multihash, (err, obj) => { - if (err) { return callback(err) } - obj.data = data - dagS.add(obj, (err) => { - if (err) { - return callback(err) - } - callback(null, obj) - }) - }) - } - }, - data: (multihash, callback) => { - this.object.get(multihash, (err, obj) => { - if (err) { - return callback(err) - } - callback(null, obj.data) - }) - }, - links: (multihash, callback) => { - this.object.get(multihash, (err, obj) => { - if (err) { - return callback(err) - } - callback(null, obj.links) - }) - }, - get: (multihash, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - dagS.get(multihash, callback) - }, - put: (dagNode, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - dagS.add(dagNode, callback) - }, - stat: (multihash, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - - this.object.get(multihash, (err, obj) => { - if (err) { - return callback(err) - } - var res = { - NumLinks: obj.links.length, - BlockSize: obj.marshal().length, - LinksSize: obj.links.reduce((prev, link) => { - return prev + link.size - }, 0), - DataSize: obj.data.length, - CumulativeSize: '' - } - callback(null, res) - }) - } - } - - const OFFLINE_ERROR = new Error('This command must be run in online mode. Try running \'ipfs daemon\' first.') - - this.libp2p = { - start: (callback) => { - libp2pNode = new libp2p.Node(peerInfo) - libp2pNode.start(() => { - // TODO connect to bootstrap nodes, it will get us more addrs - peerInfo.multiaddrs.forEach((ma) => { - console.log('Swarm listening on', ma.toString()) - }) - callback() - }) - }, - stop: (callback) => { - libp2pNode.swarm.close(callback) - }, - swarm: { - peers: (callback) => { - if (!libp2pNode) { - return callback(OFFLINE_ERROR) - } - - callback(null, peerInfoBook.getAll()) - }, - // all the addrs we know - addrs: (callback) => { - if (!libp2pNode) { - return callback(OFFLINE_ERROR) - } - // TODO - notImpl() - }, - localAddrs: (callback) => { - if (!libp2pNode) { - return callback(OFFLINE_ERROR) - } - - callback(null, peerInfo.multiaddrs) - }, - connect: (ma, callback) => { - if (!libp2pNode) { - return callback(OFFLINE_ERROR) - } - - const idStr = ma.toString().match(/\/ipfs\/(.*)/) - if (!idStr) { - return callback(new Error('invalid multiaddr')) - } - const id = peerId.createFromB58String(idStr[1]) - const peer = new PeerInfo(id) - - ma = ma.toString().replace(/\/ipfs\/(.*)/, '') // FIXME remove this when multiaddr supports ipfs - - peer.multiaddr.add(multiaddr(ma)) - peerInfoBook.put(peer) - - libp2pNode.swarm.dial(peer, (err) => { - callback(err, id) - }) - }, - disconnect: (callback) => { - if (!libp2pNode) { - return callback(OFFLINE_ERROR) - } - - notImpl() - }, - filters: notImpl // TODO - }, - routing: {}, - records: {}, - ping: notImpl - } - - this.files = { - add: (path, options, callback) => { - options.path = path - options.dagService = dagS - importer(options, callback) - } - } -} - -function notImpl () { - throw new Error('Not implemented yet') + if (!(repoInstance instanceof IPFSRepo)) { + repoInstance = defaultRepo(repoInstance) + } + + this._repo = repoInstance + this._blockS = new BlockService(this._repo) + this._dagS = new DAGService(this._blockS) + this._peerInfoBook = new PeerBook() + this._peerInfo = null + this._libp2pNode = null + + this.load = load(this) + this.version = version(this) + this.id = id(this) + this.repo = repo(this) + this.init = init(this) + this.bootstrap = bootstrap(this) + this.config = config(this) + this.block = block(this) + this.object = object(this) + this.libp2p = libp2p(this) + this.files = files(this) } diff --git a/src/core/init.js b/src/core/init.js deleted file mode 100644 index cdcfbf5bc7..0000000000 --- a/src/core/init.js +++ /dev/null @@ -1,86 +0,0 @@ -'use strict' - -const peerId = require('peer-id') -const BlockService = require('ipfs-block-service') -const DagService = require('ipfs-merkle-dag').DAGService -const path = require('path') - -module.exports = (repo, opts, callback) => { - opts = opts || {} - opts.emptyRepo = opts.emptyRepo || false - opts.bits = opts.bits || 2048 - - // Pre-set config values. - var config = require('../init-files/default-config.json') - - // Verify repo does not yet exist. - repo.exists((err, exists) => { - if (err) { - return callback(err) - } - - if (exists === true) { - return callback(new Error('repo already exists')) - } - - generateAndSetKeypair() - }) - - // Generate peer identity keypair + transform to desired format + add to config. - function generateAndSetKeypair () { - var keys = peerId.create({ - bits: opts.bits - }) - config.Identity = { - PeerID: keys.toB58String(), - PrivKey: keys.privKey.toString('base64') - } - - writeVersion() - } - - function writeVersion () { - const version = '3' - - repo.version.set(version, (err) => { - if (err) { return callback(err) } - - writeConfig() - }) - } - - // Write the config to the repo. - function writeConfig () { - repo.config.set(config, (err) => { - if (err) { return callback(err) } - - addDefaultAssets() - }) - } - - // Add the default assets to the repo. - function addDefaultAssets () { - // Skip this step on the browser, or if emptyRepo was supplied. - const isNode = !global.window - if (!isNode || opts.emptyRepo) { - return doneImport(null) - } - - const importer = require('ipfs-data-importing') - const blocks = new BlockService(repo) - const dag = new DagService(blocks) - - const initDocsPath = path.join(__dirname, '../init-files/init-docs') - - importer.import(initDocsPath, dag, { - recursive: true - }, doneImport) - - function doneImport (err, stat) { - if (err) { return callback(err) } - - // All finished! - callback(null, true) - } - } -} diff --git a/test/cli-tests/test-commands.js b/test/cli-tests/test-commands.js index aa8da00762..efb656d58a 100644 --- a/test/cli-tests/test-commands.js +++ b/test/cli-tests/test-commands.js @@ -10,7 +10,7 @@ describe('commands', () => { .run((err, stdout, exitcode) => { expect(err).to.not.exist expect(exitcode).to.equal(0) - expect(stdout.length).to.equal(45) + expect(stdout.length).to.equal(47) done() }) }) diff --git a/test/core-tests/test-files.js b/test/core-tests/test-files.js new file mode 100644 index 0000000000..4dd22d2777 --- /dev/null +++ b/test/core-tests/test-files.js @@ -0,0 +1,64 @@ +/* eslint-env mocha */ +'use strict' + +const bl = require('bl') +const expect = require('chai').expect +const Readable = require('stream').Readable +const bs58 = require('bs58') + +const IPFS = require('../../src/core') + +describe('files', () => { + let ipfs + + before((done) => { + ipfs = new IPFS(require('./repo-path')) + ipfs.load(done) + }) + + it('add', (done) => { + const buffered = new Buffer('some data') + const rs = new Readable() + rs.push(buffered) + rs.push(null) + const arr = [] + const filePair = {path: 'data.txt', stream: rs} + arr.push(filePair) + ipfs.files.add(arr, (err, res) => { + expect(err).to.not.exist + expect(res[0].path).to.equal('data.txt') + expect(res[0].size).to.equal(17) + expect(bs58.encode(res[0].multihash).toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') + done() + }) + }) + + it('cat', (done) => { + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.cat(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) + + it('get', (done) => { + // TODO create non-trival get test + const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + ipfs.files.get(hash, (err, res) => { + expect(err).to.not.exist + res.on('file', (data) => { + data.stream.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.equal('hello world\n') + done() + })) + }) + }) + }) +}) diff --git a/test/core-tests/test-init-node.js b/test/core-tests/test-init-node.js index 1e05d61b5e..ee65c3e83e 100644 --- a/test/core-tests/test-init-node.js +++ b/test/core-tests/test-init-node.js @@ -24,14 +24,14 @@ describe('init (Node.js specific)', function () { it('init docs are written', (done) => { ipfs.init({ bits: 64 }, (err) => { expect(err).to.not.exist - - // Check for default assets var multihash = new Buffer('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'hex') - ipfs.object.get(multihash, {}, (err, node) => { - expect(err).to.not.exist - expect(node.links).to.exist - done() - }) + setTimeout(() => { + ipfs.object.get(multihash, {}, (err, node) => { + expect(err).to.not.exist + expect(node.links).to.exist + done() + }) + }, 1000) }) }) diff --git a/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data new file mode 100644 index 0000000000..2965d1c457 --- /dev/null +++ b/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data @@ -0,0 +1,3 @@ + + hello world + \ No newline at end of file