diff --git a/.gitignore b/.gitignore index 8bd47460..2047f0c3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,9 @@ node_modules tmp .idea data -tests/fixtures/.dat yarn.lock -tests/fixtures/.dat -tests/fixtures/dat.json -tests/**.db -tests/.datrc-test +test/fixtures/.dat +test/fixtures/dat.json +test/**.db +test/.datrc-test package-lock.json diff --git a/README.md b/README.md index e0258752..6e11f250 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ You can also also view the files online: [datproject.org/778f8d955175c92e4ced5e4 Dat can share files from your computer to anywhere. If you have a friend going through this demo with you, try sharing to them! If not we'll see what we can do. -Find a folder on your computer to share. Inside the folder can be anything, Dat can handle all sorts of files (Dat works with really big folders too!). +Find a folder on your computer to share. Inside the folder can be anything, Dat can handle all sorts of files (Dat works with really big folders too!). First, you can create a new dat inside that folder. Using the `dat create` command also walks us through making a `dat.json` file: @@ -187,7 +187,7 @@ Get started using Dat today with the `share` and `clone` commands or read below ## Usage The first time you run a command, a `.dat` folder to store the Dat metadata. -Once a Dat is created, you can run all the commands inside that folder, similar to git. +Once a Dat is created, you can run all the commands inside that folder, similar to git. Dat keep secret keys in the `~/.dat/secret_keys` folder. These are required to write to any dats you create. @@ -232,10 +232,15 @@ Sync watched files for changes and imports updated files. #### Ignoring Files -By default, dat will ignore any files in a `.datignore` file, similar to git. Dat also ignores all hidden folders and files. +By default, dat will ignore any files in a `.datignore` file, similar to git. Each file should separated by a newline. Dat also ignores all hidden folders and files. Dat uses [dat-ignore](https://github.com/joehand/dat-ignore) to decide if a file should be ignored. +#### Selecting Files + +By default, dat will download all files. If you want to only download a subset, you can create a `.datdownload` file which downloads only the files and folders specified. Each should be separated by a newline. + + ### Downloading Start downloading by running the `clone` command. This creates a folder, download the content and metadata, and a `.dat` folder inside. Once you started the download, you can resume using `clone` or the other download commands. diff --git a/src/commands/clone.js b/src/commands/clone.js index 8c6eb0db..932cb31e 100644 --- a/src/commands/clone.js +++ b/src/commands/clone.js @@ -7,6 +7,12 @@ module.exports = { 'Usage: dat clone [download-folder]' ].join('\n'), options: [ + { + name: 'empty', + boolean: false, + default: false, + help: 'Do not download files by default. Files must be synced manually.' + }, { name: 'upload', boolean: true, @@ -40,6 +46,7 @@ function clone (opts) { opts.key = parsed.key || opts._[0] // pass other links to resolver opts.dir = parsed.dir opts.showKey = opts['show-key'] // using abbr in option makes printed help confusing + opts.sparse = opts.empty debug('clone()') debug(Object.assign({}, opts, {key: '', _: null})) // don't show key diff --git a/src/commands/pull.js b/src/commands/pull.js index 791f1092..941c6a9e 100644 --- a/src/commands/pull.js +++ b/src/commands/pull.js @@ -13,6 +13,19 @@ module.exports = { default: true, help: 'announce your address on link (improves connection capability) and upload data to other downloaders.' }, + { + name: 'selectFromFile', + boolean: false, + default: '.datdownload', + help: 'Sync only the list of selected files or directories in the given file.', + abbr: 'select-from-file' + }, + { + name: 'select', + boolean: false, + default: false, + help: 'Sync only the list of selected files or directories.' + }, { name: 'show-key', boolean: true, @@ -28,6 +41,7 @@ function pull (opts) { var neatLog = require('neat-log') var archiveUI = require('../ui/archive') var trackArchive = require('../lib/archive') + var selectiveSync = require('../lib/selective-sync') var discoveryExit = require('../lib/discovery-exit') var onExit = require('../lib/exit') var parseArgs = require('../parse-args') @@ -52,6 +66,7 @@ function pull (opts) { neat.use(onExit) neat.use(function (state, bus) { state.opts = opts + selectiveSync(state, opts) Dat(opts.dir, opts, function (err, dat) { if (err && err.name === 'MissingError') return bus.emit('exit:warn', 'No existing archive in this directory. Use clone to download a new archive.') diff --git a/src/commands/sync.js b/src/commands/sync.js index 765b21af..458a6b93 100644 --- a/src/commands/sync.js +++ b/src/commands/sync.js @@ -20,6 +20,19 @@ module.exports = { default: true, abbr: 'ignore-hidden' }, + { + name: 'selectFromFile', + boolean: false, + default: '.datdownload', + help: 'Sync only the list of selected files or directories in the given file.', + abbr: 'select-from-file' + }, + { + name: 'select', + boolean: false, + default: false, + help: 'Sync only the list of selected files or directories.' + }, { name: 'watch', boolean: true, @@ -40,6 +53,7 @@ function sync (opts) { var Dat = require('dat-node') var neatLog = require('neat-log') var archiveUI = require('../ui/archive') + var selectiveSync = require('../lib/selective-sync') var trackArchive = require('../lib/archive') var onExit = require('../lib/exit') var parseArgs = require('../parse-args') @@ -60,7 +74,7 @@ function sync (opts) { neat.use(onExit) neat.use(function (state, bus) { state.opts = opts - + selectiveSync(state, opts) Dat(opts.dir, opts, function (err, dat) { if (err && err.name === 'MissingError') return bus.emit('exit:warn', 'No existing archive in this directory.') if (err) return bus.emit('exit:error', err) diff --git a/src/lib/archive.js b/src/lib/archive.js index 3e04b836..d2fdabaa 100644 --- a/src/lib/archive.js +++ b/src/lib/archive.js @@ -1,3 +1,6 @@ +var debug = require('debug')('dat') +var path = require('path') +var EventEmitter = require('events').EventEmitter var doImport = require('./import-progress') var stats = require('./stats') var network = require('./network') @@ -14,6 +17,7 @@ module.exports = function (state, bus) { if (state.opts.http) serve(state, bus) if (state.writable && state.opts.import) doImport(state, bus) + else if (state.opts.sparse) selectiveSync(state, bus) else download(state, bus) if (state.dat.archive.content) return bus.emit('archive:content') @@ -26,3 +30,52 @@ module.exports = function (state, bus) { state.hasContent = true }) } + +function selectiveSync (state, bus) { + var archive = state.dat.archive + debug('sparse mode. downloading metadata') + var emitter = new EventEmitter() + + function download (entry) { + debug('selected', entry) + archive.stat(entry, function (err, stat) { + if (err) return state.warnings.push(err.message) + if (stat.isDirectory()) downloadDir(entry, stat) + if (stat.isFile()) downloadFile(entry, stat) + }) + } + + function downloadDir (dirname, stat) { + debug('downloading dir', dirname) + archive.readdir(dirname, function (err, entries) { + if (err) return bus.emit('exit:error', err) + entries.forEach(function (entry) { + emitter.emit('download', path.join(dirname, entry)) + }) + }) + } + + function downloadFile (entry, stat) { + var start = stat.offset + var end = stat.offset + stat.blocks + state.selectedByteLength += stat.size + bus.emit('render') + if (start === 0 && end === 0) return + debug('downloading', entry, start, end) + archive.content.download({start, end}, function () { + debug('success', entry) + }) + } + + emitter.on('download', download) + if (state.opts.selectedFiles) state.opts.selectedFiles.forEach(download) + + archive.metadata.update(function () { + return bus.emit('exit:warn', `Dat successfully created in empty mode. Download files using pull or sync.`) + }) + + archive.on('update', function () { + debug('archive update') + bus.emit('render') + }) +} diff --git a/src/lib/selective-sync.js b/src/lib/selective-sync.js new file mode 100644 index 00000000..3a27bb5e --- /dev/null +++ b/src/lib/selective-sync.js @@ -0,0 +1,32 @@ +var fs = require('fs') +var path = require('path') + +module.exports = function (state, opts) { + // selective sync stuff + var parsing = opts.selectFromFile !== '.datdownload' ? opts.selectFromFile : path.join(opts.dir, '.datdownload') + opts.selectedFiles = parseFiles(parsing) + if (opts.select && typeof opts.select === 'string') opts.selectedFiles = opts.select.split(',') + if (opts.selectedFiles) { + state.title = 'Syncing' + state.selectedByteLength = 0 + opts.sparse = true + } + return state +} + +function parseFiles (input) { + var parsed = null + + try { + if (fs.statSync(input).isFile()) { + parsed = fs.readFileSync(input).toString().trim().split(/\r?\n/) + } + } catch (err) { + if (err && !err.name === 'ENOENT') { + console.error(err) + process.exit(1) + } + } + + return parsed +} diff --git a/src/ui/archive.js b/src/ui/archive.js index 9f8af1f2..292d2bb3 100644 --- a/src/ui/archive.js +++ b/src/ui/archive.js @@ -3,6 +3,7 @@ var pretty = require('prettier-bytes') var chalk = require('chalk') var downloadUI = require('./components/download') var importUI = require('./components/import-progress') +var warningsUI = require('./components/warnings') var networkUI = require('./components/network') var sourcesUI = require('./components/sources') var keyEl = require('./elements/key') @@ -15,6 +16,7 @@ module.exports = archiveUI function archiveUI (state) { if (!state.dat) return 'Starting Dat program...' if (!state.writable && !state.hasContent) return 'Connecting to dat network...' + if (!state.warnings) state.warnings = [] var dat = state.dat var stats = dat.stats.get() @@ -27,7 +29,8 @@ function archiveUI (state) { if (state.title) title += state.title else if (state.writable) title += 'Sharing dat' else title += 'Downloading dat' - if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})` + if (state.opts.sparse) title += `: ${state.opts.selectedFiles.length} ${pluralize('file', state.opts.selectedFiles.length)} (${pretty(state.selectedByteLength)})` + else if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})` else if (stats.version === 0) title += ': (empty archive)' if (state.http && state.http.listening) title += `\nServing files over http at http://localhost:${state.http.port}` @@ -48,6 +51,7 @@ function archiveUI (state) { ${progressView} ${state.opts.sources ? sourcesUI(state) : ''} + ${state.warnings ? warningsUI(state) : ''} ${state.exiting ? 'Exiting the Dat program...' : chalk.dim('Ctrl+C to Exit')} ` } diff --git a/src/ui/components/warnings.js b/src/ui/components/warnings.js new file mode 100644 index 00000000..cdf94469 --- /dev/null +++ b/src/ui/components/warnings.js @@ -0,0 +1,9 @@ +var chalk = require('chalk') + +module.exports = function (state) { + var warning = '' + state.warnings.forEach(function (message) { + warning += `${chalk.yellow(`Warning: ${message}`)}\n` + }) + return warning +}