diff --git a/.gitignore b/.gitignore
index 8bd47460..2047f0c3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,10 +3,9 @@ node_modules
tmp
.idea
data
-tests/fixtures/.dat
yarn.lock
-tests/fixtures/.dat
-tests/fixtures/dat.json
-tests/**.db
-tests/.datrc-test
+test/fixtures/.dat
+test/fixtures/dat.json
+test/**.db
+test/.datrc-test
package-lock.json
diff --git a/README.md b/README.md
index e0258752..6e11f250 100644
--- a/README.md
+++ b/README.md
@@ -159,7 +159,7 @@ You can also also view the files online: [datproject.org/778f8d955175c92e4ced5e4
Dat can share files from your computer to anywhere. If you have a friend going through this demo with you, try sharing to them! If not we'll see what we can do.
-Find a folder on your computer to share. Inside the folder can be anything, Dat can handle all sorts of files (Dat works with really big folders too!).
+Find a folder on your computer to share. Inside the folder can be anything, Dat can handle all sorts of files (Dat works with really big folders too!).
First, you can create a new dat inside that folder. Using the `dat create` command also walks us through making a `dat.json` file:
@@ -187,7 +187,7 @@ Get started using Dat today with the `share` and `clone` commands or read below
## Usage
The first time you run a command, a `.dat` folder to store the Dat metadata.
-Once a Dat is created, you can run all the commands inside that folder, similar to git.
+Once a Dat is created, you can run all the commands inside that folder, similar to git.
Dat keep secret keys in the `~/.dat/secret_keys` folder. These are required to write to any dats you create.
@@ -232,10 +232,15 @@ Sync watched files for changes and imports updated files.
#### Ignoring Files
-By default, dat will ignore any files in a `.datignore` file, similar to git. Dat also ignores all hidden folders and files.
+By default, dat will ignore any files in a `.datignore` file, similar to git. Each file should separated by a newline. Dat also ignores all hidden folders and files.
Dat uses [dat-ignore](https://github.com/joehand/dat-ignore) to decide if a file should be ignored.
+#### Selecting Files
+
+By default, dat will download all files. If you want to only download a subset, you can create a `.datdownload` file which downloads only the files and folders specified. Each should be separated by a newline.
+
+
### Downloading
Start downloading by running the `clone` command. This creates a folder, download the content and metadata, and a `.dat` folder inside. Once you started the download, you can resume using `clone` or the other download commands.
diff --git a/src/commands/clone.js b/src/commands/clone.js
index 8c6eb0db..932cb31e 100644
--- a/src/commands/clone.js
+++ b/src/commands/clone.js
@@ -7,6 +7,12 @@ module.exports = {
'Usage: dat clone [download-folder]'
].join('\n'),
options: [
+ {
+ name: 'empty',
+ boolean: false,
+ default: false,
+ help: 'Do not download files by default. Files must be synced manually.'
+ },
{
name: 'upload',
boolean: true,
@@ -40,6 +46,7 @@ function clone (opts) {
opts.key = parsed.key || opts._[0] // pass other links to resolver
opts.dir = parsed.dir
opts.showKey = opts['show-key'] // using abbr in option makes printed help confusing
+ opts.sparse = opts.empty
debug('clone()')
debug(Object.assign({}, opts, {key: '', _: null})) // don't show key
diff --git a/src/commands/pull.js b/src/commands/pull.js
index 791f1092..941c6a9e 100644
--- a/src/commands/pull.js
+++ b/src/commands/pull.js
@@ -13,6 +13,19 @@ module.exports = {
default: true,
help: 'announce your address on link (improves connection capability) and upload data to other downloaders.'
},
+ {
+ name: 'selectFromFile',
+ boolean: false,
+ default: '.datdownload',
+ help: 'Sync only the list of selected files or directories in the given file.',
+ abbr: 'select-from-file'
+ },
+ {
+ name: 'select',
+ boolean: false,
+ default: false,
+ help: 'Sync only the list of selected files or directories.'
+ },
{
name: 'show-key',
boolean: true,
@@ -28,6 +41,7 @@ function pull (opts) {
var neatLog = require('neat-log')
var archiveUI = require('../ui/archive')
var trackArchive = require('../lib/archive')
+ var selectiveSync = require('../lib/selective-sync')
var discoveryExit = require('../lib/discovery-exit')
var onExit = require('../lib/exit')
var parseArgs = require('../parse-args')
@@ -52,6 +66,7 @@ function pull (opts) {
neat.use(onExit)
neat.use(function (state, bus) {
state.opts = opts
+ selectiveSync(state, opts)
Dat(opts.dir, opts, function (err, dat) {
if (err && err.name === 'MissingError') return bus.emit('exit:warn', 'No existing archive in this directory. Use clone to download a new archive.')
diff --git a/src/commands/sync.js b/src/commands/sync.js
index 765b21af..458a6b93 100644
--- a/src/commands/sync.js
+++ b/src/commands/sync.js
@@ -20,6 +20,19 @@ module.exports = {
default: true,
abbr: 'ignore-hidden'
},
+ {
+ name: 'selectFromFile',
+ boolean: false,
+ default: '.datdownload',
+ help: 'Sync only the list of selected files or directories in the given file.',
+ abbr: 'select-from-file'
+ },
+ {
+ name: 'select',
+ boolean: false,
+ default: false,
+ help: 'Sync only the list of selected files or directories.'
+ },
{
name: 'watch',
boolean: true,
@@ -40,6 +53,7 @@ function sync (opts) {
var Dat = require('dat-node')
var neatLog = require('neat-log')
var archiveUI = require('../ui/archive')
+ var selectiveSync = require('../lib/selective-sync')
var trackArchive = require('../lib/archive')
var onExit = require('../lib/exit')
var parseArgs = require('../parse-args')
@@ -60,7 +74,7 @@ function sync (opts) {
neat.use(onExit)
neat.use(function (state, bus) {
state.opts = opts
-
+ selectiveSync(state, opts)
Dat(opts.dir, opts, function (err, dat) {
if (err && err.name === 'MissingError') return bus.emit('exit:warn', 'No existing archive in this directory.')
if (err) return bus.emit('exit:error', err)
diff --git a/src/lib/archive.js b/src/lib/archive.js
index 3e04b836..d2fdabaa 100644
--- a/src/lib/archive.js
+++ b/src/lib/archive.js
@@ -1,3 +1,6 @@
+var debug = require('debug')('dat')
+var path = require('path')
+var EventEmitter = require('events').EventEmitter
var doImport = require('./import-progress')
var stats = require('./stats')
var network = require('./network')
@@ -14,6 +17,7 @@ module.exports = function (state, bus) {
if (state.opts.http) serve(state, bus)
if (state.writable && state.opts.import) doImport(state, bus)
+ else if (state.opts.sparse) selectiveSync(state, bus)
else download(state, bus)
if (state.dat.archive.content) return bus.emit('archive:content')
@@ -26,3 +30,52 @@ module.exports = function (state, bus) {
state.hasContent = true
})
}
+
+function selectiveSync (state, bus) {
+ var archive = state.dat.archive
+ debug('sparse mode. downloading metadata')
+ var emitter = new EventEmitter()
+
+ function download (entry) {
+ debug('selected', entry)
+ archive.stat(entry, function (err, stat) {
+ if (err) return state.warnings.push(err.message)
+ if (stat.isDirectory()) downloadDir(entry, stat)
+ if (stat.isFile()) downloadFile(entry, stat)
+ })
+ }
+
+ function downloadDir (dirname, stat) {
+ debug('downloading dir', dirname)
+ archive.readdir(dirname, function (err, entries) {
+ if (err) return bus.emit('exit:error', err)
+ entries.forEach(function (entry) {
+ emitter.emit('download', path.join(dirname, entry))
+ })
+ })
+ }
+
+ function downloadFile (entry, stat) {
+ var start = stat.offset
+ var end = stat.offset + stat.blocks
+ state.selectedByteLength += stat.size
+ bus.emit('render')
+ if (start === 0 && end === 0) return
+ debug('downloading', entry, start, end)
+ archive.content.download({start, end}, function () {
+ debug('success', entry)
+ })
+ }
+
+ emitter.on('download', download)
+ if (state.opts.selectedFiles) state.opts.selectedFiles.forEach(download)
+
+ archive.metadata.update(function () {
+ return bus.emit('exit:warn', `Dat successfully created in empty mode. Download files using pull or sync.`)
+ })
+
+ archive.on('update', function () {
+ debug('archive update')
+ bus.emit('render')
+ })
+}
diff --git a/src/lib/selective-sync.js b/src/lib/selective-sync.js
new file mode 100644
index 00000000..3a27bb5e
--- /dev/null
+++ b/src/lib/selective-sync.js
@@ -0,0 +1,32 @@
+var fs = require('fs')
+var path = require('path')
+
+module.exports = function (state, opts) {
+ // selective sync stuff
+ var parsing = opts.selectFromFile !== '.datdownload' ? opts.selectFromFile : path.join(opts.dir, '.datdownload')
+ opts.selectedFiles = parseFiles(parsing)
+ if (opts.select && typeof opts.select === 'string') opts.selectedFiles = opts.select.split(',')
+ if (opts.selectedFiles) {
+ state.title = 'Syncing'
+ state.selectedByteLength = 0
+ opts.sparse = true
+ }
+ return state
+}
+
+function parseFiles (input) {
+ var parsed = null
+
+ try {
+ if (fs.statSync(input).isFile()) {
+ parsed = fs.readFileSync(input).toString().trim().split(/\r?\n/)
+ }
+ } catch (err) {
+ if (err && !err.name === 'ENOENT') {
+ console.error(err)
+ process.exit(1)
+ }
+ }
+
+ return parsed
+}
diff --git a/src/ui/archive.js b/src/ui/archive.js
index 9f8af1f2..292d2bb3 100644
--- a/src/ui/archive.js
+++ b/src/ui/archive.js
@@ -3,6 +3,7 @@ var pretty = require('prettier-bytes')
var chalk = require('chalk')
var downloadUI = require('./components/download')
var importUI = require('./components/import-progress')
+var warningsUI = require('./components/warnings')
var networkUI = require('./components/network')
var sourcesUI = require('./components/sources')
var keyEl = require('./elements/key')
@@ -15,6 +16,7 @@ module.exports = archiveUI
function archiveUI (state) {
if (!state.dat) return 'Starting Dat program...'
if (!state.writable && !state.hasContent) return 'Connecting to dat network...'
+ if (!state.warnings) state.warnings = []
var dat = state.dat
var stats = dat.stats.get()
@@ -27,7 +29,8 @@ function archiveUI (state) {
if (state.title) title += state.title
else if (state.writable) title += 'Sharing dat'
else title += 'Downloading dat'
- if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})`
+ if (state.opts.sparse) title += `: ${state.opts.selectedFiles.length} ${pluralize('file', state.opts.selectedFiles.length)} (${pretty(state.selectedByteLength)})`
+ else if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})`
else if (stats.version === 0) title += ': (empty archive)'
if (state.http && state.http.listening) title += `\nServing files over http at http://localhost:${state.http.port}`
@@ -48,6 +51,7 @@ function archiveUI (state) {
${progressView}
${state.opts.sources ? sourcesUI(state) : ''}
+ ${state.warnings ? warningsUI(state) : ''}
${state.exiting ? 'Exiting the Dat program...' : chalk.dim('Ctrl+C to Exit')}
`
}
diff --git a/src/ui/components/warnings.js b/src/ui/components/warnings.js
new file mode 100644
index 00000000..cdf94469
--- /dev/null
+++ b/src/ui/components/warnings.js
@@ -0,0 +1,9 @@
+var chalk = require('chalk')
+
+module.exports = function (state) {
+ var warning = ''
+ state.warnings.forEach(function (message) {
+ warning += `${chalk.yellow(`Warning: ${message}`)}\n`
+ })
+ return warning
+}