Selective sync

.gitignore

@@ -3,10 +3,9 @@ node_modules
 tmp
 .idea
 data
-tests/fixtures/.dat
 yarn.lock
-tests/fixtures/.dat
-tests/fixtures/dat.json
-tests/**.db
-tests/.datrc-test
+test/fixtures/.dat
+test/fixtures/dat.json
+test/**.db
+test/.datrc-test
 package-lock.json

README.md

@@ -159,7 +159,7 @@ You can also also view the files online: [datproject.org/778f8d955175c92e4ced5e4
 
 Dat can share files from your computer to anywhere. If you have a friend going through this demo with you, try sharing to them! If not we'll see what we can do.
 
-Find a folder on your computer to share. Inside the folder can be anything, Dat can handle all sorts of files (Dat works with really big folders too!). 
+Find a folder on your computer to share. Inside the folder can be anything, Dat can handle all sorts of files (Dat works with really big folders too!).
 
 First, you can create a new dat inside that folder. Using the `dat create` command also walks us through making a `dat.json` file:
 
@@ -187,7 +187,7 @@ Get started using Dat today with the `share` and `clone` commands or read below
 ## Usage
 
 The first time you run a command, a `.dat` folder to store the Dat metadata.
-Once a Dat is created, you can run all the commands inside that folder, similar to git. 
+Once a Dat is created, you can run all the commands inside that folder, similar to git.
 
 Dat keep secret keys in the `~/.dat/secret_keys` folder. These are required to write to any dats you create.
 
@@ -232,10 +232,15 @@ Sync watched files for changes and imports updated files.
 
 #### Ignoring Files
 
-By default, dat will ignore any files in a `.datignore` file, similar to git. Dat also ignores all hidden folders and files.
+By default, dat will ignore any files in a `.datignore` file, similar to git. Each file should separated by a newline. Dat also ignores all hidden folders and files.
 
 Dat uses [dat-ignore](https://github.com/joehand/dat-ignore) to decide if a file should be ignored.
 
+#### Selecting Files
+
+By default, dat will download all files. If you want to only download a subset, you can create a `.datdownload` file which downloads only the files and folders specified. Each should be separated by a newline.
+
+
 ### Downloading
 
 Start downloading by running the `clone` command. This creates a folder, download the content and metadata, and a `.dat` folder inside. Once you started the download, you can resume using `clone` or the other download commands.

src/commands/clone.js

@@ -7,6 +7,12 @@ module.exports = {
     'Usage: dat clone <link> [download-folder]'
   ].join('\n'),
   options: [
+    {
+      name: 'empty',
+      boolean: false,
+      default: false,
+      help: 'Do not download files by default. Files must be synced manually.'
+    },
     {
       name: 'upload',
       boolean: true,
@@ -40,6 +46,7 @@ function clone (opts) {
   opts.key = parsed.key || opts._[0] // pass other links to resolver
   opts.dir = parsed.dir
   opts.showKey = opts['show-key'] // using abbr in option makes printed help confusing
+  opts.sparse = opts.empty
 
   debug('clone()')
   debug(Object.assign({}, opts, {key: '<private>', _: null})) // don't show key

src/commands/pull.js

@@ -13,6 +13,19 @@ module.exports = {
       default: true,
       help: 'announce your address on link (improves connection capability) and upload data to other downloaders.'
     },
+    {
+      name: 'selectFromFile',
+      boolean: false,
+      default: '.datdownload',
+      help: 'Sync only the list of selected files or directories in the given file.',
+      abbr: 'select-from-file'
+    },
+    {
+      name: 'select',
+      boolean: false,
+      default: false,
+      help: 'Sync only the list of selected files or directories.'
+    },
     {
       name: 'show-key',
       boolean: true,
@@ -28,6 +41,7 @@ function pull (opts) {
   var neatLog = require('neat-log')
   var archiveUI = require('../ui/archive')
   var trackArchive = require('../lib/archive')
+  var selectiveSync = require('../lib/selective-sync')
   var discoveryExit = require('../lib/discovery-exit')
   var onExit = require('../lib/exit')
   var parseArgs = require('../parse-args')
@@ -52,6 +66,7 @@ function pull (opts) {
   neat.use(onExit)
   neat.use(function (state, bus) {
     state.opts = opts
+    selectiveSync(state, opts)
 
     Dat(opts.dir, opts, function (err, dat) {
       if (err && err.name === 'MissingError') return bus.emit('exit:warn', 'No existing archive in this directory. Use clone to download a new archive.')

src/commands/sync.js

@@ -20,6 +20,19 @@ module.exports = {
       default: true,
       abbr: 'ignore-hidden'
     },
+    {
+      name: 'selectFromFile',
+      boolean: false,
+      default: '.datdownload',
+      help: 'Sync only the list of selected files or directories in the given file.',
+      abbr: 'select-from-file'
+    },
+    {
+      name: 'select',
+      boolean: false,
+      default: false,
+      help: 'Sync only the list of selected files or directories.'
+    },
     {
       name: 'watch',
       boolean: true,
@@ -40,6 +53,7 @@ function sync (opts) {
   var Dat = require('dat-node')
   var neatLog = require('neat-log')
   var archiveUI = require('../ui/archive')
+  var selectiveSync = require('../lib/selective-sync')
   var trackArchive = require('../lib/archive')
   var onExit = require('../lib/exit')
   var parseArgs = require('../parse-args')
@@ -60,7 +74,7 @@ function sync (opts) {
   neat.use(onExit)
   neat.use(function (state, bus) {
     state.opts = opts
-
+    selectiveSync(state, opts)
     Dat(opts.dir, opts, function (err, dat) {
       if (err && err.name === 'MissingError') return bus.emit('exit:warn', 'No existing archive in this directory.')
       if (err) return bus.emit('exit:error', err)

src/lib/archive.js

@@ -1,3 +1,6 @@
+var debug = require('debug')('dat')
+var path = require('path')
+var EventEmitter = require('events').EventEmitter
 var doImport = require('./import-progress')
 var stats = require('./stats')
 var network = require('./network')
@@ -14,6 +17,7 @@ module.exports = function (state, bus) {
     if (state.opts.http) serve(state, bus)
 
     if (state.writable && state.opts.import) doImport(state, bus)
+    else if (state.opts.sparse) selectiveSync(state, bus)
     else download(state, bus)
 
     if (state.dat.archive.content) return bus.emit('archive:content')
@@ -26,3 +30,52 @@ module.exports = function (state, bus) {
     state.hasContent = true
   })
 }
+
+function selectiveSync (state, bus) {
+  var archive = state.dat.archive
+  debug('sparse mode. downloading metadata')
+  var emitter = new EventEmitter()
+
+  function download (entry) {
+    debug('selected', entry)
+    archive.stat(entry, function (err, stat) {
+      if (err) return state.warnings.push(err.message)
+      if (stat.isDirectory()) downloadDir(entry, stat)
+      if (stat.isFile()) downloadFile(entry, stat)
+    })
+  }
+
+  function downloadDir (dirname, stat) {
+    debug('downloading dir', dirname)
+    archive.readdir(dirname, function (err, entries) {
+      if (err) return bus.emit('exit:error', err)
+      entries.forEach(function (entry) {
+        emitter.emit('download', path.join(dirname, entry))
+      })
+    })
+  }
+
+  function downloadFile (entry, stat) {
+    var start = stat.offset
+    var end = stat.offset + stat.blocks
+    state.selectedByteLength += stat.size
+    bus.emit('render')
+    if (start === 0 && end === 0) return
+    debug('downloading', entry, start, end)
+    archive.content.download({start, end}, function () {
+      debug('success', entry)
+    })
+  }
+
+  emitter.on('download', download)
+  if (state.opts.selectedFiles) state.opts.selectedFiles.forEach(download)
+
+  archive.metadata.update(function () {
+    return bus.emit('exit:warn', `Dat successfully created in empty mode. Download files using pull or sync.`)
+  })
+
+  archive.on('update', function () {
+    debug('archive update')
+    bus.emit('render')
+  })
+}

src/lib/selective-sync.js

@@ -0,0 +1,32 @@
+var fs = require('fs')
+var path = require('path')
+
+module.exports = function (state, opts) {
+  // selective sync stuff
+  var parsing = opts.selectFromFile !== '.datdownload' ? opts.selectFromFile : path.join(opts.dir, '.datdownload')
+  opts.selectedFiles = parseFiles(parsing)
+  if (opts.select && typeof opts.select === 'string') opts.selectedFiles = opts.select.split(',')
+  if (opts.selectedFiles) {
+    state.title = 'Syncing'
+    state.selectedByteLength = 0
+    opts.sparse = true
+  }
+  return state
+}
+
+function parseFiles (input) {
+  var parsed = null
+
+  try {
+    if (fs.statSync(input).isFile()) {
+      parsed = fs.readFileSync(input).toString().trim().split(/\r?\n/)
+    }
+  } catch (err) {
+    if (err && !err.name === 'ENOENT') {
+      console.error(err)
+      process.exit(1)
+    }
+  }
+
+  return parsed
+}

src/ui/archive.js

@@ -3,6 +3,7 @@ var pretty = require('prettier-bytes')
 var chalk = require('chalk')
 var downloadUI = require('./components/download')
 var importUI = require('./components/import-progress')
+var warningsUI = require('./components/warnings')
 var networkUI = require('./components/network')
 var sourcesUI = require('./components/sources')
 var keyEl = require('./elements/key')
@@ -15,6 +16,7 @@ module.exports = archiveUI
 function archiveUI (state) {
   if (!state.dat) return 'Starting Dat program...'
   if (!state.writable && !state.hasContent) return 'Connecting to dat network...'
+  if (!state.warnings) state.warnings = []
 
   var dat = state.dat
   var stats = dat.stats.get()
@@ -27,7 +29,8 @@ function archiveUI (state) {
   if (state.title) title += state.title
   else if (state.writable) title += 'Sharing dat'
   else title += 'Downloading dat'
-  if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})`
+  if (state.opts.sparse) title += `: ${state.opts.selectedFiles.length} ${pluralize('file', state.opts.selectedFiles.length)} (${pretty(state.selectedByteLength)})`
+  else if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})`
   else if (stats.version === 0) title += ': (empty archive)'
   if (state.http && state.http.listening) title += `\nServing files over http at http://localhost:${state.http.port}`
 
@@ -48,6 +51,7 @@ function archiveUI (state) {
 
     ${progressView}
     ${state.opts.sources ? sourcesUI(state) : ''}
+    ${state.warnings ? warningsUI(state) : ''}
     ${state.exiting ? 'Exiting the Dat program...' : chalk.dim('Ctrl+C to Exit')}
   `
 }

src/ui/components/warnings.js

@@ -0,0 +1,9 @@
+var chalk = require('chalk')
+
+module.exports = function (state) {
+  var warning = ''
+  state.warnings.forEach(function (message) {
+    warning += `${chalk.yellow(`Warning: ${message}`)}\n`
+  })
+  return warning
+}