add selective sync

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dat",
-  "version": "13.7.0",
+  "version": "13.8.0",
   "description": "Dat is the package manager for data. Easily share and version control data.",
   "keywords": [
     "dat",

src/commands/clone.js

@@ -7,6 +7,11 @@ module.exports = {
     'Usage: dat clone <link> [download-folder]'
   ].join('\n'),
   options: [
+    {
+      name: 'empty',
+      default: false,
+      help: 'Do not download files by default. Files must be synced manually.'
+    },
     {
       name: 'upload',
       boolean: true,
@@ -40,6 +45,7 @@ function clone (opts) {
   opts.key = parsed.key || opts._[0] // pass other links to resolver
   opts.dir = parsed.dir
   opts.showKey = opts['show-key'] // using abbr in option makes printed help confusing
+  opts.sparse = opts.empty
 
   debug('clone()')
   debug(Object.assign({}, opts, {key: '<private>', _: null})) // don't show key

src/commands/sync.js

@@ -20,6 +20,12 @@ module.exports = {
       default: true,
       abbr: 'ignore-hidden'
     },
+    {
+      name: 'select',
+      boolean: false,
+      default: false,
+      help: 'Sync only the list of selected files or directories.'
+    },
     {
       name: 'watch',
       boolean: true,
@@ -42,6 +48,7 @@ function sync (opts) {
   var archiveUI = require('../ui/archive')
   var trackArchive = require('../lib/archive')
   var onExit = require('../lib/exit')
+  var parseFiles = require('../parse-files')
   var parseArgs = require('../parse-args')
   var debug = require('debug')('dat')
 
@@ -61,6 +68,13 @@ function sync (opts) {
   neat.use(function (state, bus) {
     state.opts = opts
 
+    if (opts.select) {
+      state.title = 'Syncing'
+      state.selectedByteLength = 0
+      opts.select = parseFiles(opts.select)
+      opts.sparse = true
+    }
+
     Dat(opts.dir, opts, function (err, dat) {
       if (err && err.name === 'MissingError') return bus.emit('exit:warn', 'No existing archive in this directory.')
       if (err) return bus.emit('exit:error', err)

src/lib/archive.js

@@ -1,3 +1,4 @@
+var selectiveSync = require('./selective-sync')
 var doImport = require('./import-progress')
 var stats = require('./stats')
 var network = require('./network')
@@ -14,6 +15,7 @@ module.exports = function (state, bus) {
     if (state.opts.http) serve(state, bus)
 
     if (state.writable && state.opts.import) doImport(state, bus)
+    else if (state.opts.sparse) selectiveSync(state, bus)
     else download(state, bus)
 
     if (state.dat.archive.content) return bus.emit('archive:content')

src/lib/selective-sync.js

@@ -0,0 +1,52 @@
+var debug = require('debug')('dat')
+var path = require('path')
+var EventEmitter = require('events').EventEmitter
+
+module.exports = function selectiveSync (state, bus) {
+  var archive = state.dat.archive
+  debug('sparse mode. downloading metadata')
+  var emitter = new EventEmitter()
+
+  function download (entry) {
+    debug('selected', entry)
+    archive.stat(entry, function (err, stat) {
+      if (err) return bus.emit('exit:error', err)
+      if (stat.isDirectory()) downloadDir(entry, stat)
+      if (stat.isFile()) downloadFile(entry, stat)
+    })
+  }
+
+  function downloadDir (dirname, stat) {
+    debug('downloading dir', dirname)
+    archive.readdir(dirname, function (err, entries) {
+      if (err) return bus.emit('exit:error', err)
+      entries.forEach(function (entry) {
+        emitter.emit('download', path.join(dirname, entry))
+      })
+    })
+  }
+
+  function downloadFile (entry, stat) {
+    var start = stat.offset
+    var end = stat.offset + stat.blocks
+    state.selectedByteLength += stat.size
+    bus.emit('render')
+    if (start === 0 && end === 0) return 
+    debug('downloading', entry, start, end)
+    archive.content.download({start, end}, function () {
+      debug('success', entry)
+    })
+  }
+
+  emitter.on('download', download)
+  if (state.opts.select) state.opts.select.forEach(download)
+
+  archive.metadata.update(function () {
+    return bus.emit('exit:warn', `Dat successfully created in empty mode. Download files using pull or sync.`)
+  })
+
+  archive.on('update', function () {
+    debug('archive update')
+    bus.emit('render')
+  })
+}

src/parse-files.js

@@ -0,0 +1,18 @@
+var fs = require('fs')
+
+module.exports = function (input) {
+  var parsed = input.split(',')
+
+  try {
+    if (fs.statSync(input).isFile()) {
+      parsed = fs.readFileSync(input).toString().trim().split('\n')
+    }
+  } catch (err) {
+    if (err && !err.name === 'ENOENT') {
+      console.error(err)
+      process.exit(1)
+    }
+  }
+
+  return parsed
+}

src/ui/archive.js

@@ -27,7 +27,8 @@ function archiveUI (state) {
   if (state.title) title += state.title
   else if (state.writable) title += 'Sharing dat'
   else title += 'Downloading dat'
-  if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})`
+  if (state.opts.sparse) title += `: ${state.opts.select.length} ${pluralize('file', state.opts.select.length)} (${pretty(state.selectedByteLength)})`
+  else if (stats.version > 0) title += `: ${stats.files} ${pluralize('file', stats.file)} (${pretty(stats.byteLength)})`
   else if (stats.version === 0) title += ': (empty archive)'
   if (state.http && state.http.listening) title += `\nServing files over http at http://localhost:${state.http.port}`