From 4be290281450a7fb5052db7896723fd9f52cffd6 Mon Sep 17 00:00:00 2001 From: Stephen Sawchuk Date: Mon, 13 Apr 2015 08:50:48 -0400 Subject: [PATCH] accept a directory to `upload` --- lib/storage/bucket.js | 171 +++++++++++++++++++++++++++++++----------- package.json | 1 + 2 files changed, 130 insertions(+), 42 deletions(-) diff --git a/lib/storage/bucket.js b/lib/storage/bucket.js index 248b4fc6b86..4195c469769 100644 --- a/lib/storage/bucket.js +++ b/lib/storage/bucket.js @@ -23,6 +23,7 @@ var async = require('async'); var extend = require('extend'); var fs = require('fs'); +var globby = require('globby'); var mime = require('mime-types'); var path = require('path'); @@ -44,6 +45,12 @@ var File = require('./file.js'); */ var util = require('../common/util.js'); +/** + * @const {number} + * @private + */ +var MAX_PARALLEL_UPLOADS = 5; + /** * @const {string} * @private @@ -664,8 +671,8 @@ Bucket.prototype.setMetadata = function(metadata, callback) { * will be uploaded to the File object's bucket and under the File object's * name. Lastly, when this argument is omitted, the file is uploaded to your * bucket using the name of the local file. - * @param {object=} options.metadata - Metadata to set for your file. - * @param {boolean=} options.resumable - Force a resumable upload. (default: + * @param {object} options.metadata - Metadata to set for your file. + * @param {boolean} options.resumable - Force a resumable upload. (default: * true for files larger than 5MB). Read more about resumable uploads * [here](http://goo.gl/1JWqCF). NOTE: This behavior is only possible with * this method, and not {module:storage/file#createWriteStream}. When @@ -730,61 +737,141 @@ Bucket.prototype.setMetadata = function(metadata, callback) { * }); */ Bucket.prototype.upload = function(localPath, options, callback) { + var self = this; + + var errors = []; + var files = []; + if (util.is(options, 'function')) { callback = options; options = {}; } - var newFile; - if (options.destination instanceof File) { - newFile = options.destination; - } else if (util.is(options.destination, 'string')) { - // Use the string as the name of the file. - newFile = this.file(options.destination); - } else { - // Resort to using the name of the incoming file. - newFile = this.file(path.basename(localPath)); - } + options = options || {}; - var metadata = options.metadata || {}; - var contentType = mime.contentType(path.basename(localPath)); + var globOptions = extend({}, options.globOptions, { nodir: true }); - if (contentType && !metadata.contentType) { - metadata.contentType = contentType; - } + globby(localPath, globOptions, function(err, filePaths) { + if (err) { + callback(err); + return; + } - var resumable; - if (util.is(options.resumable, 'boolean')) { - resumable = options.resumable; - upload(); - } else { - // Determine if the upload should be resumable if it's over the threshold. - fs.stat(localPath, function(err, fd) { - if (err) { - callback(err); - return; - } + var uploadFileFns = filePaths.map(function(filePath) { + return function(done) { + var fileName = path.basename(filePath); + + if (options.basePath) { + fileName = path.relative(options.basePath, filePath); + } + + var opts = extend({ destination: fileName }, options); - resumable = fd.size > RESUMABLE_THRESHOLD; + self.uploadFile(filePath, opts, function(err, file) { + if (err) { + errors.push(err); + } else { + files.push(file); + } - upload(); + done(options.force ? null : err || null); + }); + }; }); + + async.parallelLimit(uploadFileFns, MAX_PARALLEL_UPLOADS, function() { + if (options.force) { + callback(errors, files); + } else { + callback(errors[0], files); + } + }); + }); +}; + +Bucket.prototype.uploadDirectory = function(directoryPath, options, callback) { + if (util.is(options, 'function')) { + callback = options; + options = {}; } - function upload() { - fs.createReadStream(localPath) - .pipe(newFile.createWriteStream({ - validation: options.validation, - resumable: resumable, - metadata: metadata - })) - .on('error', function(err) { - callback(err); - }) - .on('complete', function() { - callback(null, newFile); + options = options || {}; + options.basePath = directoryPath; + + this.upload(path.join(directoryPath, '**/*'), options, callback); +}; + +Bucket.prototype.uploadFile = function(filePath, options, callback) { + var self = this; + + if (util.is(options, 'function')) { + callback = options; + options = {}; + } + + options = options || {}; + + if (!util.is(options.resumable, 'boolean')) { + // User didn't specify a preference of resumable or simple upload. Check the + // file's size to determine which to use. + if (!util.is(options.size, 'number')) { + fs.stat(filePath, function(err, stats) { + if (err) { + callback(err); + return; + } + + options.size = stats.size; + self.uploadFile(filePath, options, callback); }); + return; + } + + options.resumable = options.size > RESUMABLE_THRESHOLD; + } + + if (util.is(options.destination, 'string')) { + options.destination = this.file(options.destination); + } + + if (!options.destination) { + options.destination = this.file(path.basename(filePath)); } + + this.uploadFile_(filePath, options, callback); +}; + +/** + * Same signature as {module:storage/bucket#upload}, but simply uploads the file + * after determining its name. + * + * The `upload` function is a public-facing, pre-processor which can read files + * from a directory, then send them to this method. + * + * @private + * @borrows {module:storage/bucket#upload} as uploadFile_ + * + * @param {module:storage/file} options.destination - File destination. + */ +Bucket.prototype.uploadFile_ = function(filePath, options, callback) { + var file = options.destination; + var metadata = options.metadata || {}; + var contentType = mime.contentType(path.basename(filePath)); + + if (contentType && !metadata.contentType) { + metadata.contentType = contentType; + } + + fs.createReadStream(filePath) + .pipe(file.createWriteStream({ + validation: options.validation, + resumable: options.resumable, + metadata: metadata + })) + .on('error', callback) + .on('complete', function() { + callback(null, file); + }); }; /** diff --git a/package.json b/package.json index a50b0a9a72b..0485f527def 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "duplexify": "^3.2.0", "extend": "^2.0.0", "fast-crc32c": "^0.1.3", + "globby": "^2.0.0", "google-auth-library": "^0.9.4", "mime-types": "^2.0.8", "node-uuid": "^1.4.2",