Skip to content

Commit

Permalink
storage: add gzip option
Browse files Browse the repository at this point in the history
  • Loading branch information
stephenplusplus committed Jul 28, 2015
1 parent 7efd2a8 commit ef9a138
Show file tree
Hide file tree
Showing 7 changed files with 126 additions and 20 deletions.
16 changes: 15 additions & 1 deletion lib/storage/bucket.js
Original file line number Diff line number Diff line change
Expand Up @@ -875,6 +875,8 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
* will be uploaded to the File object's bucket and under the File object's
* name. Lastly, when this argument is omitted, the file is uploaded to your
* bucket using the name of the local file.
* @param {boolean} options.gzip - Automatically gzip the file. This will set
* `options.metadata.contentEncoding` to `gzip`.
* @param {object=} options.metadata - Metadata to set for your file.
* @param {boolean=} options.resumable - Force a resumable upload. (default:
* true for files larger than 5MB). Read more about resumable uploads
Expand Down Expand Up @@ -927,6 +929,17 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
* });
*
* //-
* // You can also have a file gzip'd on the fly.
* //-
* bucket.upload('index.html', { gzip: true }, function(err, file) {
* // Your bucket now contains:
* // - "index.html" (automatically compressed with gzip)
*
* // Downloading the file with `file.download` will automatically decode the
* // file.
* });
*
* //-
* // You may also re-use a File object, {module:storage/file}, that references
* // the file you wish to create or overwrite.
* //-
Expand Down Expand Up @@ -990,7 +1003,8 @@ Bucket.prototype.upload = function(localPath, options, callback) {
.pipe(newFile.createWriteStream({
validation: options.validation,
resumable: resumable,
metadata: metadata
metadata: metadata,
gzip: options.gzip
}))
.on('error', function(err) {
callback(err);
Expand Down
72 changes: 55 additions & 17 deletions lib/storage/file.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ var request = require('request').defaults({
});
var streamEvents = require('stream-events');
var through = require('through2');
var zlib = require('zlib');

/**
* @type {module:storage/acl}
Expand Down Expand Up @@ -611,8 +612,10 @@ File.prototype.createReadStream = function(options) {
* uploaded.
*
* @param {object=} options - Configuration object.
* @param {object=} options.metadata - Set the metadata for this file.
* @param {boolean=} options.resumable - Force a resumable upload. NOTE: When
* @param {boolean} options.gzip - Automatically gzip the file. This will set
* `options.metadata.contentEncoding` to `gzip`.
* @param {object} options.metadata - Set the metadata for this file.
* @param {boolean} options.resumable - Force a resumable upload. NOTE: When
* working with streams, the file format and size is unknown until it's
* completely consumed. Because of this, it's best for you to be explicit
* for what makes sense given your input. Read more about resumable uploads
Expand Down Expand Up @@ -642,6 +645,24 @@ File.prototype.createReadStream = function(options) {
* });
*
* //-
* // <h4>Uploading a File with gzip compression</h4>
* //-
* var fs = require('fs');
* var htmlFile = myBucket.file('index.html');
*
* fs.createReadStream('/Users/stephen/site/index.html')
* .pipe(htmlFile.createWriteStream({ gzip: true }))
* .on('error', function(err) {})
* .on('complete', function(metadata) {
* // The file upload is complete.
* });
*
* //-
* // Downloading the file with `createReadStream` will automatically decode the
* // file.
* //-
*
* //-
* // <h4>Uploading a File with Metadata</h4>
* //
* // One last case you may run into is when you want to upload a file to your
Expand All @@ -661,13 +682,23 @@ File.prototype.createReadStream = function(options) {
* }
* }
* }))
* .on('error', function(err) {});
* .on('error', function(err) {})
* .on('complete', function(metadata) {
* // The file upload is complete.
* });
*/
File.prototype.createWriteStream = function(options) {
options = options || {};

var that = this;

var gzip = options.gzip;

var metadata = options.metadata || {};
if (gzip) {
metadata.contentEncoding = 'gzip';
}

var validations = ['crc32c', 'md5'];
var validation;

Expand All @@ -693,9 +724,11 @@ File.prototype.createWriteStream = function(options) {
var localCrc32cHash;
var localMd5Hash = crypto.createHash('md5');

var dup = streamEvents(duplexify());
var writableStream = streamEvents(duplexify());

var throughStream = through(function(chunk, enc, next) {
var throughStream = through();

var validationStream = through(function(chunk, enc, next) {
if (crc32c) {
localCrc32cHash = crc.calculate(chunk, localCrc32cHash);
}
Expand All @@ -708,25 +741,30 @@ File.prototype.createWriteStream = function(options) {
next();
});

validationStream.on('end', function() {
if (crc32c) {
localCrc32cHash = new Buffer([localCrc32cHash]).toString('base64');
}

if (md5) {
localMd5Hash = localMd5Hash.digest('base64');
}
});

throughStream
.on('end', function() {
if (crc32c) {
localCrc32cHash = new Buffer([localCrc32cHash]).toString('base64');
}

if (md5) {
localMd5Hash = localMd5Hash.digest('base64');
}
})
.pipe(gzip ? zlib.createGzip() : through())

.pipe(validationStream)

.pipe(dup)
.pipe(writableStream)

// Wait until we've received data to determine what upload technique to use.
.once('writing', function() {
if (util.is(options.resumable, 'boolean') && !options.resumable) {
that.startSimpleUpload_(dup, metadata);
if (options.resumable === false) {
that.startSimpleUpload_(writableStream, metadata);
} else {
that.startResumableUpload_(dup, metadata);
that.startResumableUpload_(writableStream, metadata);
}
})

Expand Down
Binary file modified system-test/data/long-html-file.html
Binary file not shown.
Binary file added system-test/data/long-html-file.html.gz
Binary file not shown.
32 changes: 30 additions & 2 deletions system-test/storage.js
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,11 @@ var files = {
big: {
path: 'system-test/data/three-mb-file.tif'
},
gzip: {
html: {
path: 'system-test/data/long-html-file.html'
},
gzip: {
path: 'system-test/data/long-html-file.html.gz'
}
};

Expand Down Expand Up @@ -535,16 +538,41 @@ describe('storage', function() {
});
});

it('should gzip a file on the fly and download it', function(done) {
var options = {
gzip: true
};

var expectedContents = fs.readFileSync(files.html.path, 'utf-8');

bucket.upload(files.html.path, options, function(err, file) {
assert.ifError(err);

file.download(function(err, contents) {
assert.ifError(err);
assert.strictEqual(contents.toString(), expectedContents);
file.delete(done);
});
});
});

it('should upload a gzipped file and download it', function(done) {
var options = {
metadata: {
contentEncoding: 'gzip'
}
};

var expectedContents = fs.readFileSync(files.html.path, 'utf-8');

bucket.upload(files.gzip.path, options, function(err, file) {
assert.ifError(err);
file.download(done);

file.download(function(err, contents) {
assert.ifError(err);
assert.strictEqual(contents.toString(), expectedContents);
file.delete(done);
});
});
});

Expand Down
15 changes: 15 additions & 0 deletions test/storage/bucket.js
Original file line number Diff line number Diff line change
Expand Up @@ -960,6 +960,21 @@ describe('Bucket', function() {
bucket.upload(filepath, options, assert.ifError);
});

it('should allow specifying options.gzip', function(done) {
var fakeFile = new FakeFile(bucket, 'file-name');
var options = { destination: fakeFile, gzip: true };
fakeFile.createWriteStream = function(options) {
var ws = new stream.Writable();
ws.write = util.noop;
setImmediate(function() {
assert.strictEqual(options.gzip, true);
done();
});
return ws;
};
bucket.upload(filepath, options, assert.ifError);
});

it('should allow specifying options.resumable', function(done) {
var fakeFile = new FakeFile(bucket, 'file-name');
var options = { destination: fakeFile, resumable: false };
Expand Down
11 changes: 11 additions & 0 deletions test/storage/file.js
Original file line number Diff line number Diff line change
Expand Up @@ -920,6 +920,17 @@ describe('File', function() {
writable.write('data');
});

it('should set metadata.contentEncoding with gzip', function(done) {
var writable = file.createWriteStream({ gzip: true });

file.startResumableUpload_ = function(stream, metadata) {
assert.strictEqual(metadata.contentEncoding, 'gzip');
done();
};

writable.write('data');
});

describe('validation', function() {
var data = 'test';

Expand Down

0 comments on commit ef9a138

Please sign in to comment.