From f9b7f4efb242a9af7ccec05b4521462a71fd4b39 Mon Sep 17 00:00:00 2001 From: Stephen Sawchuk Date: Mon, 18 Aug 2014 15:30:12 -0400 Subject: [PATCH] storage: docs. --- lib/storage/index.js | 384 ++++++++++++++++++++++++++++--------------- 1 file changed, 256 insertions(+), 128 deletions(-) diff --git a/lib/storage/index.js b/lib/storage/index.js index ce89cfa8083..f36c5cd2a42 100644 --- a/lib/storage/index.js +++ b/lib/storage/index.js @@ -14,6 +14,10 @@ * limitations under the License. */ +/** + * @module storage + */ + 'use strict'; var events = require('events'); @@ -22,45 +26,46 @@ var nodeutil = require('util'); var stream = require('stream'); var uuid = require('node-uuid'); +/** @type {module:common/connection} */ var conn = require('../common/connection.js'); + +/** @type {module:common/util} */ var util = require('../common/util.js'); /** * Required scopes for Google Cloud Storage API. - * @type {Array} + * @const {array} */ var SCOPES = ['https://www.googleapis.com/auth/devstorage.full_control']; +/** + * @const {string} + */ var STORAGE_BASE_URL = 'https://www.googleapis.com/storage/v1/b'; -var STORAGE_UPLOAD_BASE_URL = 'https://www.googleapis.com/upload/storage/v1/b'; -var reqStreamToCallback = function(st, callback) { - st.callback = util.noop; - st.on('error', function(err) { - callback(err); - }); - st.on('complete', function(resp) { - util.handleResp(null, resp, resp.body, callback); - }); -}; +/** + * @const {string} + */ +var STORAGE_UPLOAD_BASE_URL = 'https://www.googleapis.com/upload/storage/v1/b'; /** - * BufferStream is a readable stream implementation - * that stream the given buffer. + * Readable stream implementation to stream the given buffer. + * + * @private + * + * @constructor + * + * @param {buffer} buffer - The buffer to stream. */ function BufferStream(buffer) { stream.Readable.call(this); this.data = buffer; } -/** - * Inherit from stream.Readable. - */ nodeutil.inherits(BufferStream, stream.Readable); /** - * @private - * Pushes the provided buffer to the stream. + * Push the provided buffer to the stream. */ BufferStream.prototype._read = function() { this.push(this.data); @@ -68,10 +73,23 @@ BufferStream.prototype._read = function() { }; /** - * Read stream is a readable stream that streams the - * contents of a file. - * @param {Bucket} bucket Bucket the source file belongs to. - * @param {String} name Name of the file to read from. + * A readable stream that streams the contents of a file. + * + * @private + * + * @constructor + * @mixes {stream#Readable} + * + * @param {module:storage~Bucket} bucket - Bucket the source file belongs to. + * @param {string} name - Name of the file to read from. + * + * @example + * ```js + * var myBucket = new Bucket({ + * bucketName: 'my-bucket' + * }); + * var readStream = new ReadStream(myBucket, 'file/to/fetch.pdf'); + * ``` */ function ReadStream(bucket, name) { events.EventEmitter.call(this); @@ -83,40 +101,37 @@ function ReadStream(bucket, name) { this.open(); } -/** - * Inherits EventEmitter. - */ nodeutil.inherits(ReadStream, events.EventEmitter); +/** + * Open a connection to retrieve a file. + */ ReadStream.prototype.open = function() { var that = this; - var bucket = this.bucket; - var callback = function(err, req) { + this.bucket.stat(this.name, function(err, metadata) { if (err) { that.emit('error', err); return; } - that.remoteStream = bucket.conn.requester(req); - that.remoteStream.on('complete', function(resp) { - that.emit('complete', resp); + that.bucket.conn.createAuthorizedReq( + { uri: metadata.mediaLink }, function(err, req) { + if (err) { + that.emit('error', err); + return; + } + that.remoteStream = that.bucket.conn.requester(req); + that.remoteStream.on('complete', that.emit.bind(that, 'complete')); + that.emit('readable'); }); - that.emit('readable'); - }; - bucket.stat(this.name, function(err, metadata) { - if (err) { - that.emit('error', err); - return; - } - bucket.conn.createAuthorizedReq({ uri: metadata.mediaLink }, callback); }); }; /** - * Pipes the output to the destination stream - * with the provided options. - * @param {Stream} dest Destination stream to write to. - * @param {Object} opts Piping options. - * @return {Stream} Returns the destination stream. + * Pipe the output to the destination stream with the provided options. + * + * @param {stream} dest - Destination stream to write to. + * @param {object} opts - Piping options. + * @return {stream} */ ReadStream.prototype.pipe = function(dest, opts) { var that = this; @@ -125,11 +140,10 @@ ReadStream.prototype.pipe = function(dest, opts) { that.pipe(dest, opts); }); } - // Register an on-data listener instead of - // piping, so we can avoid writing if request - // ends up with a non-200 response. + // Register an on-data listener instead of piping, so we can avoid writing if + // the request ends up with a non-200 response. that.remoteStream.on('data', function(data) { - if(!that.errored) { + if (!that.errored) { that.emit('data', data); dest.write(data); } @@ -138,39 +152,74 @@ ReadStream.prototype.pipe = function(dest, opts) { }; /** - * A bucket is a Google Cloud Storage bucket. See the - * guide on https://developers.google.com/storage to - * create a bucket. + * Google Cloud Storage allows you to store data on Google infrastructure. See + * the guide on {@link https://developers.google.com/storage} to create a + * bucket. + * + * @throws if a bucket name isn't provided. + * + * @param {object} options - Configuration options. + * @param {string} options.bucketName - Name of the bucket. + * @param {string} options.keyFileName - Full path to the JSON key downloaded + * from the Google Developers Console. + * + * @example + * ```js + * var gcloud = require('gcloud'); + * var bucket; + * + * // From Google Compute Engine + * bucket = new gcloud.storage.Bucket({ + * bucketName: YOUR_BUCKET_NAME + * }); + * + * // From elsewhere + * bucket = new gcloud.storage.Bucket({ + * bucketName: YOUR_BUCKET_NAME, + * keyFilename: '/path/to/the/key.json' + * }); + * ``` */ -function Bucket(opts) { - if (!opts.bucketName) { +function Bucket(options) { + if (!options.bucketName) { throw Error('A bucket name is needed to use Google Cloud Storage'); } - this.bucketName = opts.bucketName; + this.bucketName = options.bucketName; this.conn = new conn.Connection({ - keyFilename: opts.keyFilename, + keyFilename: options.keyFilename, scopes: SCOPES }); } /** - * Lists files from the current bucket. - * @param {String} query.delimeter Results will contain only objects - * whose names, aside from the prefix, - * do not contain delimiter. Objects whose - * names, aside from the prefix, contain - * delimiter will have their name, - * truncated after the delimiter, returned - * in prefixes. Duplicate prefixes - * are omitted. - * @param {String} query.prefix Filters results to objects whose names - * begin with this prefix. - * @param {Number} query.maxResults Maximum number of items plus prefixes - * to return. - * @param {String} query.pageToken A previously-returned page token - * representing part of the larger set of - * results to view. - * @param {Function} callback Callback function. + * List files from the current bucket. + * + * @param {object=} query - Query object. + * @param {string} query.delimeter - Results will contain only objects whose + * names, aside from the prefix, do not contain delimiter. Objects whose + * names, aside from the prefix, contain delimiter will have their name + * truncated after the delimiter, returned in prefixes. Duplicate prefixes + * are omitted. + * @param {string} query.prefix - Filters results to objects whose names begin + * with this prefix. + * @param {number} query.maxResults - Maximum number of items plus prefixes to + * return. + * @param {string} query.pageToken - A previously-returned page token + * representing part of the larger set of results to view. + * @param {function} callback - The callback function. + * + * @example + * ```js + * bucket.list(function(err, files, nextQuery) { + * if (nextQuery) { + * // nextQuery will be non-null if there are more results. + * bucket.list(nextQuery, function(err, files, nextQuery) {}); + * } + * }); + * + * // Fetch using a query. + * bucket.list({ maxResults: 5 }, function(err, files, nextQuery) {}); + * ``` */ Bucket.prototype.list = function(query, callback) { if (arguments.length === 1) { @@ -190,12 +239,16 @@ Bucket.prototype.list = function(query, callback) { }); }; -// TODO: Bucket should implement writable Streaming API. - /** - * Stats a file. - * @param {String} name Name of the remote file. - * @param {Function} callback Callback. + * Stat a file. + * + * @param {string} name - Name of the remote file. + * @param {function} callback - The callback function. + * + * @example + * ```js + * bucket.stat('filename', function(err, metadata){}); + * ``` */ Bucket.prototype.stat = function(name, callback) { var path = util.format('o/{name}', { name: name }); @@ -203,13 +256,25 @@ Bucket.prototype.stat = function(name, callback) { }; /** - * Copies an existing file. - * @param {String} name Name of the existing file. - * @param {String} metadata.name Name of the destination file. - * @param {String?} metadata.bucket Optional destination bucket for - * the file. If none is provided, - * source's bucket name is used. - * @param {Function=} callback Optional callback. + * Copy an existing file. If no bucket name is provided for the destination + * file, the current bucket name will be used. + * + * @throws if the destination filename is not provided. + * + * @param {string} name - Name of the existing file. + * @param {object} metadata - Destination file metadata object. + * @param {string} metadata.name - Name of the destination file. + * @param {string=} metadata.bucket - Destination bucket for the file. If none + * is provided, the source's bucket name is used. + * @param {function=} callback - The callback function. + * + * @example + * ```js + * bucket.copy('filename', { + * bucket: 'destination-bucket', + * name: 'destination-filename' + * }, function(err) {}); + * ``` */ Bucket.prototype.copy = function(name, metadata, callback) { callback = callback || util.noop; @@ -230,57 +295,116 @@ Bucket.prototype.copy = function(name, metadata, callback) { }; /** - * Removes a file. - * @param {String} name Name of the file to remove. - * @param {Function} callback Callback function. + * Remove a file. + * + * @param {string} name - Name of the file to remove. + * @param {function} callback - The callback function. + * + * @example + * ```js + * bucket.remove('filename', function(err) {}); + * ``` */ Bucket.prototype.remove = function(name, callback) { var path = util.format('o/{name}', { name: name }); - this.makeReq('DELETE', path, null, true, function(err) { - callback(err); - }); + this.makeReq('DELETE', path, null, true, callback); }; /** - * Creates a readable stream to read contents of the - * provided remote file. - * @param {String} name Name of the remote file. - * @return {ReadStream} Readable stream. + * Create a readable stream to read contents of the provided remote file. It + * can be piped to a write stream, or listened to for 'data' and `complete` + * events to read a file's contents. + * + * @param {string} name - Name of the remote file. + * @return {ReadStream} + * + * @example + * ```js + * // Create a readable stream and write the file contents to "/path/to/file" + * bucket.createReadStream('filename') + * .pipe(fs.createWriteStream('/path/to/file')); + * ``` */ Bucket.prototype.createReadStream = function(name) { return new ReadStream(this, name); }; /** - * Writes the provided stream to the destination - * with optional metadata. - * @param {String} name Name of the remote file. - * @param {Object=} opts.data A string, buffer or readable stream. - * @param {string=} opts.filename Path of the source file. - * @param {Object=} opts.metadata Optional metadata. - * @param {Function} callback Callback function. + * Write the provided data to the destination with optional metadata. + * + * @param {string} name - Name of the remote file. + * @param {object} options - Configuration object. + * @param {String|Buffer|ReadableStream=} options.data - Data. + * @param {string=} options.filename - Path of the source file. + * @param {object=} options.metadata - Optional metadata. + * @param {function} callback - The callback function. + * + * @example + * ```js + * // Upload file.pdf + * bucket.write('filename', { + * filename: '/path/to/file.pdf', + * metadata: { + * // optional metadata + * } + * }, function(err) {}); + * + * // Upload a readable stream + * bucket.write('filename', { + * data: fs.createReadStream('/path/to/file.pdf') + * }, function(err) {}); + * + * // Upload "Hello World" as file contents. `data` can be any string or buffer + * bucket.write('filename', { + * data: 'Hello World' + * }, function(err) {}); + * ``` */ -Bucket.prototype.write = function(name, opts, callback) { - var that = this; - +Bucket.prototype.write = function(name, options, callback) { callback = callback || util.noop; - var metadata = opts.metadata || {}; - var stream = opts.data; - var isStringOrBuffer = typeof stream === 'string' || stream instanceof Buffer; + var metadata = options.metadata || {}; + var readStream = options.data; + var isStringOrBuffer = + typeof readStream === 'string' || readStream instanceof Buffer; - if (opts.filename) { - stream = fs.createReadStream(opts.filename); - } else if (stream && isStringOrBuffer) { - stream = new BufferStream(stream); + if (options.filename) { + readStream = fs.createReadStream(options.filename); + } else if (readStream && isStringOrBuffer) { + readStream = new BufferStream(readStream); } - if (!stream) { + if (!readStream) { // metadata only write - this.makeReq('PATCH', 'o/' + name, null, opts.metadata, callback); + this.makeReq('PATCH', 'o/' + name, null, metadata, callback); return; } + this.getRemoteStream_(name, metadata, function(err, remoteStream) { + if (err) { + callback(err); + return; + } + // TODO(jbd): High potential of multiple callback invokes. + readStream.pipe(remoteStream) + .on('error', callback); + remoteStream + .on('error', callback) + .on('complete', function(resp) { + util.handleResp(null, resp, resp.body, callback); + }); + }); +}; + +/** + * Get a remote stream to begin piping a readable stream to. + * + * @param {string} name - The desired name of the file. + * @param {object} metadata - File descriptive metadata. + * @param {function} callback - The callback function. + */ +Bucket.prototype.getRemoteStream_ = function(name, metadata, callback) { var boundary = uuid.v4(); + var that = this; metadata.contentType = metadata.contentType || 'text/plain'; this.conn.createAuthorizedReq({ method: 'POST', @@ -289,7 +413,8 @@ Bucket.prototype.write = function(name, opts, callback) { bucket: this.bucketName }), qs: { - name: name, uploadType: 'multipart' + name: name, + uploadType: 'multipart' }, headers: { 'Content-Type': 'multipart/related; boundary="' + boundary + '"' @@ -300,6 +425,13 @@ Bucket.prototype.write = function(name, opts, callback) { return; } var remoteStream = that.conn.requester(req); + remoteStream.callback = util.noop; + remoteStream.write('--' + boundary + '\n'); + remoteStream.write('Content-Type: application/json\n\n'); + remoteStream.write(JSON.stringify(metadata)); + remoteStream.write('\n\n'); + remoteStream.write('--' + boundary + '\n'); + remoteStream.write('Content-Type: ' + metadata.contentType + '\n\n'); var oldEndFn = remoteStream.end; remoteStream.end = function(data, encoding, callback) { data = data || ''; @@ -307,28 +439,24 @@ Bucket.prototype.write = function(name, opts, callback) { remoteStream.write(data, encoding, callback); oldEndFn.apply(this); }; - remoteStream.write('--' + boundary + '\n'); - remoteStream.write('Content-Type: application/json\n\n'); - remoteStream.write(JSON.stringify(metadata)); - remoteStream.write('\n\n'); - remoteStream.write('--' + boundary + '\n'); - remoteStream.write('Content-Type: ' + metadata.contentType + '\n\n'); - stream.pipe(remoteStream); - // TODO(jbd): High potential of multiple callback invokes. - stream.on('error', callback); - reqStreamToCallback(remoteStream, callback); + callback(null, remoteStream); }); }; /** - * Makes a new request object from the provided - * arguments, and wraps the callback to intercept - * non-successful responses. + * Make a new request object from the provided arguments and wrap the callback + * to intercept non-successful responses. + * + * @param {string} method - Action. + * @param {string} path - Request path. + * @param {*} query - Request query object. + * @param {*} body - Request body contents. + * @param {function} callback - The callback function. */ -Bucket.prototype.makeReq = function(method, path, q, body, callback) { +Bucket.prototype.makeReq = function(method, path, query, body, callback) { var reqOpts = { method: method, - qs: q, + qs: query, uri: util.format('{base}/{bucket}/{path}', { base: STORAGE_BASE_URL, bucket: this.bucketName,