From 009ae901986f0642f3e8372f3eba38b7148703b6 Mon Sep 17 00:00:00 2001 From: Stephen Sawchuk Date: Mon, 15 Sep 2014 11:37:08 -0400 Subject: [PATCH] support global configuration. --- docs/components/docs/docs.html | 76 ++++++++++----- docs/components/docs/docs.js | 5 +- docs/css/main.css | 15 +++ lib/common/util.js | 56 +++++++---- lib/datastore/dataset.js | 18 ++-- lib/datastore/index.js | 95 ++++++++++++++++-- lib/datastore/query.js | 73 +++++++------- lib/index.js | 106 ++++++++++++++++---- lib/pubsub/index.js | 5 +- lib/storage/index.js | 144 ++++++++++++++++++++++++---- package.json | 1 + regression/datastore.js | 2 +- regression/storage.js | 4 +- test/common/util.js | 36 ++++--- test/datastore/dataset.js | 57 ++++++++--- test/datastore/entity.js | 2 +- test/datastore/index.js | 2 +- test/datastore/transaction.js | 2 +- test/storage/index.js | 48 ++++++---- test/testdata/privateKeyFile-2.json | 7 ++ 20 files changed, 562 insertions(+), 192 deletions(-) create mode 100644 test/testdata/privateKeyFile-2.json diff --git a/docs/components/docs/docs.html b/docs/components/docs/docs.html index 87805578a3a8..893b9f923aa5 100644 --- a/docs/components/docs/docs.html +++ b/docs/components/docs/docs.html @@ -13,39 +13,65 @@

Node.js

-
-

{{module[0].toUpperCase() + module.substr(1)}}

+

{{module[0].toUpperCase() + module.substr(1)}}

+

+
+ ▹ + ▿ +
+ Getting Started with gcloud +

+

First, install gcloud with npm and require it into your project:

$ npm install --save gcloud
var gcloud = require('gcloud');
+

+ There are a couple of ways to use the gcloud module. +

+

+ If you are running your app on Google App Engine or Google Compute Engine, you won't need to worry about supplying connection configuration options to gcloud— we figure that out for you. +

+

+ However, if you're running your app elsewhere, you will need to provide this information. +

+
+// App Engine and Compute Engine +var gcloud = require('gcloud'); -
-

- The gcloud.datastore object gives you some convenience methods, as well as exposes a Dataset function. This will allow you to create a Dataset, which is the object from which you will interact with the Google Cloud Datastore. -

-
+// Elsewhere +var gcloud = require('gcloud')({ + keyFilename: '/path/to/keyfile.json' +});
+

+ In any environment, you are free to provide these and other default properties, which eventually will be passed to the gcloud sub-modules (Datastore, Storage, etc.). +

+
+
+
+

Overview

+

+ The gcloud.datastore object gives you some convenience methods, as well as exposes a dataset function. This will allow you to create a dataset, which is the object from which you will interact with the Google Cloud Datastore. +

+
var datastore = gcloud.datastore; -var dataset = new datastore.Dataset();
-

- See the Dataset documentation for examples of how to query the datastore, save entities, run a transaction, and others. -

-
- -
-

- The gcloud.storage object contains a Bucket object, which is how you will interact with your Google Cloud Storage bucket. -

-
-var storage = gcloud.storage; -var bucket = new storage.Bucket({ - bucketName: 'MyBucket' +var dataset = datastore.dataset({ + projectId: 'myProject', + keyFilename: '/path/to/keyfile.json' });
-

- See examples below for more on how to upload a file, read from your bucket's files, create signed URLs, and more. -

-
+

+ See the Dataset documentation for examples of how to query the datastore, save entities, run a transaction, and others. +

+
+
+

Overview

+

+ The gcloud.storage object contains a bucket object, which is how you will interact with your Google Cloud Storage bucket. See the guide on Google Cloud Storage to create a bucket. +

+

+ See examples below for more on how to access your bucket to upload a file, read its files, create signed URLs, and more. +

b.name; + return a.constructor ? -1: a.name > b.name; }); }; } diff --git a/docs/css/main.css b/docs/css/main.css index 7870e6a3968a..12a46ff478e0 100755 --- a/docs/css/main.css +++ b/docs/css/main.css @@ -625,6 +625,21 @@ h2, h3 { display: block; } +.sub-heading { + color: #5d6061; + margin: 0; +} + +.toggler { + float: left; + min-width: 15px; + margin: auto; +} + +.toggle { + cursor: pointer; +} + /* Page Title */ diff --git a/lib/common/util.js b/lib/common/util.js index 5b1b8f53e0d7..8158afce99df 100644 --- a/lib/common/util.js +++ b/lib/common/util.js @@ -21,36 +21,50 @@ * @module common/util */ +var extend = require('extend'); var util = require('util'); /** - * Extend a base object with properties from another. + * Extend a global configuration object with user options provided at the time + * of sub-module instantiation. * - * @param {object} from - The base object. - * @param {object} to - The object to extend with. + * Connection details currently come in two ways: `credentials` or + * `keyFilename`. Because of this, we have a special exception when overriding a + * global configuration object. If a user provides either to the global + * configuration, then provides another at submodule instantiation-time, the + * latter is preferred. + * + * @param {object} globalConfig - The global configuration object. + * @param {object} overrides - The instantiation-time configuration object. * @return {object} - * ``` + * + * @example + * // globalConfig = { + * // credentials: {...} + * // } + * Datastore.prototype.dataset = function(options) { + * // options = { + * // keyFilename: 'keyfile.json' + * // } + * return extendGlobalConfig(this.config, options); + * // returns: + * // { + * // keyFilename: 'keyfile.json' + * // } + * }; */ -function extend(from, to) { - if (from === null || typeof from !== 'object') { - return from; - } - if (from.constructor === Date || from.constructor === Function || - from.constructor === String || from.constructor === Number || - from.constructor === Boolean) { - return new from.constructor(from); - } - if (from.constructor !== Object && from.constructor !== Array) { - return from; - } - to = to || new from.constructor(); - for (var name in from) { - to[name] = to[name] ? extend(from[name], null) : to[name]; +function extendGlobalConfig(globalConfig, overrides) { + var options = extend({}, globalConfig); + var hasGlobalConnection = options.credentials || options.keyFilename; + var isOverridingConnection = overrides.credentials || overrides.keyFilename; + if (hasGlobalConnection && isOverridingConnection) { + delete options.credentials; + delete options.keyFilename; } - return to; + return extend(true, {}, options, overrides); } -module.exports.extend = extend; +module.exports.extendGlobalConfig = extendGlobalConfig; /** * Wrap an array around a non-Array object. If given an Array, it is returned. diff --git a/lib/datastore/dataset.js b/lib/datastore/dataset.js index ed0e71b3db8d..5ef2b8609b9b 100644 --- a/lib/datastore/dataset.js +++ b/lib/datastore/dataset.js @@ -74,7 +74,7 @@ var SCOPES = [ * @alias module:datastore/dataset * * @param {object=} options - * @param {string} options.projectId - Dataset ID. This is your project ID from + * @param {string=} options.projectId - Dataset ID. This is your project ID from * the Google Developers Console. * @param {string=} options.keyFilename - Full path to the JSON key downloaded * from the Google Developers Console. Alternatively, you may provide a @@ -84,12 +84,16 @@ var SCOPES = [ * @param {string} options.namespace - Namespace to isolate transactions to. * * @example - * var dataset = new datastore.Dataset({ + * var dataset = datastore.dataset({ * projectId: 'my-project', * keyFilename: '/path/to/keyfile.json' * }); */ function Dataset(options) { + if (!(this instanceof Dataset)) { + return new Dataset(options); + } + options = options || {}; this.connection = new conn.Connection({ @@ -97,7 +101,7 @@ function Dataset(options) { keyFilename: options.keyFilename, scopes: SCOPES }); - this.id = options.projectId; + this.projectId = options.projectId; this.namespace = options.namespace; this.transaction = this.createTransaction_(); } @@ -108,13 +112,11 @@ function Dataset(options) { * You may also specify a configuration object to define a namespace and path. * * @example - * var key; - * * // Create a key from the dataset's namespace. - * key = dataset.key('Company', 123); + * var company123 = dataset.key('Company', 123); * * // Create a key from a provided namespace and path. - * key = dataset.key({ + * var nsCompany123 = dataset.key({ * namespace: 'My-NS', * path: ['Company', 123] * }); @@ -347,7 +349,7 @@ Dataset.prototype.allocateIds = function(incompleteKey, n, callback) { * @private */ Dataset.prototype.createTransaction_ = function() { - return new Transaction(this.connection, this.id); + return new Transaction(this.connection, this.projectId); }; module.exports = Dataset; diff --git a/lib/datastore/index.js b/lib/datastore/index.js index 1ee836ac2e6d..9bcd7044409e 100644 --- a/lib/datastore/index.js +++ b/lib/datastore/index.js @@ -26,22 +26,97 @@ */ var entity = require('./entity'); -/*! - * @alias module:datastore +/** + * @type module:common/util + * @private + */ +var util = require('../common/util.js'); + +/** + * @type module:datastore/dataset + * @private */ -var datastore = {}; +var Dataset = require('./dataset'); +/*! Developer Documentation + * + * Invoking the Datastore class allows you to provide configuration up-front. + * This configuration will be used for future invokations of the returned + * `dataset` method. + * + * @example + * var datastore = require('gcloud/lib/datastore')({ + * keyFilename: '/path/to/keyfile.json' + * }); + * + * var dataset = datastore.dataset(); + * // equal to: + * // datastore.dataset({ + * // keyFilename: '/path/to/keyfile.json' + * // }); + */ /** - * @see {module:datastore/dataset} + * The example below will demonstrate the different usage patterns your app may + * need to support to retrieve a datastore object. + * + * @alias module:datastore + * @constructor * * @example * var gcloud = require('gcloud'); - * var datastore = gcloud.datastore; + * + * // Providing configuration details up-front. + * var myProject = gcloud({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'my-project' + * }); + * + * var dataset = myProject.datastore.dataset(); + * + * + * // Overriding default configuration details. + * var anotherDataset = myProject.datastore.dataset({ + * keyFilename: '/path/to/another/keyfile.json' + * }); + * + * + * // Not using a default configuration. + * var myOtherProject = gcloud.datastore.dataset({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'my-project' + * }); + */ +function Datastore(config) { + this.config = config || {}; +} + +/*! Developer Documentation + * + * Use this static method to create a dataset without any pre-configured + * options. + * + * @example + * var datastore = require('gcloud/lib/datastore'); * * // Create a Dataset object. - * var dataset = new datastore.Dataset(); + * var dataset = datastore.dataset({ + * keyFilename: '/path/to/keyfile.json' + * }); */ -datastore.Dataset = require('./dataset'); +Datastore.dataset = Dataset; + +/*! Developer Documentation + * + * Create a dataset using the instance method when you want to use your + * pre-configured options from the Datastore instance. + * + * @param {object=} options - Configuration object. + * @return {module:datastore/dataset} + */ +Datastore.prototype.dataset = function(options) { + // Mix in global config data to the provided options. + return new Dataset(util.extendGlobalConfig(this.config, options)); +}; /** * Helper function to get a Datastore Integer object. @@ -55,7 +130,7 @@ datastore.Dataset = require('./dataset'); * // Create an Integer. * var sevenInteger = gcloud.datastore.int(7); */ -datastore.int = function(value) { +Datastore.int = function(value) { return new entity.Int(value); }; @@ -71,8 +146,8 @@ datastore.int = function(value) { * // Create a Double. * var threeDouble = gcloud.datastore.double(3.0); */ -datastore.double = function(value) { +Datastore.double = function(value) { return new entity.Double(value); }; -module.exports = datastore; +module.exports = Datastore; diff --git a/lib/datastore/query.js b/lib/datastore/query.js index f5fac2baf110..4258c977110b 100644 --- a/lib/datastore/query.js +++ b/lib/datastore/query.js @@ -20,6 +20,8 @@ 'use strict'; +var extend = require('extend'); + /** * @type {module:common/util} * @private @@ -42,17 +44,15 @@ var util = require('../common/util.js'); * @param {string[]} kinds - Kinds to query. * * @example - * var query; - * * // If your dataset was scoped to a namespace at initialization, your query * // will likewise be scoped to that namespace. - * query = dataset.createQuery(['Lion', 'Chimp']); + * dataset.createQuery(['Lion', 'Chimp']); * * // However, you may override the namespace per query. - * query = dataset.createQuery('AnimalNamespace', ['Lion', 'Chimp']); + * dataset.createQuery('AnimalNamespace', ['Lion', 'Chimp']); * * // You may also remove the namespace altogether. - * query = dataset.createQuery(null, ['Lion', 'Chimp']); + * dataset.createQuery(null, ['Lion', 'Chimp']); */ function Query(namespace, kinds) { if (!kinds) { @@ -100,13 +100,17 @@ function Query(namespace, kinds) { */ Query.prototype.filter = function(filter, value) { // TODO: Add filter validation. - var q = util.extend(this, new Query()); + var query = extend(new Query(), this); filter = filter.trim(); var fieldName = filter.replace(/[>|<|=|>=|<=]*$/, '').trim(); var op = filter.substr(fieldName.length, filter.length).trim(); - q.filters = q.filters || []; - q.filters.push({ name: fieldName, op: op, val: value }); - return q; + query.filters = query.filters || []; + query.filters.push({ + name: fieldName, + op: op, + val: value + }); + return query; }; /** @@ -121,9 +125,9 @@ Query.prototype.filter = function(filter, value) { * var ancestoryQuery = query.hasAncestor(dataset.key('Parent', 123)); */ Query.prototype.hasAncestor = function(key) { - var q = util.extend(this, new Query()); - this.filters.push({ name: '__key__', op: 'HAS_ANCESTOR', val: key }); - return q; + var query = extend(new Query(), this); + query.filters.push({ name: '__key__', op: 'HAS_ANCESTOR', val: key }); + return query; }; /** @@ -143,15 +147,15 @@ Query.prototype.hasAncestor = function(key) { * var companiesDescending = companyQuery.order('-size'); */ Query.prototype.order = function(property) { - var q = util.extend(this, new Query()); + var query = extend(new Query(), this); var sign = '+'; if (property[0] === '-' || property[0] === '+') { sign = property[0]; property = property.substr(1); } - q.orders = q.orders || []; - q.orders.push({ name: property, sign: sign }); - return q; + query.orders = query.orders || []; + query.orders.push({ name: property, sign: sign }); + return query; }; /** @@ -164,10 +168,9 @@ Query.prototype.order = function(property) { * var groupedQuery = companyQuery.groupBy(['name', 'size']); */ Query.prototype.groupBy = function(fieldNames) { - var fields = util.arrayize(fieldNames); - var q = util.extend(this, new Query()); - q.groupByVal = fields; - return q; + var query = extend(new Query(), this); + query.groupByVal = util.arrayize(fieldNames); + return query; }; /** @@ -183,9 +186,9 @@ Query.prototype.groupBy = function(fieldNames) { * var selectQuery = companyQuery.select(['name', 'size']); */ Query.prototype.select = function(fieldNames) { - var q = util.extend(this, new Query()); - q.selectVal = fieldNames; - return q; + var query = extend(new Query(), this); + query.selectVal = fieldNames; + return query; }; /** @@ -203,9 +206,9 @@ Query.prototype.select = function(fieldNames) { * var startQuery = companyQuery.start(cursorToken); */ Query.prototype.start = function(start) { - var q = util.extend(this, new Query()); - q.startVal = start; - return q; + var query = extend(new Query(), this); + query.startVal = start; + return query; }; /** @@ -223,9 +226,9 @@ Query.prototype.start = function(start) { * var endQuery = companyQuery.end(cursorToken); */ Query.prototype.end = function(end) { - var q = util.extend(this, new Query()); - q.endVal = end; - return q; + var query = extend(new Query(), this); + query.endVal = end; + return query; }; /** @@ -241,9 +244,9 @@ Query.prototype.end = function(end) { * var limitQuery = companyQuery.limit(10); */ Query.prototype.limit = function(n) { - var q = util.extend(this, new Query()); - q.limitVal = n; - return q; + var query = extend(new Query(), this); + query.limitVal = n; + return query; }; /** @@ -259,9 +262,9 @@ Query.prototype.limit = function(n) { * var offsetQuery = companyQuery.offset(100); */ Query.prototype.offset = function(n) { - var q = util.extend(this, new Query()); - q.offsetVal = n; - return q; + var query = extend(new Query(), this); + query.offsetVal = n; + return query; }; module.exports = Query; diff --git a/lib/index.js b/lib/index.js index c693970ce710..530f42352d9c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -20,15 +20,87 @@ 'use strict'; -/*! +/** + * @type {module:datastore} + * @private + */ +var Datastore = require('./datastore'); + +/** + * @type {module:pubsub} + * @private + */ +var pubsub = require('./pubsub'); + +/** + * @type {module:storage} + * @private + */ +var Storage = require('./storage'); + +/** + * There are two key ways to use the `gcloud` module. + * + * 1. Provide connection & configuration details up-front. + * + * 2. Provide them at the time of instantiation of sub-modules, e.g. a Datastore + * dataset, a Cloud Storage bucket, etc. + * + * If you are using Google App Engine or Google Compute Engine, your connection + * details are handled for you. You won't have to worry about specifying these, + * however you may find it advantageous to provide a `projectId` at + * instantiation. + * + * To specify the configuration details up-front, invoke the gcloud module, + * passing in an object. The properties defined on this object will be persisted + * to the instantiation of every sub-module. It's important to note that you can + * override any of these defaults when you invoke a sub-module later. + * + * You can invoke this module as many times as your project warrants. Each time, + * your provided configuration will remain isolated to the returned gcloud + * module. + * * @alias module:gcloud + * @constructor + * + * @param {object} config - Connection configuration options. + * @param {string=} config.keyFilename - Full path to the JSON key downloaded + * from the Google Developers Console. Alternatively, you may provide a + * `credentials` object. + * @param {object=} config.credentials - Credentials object. + * @param {string} config.credentials.client_email + * @param {string} config.credentials.private_key + * + * @example + * var gcloud = require('gcloud')({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'my-project' + * }); + * + * var dataset = gcloud.datastore.dataset(); + * // equal to: + * // gcloud.datastore.dataset({ + * // keyFilename: '/path/to/keyfile.json', + * // projectId: 'my-project' + * // }); + * + * var bucket = gcloud.storage.bucket({ + * bucketName: 'PhotosBucket', + * // properties may be overriden: + * keyFilename: '/path/to/other/keyfile.json' + * }); */ -var gcloud = {}; +function gcloud(config) { + return { + datastore: new Datastore(config), + storage: new Storage(config) + }; +} /** - * [Google Cloud Datastore]{@link https://developers.google.com/datastore/} is a - * fully managed, schemaless database for storing non-relational data. Use this - * object to create a Dataset to interact with your data, an "Int", and a + * [Google Cloud Datastore]{@link https://developers.google.com/datastore/} is + * a fully managed, schemaless database for storing non-relational data. Use + * this object to create a Dataset to interact with your data, an "Int", and a * "Double" representation. * * @type {module:datastore} @@ -46,20 +118,20 @@ var gcloud = {}; * // int: function() {} * // } */ -gcloud.datastore = require('./datastore'); +gcloud.datastore = Datastore; /** * **Experimental** * * [Google Cloud Pub/Sub]{@link https://developers.google.com/pubsub/overview} - * is a reliable, many-to-many, asynchronous messaging service from Google Cloud - * Platform. + * is a reliable, many-to-many, asynchronous messaging service from Google + * Cloud Platform. * * Note: Google Cloud Pub/Sub API is available as a Limited Preview and the * client library we provide is currently experimental. The API and/or the * client might be changed in backward-incompatible ways. This API is not - * subject to any SLA or deprecation policy. Request to be whitelisted to use it - * by filling the + * subject to any SLA or deprecation policy. Request to be whitelisted to use + * it by filling the * [Limited Preview application form]{@link http://goo.gl/sO0wTu}. * * @type {module:pubsub} @@ -75,17 +147,19 @@ gcloud.datastore = require('./datastore'); * keyFilename: '/path/to/the/key.json' * }); */ -gcloud.pubsub = require('./pubsub'); +gcloud.pubsub = pubsub; /** - * Google Cloud Storage allows you to store data on Google infrastructure. Read + * Google Cloud Storage allows you to store data on Google infrastructure. + * Read * [Google Cloud Storage API docs]{@link https://developers.google.com/storage/} * for more information. * - * You need to create a Google Cloud Storage bucket to use this client library. + * You need to create a Google Cloud Storage bucket to use this client + * library. * Follow the steps on - * [Google Cloud Storage docs]{@link https://developers.google.com/storage/} to - * create a bucket. + * [Google Cloud Storage docs]{@link https://developers.google.com/storage/} + * to create a bucket. * @type {module:storage} * @@ -100,6 +174,6 @@ gcloud.pubsub = require('./pubsub'); * // Bucket: function() {} * // } */ -gcloud.storage = require('./storage'); +gcloud.storage = Storage; module.exports = gcloud; diff --git a/lib/pubsub/index.js b/lib/pubsub/index.js index facc8c52bd48..31d67eec1e32 100644 --- a/lib/pubsub/index.js +++ b/lib/pubsub/index.js @@ -21,6 +21,7 @@ 'use strict'; var events = require('events'); +var extend = require('extend'); var nodeutil = require('util'); /** @type {module:common/connection} */ @@ -252,7 +253,7 @@ Connection.prototype.listSubscriptions = function(query, callback) { callback = query; query = {}; } - var q = util.extend({}, query); + var q = extend({}, query); if (q.filterByTopic) { q.query = 'pubsub.googleapis.com/topic in (' + @@ -350,7 +351,7 @@ Connection.prototype.listTopics = function(query, callback) { callback = query; query = {}; } - var q = util.extend({}, query); + var q = extend({}, query); q.query = 'cloud.googleapis.com/project in (' + this.fullProjectName_() + ')'; this.makeReq('GET', 'topics', q, true, function(err, result) { if (err) { diff --git a/lib/storage/index.js b/lib/storage/index.js index 6f3eb8c5d9b0..93b22a389735 100644 --- a/lib/storage/index.js +++ b/lib/storage/index.js @@ -22,6 +22,7 @@ var crypto = require('crypto'); var duplexify = require('duplexify'); +var extend = require('extend'); var stream = require('stream'); var uuid = require('node-uuid'); @@ -56,13 +57,115 @@ var STORAGE_BASE_URL = 'https://www.googleapis.com/storage/v1/b'; */ var STORAGE_UPLOAD_BASE_URL = 'https://www.googleapis.com/upload/storage/v1/b'; + +/*! Developer Documentation + * + * Invoke this method to create a new Storage object bound with pre-determined + * configuration options. For each object that can be created (e.g., a bucket), + * there is an equivalent static and instance method. While they are classes, + * they can be instantiated without use of the `new` keyword. + * @param {object} config - Configuration object. + */ /** - * Google Cloud Storage allows you to store data on Google infrastructure. See - * the guide on {@link https://developers.google.com/storage} to create a - * bucket. + * To access your Cloud Storage buckets, you will use the `bucket` function + * returned from this `storage` object. + * + * The example below will demonstrate the different usage patterns your app may + * need to connect to `gcloud` and access your bucket. * * @alias module:storage + * @constructor + * + * @example + * var gcloud = require('gcloud'); + * + * // From Google Compute Engine and Google App Engine: + * + * // Access `storage` through the `gcloud` module directly. + * var musicBucket = gcloud.storage.bucket({ + * bucketName: 'MusicBucket' + * }); + * + * // Elsewhere: + * + * // Provide configuration details up-front. + * var myProject = gcloud({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'my-project' + * }); + * + * var albums = myProject.storage.bucket({ + * bucketName: 'AlbumsBucket' + * }); + * + * var photos = myProject.storage.bucket({ + * bucketName: 'PhotosBucket' + * }); + * + * + * // Override default configuration details. + * var records = myProject.storage.bucket({ + * bucketName: 'RecordsBucket', + * keyFilename: '/path/to/another/keyfile.json', + * }); + * + * + * // Specify all options at instantiation. + * var misc = gcloud.storage.bucket({ + * keyFilename: '/path/to/keyfile.json', + * bucketName: 'MiscBucket' + * }); + */ +function Storage(config) { + this.config = config || {}; +} + +/*! Developer Documentation + * + * Static method to create a Bucket without any pre-configured options. + * + * @example + * var gcloud = require('gcloud'); + * + * var Albums = gcloud.storage.bucket({ + * bucketName: 'AlbumsBucket', + * keyFilename: '/path/to/keyfile.json' + * }); + */ +Storage.bucket = Bucket; + +/*! Developer Documentation + * + * Instance method for creating a Bucket. Options configured at instantiation of + * the Storage class will be passed through, allowing for overridden options + * specified here. + * + * @param {object} options + * @return {Bucket} + * + * @example + * var gcloud = require('gcloud')({ + * keyFilename: '/path/to/keyfile.json' + * }); + * + * var Albums = gcloud.storage.bucket({ + * bucketName: 'AlbumsBucket' + * }); + * + * var Photos = gcloud.storage.bucket({ + * bucketName: 'PhotosBucket' + * }); + */ +Storage.prototype.bucket = function(options) { + // Mix in instance config data to the provided options. + return new Bucket(util.extendGlobalConfig(this.config, options)); +}; + +module.exports = Storage; +/** + * Create a Bucket object to interact with a Google Cloud Storage bucket. + * * @throws if a bucket name isn't provided. * * @param {object} options - Configuration options. @@ -75,26 +178,29 @@ var STORAGE_UPLOAD_BASE_URL = 'https://www.googleapis.com/upload/storage/v1/b'; * * @example * var gcloud = require('gcloud'); - * var storage = gcloud.storage; - * var bucket; * * // From Google Compute Engine - * bucket = new storage.Bucket({ - * bucketName: YOUR_BUCKET_NAME + * var Albums = gcloud.storage.bucket({ + * bucketName: 'Albums' * }); * * // From elsewhere - * bucket = new storage.Bucket({ - * bucketName: YOUR_BUCKET_NAME, - * keyFilename: '/path/to/the/key.json' + * var Photos = gcloud.storage.bucket({ + * bucketName: 'PhotosBucket', + * keyFilename: '/path/to/keyfile.json' * }); */ function Bucket(options) { + if (!(this instanceof Bucket)) { + return new Bucket(options); + } + if (!options.bucketName) { throw Error('A bucket name is needed to use Google Cloud Storage'); } + this.bucketName = options.bucketName; - this.conn = new conn.Connection({ + this.connection = new conn.Connection({ credentials: options.credentials, keyFilename: options.keyFilename, scopes: SCOPES @@ -141,7 +247,7 @@ Bucket.prototype.list = function(query, callback) { } var nextQuery = null; if (resp.nextPageToken) { - nextQuery = util.extend({}, query); + nextQuery = extend({}, query); nextQuery.pageToken = resp.nextPageToken; } callback(null, resp.items, nextQuery); @@ -250,7 +356,7 @@ Bucket.prototype.getSignedUrl = function(options, callback) { resource: options.resource }); - this.conn.getCredentials(function(err, credentials) { + this.connection.getCredentials(function(err, credentials) { if (err) { callback(err); return; @@ -299,13 +405,13 @@ Bucket.prototype.createReadStream = function(name) { dup.emit('error', err); return; } - that.conn.createAuthorizedReq( + that.connection.createAuthorizedReq( { uri: metadata.mediaLink }, function(err, req) { if (err) { dup.emit('error', err); return; } - dup.setReadable(that.conn.requester(req)); + dup.setReadable(that.connection.requester(req)); }); }); return dup; @@ -401,7 +507,7 @@ Bucket.prototype.getWritableStream_ = function(name, metadata, callback) { var boundary = uuid.v4(); var that = this; metadata.contentType = metadata.contentType || 'text/plain'; - this.conn.createAuthorizedReq({ + this.connection.createAuthorizedReq({ method: 'POST', uri: util.format('{base}/{bucket}/o', { base: STORAGE_UPLOAD_BASE_URL, @@ -419,7 +525,7 @@ Bucket.prototype.getWritableStream_ = function(name, metadata, callback) { callback(err); return; } - var remoteStream = that.conn.requester(req); + var remoteStream = that.connection.requester(req); remoteStream.callback = util.noop; remoteStream.write('--' + boundary + '\n'); remoteStream.write('Content-Type: application/json\n\n'); @@ -462,9 +568,7 @@ Bucket.prototype.makeReq_ = function(method, path, query, body, callback) { if (body) { reqOpts.json = body; } - this.conn.req(reqOpts, function(err, res, body) { + this.connection.req(reqOpts, function(err, res, body) { util.handleResp(err, res, body, callback); }); }; - -module.exports.Bucket = Bucket; diff --git a/package.json b/package.json index 8e514c96c45b..b5a6d8fda14f 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ ], "dependencies": { "duplexify": "^3.1.2", + "extend": "^1.3.0", "gapitoken": "^0.1.3", "node-uuid": "^1.4.1", "protobufjs": "^3.4.0", diff --git a/regression/datastore.js b/regression/datastore.js index 3a1ed3f86ecb..0179bdacd71b 100644 --- a/regression/datastore.js +++ b/regression/datastore.js @@ -22,7 +22,7 @@ var env = require('./env.js'); var assert = require('assert'); var datastore = require('../lib/datastore'); -var ds = new datastore.Dataset(env); +var ds = datastore.dataset(env); var entity = require('../lib/datastore/entity.js'); describe('datastore', function() { diff --git a/regression/storage.js b/regression/storage.js index 6789f2d666a5..ab39409b21e8 100644 --- a/regression/storage.js +++ b/regression/storage.js @@ -26,9 +26,9 @@ var request = require('request'); var tmp = require('tmp'); var env = require('./env.js'); -var gcloud = require('../lib'); +var storage = require('../lib/storage'); -var bucket = new gcloud.storage.Bucket(env); +var bucket = storage.bucket(env); var files = { logo: { diff --git a/test/common/util.js b/test/common/util.js index 8eb99d463cd0..c56494ab7804 100644 --- a/test/common/util.js +++ b/test/common/util.js @@ -21,20 +21,6 @@ var assert = require('assert'); var util = require('../../lib/common/util.js'); -describe('extend', function() { - it ('should return null for null input', function() { - var copy = util.extend(null, {}); - assert.strictEqual(copy, null); - }); - - it('should return a new Date for Date input', function() { - var now = new Date(); - var copy = util.extend(now, {}); - assert.notStrictEqual(copy, now); - assert.strictEqual(copy.toString(), now.toString()); - }); -}); - describe('arrayize', function() { it('should arrayize if the input is not an array', function(done) { var o = util.arrayize('text'); @@ -43,6 +29,28 @@ describe('arrayize', function() { }); }); +describe('extendGlobalConfig', function() { + it('should favor `keyFilename` when `credentials` is global', function() { + var globalConfig = { credentials: {} }; + var options = util.extendGlobalConfig(globalConfig, { + keyFilename: 'key.json' + }); + assert.deepEqual(options, { keyFilename: 'key.json' }); + }); + + it('should favor `credentials` when `keyFilename` is global', function() { + var globalConfig = { keyFilename: 'key.json' }; + var options = util.extendGlobalConfig(globalConfig, { credentials: {} }); + assert.deepEqual(options, { credentials: {} }); + }); + + it('should not modify original object', function() { + var globalConfig = { keyFilename: 'key.json' }; + util.extendGlobalConfig(globalConfig, { credentials: {} }); + assert.deepEqual(globalConfig, { keyFilename: 'key.json' }); + }); +}); + describe('handleResp', function() { it('should handle errors', function(done) { var defaultErr = new Error('new error'); diff --git a/test/datastore/dataset.js b/test/datastore/dataset.js index e442928f8fe0..ae2325e9bb58 100644 --- a/test/datastore/dataset.js +++ b/test/datastore/dataset.js @@ -20,21 +20,48 @@ var assert = require('assert'); var ByteBuffer = require('bytebuffer'); +var gcloud = require('../../lib'); var datastore = require('../../lib').datastore; var entity = require('../../lib/datastore/entity.js'); var mockRespGet = require('../testdata/response_get.json'); var Transaction = require('../../lib/datastore/transaction.js'); describe('Dataset', function() { + it('should not require connection details', function() { + var credentials = require('../testdata/privateKeyFile.json'); + var project = gcloud({ + projectId: 'test-project', + credentials: credentials + }); + var ds = project.datastore.dataset({ hi: 'there' }); + assert.equal(ds.projectId, 'test-project'); + assert.deepEqual(ds.connection.credentials, credentials); + }); + + it('should allow overriding connection details', function() { + var projectCredentials = require('../testdata/privateKeyFile.json'); + var uniqueCredentials = require('../testdata/privateKeyFile-2.json'); + var project = gcloud({ + projectId: 'test-project', + credentials: projectCredentials + }); + var ds = project.datastore.dataset({ + projectId: 'another-project', + credentials: uniqueCredentials + }); + assert.equal(ds.projectId, 'another-project'); + assert.deepEqual(ds.connection.credentials, uniqueCredentials); + }); + it('should return a key scoped by namespace', function() { - var ds = new datastore.Dataset({ projectId: 'test', namespace: 'my-ns' }); + var ds = datastore.dataset({ projectId: 'test', namespace: 'my-ns' }); var key = ds.key('Company', 1); assert.equal(key.namespace, 'my-ns'); assert.deepEqual(key.path, ['Company', 1]); }); it('should allow namespace specification when creating a key', function() { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); var key = ds.key({ namespace: 'custom-ns', path: ['Company', 1] @@ -44,7 +71,7 @@ describe('Dataset', function() { }); it('should get by key', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); ds.transaction.makeReq = function(method, proto, typ, callback) { assert.equal(method, 'lookup'); assert.equal(proto.key.length, 1); @@ -61,7 +88,7 @@ describe('Dataset', function() { }); it('should multi get by keys', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); ds.transaction.makeReq = function(method, proto, typ, callback) { assert.equal(method, 'lookup'); assert.equal(proto.key.length, 1); @@ -80,7 +107,7 @@ describe('Dataset', function() { }); it('should continue looking for deferred results', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); var key = ds.key('Kind', 5732568548769792); var key2 = ds.key('Kind', 5732568548769792); var lookupCount = 0; @@ -103,7 +130,7 @@ describe('Dataset', function() { }); it('should delete by key', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); ds.transaction.makeReq = function(method, proto, typ, callback) { assert.equal(method, 'commit'); assert.equal(!!proto.mutation.delete, true); @@ -113,7 +140,7 @@ describe('Dataset', function() { }); it('should multi delete by keys', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); ds.transaction.makeReq = function(method, proto, typ, callback) { assert.equal(method, 'commit'); assert.equal(proto.mutation.delete.length, 2); @@ -126,7 +153,7 @@ describe('Dataset', function() { }); it('should save with incomplete key', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); ds.transaction.makeReq = function(method, proto, typ, callback) { assert.equal(method, 'commit'); assert.equal(proto.mutation.insert_auto_id.length, 1); @@ -137,7 +164,7 @@ describe('Dataset', function() { }); it('should save with keys', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); ds.transaction.makeReq = function(method, proto, typ, callback) { assert.equal(method, 'commit'); assert.equal(proto.mutation.upsert.length, 2); @@ -153,7 +180,7 @@ describe('Dataset', function() { }); it('should produce proper allocate IDs req protos', function(done) { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); ds.transaction.makeReq = function(method, proto, typ, callback) { assert.equal(method, 'allocateIds'); assert.equal(proto.key.length, 1); @@ -174,7 +201,7 @@ describe('Dataset', function() { }); it('should throw if trying to allocate IDs with complete keys', function() { - var ds = new datastore.Dataset({ projectId: 'test' }); + var ds = datastore.dataset({ projectId: 'test' }); assert.throws(function() { ds.allocateIds(ds.key('Kind', 123)); }); @@ -185,7 +212,7 @@ describe('Dataset', function() { var transaction; beforeEach(function() { - ds = new datastore.Dataset({ projectId: 'test' }); + ds = datastore.dataset({ projectId: 'test' }); ds.createTransaction_ = function() { transaction = new Transaction(); transaction.makeReq = function(method, proto, typ, callback) { @@ -221,8 +248,8 @@ describe('Dataset', function() { var dsWithNs; beforeEach(function() { - ds = new datastore.Dataset({ projectId: 'test' }); - dsWithNs = new datastore.Dataset({ + ds = datastore.dataset({ projectId: 'test' }); + dsWithNs = datastore.dataset({ projectId: 'test', namespace: 'my-ns' }); @@ -269,7 +296,7 @@ describe('Dataset', function() { }; beforeEach(function() { - ds = new datastore.Dataset({ projectId: 'test' }); + ds = datastore.dataset({ projectId: 'test' }); query = ds.createQuery('Kind'); }); diff --git a/test/datastore/entity.js b/test/datastore/entity.js index f3a789b36777..2ac8fa9ea305 100644 --- a/test/datastore/entity.js +++ b/test/datastore/entity.js @@ -322,7 +322,7 @@ describe('entityToEntityProto', function() { describe('queryToQueryProto', function() { it('should support filters and ancestory filtering', function(done) { - var ds = new datastore.Dataset({ projectId: 'project-id' }); + var ds = datastore.dataset({ projectId: 'project-id' }); var q = ds.createQuery('Kind1') .filter('name =', 'John') .hasAncestor(new entity.Key({ path: [ 'Kind2', 'somename' ] })); diff --git a/test/datastore/index.js b/test/datastore/index.js index a71fa6e3913e..e39377c7f8b5 100644 --- a/test/datastore/index.js +++ b/test/datastore/index.js @@ -39,7 +39,7 @@ var datastore = require('sandboxed-module') describe('Datastore', function() { it('should expose Dataset class', function() { - assert.equal(typeof datastore.Dataset, 'function'); + assert.equal(typeof datastore.dataset, 'function'); }); it('should expose Int builder', function() { diff --git a/test/datastore/transaction.js b/test/datastore/transaction.js index 4b9097dae516..c273c7a1e1de 100644 --- a/test/datastore/transaction.js +++ b/test/datastore/transaction.js @@ -26,7 +26,7 @@ describe('Transaction', function() { var transaction; beforeEach(function() { - ds = new datastore.Dataset({ projectId: 'test' }); + ds = datastore.dataset({ projectId: 'test' }); transaction = ds.createTransaction_(null, 'test'); }); diff --git a/test/storage/index.js b/test/storage/index.js index dd1c8548cf56..eda3489ca406 100644 --- a/test/storage/index.js +++ b/test/storage/index.js @@ -14,30 +14,48 @@ * limitations under the License. */ -/*global describe, it */ +/*global describe, it, beforeEach */ 'use strict'; var assert = require('assert'); -var storage = require('../../lib').storage; +var gcloud = require('../../lib'); +var storage = require('../../lib/storage'); +var credentials = require('../testdata/privateKeyFile.json'); var noop = function() {}; -function createBucket() { - return new storage.Bucket({ - bucketName: 'bucket-name', - email: 'xxx@email.com', - credentials: require('../testdata/privateKeyFile.json') +describe('Bucket', function() { + var bucket; + + beforeEach(function() { + bucket = storage.bucket({ + bucketName: 'bucket-name', + credentials: credentials + }); + }); + + it('should not require connection details', function() { + var project = gcloud({ credentials: credentials }); + var aBucket = project.storage.bucket({ bucketName: 'test' }); + assert.deepEqual(aBucket.connection.credentials, credentials); + }); + + it('should allow overriding connection details', function() { + var uniqueCredentials = require('../testdata/privateKeyFile-2.json'); + var project = gcloud({ credentials: credentials }); + var aBucket = project.storage.bucket({ + bucketName: 'another-bucket', + credentials: uniqueCredentials + }); + assert.deepEqual(aBucket.connection.credentials, uniqueCredentials); }); -} -describe('Bucket', function() { it('should throw if a bucket name is not passed', function() { - assert.throws(function() { new storage.Bucket(); }, Error); + assert.throws(storage.bucket, Error); }); it('should list without a query', function(done) { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body) { assert.strictEqual(method, 'GET'); assert.strictEqual(path, 'o'); @@ -49,7 +67,6 @@ describe('Bucket', function() { }); it('should list with a query', function(done) { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body) { assert.strictEqual(method, 'GET'); assert.strictEqual(path, 'o'); @@ -61,7 +78,6 @@ describe('Bucket', function() { }); it('should return nextQuery if more results', function() { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body, callback) { callback(null, { nextPageToken: 'next-page-token', items: [] }); }; @@ -72,7 +88,6 @@ describe('Bucket', function() { }); it('should return no nextQuery if no more results', function() { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body, callback) { callback(null, { items: [] }); }; @@ -82,7 +97,6 @@ describe('Bucket', function() { }); it('should stat a file', function(done) { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body) { assert.strictEqual(method, 'GET'); assert.strictEqual(path, 'o/file-name'); @@ -94,7 +108,6 @@ describe('Bucket', function() { }); it('should copy a file', function(done) { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body) { assert.strictEqual(method, 'POST'); assert.strictEqual(path, 'o/file-name/copyTo/b/new-bucket/o/new-name'); @@ -107,7 +120,6 @@ describe('Bucket', function() { }); it('should use the same bucket if nothing else is provided', function(done) { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body) { assert.strictEqual(method, 'POST'); assert.strictEqual(path, 'o/file-name/copyTo/b/bucket-name/o/new-name'); @@ -120,7 +132,6 @@ describe('Bucket', function() { }); it('should remove a file', function(done) { - var bucket = createBucket(); bucket.makeReq_ = function(method, path, q, body) { assert.strictEqual(method, 'DELETE'); assert.strictEqual(path, 'o/file-name'); @@ -132,7 +143,6 @@ describe('Bucket', function() { }); it('should create a signed url', function(done) { - var bucket = createBucket(); bucket.getSignedUrl({ action: 'read', resource: 'filename', diff --git a/test/testdata/privateKeyFile-2.json b/test/testdata/privateKeyFile-2.json new file mode 100644 index 000000000000..f462a3ea951b --- /dev/null +++ b/test/testdata/privateKeyFile-2.json @@ -0,0 +1,7 @@ +{ + "private_key_id": "8", + "private_key": "-----BEGIN RSA PRIVATE KEY-----IIBOgIBAAJBAK8Q+ToR4tWGshaKYRHKJ3ZmMUF6jjwCS/u1A8v1tFbQiVpBlxYB\npaNcT2ENEXBGdmWqr8VwSl0NBIKyq4p0rhsCAQMCQHS1+3wL7I5ZzA8G62Exb6RE\nINZRtCgBh/0jV91OeDnfQUc07SE6vs31J8m7qw/rxeB3E9h6oGi9IVRebVO+9zsC\nIQDWb//KAzrSOo0P0yktnY57UF9Q3Y26rulWI6LqpsxZDwIhAND/cmlg7rUz34Pf\nSmM61lJEmMEjKp8RB/xgghzmCeI1AiEAjvVVMVd8jCcItTdwyRO0UjWU4JOz0cnw\n5BfB8cSIO18CIQCLVPbw60nOIpUClNxCJzmMLbsrbMcUtgVS6wFomVvsIwIhAK+A\nYqT6WwsMW2On5l9di+RPzhDT1QdGyTI5eFNS+GxY\n-----END RSA PRIVATE KEY-----", + "client_email": "secondpart@firstpart.com", + "client_id": "9", + "type": "service_account" +}