test(storage): add regression tests.

CONTRIBUTING.md

@@ -13,6 +13,7 @@ To run the regression tests, first create and configure a project following the
 After that, set the following environment variables:
 
 - **GCLOUD_TESTS_PROJECT_ID**: Developers Console project's ID (e.g. bamboo-shift-455)
+- **GCLOUD_TESTS_BUCKET_NAME**: The name of the bucket to use for the Cloud Storage API tests
 - **GCLOUD_TESTS_KEY**: The path to the JSON key file.
 
 Lastly, create the indexes used in the datastore regression tests using the [gcloud command-line tool](https://developers.google.com/cloud/sdk/gcloud/) and the indexes that you can find in `regression/data/index/yaml`:

README.md

@@ -6,7 +6,7 @@ Node idiomatic client for Google Cloud services. Work in progress... Watch the r
 
 This client supports the following Google Cloud services:
 
-* [Google Cloud Datastore](https://developers.google.com/datastore/) (also allows access to the collections of your existing AppEngine apps)
+* [Google Cloud Datastore](https://developers.google.com/datastore/)
 * [Google Cloud Storage](https://cloud.google.com/products/cloud-storage/)
 * [Google Cloud Pub/Sub](https://developers.google.com/pubsub/)
 * Planned but not yet started: [Google Compute Engine](https://developers.google.com/compute), and [Google BigQuery](https://developers.google.com/bigquery/)
@@ -67,11 +67,7 @@ The downloaded file contains credentials you'll need for authorization.
 
 ### Google Cloud Datastore
 
-Google Cloud Datastore is a NoSQL database with the
-convenience of a traditional RDBMS in terms of strong
-consistency guarantees and high availability. It's also known
-as Megastore. Its performance characteristics are explained
-in detail on [Megastore: Providing Scalable, Highly Available Storage for Interactive Services](http://www.cidrdb.org/cidr2011/Papers/CIDR11_Paper32.pdf).
+[Google Cloud Datastore](https://developers.google.com/datastore/) is a fully managed, schemaless database for storing non-relational data. Cloud Datastore automatically scales with your users and supports ACID transactions, high availability of reads and writes, strong consistency for reads and ancestor queries, and eventual consistency for all other queries.
 
 #### Configuration
 
@@ -101,11 +97,11 @@ TODO
 Get operations require a valid key to retrieve the key identified entity from Datastore. Skip to the "Querying" section if you'd like to learn more about querying against Datastore.
 
 ~~~~ js
-ds.get(['Company', 123], function(err, key, obj) {
+ds.get(['Company', 123], function(err, entities) {
 
 });
 // alternatively, you can retrieve multiple entities at once.
-ds.getAll([key1, key2, ...], function(err, keys, objs) {
+ds.getAll([key1, key2, ...], function(err, entities) {
 
 });
 ~~~~
@@ -150,7 +146,7 @@ also supported.
 ~~~~ js
 // retrieves 5 companies
 var q = ds.createQuery('Company').limit(5);
-ds.runQuery(q, function(err, keys, objs, nextQuery) {
+ds.runQuery(q, function(err, entities, nextQuery) {
     // nextQuery is not null if there are more results.
     if (nextQuery) {
         ds.runQuery(nextQuery, callback);
@@ -172,6 +168,14 @@ var q = ds.createQuery('Company')
     .filter('size <', 400);
 ~~~~
 
+To filter by key, use `__key__` for the property name. Filtering on keys
+stored as properties is not currently supported.
+
+~~~~ js
+var q = ds.createQuery('Company')
+    .filter('__key__ =', ['Company', 'Google'])
+~~~~
+
 In order to filter by ancestors, use `hasAncestor` helper.
 
 ~~~ js
@@ -341,16 +345,21 @@ A bucket object allows you to write a readable stream, a file and a buffer
 as file contents.
 
 ~~~~ js
-// Uploads file.pdf
-bucket.writeFile(
-    filename, '/path/to/file.pdf', { contentType: 'application/pdf' }, callback);
+// Uploads file.pdf.
+bucket.write(name, {
+    filename: '/path/to/file.pdf',
+    metadata: { /* metadata properties */ }
+}, callback);
 
-// Reads the stream and uploads it as file contents
-bucket.writeStream(
-    filename, fs.createReadStream('/path/to/file.pdf'), metadata, callback);
+// Uploads the readable stream.
+bucket.write(name, {
+    data: anyReadableStream,
+    metadata: { /* metadata properties */ }
+}, callback);
 
-// Uploads 'Hello World' as file contents
-bucket.writeBuffer(filename, 'Hello World', callback);
+// Uploads 'Hello World' as file contents.
+// data could be any string or buffer.
+bucket.write(name, { data: 'Hello World' }, callback);
 ~~~~
 
 #### Copy files

lib/common/util.js

@@ -49,14 +49,38 @@ module.exports.format = function(templ, args) {
   return templ;
 };
 
-module.exports.noop = function(){};
+var noop = function() {};
 
-module.exports.ApiError = function (errorBody) {
+module.exports.noop = noop;
+
+function ApiError (errorBody) {
   Error.call(this);
   Error.captureStackTrace(this, arguments.callee);
   this.errors = errorBody.errors;
   this.code = errorBody.code;
   this.message = errorBody.message;
-}
+};
 
-util.inherits(module.exports.ApiError, Error);
+util.inherits(ApiError, Error);
+
+module.exports.handleResp = function(err, resp, body, opt_callback) {
+  var callback = opt_callback || noop;
+  if (err) {
+    callback(err);
+    return;
+  }
+  if (typeof body === 'string') {
+    try {
+      body = JSON.parse(body);
+    } catch(err) {}
+  }
+  if (body && body.error) {
+    callback(new ApiError(body.error));
+    return;
+  }
+  if (resp && (resp.statusCode < 200 || resp.statusCode > 299)) {
+    callback(new Error('error during request, statusCode: ' + resp.statusCode));
+    return;
+  }
+  callback(null, body, resp);
+};

lib/datastore/entity.js

@@ -110,6 +110,15 @@ var keyToKeyProto = function(datasetId, key) {
 
 module.exports.keyToKeyProto = keyToKeyProto;
 
+module.exports.formatArray = function(results) {
+  return results.map(function (result) {
+    return {
+      key: keyFromKeyProto(result.entity.key),
+      data: entityFromEntityProto(result.entity)
+    };
+  });
+};
+
 module.exports.isKeyComplete = function(key) {
   var proto = keyToKeyProto(null, key);
   for (var i = 0; i < proto.path.length; i++) {

lib/datastore/index.js

@@ -99,11 +99,11 @@ Transaction.prototype.finalize = function(callback) {
  * @param  {Function} callback
  */
 Transaction.prototype.get = function(key, callback) {
-  this.getAll([key], function(err, keys, objs) {
-    if (err || objs.length < 0) {
+  this.getAll([key], function(err, results) {
+    if (err) {
       return callback(err);
     }
-    return callback(null, keys[0], objs[0]);
+    return callback(null, results[0]);
   });
 };
 
@@ -114,6 +114,7 @@ Transaction.prototype.get = function(key, callback) {
  * @param  {Function}       callback
  */
 Transaction.prototype.getAll = function(keys, callback) {
+  callback = callback || util.noop;
   keysProto = [];
   keys.forEach(function(k) {
     keysProto.push(entity.keyToKeyProto(this.id, k));
@@ -127,12 +128,8 @@ Transaction.prototype.getAll = function(keys, callback) {
     if (err) {
       return callback(err);
     }
-    var results = [], keys = [];
-    resp.found.forEach(function(f) {
-      keys.push(entity.keyFromKeyProto(f.entity.key));
-      results.push(entity.entityFromEntityProto(f.entity));
-    });
-    callback && callback(null, keys, results);
+
+    callback(null, entity.formatArray(resp.found));
   });
 };
 
@@ -245,7 +242,7 @@ Transaction.prototype.delAll = function(keys, callback) {
  * available, a query to retrieve the next page is provided to
  * the callback function. Example:
  *
- * t.runQuery(q, function(err, keys, objs, nextQuery) {
+ * t.runQuery(q, function(err, entities, nextQuery) {
  *   if (err) return;
  *   // if next page is available, retrieve more results
  *   if (nextQuery) {
@@ -256,6 +253,7 @@ Transaction.prototype.delAll = function(keys, callback) {
  * @param  {Function} opt_callback
  */
 Transaction.prototype.runQuery = function(q, opt_callback) {
+  opt_callback = opt_callback || util.noop;
   var req = {
     readOptions: {
       transaction: this.id
@@ -269,19 +267,13 @@ Transaction.prototype.runQuery = function(q, opt_callback) {
   }
   this.makeReq('runQuery', req, function(err, resp) {
     if (err || !resp.batch || !resp.batch.entityResults) {
-      return opt_callback && opt_callback(err);
+      return opt_callback(err);
     }
-    var results = resp.batch.entityResults,
-        keys = [], objs = [];
-    results.forEach(function(r) {
-      keys.push(entity.keyFromKeyProto(r.entity.key));
-      objs.push(entity.entityFromEntityProto(r.entity));
-    });
     var nextQuery = null;
     if (resp.batch.endCursor && resp.batch.endCursor != q.startVal) {
       nextQuery = q.start(resp.batch.endCursor);
     }
-    opt_callback && opt_callback(err, keys, objs, nextQuery);
+    opt_callback(null, entity.formatArray(resp.batch.entityResults), nextQuery);
   });
 };
 
@@ -301,11 +293,7 @@ Transaction.prototype.makeReq = function(method, req, callback) {
     uri: DATASTORE_BASE_URL + '/' + this.datasetId + '/' + method,
     json: req
   }, function(err, res, body) {
-    if (body && body.error) {
-      var error = new util.ApiError(body.error);
-      return callback(error, null);
-    }
-    callback(err, body);
+    util.handleResp(err, res, body, callback);
   });
 };
 

lib/pubsub/index.js

@@ -398,7 +398,6 @@ Connection.prototype.fullProjectName_ = function() {
   });
 };
 
-// TOOD(jbd): Don't duplicate, unify this with bucket.makeReq.
 Connection.prototype.makeReq = function(method, path, q, body, callback) {
   var reqOpts = {
     method: method,
@@ -412,13 +411,7 @@ Connection.prototype.makeReq = function(method, path, q, body, callback) {
     reqOpts.json = body;
   }
   this.conn.req(reqOpts, function(err, res, body) {
-    if (body && body.error) {
-      callback(new util.ApiError(body.error)); return;
-    }
-    if (res && (res.statusCode < 200 || res.statusCode > 299)) {
-      callback(new Error('error during request, statusCode: ' + res.statusCode)); return;
-    }
-    callback(null, body);
+    util.handleResp(err, res, body, callback);
   });
 };
 

lib/storage/index.js

@@ -33,16 +33,12 @@ var STORAGE_BASE_URL = 'https://www.googleapis.com/storage/v1/b';
     STORAGE_UPLOAD_BASE_URL = 'https://www.googleapis.com/upload/storage/v1/b';
 
 var reqStreamToCallback = function(st, callback) {
+  st.callback = util.noop;
   st.on('error', function(err) {
     callback(err);
   });
   st.on('complete', function(resp) {
-    // TODO(jbd): Buffer the response to pass the resp body
-    // to the callback.
-    if (resp.statusCode < 200 || resp.statusCode > 299) {
-      return callback(new Error('error during request, statusCode: ' + resp.statusCode), resp);
-    }
-    callback(null, resp);
+    util.handleResp(null, resp, resp.body, callback);
   });
 };
 
@@ -254,17 +250,25 @@ Bucket.prototype.createReadStream = function(name) {
 /**
  * Writes the provided stream to the destination
  * with optional metadata.
- * @param  {String}   name     Name of the remote file.
- * @param  {Stream}   stream   A readable stream.
- * @param  {Object?}  metadata Optional metadata.
+ * @param  {String}    name           Name of the remote file.
+ * @param  {Object=}   opts.data     A string, buffer or readable stream.
+ * @param  {string=}   opts.filename Path of the source file.
+ * @param  {Object=}   opts.metadata Optional metadata.
  * @param  {Function} callback Callback function.
  */
-Bucket.prototype.writeStream = function(name, stream, metadata, callback) {
-  if (!callback) {
-    callback = metadata, metadata = {};
+Bucket.prototype.write = function(name, opts, callback) {
+  // TODO(jbd): Support metadata only requests.
+  var that = this;
+
+  var metadata = opts.metadata || {};
+  var stream = opts.data;
+
+  if (opts.filename) {
+    stream = fs.createReadStream(opts.filename);
+  } else if (opts.data && (typeof opts.data === 'string' || opts.data instanceof Buffer)) {
+    stream = new BufferStream(opts.data);
   }
 
-  var that = this;
   var boundary = uuid.v4();
   metadata.contentType = metadata.contentType || 'text/plain'
   this.conn.createAuthorizedReq({
@@ -298,34 +302,11 @@ Bucket.prototype.writeStream = function(name, stream, metadata, callback) {
     remoteStream.write('Content-Type: ' + metadata.contentType + '\n\n');
     stream.pipe(remoteStream);
     // TODO(jbd): High potential of multiple callback invokes.
-    reqStreamToCallback(stream, callback);
+    stream.on('error', callback);
     reqStreamToCallback(remoteStream, callback);
   });
 };
 
-/**
- * Writes the source file to the destination with
- * optional metadata.
- * @param  {String}   name     Name of the remote file.
- * @param  {String}   filename Path to the source file.
- * @param  {object?}  metadata Optional metadata.
- * @param  {Function} callback Callback function.
- */
-Bucket.prototype.writeFile = function(name, filename, metadata, callback) {
-  this.writeStream(name, fs.createReadStream(filename), metadata, callback);
-};
-
-/**
- * Writes the provided buffer to the destination file.
- * @param  {String}   name     Name of the remote file resource.
- * @param  {Buffer}   buffer   Buffer contents to be written.
- * @param  {Object?}  metadata Optional metadata.
- * @param  {Function} callback Callback function.
- */
-Bucket.prototype.writeBuffer = function(name, buffer, metadata, callback) {
-  this.writeStream(name, new BufferStream(buffer), metadata, callback);
-};
-
 /**
  * Makes a new request object from the provided
  * arguments, and wraps the callback to intercept
@@ -345,13 +326,7 @@ Bucket.prototype.makeReq = function(method, path, q, body, callback) {
     reqOpts.json = body;
   }
   this.conn.req(reqOpts, function(err, res, body) {
-    if (body && body.error) {
-      return callback(body.error);
-    }
-    if (res && (res.statusCode < 200 || res.statusCode > 299)) {
-      return callback(new Error('error during request, statusCode: ' + res.statusCode));
-    }
-    callback(err, body);
+    util.handleResp(err, res, body, callback);
   });
 };
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gcloud",
-  "version": "0.0.6",
+  "version": "0.1.0",
   "author": "Google Inc.",
   "description": "Google Cloud APIs Client Library for Node.js",
   "contributors": [