From 27fbcf9f0300fd5691d27a6dcd02bff5df75b3d8 Mon Sep 17 00:00:00 2001 From: Stephen Sawchuk Date: Thu, 18 Jun 2015 14:30:46 -0400 Subject: [PATCH 1/2] search: introduce api --- README.md | 48 ++ docs/json/master/search/.gitkeep | 0 docs/site/components/docs/docs-values.js | 25 +- docs/site/components/docs/docs.html | 2 +- docs/site/components/docs/docs.js | 3 + .../site/components/docs/search-overview.html | 7 + lib/common/stream-router.js | 118 +++++ lib/common/util.js | 2 +- lib/index.js | 45 +- lib/search/document.js | 213 +++++++++ lib/search/field.js | 175 +++++++ lib/search/index-class.js | 385 ++++++++++++++++ lib/search/index.js | 232 ++++++++++ lib/storage/bucket.js | 2 +- scripts/docs.sh | 5 + system-test/search.js | 301 +++++++++++++ test/common/stream-router.js | 224 +++++++++ test/docs.js | 14 +- test/search.index.js | 0 test/search/document.js | 304 +++++++++++++ test/search/field.js | 149 ++++++ test/search/index-class.js | 426 ++++++++++++++++++ test/search/index.js | 281 ++++++++++++ test/testdata/search-document.json | 31 ++ 24 files changed, 2979 insertions(+), 13 deletions(-) create mode 100644 docs/json/master/search/.gitkeep create mode 100644 docs/site/components/docs/search-overview.html create mode 100644 lib/common/stream-router.js create mode 100644 lib/search/document.js create mode 100644 lib/search/field.js create mode 100644 lib/search/index-class.js create mode 100644 lib/search/index.js create mode 100644 system-test/search.js create mode 100644 test/common/stream-router.js create mode 100644 test/search.index.js create mode 100644 test/search/document.js create mode 100644 test/search/field.js create mode 100644 test/search/index-class.js create mode 100644 test/search/index.js create mode 100644 test/testdata/search-document.json diff --git a/README.md b/README.md index 80778778254..cf1267c428b 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This client supports the following Google Cloud Platform services: * [Google Cloud Datastore](#google-cloud-datastore) * [Google Cloud Storage](#google-cloud-storage) * [Google Cloud Pub/Sub](#google-cloud-pubsub-beta) (Beta) +* [Google Cloud Search](#google-cloud-search-alpha) (Alpha) If you need support for other Google APIs, check out the [Google Node.js API Client library][googleapis]. @@ -238,6 +239,48 @@ topic.subscribe('new-subscription', function(err, subscription) { }); ``` +## Google Cloud Search (Alpha) +> This is an *Alpha* release of Google Cloud Search. This feature is not covered by any SLA or deprecation policy and may be subject to backward-incompatible changes. + +[Google Cloud Search][cloud-search] ([docs][cloud-search-docs]) allows you to quickly perform full-text and geospatial searches against your data without having to spin up your own instances and without the hassle of managing and maintaining a search service. + +See the [gcloud-node Search API documentation][gcloud-search-docs] to learn how to store and query your indexes and documents using this library. + +```js +var gcloud = require('gcloud'); + +// Authorizing on a per-API-basis. You don't need to do this if you auth on a +// global basis (see Authorization section above). + +var search = gcloud.search({ + keyFilename: '/path/to/keyfile.json', + projectId: 'my-project' +}); + +// Create a document in a new index. +var index = search.index('memberData'); + +var document = index.document('member-id-34211'); +document.addField('preferredContactForm').addTextValue('phone'); + +index.createDocument(document, function(err, document) { + console.log(err || document); +}); + +// Search an index and get the results as a readable object stream. +var index = search.index('memberData'); + +index.search('preferredContactForm:phone') + .on('error', console.error) + .on('data', function(document) { + // document.id = 'member-id-34211'; + }) + .on('end', function() { + // All results consumed. + }); +``` + + ## Contributing Contributions to this library are always welcome and highly encouraged. @@ -253,6 +296,7 @@ Apache 2.0 - See [COPYING](COPYING) for more information. [gcloud-bigquery-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/bigquery [gcloud-datastore-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/datastore [gcloud-pubsub-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/pubsub +[gcloud-search-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/search [gcloud-storage-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/storage [gcloud-todos]: https://github.com/GoogleCloudPlatform/gcloud-node-todos [gitnpm]: https://github.com/stephenplusplus/gitnpm @@ -273,8 +317,12 @@ Apache 2.0 - See [COPYING](COPYING) for more information. [cloud-pubsub]: https://cloud.google.com/pubsub/ [cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs +[cloud-search]: https://cloud.google.com/search/ +[cloud-search-docs]: https://cloud.google.com/search/ + [cloud-storage]: https://cloud.google.com/storage/ [cloud-storage-docs]: https://cloud.google.com/storage/docs/overview [cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets + [hya-wave]: https://wav.hya.io [hya-io]: https://hya.io diff --git a/docs/json/master/search/.gitkeep b/docs/json/master/search/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/site/components/docs/docs-values.js b/docs/site/components/docs/docs-values.js index 4767cc9c66f..2b30f1da72e 100644 --- a/docs/site/components/docs/docs-values.js +++ b/docs/site/components/docs/docs-values.js @@ -88,6 +88,25 @@ angular.module('gcloud.docs') ] }, + search: { + title: 'Search', + _url: '{baseUrl}/search', + pages: [ + { + title: 'Index', + url: '/index' + }, + { + title: 'Document', + url: '/document' + }, + { + title: 'Field', + url: '/field' + } + ] + }, + storage: { title: 'Storage', _url: '{baseUrl}/storage' @@ -135,6 +154,10 @@ angular.module('gcloud.docs') // introduce new storage api. '>=0.9.0': ['storageWithFiles'], - '>=0.10.0': ['bigquery'] + // introduce bigquery api. + '>=0.10.0': ['bigquery'], + + // introduce search api. + '>=0.16.0': ['search'] } }); diff --git a/docs/site/components/docs/docs.html b/docs/site/components/docs/docs.html index def79b9928e..0b5d4c650ce 100644 --- a/docs/site/components/docs/docs.html +++ b/docs/site/components/docs/docs.html @@ -44,7 +44,7 @@

-
diff --git a/docs/site/components/docs/docs.js b/docs/site/components/docs/docs.js index b78f3c3d02f..c166986a09d 100644 --- a/docs/site/components/docs/docs.js +++ b/docs/site/components/docs/docs.js @@ -244,6 +244,9 @@ angular path.push('index.json'); } else if (module && cl) { path.push(module); + if (cl === 'index') { + cl = 'index-class'; + } path.push(cl + '.json'); } return $http.get(path.join('/')) diff --git a/docs/site/components/docs/search-overview.html b/docs/site/components/docs/search-overview.html new file mode 100644 index 00000000000..a1b30c332df --- /dev/null +++ b/docs/site/components/docs/search-overview.html @@ -0,0 +1,7 @@ +

Search Overview

+

+ The object returned from gcloud.search gives you complete access to store your documents and search your indexes. +

+

+ To learn more about Search, see What is Google Cloud Search? +

diff --git a/lib/common/stream-router.js b/lib/common/stream-router.js new file mode 100644 index 00000000000..12490d2288d --- /dev/null +++ b/lib/common/stream-router.js @@ -0,0 +1,118 @@ +/*! + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module common/streamrouter + */ + +'use strict'; + +var streamEvents = require('stream-events'); +var through = require('through2'); + +/** + * @type {module:common/util} + * @private + */ +var util = require('../common/util.js'); + +/*! Developer Documentation + * + * streamRouter is used to extend `nextQuery`+callback methods with stream + * functionality. + * + * Before: + * + * search.query('done=true', function(err, results, nextQuery) {}); + * + * After: + * + * search.query('done=true').on('data', function(result) {}); + * + * Methods to extend should be written to accept callbacks and return a + * `nextQuery`. All stream logic is handled in `streamRouter.router_`. + */ +var streamRouter = {}; + +/** + * Cache the original method, then overwrite it on the Class's prototype. + * + * @param {function} Class - The parent class of the methods to extend. + * @param {array|string} methodNames - Name(s) of the methods to extend. + */ +streamRouter.extend = function(Class, methodNames) { + methodNames = util.arrayize(methodNames); + + methodNames.forEach(function(methodName) { + var originalMethod = Class.prototype[methodName]; + + Class.prototype[methodName] = function() { + return streamRouter.router_(arguments, originalMethod.bind(this)); + }; + }); +}; + +/** + * The router accepts all incoming arguments to the overwritten method. If the + * last argument is a function, simply pass them through to the original method. + * If the last argument is not a function, activate stream mode. + * + * Stream mode simply calls the nextQuery recursively. The stream ends when + * `nextQuery` is null. + * + * @param {array} args - The original `arguments` pseudo-array as it was + * received by the original method. + * @param {function} originalMethod - The cached method that accepts a callback + * and returns `nextQuery` to receive more results. + * @return {undefined|stream} + */ +streamRouter.router_ = function(args, originalMethod) { + args = util.toArray(args); + var callback = args[args.length - 1]; + var isStreamMode = !util.is(callback, 'function'); + + if (isStreamMode) { + var stream = streamEvents(through.obj()); + + var onResultSet = function(err, results, nextQuery) { + if (err) { + stream.emit('error', err); + stream.end(); + return; + } + + results.forEach(function(result) { + stream.push(result); + }); + + if (nextQuery) { + originalMethod(nextQuery, onResultSet); + } else { + stream.end(); + } + }; + + stream.once('reading', function() { + originalMethod.apply(null, args.concat(onResultSet)); + }); + + return stream; + } else { + originalMethod.apply(null, args); + } +}; + +module.exports = streamRouter; diff --git a/lib/common/util.js b/lib/common/util.js index 0d25bb8aebb..e58e6113758 100644 --- a/lib/common/util.js +++ b/lib/common/util.js @@ -23,12 +23,12 @@ var extend = require('extend'); var GoogleAuth = require('google-auth-library'); +var nodeutil = require('util'); var request = require('request').defaults({ pool: { maxSockets: Infinity } }); -var nodeutil = require('util'); var uuid = require('node-uuid'); /** @const {object} gcloud-node's package.json file. */ diff --git a/lib/index.js b/lib/index.js index 38a751d90a7..fbeab901aa3 100644 --- a/lib/index.js +++ b/lib/index.js @@ -38,6 +38,12 @@ var Datastore = require('./datastore'); */ var PubSub = require('./pubsub'); +/** + * @type {module:search} + * @private + */ +var Search = require('./search'); + /** * @type {module:storage} * @private @@ -120,6 +126,10 @@ function gcloud(config) { options = options || {}; return new PubSub(util.extendGlobalConfig(config, options)); }, + search: function(options) { + options = options || {}; + return new Search(util.extendGlobalConfig(config, options)); + }, storage: function(options) { options = options || {}; return new Storage(util.extendGlobalConfig(config, options)); @@ -173,11 +183,9 @@ gcloud.datastore = Datastore; * reliable, many-to-many, asynchronous messaging service from Google Cloud * Platform. * - * Note: Google Cloud Pub/Sub API is available as a Limited Preview and the - * client library we provide is currently experimental. The API and/or the - * client might be changed in backward-incompatible ways. This API is not - * subject to any SLA or deprecation policy. Request to be whitelisted to use it - * by filling the [Limited Preview application form](http://goo.gl/sO0wTu). + * Note: This is a *Beta* release of Google Cloud Pub/Sub. This feature is not + * covered by any SLA or deprecation policy and may be subject to backward- + * incompatible changes. * * @type {module:pubsub} * @@ -194,6 +202,33 @@ gcloud.pubsub = function(config) { return new PubSub(config); }; +/** + * **Experimental** + * + * [Google Cloud Search](https://cloud.google.com/search/) allows you to quickly + * perform full-text and geospatial searches against your data without having to + * spin up your own instances and without the hassle of managing and maintaining + * a search service. + * + * Note: This is an *Alpha* release of Google Cloud Search. This feature is not + * covered by any SLA or deprecation policy and may be subject to backward- + * incompatible changes. + * + * @type {module:search} + * + * @return {module:search} + * + * @example + * var gcloud = require('gcloud'); + * var search = gcloud.search({ + * projectId: 'project-id', + * keyFilename: '/path/to/keyfile.json' + * }); + */ +gcloud.search = function (config) { + return new Search(config); +}; + /** * Google Cloud Storage allows you to store data on Google infrastructure. * Read [Google Cloud Storage API docs](https://developers.google.com/storage/) diff --git a/lib/search/document.js b/lib/search/document.js new file mode 100644 index 00000000000..39add68b8b2 --- /dev/null +++ b/lib/search/document.js @@ -0,0 +1,213 @@ +/*! + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module search/document + */ + +'use strict'; + +/** + * @type {module:search/field} + * @private + */ +var Field = require('./field.js'); + +/** + * @type {module:common/util} + * @private + */ +var util = require('../common/util.js'); + +/** + * Create a Document object to create or manipulate a document from your index. + * + * @constructor + * @alias module:search/document + * + * @param {string=} id - ID of the document. + * + * @example + * var gcloud = require('gcloud'); + * + * var search = gcloud.search({ + * projectId: 'grape-spaceship-123' + * }); + * + * var document = search.index('records').document('stephen'); + */ +function Document(index, id) { + this.search_ = index.search_; + this.index_ = index; + + this.id = id; + this.fields = {}; +} + +/** + * Add a field to this document. + * + * @throws {error} if a name is not provided. + * + * @param {string} name - This field's name. + * @return {module:search/field} + * + * @example + * var scoreField = document.addField('score'); + * // scoreField is a Field object. + */ +Document.prototype.addField = function(name) { + if (!util.is(name, 'string')) { + throw new Error('A name is required to add a field to this document.'); + } + + this.fields[name] = new Field(); + + return this.fields[name]; +}; + +/** + * Delete this document. + * + * @param {function=} callback - The callback function. + * + * @example + * document.delete(function(err, apiResponse) {}); + */ +Document.prototype.delete = function(callback) { + this.makeReq_('DELETE', '', null, null, function(err, resp) { + (callback || util.noop)(err, resp); + }); +}; + +/** + * Get the details of this document. After running, the Document instance will + * update the `fields` and `rank` properties to their most recent values at the + * time of this call. + * + * @param {function} callback - The callback function. + * + * @example + * document.getMetadata(function(err, doc, apiResponse) { + * if (err) { + * console.error(err); + * return; + * } + * + * // `doc` is a reference to `document`, both of which are now up to date. + * // + * // document.fields = Array of Field objects. + * // document.rank = Document's numeric rank. + * }); + */ +Document.prototype.getMetadata = function(callback) { + var self = this; + + callback = callback || util.noop; + + this.makeReq_('GET', '/', null, null, function(err, resp) { + if (err) { + callback(err, null, resp); + return; + } + + self.fields = {}; + delete self.rank; + + if (util.is(resp.fields, 'object')) { + Object.keys(resp.fields).forEach(function(fieldName) { + var fieldInstance = self.addField(fieldName); + fieldInstance.values = resp.fields[fieldName].values; + }); + } + + if (util.is(resp.rank, 'number')) { + self.rank = resp.rank; + } + + callback(null, self, resp); + }); +}; + +/** + * Set the rank for this document. The rank of a document is a positive integer + * which determines the default ordering of documents returned from a search. By + * default, the rank is set at the time the document is created to the number of + * seconds since January 1, 2011. + * + * @throws {error} If a rank is not a number. + * + * @param {number} rank - The rank of this document. + * + * @example + * document.setRank(8); + */ +Document.prototype.setRank = function(rank) { + if (!util.is(rank, 'number') || rank < 0) { + throw new Error('rank should be a positive integer.'); + } + + this.rank = rank; +}; + +/** + * Return just the document detail properties of this Document instance. + * + * @example + * document.toJSON(); + * // { + * // docId: 'this-document-id', + * // fields: { + * // // ... + * // }, + * // rank: 8 + * // } + */ +Document.prototype.toJSON = function() { + var documentObject = { + fields: this.fields + }; + + if (util.is(this.id, 'string')) { + documentObject.docId = this.id; + } + + if (util.is(this.rank, 'number')) { + documentObject.rank = this.rank; + } + + return documentObject; +}; + +/** + * Make a new request object from the provided arguments and wrap the callback + * to intercept non-successful responses. + * + * @private + * + * @param {string} method - Action. + * @param {string} path - Request path. + * @param {*} query - Request query object. + * @param {*} body - Request body contents. + * @param {function} callback - The callback function. + */ +Document.prototype.makeReq_ = function(method, path, query, body, callback) { + path = '/documents/' + this.id + path; + + this.index_.makeReq_(method, path, query, body, callback); +}; + +module.exports = Document; diff --git a/lib/search/field.js b/lib/search/field.js new file mode 100644 index 00000000000..3ffc41c7aa7 --- /dev/null +++ b/lib/search/field.js @@ -0,0 +1,175 @@ +/*! + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module search/field + */ + +'use strict'; + +/** + * @type {module:common/util} + * @private + */ +var util = require('../common/util.js'); + +/** + * Create a Field object to easily format a Cloud Search index's field. + * + * @constructor + * @alias module:search/field + * + * @example + * var gcloud = require('gcloud'); + * + * var search = gcloud.search({ + * projectId: 'grape-spaceship-123' + * }); + * + * var document = search.index('records').document('stephen'); + * var field = document.addField('alias'); + */ +function Field() { + this.values = []; +} + +/** + * An atom value is a string value that is treated as a single token. A query + * will not match if it includes only a substring rather than the full field + * value. + * + * @param {string} atom - String value. + * + * @example + * field.addAtomValue('ryanseys'); + */ +Field.prototype.addAtomValue = function(atom) { + this.addTextValue(atom, { format: 'ATOM' }); +}; + +/** + * A geo value is a point on earth with latitude and longitude coordinates. + * + * @param {string|object} geo - Coordinate value as a string or object. String + * format: `'40.6894, -74.0447'`. + * @param {number} geo.latitude - Latitudinal value. + * @param {number} geo.longitude - Longitudinal value. + * + * @example + * var coordinates = '40.6894, -74.0447'; + * field.addGeoValue(coordinates); + * + * //- + * // Use an object. + * //- + * var coordinates = { + * latitude: 40.6894, + * longitude: -74.0447 + * }; + * + * field.addGeoValue(coordinates); + */ +Field.prototype.addGeoValue = function(geo) { + if (util.is(geo, 'object')) { + geo = util.format('{latitude}, {longitude}', geo); + } + + this.values.push({ + geoValue: geo + }); +}; + +/** + * An HTML value is an HTML-formatted string. Text out of markup tags are + * tokenized and markup tags are considered metadata. + * + * @param {string} html - HTML value. + * + * @example + * var html = 'hello, world'; + * field.addHtmlValue(html); + */ +Field.prototype.addHtmlValue = function(html) { + this.addTextValue(html, { format: 'HTML' }); +}; + +/** + * A number value is a double precision floating point value. + * + * @throws {error} If a number is not provided. + * + * @param {number} number - Number value. + * + * @example + * field.addNumberValue(8); + */ +Field.prototype.addNumberValue = function(number) { + if (!util.is(number, 'number')) { + throw new Error('number must be a number.'); + } + + this.values.push({ + numberValue: number + }); +}; + +/** + * A text value is a string that is tokenized as plain text. + * + * @param {string} text - Text value. + * @param {object=} options - Configuration object. + * @param {string} options.format - One of `ATOM`, `HTML`, or `TEXT`. (Default: + * `TEXT`). + * + * @example + * field.addTextValue('Hello!'); + */ +Field.prototype.addTextValue = function(text, options) { + options = options || {}; + var format = options.format || 'TEXT'; + + this.values.push({ + stringValue: text.toLowerCase(), + stringFormat: format.toUpperCase() + }); +}; + +/** + * A timestamp value is a date-time value with millisecond precision. + * + * @param {string|date} timestamp - Timestamp value. + * + * @example + * var timestamp = '2014-08-18T21:19:55.000Z'; + * field.addTimestampValue(timestamp); + * + * //- + * // Use a Date object. + * //- + * var now = Date.now(); + * field.addTimestampValue(now); + */ +Field.prototype.addTimestampValue = function(timestamp) { + if (timestamp instanceof Date) { + timestamp = timestamp.toJSON(); + } + + this.values.push({ + timestampValue: timestamp + }); +}; + +module.exports = Field; diff --git a/lib/search/index-class.js b/lib/search/index-class.js new file mode 100644 index 00000000000..3226e075489 --- /dev/null +++ b/lib/search/index-class.js @@ -0,0 +1,385 @@ +/*! + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module search/index + */ + +'use strict'; + +var extend = require('extend'); + +/** + * @type {module:search/document} + * @private + */ +var Document = require('./document.js'); + +/** + * @type {module:common/streamrouter} + * @private + */ +var streamRouter = require('../common/stream-router.js'); + +/** + * @type {module:common/util} + * @private + */ +var util = require('../common/util.js'); + +/** + * Create an Index object to interact with a Google Cloud Search index. + * + * @constructor + * @alias module:search/index + * + * @param {string} id - ID of the index. + * + * @example + * var gcloud = require('gcloud'); + * + * var search = gcloud.search({ + * projectId: 'grape-spaceship-123' + * }); + * + * var index = search.index('records'); + */ +function Index(search, id) { + this.search_ = search; + this.id = id; + + if (!this.id) { + throw new Error('An ID is needed to access a Google Cloud Search index.'); + } +} + +/** + * Create a document in the index. + * + * @param {object|module:search/document} documentObj - A properly-formed + * Document object as outlined in the + * [official docs](https://goo.gl/AYhSgI). + * @param {function} callback - The callback function. + * + * @example + * //- + * // A document can be built using the {module:search/document} object. + * //- + * var newDocument = index.document('new-document-id'); + * newDocument.addField('person').addTextValue('Stephen'); + * + * index.createDocument(newDocument, function(err, document, apiResponse) {}); + * + * //- + * // A document can also be created from a properly-formed object as outlined + * // in the official docs. + * // + * // This will create the same resulting document object as the example above. + * //- + * var newDocument = { + * docId: 'new-document-id', + * fields: { + * person: { + * values: [ + * { + * stringFormat: 'TEXT', + * stringValue: 'Stephen' + * } + * ] + * } + * } + * }; + * + * index.createDocument(newDocument, function(err, document, apiResponse) {}); + * + * //- + * // Specifying an ID for your new document isn't required. In both of the + * // scenarios above, simply don't specify 'new-document-id' and one will be + * // generated for you. + * //- + */ +Index.prototype.createDocument = function(documentObj, callback) { + var document; + + if (documentObj instanceof Document) { + document = documentObj; + documentObj = document.toJSON(); + } else { + document = this.documentFromObject_(documentObj); + } + + this.makeReq_('POST', '/documents', null, documentObj, function(err, resp) { + if (err) { + callback(err, null, resp); + return; + } + + callback(null, document, resp); + }); +}; + +/** + * Access a {module:search/document} object. + * + * @param {string} id - The id of the document. + * @return {module:search/document} + * + * @example + * var myDocument = index.document('my-document'); + */ +Index.prototype.document = function(id) { + return new Document(this, id); +}; + +/** + * Get {module:search/document} objects for all of the documents in this index. + * + * @param {object=} query - Query object. + * @param {string} query.pageSize - The maximum number of documents to return + * per page. If not specified, 100 documents are returned per page. + * @param {string} query.pageToken - A previously-returned page token + * representing part of the larger set of results to view. + * @param {string} query.view - One of `INDEX_VIEW_UNSPECIFIED`, `ID_ONLY`, or + * `FULL`. See [this table](https://goo.gl/sY6Lpt) for more details. + * @param {function} callback - The callback function. + * + * @example + * function onApiResponse(err, documents, nextQuery, apiResponse) { + * if (err) { + * console.error(err); + * return; + * } + * + * // `documents` is an array of Document objects in this index. + * + * if (nextQuery) { + * index.getDocuments(nextQuery, onApiResponse); + * } + * } + * + * index.getDocuments(onApiResponse); + * + * //- + * // Customize the request using a query object. + * //- + * index.getDocuments({ + * pageSize: 10 + * }, onApiResponse); + * + * //- + * // Get the documents as a readable object stream. + * //- + * index.getDocuments() + * .on('error', console.error) + * .on('data', function(document) { + * // document is a Document object. + * }) + * .on('end', function() { + * // All documents retrieved. + * }); + */ +Index.prototype.getDocuments = function(query, callback) { + var self = this; + + if (util.is(query, 'function')) { + callback = query; + query = {}; + } + + this.makeReq_('GET', '/documents', query, null, function(err, resp) { + if (err) { + callback(err, null, null, resp); + return; + } + + var nextQuery = null; + + if (resp.nextPageToken) { + nextQuery = extend({}, query, { + pageToken: resp.nextPageToken + }); + } + + var documents = (resp.documents || []) + .map(self.documentFromObject_.bind(self)); + + callback(null, documents, nextQuery, resp); + }); +}; + +/** + * Run a query against the documents in this index. + * + * For a full list of supported query parameters, see the + * [JSON API documentation](https://goo.gl/706zrP). + * + * @throws {error} If a query string or object isn't provided. + * + * @param {string|object} query - A query object or simply a string query. + * @param {string} query.pageSize - The maximum number of documents to return + * per page. If not specified, 100 documents are returned per page. + * @param {string} query.pageToken - A previously-returned page token + * representing part of the larger set of results to view. + * @param {string} query.query = A query string using the syntax described by + * the [official docs](https://goo.gl/2SYl3S). + * @param {function} callback - The callback function. + * + * @example + * function onApiResponse(err, documents, nextQuery, apiResponse) { + * if (err) { + * console.error(err); + * return; + * } + * + * // `documents` is an array of Document objects that matched your query. + * + * if (nextQuery) { + * index.search(nextQuery, onApiResponse); + * } + * } + * + * //- + * // Run a simple query against all documents. + * //- + * var query = 'person:stephen'; + * + * index.search(query, onApiResponse); + * + * //- + * // Configure the query. + * //- + * var query = { + * query: 'person:stephen', + * pageSize: 10 + * }; + * + * index.search(query, onApiResponse); + * + * //- + * // Get the documents that match your query as a readable object stream. + * //- + * index.search('person:stephen') + * .on('error', console.error) + * .on('data', function(document) { + * // document is a Document object. + * }) + * .on('end', function() { + * // All search results retrieved. + * }); + */ +Index.prototype.search = function(query, callback) { + var self = this; + + if (util.is(query, 'string')) { + query = { + query: query + }; + } + + if (!util.is(query, 'object')) { + throw new Error('A query must be either a string or object.'); + } + + this.makeReq_('GET', '/search', query, null, function(err, resp) { + if (err) { + callback(err, null, null, resp); + return; + } + + var nextQuery = null; + + if (resp.nextPageToken) { + nextQuery = extend({}, query, { + pageToken: resp.nextPageToken + }); + } + + var documents = (resp.results || []) + .map(self.documentFromObject_.bind(self)); + + callback(null, documents, nextQuery, resp); + }); +}; + +/** + * Convert an object to a {module:search/document} object. + * + * @private + * + * @param {object} documentObj - Object describing a document. + * @param {object} documentObj.fields - Fields the document contains. + * @param {number=} documentObj.rank - The rank of the document. + * @return {module:search/document} + * + * @example + * var documentObject = { + * docId: 'new-document-id', + * fields: { + * person: { + * values: [ + * { + * stringFormat: 'TEXT', + * stringValue: 'Stephen' + * } + * ] + * } + * } + * }; + * + * var document = index.documentFromObject_(documentObject); + * // document is a {module:search/document} object. + */ +Index.prototype.documentFromObject_ = function(documentObj) { + var document = this.document(documentObj.docId); + + if (util.is(documentObj.fields, 'object')) { + document.fields = documentObj.fields; + } + + if (util.is(documentObj.rank, 'number')) { + document.rank = documentObj.rank; + } + + return document; +}; + +/** + * Make a new request object from the provided arguments and wrap the callback + * to intercept non-successful responses. + * + * @private + * + * @param {string} method - Action. + * @param {string} path - Request path. + * @param {*} query - Request query object. + * @param {*} body - Request body contents. + * @param {function} callback - The callback function. + */ +Index.prototype.makeReq_ = function(method, path, query, body, callback) { + path = '/indexes/' + this.id + path; + + this.search_.makeReq_(method, path, query, body, callback); +}; + +/*! Developer Documentation + * + * {module:search/index#getDocuments} and {module:search/index#search} can be + * used with either a callback or as a readable object stream. `streamRouter` is + * used to add this dual behavior to these methods. + */ +streamRouter.extend(Index, ['getDocuments', 'search']); + +module.exports = Index; diff --git a/lib/search/index.js b/lib/search/index.js new file mode 100644 index 00000000000..15e16f3ab71 --- /dev/null +++ b/lib/search/index.js @@ -0,0 +1,232 @@ +/*! + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module search + */ + +'use strict'; + +var extend = require('extend'); + +/** + * @type {module:search/index} + * @private + */ +var Index = require('./index-class.js'); + +/** + * @type {module:common/streamrouter} + * @private + */ +var streamRouter = require('../common/stream-router.js'); + +/** + * @type {module:common/util} + * @private + */ +var util = require('../common/util.js'); + +/** + * @const {array} Required scopes for the Search API. + * @private + */ +var SCOPES = [ + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/cloudsearch', + 'https://www.googleapis.com/auth/userinfo.email' +]; + +/** + * @const {string} Base URL for the Search API. + * @private + */ +var SEARCH_BASE_URL = 'https://cloudsearch.googleapis.com/v1/'; + +/** + * Create a Search object to Interact with the Cloud Search API. Using this + * object, you can access your indexes with {module:search/index} and documents + * with {module:search/document}. + * + * Follow along with the examples to see how to do everything from creating + * documents to searching indexes. + * + * @alias module:search + * @constructor + * + * @param {object} options - [Configuration object](#/docs/?method=gcloud). + * + * @example + * var gcloud = require('gcloud')({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'grape-spaceship-123' + * }); + * + * var search = gcloud.search(); + */ +function Search(options) { + if (!options || !options.projectId) { + throw util.missingProjectIdError; + } + + this.makeAuthorizedRequest_ = util.makeAuthorizedRequestFactory({ + credentials: options.credentials, + email: options.email, + keyFile: options.keyFilename, + scopes: SCOPES + }); + + this.projectId_ = options.projectId; +} + +/** + * Get {module:search/index} objects for all of the indexes in your project. + * + * @param {object=} query - Query object. + * @param {string} query.pageSize - The maximum number of indexes to return per + * page. If not specified, 100 indexes are returned per page. + * @param {string} query.pageToken - A previously-returned page token + * representing part of the larger set of results to view. + * @param {string} query.prefix - The prefix of the index name. It is used to + * list all indexes with names that have this prefix. + * @param {string} query.view - See [this table](https://goo.gl/sY6Lpt) for a + * list of accepted values and what each will do. + * @param {function} callback - The callback function. + * + * @example + * function onApiResponse(err, indexes, nextQuery, apiResponse) { + * if (err) { + * console.error(err); + * return; + * } + * + * if (nextQuery) { + * search.getIndexes(nextQuery, onApiResponse); + * } + * } + * + * search.getIndexes(onApiResponse); + * + * //- + * // Customize the request using a query object. + * //- + * search.getIndexes({ + * pageSize: 10 + * }, onApiResponse); + * + * //- + * // Get the indexes as a readable object stream. + * //- + * search.getIndexes() + * .on('error', console.error) + * .on('data', function(index) { + * // index is an Index object. + * }) + * .on('end', function() { + * // All indexes retrieved. + * }); + */ +Search.prototype.getIndexes = function(query, callback) { + var self = this; + + if (util.is(query, 'function')) { + callback = query; + query = {}; + } + + query = extend({}, query); + + if (query.prefix) { + query.indexNamePrefix = query.prefix; + delete query.prefix; + } + + this.makeReq_('GET', '/indexes', query, null, function(err, resp) { + if (err) { + callback(err, null, null, resp); + return; + } + + var nextQuery = null; + + if (resp.nextPageToken) { + nextQuery = extend({}, query, { + pageToken: resp.nextPageToken + }); + } + + var indexes = (resp.indexes || []).map(function(indexObject) { + var index = self.index(indexObject.indexId); + + if (util.is(resp.indexedField, 'object')) { + index.fields = resp.indexedField; + } + + return index; + }); + + callback(null, indexes, nextQuery, resp); + }); +}; + +/** + * Get a reference to a Google Cloud Search index. + * + * @param {string} id - Name of the index. + * @return {module:search/index} + */ +Search.prototype.index = function(id) { + return new Index(this, id); +}; + +/** + * Make a new request object from the provided arguments and wrap the callback + * to intercept non-successful responses. + * + * @private + * + * @param {string} method - Action. + * @param {string} path - Request path. + * @param {*} query - Request query object. + * @param {*} body - Request body contents. + * @param {function} callback - The callback function. + */ +Search.prototype.makeReq_ = function(method, path, query, body, callback) { + var reqOpts = { + method: method, + qs: query, + uri: util.format('{base}projects/{projectId}{path}', { + base: SEARCH_BASE_URL, + projectId: this.projectId_, + path: path + }) + }; + + if (body) { + reqOpts.json = body; + } + + this.makeAuthorizedRequest_(reqOpts, callback); +}; + +/*! Developer Documentation + * + * {module:search#getIndexes} can be used with either a callback or as a + * readable object stream. `streamRouter` is used to add this dual behavior. + */ +streamRouter.extend(Search, 'getIndexes'); + +module.exports = Search; diff --git a/lib/storage/bucket.js b/lib/storage/bucket.js index 742647c0ce0..4e845386123 100644 --- a/lib/storage/bucket.js +++ b/lib/storage/bucket.js @@ -90,7 +90,7 @@ function Bucket(storage, name) { this.storage = storage; if (!this.name) { - throw Error('A bucket name is needed to use Google Cloud Storage.'); + throw new Error('A bucket name is needed to use Google Cloud Storage.'); } /** diff --git a/scripts/docs.sh b/scripts/docs.sh index b797fa86140..24a477ad27c 100755 --- a/scripts/docs.sh +++ b/scripts/docs.sh @@ -31,6 +31,11 @@ ./node_modules/.bin/dox < lib/pubsub/subscription.js > docs/json/master/pubsub/subscription.json & ./node_modules/.bin/dox < lib/pubsub/topic.js > docs/json/master/pubsub/topic.json & +./node_modules/.bin/dox < lib/search/index.js > docs/json/master/search/index.json & +./node_modules/.bin/dox < lib/search/index-class.js > docs/json/master/search/index-class.json & +./node_modules/.bin/dox < lib/search/document.js > docs/json/master/search/document.json & +./node_modules/.bin/dox < lib/search/field.js > docs/json/master/search/field.json & + ./node_modules/.bin/dox < lib/storage/acl.js > docs/json/master/storage/acl.json & ./node_modules/.bin/dox < lib/storage/bucket.js > docs/json/master/storage/bucket.json & ./node_modules/.bin/dox < lib/storage/file.js > docs/json/master/storage/file.json & diff --git a/system-test/search.js b/system-test/search.js new file mode 100644 index 00000000000..4d059633e1c --- /dev/null +++ b/system-test/search.js @@ -0,0 +1,301 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var async = require('async'); +var uuid = require('node-uuid'); + +var env = require('./env.js'); +var gcloud = require('../lib')(env); + +var TEST_DOCUMENT_JSON = require('../test/testdata/search-document.json'); + +var MAX_PARALLEL = 10; + +var search = gcloud.search(); + +function deleteDocument(document, callback) { + document.delete(callback); +} + +function deleteIndexContents(index, callback) { + function handleResp(err, documents, nextQuery) { + if (err) { + callback(err); + return; + } + + async.eachLimit(documents, MAX_PARALLEL, deleteDocument, function(err) { + if (err) { + callback(err); + return; + } + + if (nextQuery) { + index.getDocuments(nextQuery, handleResp); + return; + } + + callback(); + }); + } + + index.getDocuments(handleResp); +} + +function deleteAllDocuments(callback) { + function handleResp(err, indexes, nextQuery) { + if (err) { + callback(err); + return; + } + + async.eachLimit(indexes, MAX_PARALLEL, deleteIndexContents, function(err) { + if (err) { + callback(err); + return; + } + + if (nextQuery) { + search.getIndexes(nextQuery, handleResp); + return; + } + + callback(); + }); + } + + search.getIndexes(handleResp); +} + +function generateIndexName() { + return 'gcloud-test-index-' + uuid.v1(); +} + +function generateDocumentName() { + return 'gcloud-test-document-' + uuid.v1(); +} + +describe('Search', function() { + var INDEX_NAME = generateIndexName(); + var index = search.index(INDEX_NAME); + + before(function(done) { + deleteAllDocuments(done); + }); + + after(function(done) { + deleteAllDocuments(done); + }); + + describe('creating an index', function() { + it('should create a document in a new index', function(done) { + var newIndexName = generateIndexName(); + var newIndex = search.index(newIndexName); + + newIndex.createDocument(TEST_DOCUMENT_JSON, function(err, document) { + assert.ifError(err); + document.delete(done); + }); + }); + }); + + describe('listing indexes', function() { + before(function(done) { + // Creating a new document in a new index will create the index at the + // same time. Immediately delete the document, as we just need the index + // to exist. + var newIndexName = generateIndexName(); + var newIndex = search.index(newIndexName); + + newIndex.createDocument(TEST_DOCUMENT_JSON, function(err, document) { + if (err) { + done(err); + return; + } + + document.delete(done); + }); + }); + + it('should get all indexes', function(done) { + search.getIndexes(function(err, indexes) { + assert.ifError(err); + assert(indexes.length > 0); + done(); + }); + }); + + it('should get all indexes in stream mode', function(done) { + var resultsMatched = 0; + + search.getIndexes() + .on('error', done) + .on('data', function() { resultsMatched++; }) + .on('end', function() { + assert(resultsMatched > 0); + done(); + }); + }); + }); + + describe('listing documents', function() { + var document; + + before(function(done) { + async.series([ + deleteAllDocuments, + + function(next) { + index.createDocument(TEST_DOCUMENT_JSON, function(err, doc) { + if (err) { + next(err); + return; + } + + document = doc; + next(); + }); + } + ], done); + }); + + after(function(done) { + document.delete(done); + }); + + it('should get all documents', function(done) { + index.getDocuments(function(err, documents) { + assert.ifError(err); + assert.strictEqual(documents.length, 1); + done(); + }); + }); + + it('should get all documents in stream mode', function(done) { + var resultsMatched = 0; + + index.getDocuments() + .on('error', done) + .on('data', function() { resultsMatched++; }) + .on('end', function() { + assert.strictEqual(resultsMatched, 1); + done(); + }); + }); + }); + + describe('creating documents', function() { + it('should create a document from the doc builder', function(done) { + // This recreates the test JSON file with the document builder. + var newDocument = index.document(TEST_DOCUMENT_JSON.docId); + + newDocument.setRank(TEST_DOCUMENT_JSON.rank); + + Object.keys(TEST_DOCUMENT_JSON.fields).forEach(function(fieldName) { + var field = newDocument.addField(fieldName); + + TEST_DOCUMENT_JSON.fields[fieldName].values.forEach(function(value) { + if (value.geoValue) { + field.addGeoValue(value.geoValue); + } + + if (value.numberValue) { + field.addNumberValue(value.numberValue); + } + + if (value.stringValue) { + var options = {}; + if (value.stringFormat) { + options.format = value.stringFormat; + } + field.addTextValue(value.stringValue, options); + } + + if (value.timestampValue) { + field.addTimestampValue(value.timestampValue); + } + }); + }); + + index.createDocument(newDocument, function(err, document) { + assert.ifError(err); + document.getMetadata(function(err) { + assert.ifError(err); + assert.deepEqual(document.toJSON(), TEST_DOCUMENT_JSON); + done(); + }); + }); + }); + + it('should create a document from JSON', function(done) { + index.createDocument(TEST_DOCUMENT_JSON, function(err, document) { + assert.ifError(err); + document.getMetadata(function(err) { + assert.ifError(err); + assert.deepEqual(document.toJSON(), TEST_DOCUMENT_JSON); + done(); + }); + }); + }); + }); + + describe('search', function() { + var query = 'ryan'; + var DOCUMENT_NAME = generateDocumentName(); + var document; + + before(function(done) { + document = index.document(DOCUMENT_NAME); + + var questions = document.addField('question'); + questions.addTextValue('Where did Ryan go?'); + questions.addTextValue('Where did Silvano go?'); + + index.createDocument(document, done); + }); + + after(function(done) { + document.delete(done); + }); + + it('should search document', function(done) { + index.search(query, function(err, results) { + assert.ifError(err); + assert.equal(results.length, 1); + assert.equal(results[0].id, DOCUMENT_NAME); + done(); + }); + }); + + it('should search document in stream mode', function(done) { + var results = []; + + index.search(query) + .on('error', done) + .on('data', function(result) { + results.push(result); + }) + .on('end', function() { + assert.equal(results.length, 1); + assert.equal(results[0].id, DOCUMENT_NAME); + done(); + }); + }); + }); +}); diff --git a/test/common/stream-router.js b/test/common/stream-router.js new file mode 100644 index 00000000000..b25986ace1c --- /dev/null +++ b/test/common/stream-router.js @@ -0,0 +1,224 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var extend = require('extend'); +var mockery = require('mockery'); +var util = require('../../lib/common/util.js'); +var uuid = require('node-uuid'); + +describe('streamRouter', function() { + var streamRouter; + var streamRouterOverrides = {}; + + var UUID = uuid.v1(); + + function FakeClass() {} + + before(function() { + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + streamRouter = require('../../lib/common/stream-router.js'); + var streamRouter_Cached = extend(true, {}, streamRouter); + + Object.keys(streamRouter).forEach(function(streamRouterMethod) { + if (typeof streamRouter[streamRouterMethod] !== 'function') { + return; + } + + streamRouter[streamRouterMethod] = function() { + var args = arguments; + + if (streamRouterOverrides[streamRouterMethod]) { + return streamRouterOverrides[streamRouterMethod].apply(this, args); + } else { + return streamRouter_Cached[streamRouterMethod].apply(this, args); + } + }; + }); + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + beforeEach(function() { + FakeClass.prototype = { methodToExtend: function() { return UUID; } }; + streamRouterOverrides = {}; + }); + + describe('extend', function() { + it('should overwrite a method on a class', function() { + var originalMethod = FakeClass.prototype.methodToExtend; + streamRouter.extend(FakeClass, 'methodToExtend'); + var overwrittenMethod = FakeClass.prototype.methodToExtend; + + assert.notEqual(originalMethod, overwrittenMethod); + }); + + it('should accept an array or string method names', function() { + var originalMethod = FakeClass.prototype.methodToExtend; + + FakeClass.prototype.anotherMethodToExtend = function() {}; + var anotherMethod = FakeClass.prototype.anotherMethodToExtend; + + var methodsToExtend = ['methodToExtend', 'anotherMethodToExtend']; + streamRouter.extend(FakeClass, methodsToExtend); + + assert.notEqual(originalMethod, FakeClass.prototype.methodToExtend); + assert.notEqual(anotherMethod, FakeClass.prototype.anotherMethodToExtend); + }); + + it('should call router when the original method is called', function(done) { + var expectedReturnValue = FakeClass.prototype.methodToExtend(); + + streamRouterOverrides.router_ = function(args, originalMethod) { + assert.deepEqual([].slice.call(args), [1, 2, 3]); + assert.equal(originalMethod(), expectedReturnValue); + done(); + }; + + streamRouter.extend(FakeClass, 'methodToExtend'); + FakeClass.prototype.methodToExtend(1, 2, 3); + }); + + it('should maintain `this` context', function(done) { + FakeClass.prototype.methodToExtend = function() { return this.uuid; }; + + var cls = new FakeClass(); + cls.uuid = uuid.v1(); + + streamRouterOverrides.router_ = function(args, originalMethod) { + assert.equal(originalMethod(), cls.uuid); + done(); + }; + + streamRouter.extend(FakeClass, 'methodToExtend'); + cls.methodToExtend(); + }); + + it('should return what the router returns', function() { + var uniqueValue = 234; + streamRouterOverrides.router_ = function() { + return uniqueValue; + }; + + streamRouter.extend(FakeClass, 'methodToExtend'); + assert.equal(FakeClass.prototype.methodToExtend(), uniqueValue); + }); + }); + + describe('router_', function() { + var ARGS_WITHOUT_CALLBACK = [1, 2, 3]; + var ARGS_WITH_CALLBACK = ARGS_WITHOUT_CALLBACK.concat(util.noop); + + describe('stream mode', function() { + it('should call original method when stream opens', function(done) { + function originalMethod() { + var args = arguments; + + ARGS_WITHOUT_CALLBACK.forEach(function(arg, index) { + assert.strictEqual(args[index], arg); + }); + + done(); + } + + var rs = streamRouter.router_(ARGS_WITHOUT_CALLBACK, originalMethod); + rs.on('data', util.noop); // Trigger the underlying `_read` event. + }); + + it('should emit an error if one occurs', function(done) { + var error = new Error('Error.'); + + function originalMethod() { + var callback = [].slice.call(arguments).pop(); + setImmediate(function() { + callback(error); + }); + } + + var rs = streamRouter.router_(ARGS_WITHOUT_CALLBACK, originalMethod); + rs.on('data', util.noop); // Trigger the underlying `_read` event. + rs.on('error', function(err) { + assert.deepEqual(err, error); + done(); + }); + }); + + it('should push results onto the stream', function(done) { + var results = ['a', 'b', 'c']; + var resultsReceived = []; + + function originalMethod() { + var callback = [].slice.call(arguments).pop(); + setImmediate(function() { + callback(null, results); + }); + } + + var rs = streamRouter.router_(ARGS_WITHOUT_CALLBACK, originalMethod); + rs.on('data', function(result) { + resultsReceived.push(result); + }); + rs.on('end', function() { + assert.deepEqual(results, resultsReceived); + done(); + }); + }); + + it('should get more results if nextQuery exists', function(done) { + var nextQuery = { a: 'b', c: 'd' }; + var nextQuerySent = false; + + function originalMethod() { + var query = arguments[0]; + var callback = [].slice.call(arguments).pop(); + + if (nextQuerySent) { + assert.deepEqual(query, nextQuery); + done(); + return; + } + + setImmediate(function() { + nextQuerySent = true; + callback(null, [], nextQuery); + }); + } + + var rs = streamRouter.router_(ARGS_WITHOUT_CALLBACK, originalMethod); + rs.on('data', util.noop); // Trigger the underlying `_read` event. + }); + }); + + describe('callback mode', function() { + it('should call original method', function(done) { + function originalMethod() { + assert.deepEqual([].slice.call(arguments), ARGS_WITH_CALLBACK); + done(); + } + + streamRouter.router_(ARGS_WITH_CALLBACK, originalMethod); + }); + }); + }); +}); diff --git a/test/docs.js b/test/docs.js index dc145939657..93faf64487d 100644 --- a/test/docs.js +++ b/test/docs.js @@ -14,16 +14,16 @@ * limitations under the License. */ -/*global describe, it */ - 'use strict'; -var vm = require('vm'); var assert = require('assert'); +var fs = require('fs'); var gcloud = require('../'); var glob = require('glob'); var mitm = require('mitm'); -var fs = require('fs'); +var vm = require('vm'); + +var util = require('../lib/common/util.js'); function runCodeInSandbox(code, sandbox) { vm.createContext(sandbox); @@ -74,10 +74,16 @@ describe('documentation', function() { ].join('\n')); } + var mockConsole = Object.keys(console).reduce(function(console, method) { + console[method] = util.noop; + return console; + }, {}); + var sandbox = { gcloud: gcloud, require: require, process: process, + console: mockConsole, Buffer: Buffer, Date: Date, Array: Array diff --git a/test/search.index.js b/test/search.index.js new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/search/document.js b/test/search/document.js new file mode 100644 index 00000000000..c5c937e24c8 --- /dev/null +++ b/test/search/document.js @@ -0,0 +1,304 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); + +var Document = require('../../lib/search/document.js'); +var Field = require('../../lib/search/field.js'); + +var DOCUMENT_JSON = require('../testdata/search-document.json'); + +describe('Document', function() { + var document; + + var SEARCH_INSTANCE = { + projectId: 'project-id' + }; + + var INDEX_INSTANCE = { + search_: SEARCH_INSTANCE + }; + + var ID = 'document-id'; + + beforeEach(function() { + document = new Document(INDEX_INSTANCE, ID); + }); + + describe('instantiation', function() { + it('should localize the Search instance', function() { + assert.deepEqual(document.search_, SEARCH_INSTANCE); + }); + + it('should localize the Index instance', function() { + assert.deepEqual(document.index_, INDEX_INSTANCE); + }); + + it('should localize the id', function() { + assert.equal(document.id, ID); + }); + }); + + describe('addField', function() { + var FIELD_NAME = 'field-name'; + + it('should throw if a name is not provided', function() { + assert.throws(function() { + document.addField(); + }, /A name is required/); + }); + + it('should return a Field instance', function() { + var field = document.addField(FIELD_NAME); + + assert(field instanceof Field); + }); + + it('should localize the Field instance', function() { + var field = document.addField(FIELD_NAME); + + assert.deepEqual(document.fields[FIELD_NAME], field); + }); + }); + + describe('delete', function() { + it('should delete the document', function(done) { + document.makeReq_ = function(method, path, query, body) { + assert.equal(method, 'DELETE'); + assert.equal(path, ''); + assert.strictEqual(query, null); + assert.strictEqual(body, null); + done(); + }; + + document.delete(assert.ifError); + }); + + it('should pass an error if one occurred', function(done) { + var error = new Error('Error.'); + var apiResponse = { a: 'b', c: 'd' }; + + document.makeReq_ = function(method, path, query, body, callback) { + callback(error, apiResponse); + }; + + document.delete(function(err, apiResponse_) { + assert.deepEqual(err, error); + assert.deepEqual(apiResponse_, apiResponse); + done(); + }); + }); + + it('should pass the API response to the callback', function(done) { + var apiResponse = { a: 'b', c: 'd' }; + + document.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + document.delete(function(err, apiResponse_) { + assert.ifError(err); + assert.deepEqual(apiResponse_, apiResponse); + done(); + }); + }); + }); + + describe('getMetadata', function() { + it('should get the document from the API', function(done) { + document.makeReq_ = function(method, path, query, body) { + assert.equal(method, 'GET'); + assert.equal(path, '/'); + assert.strictEqual(query, null); + assert.strictEqual(body, null); + done(); + }; + + document.getMetadata(assert.ifError); + }); + + it('should execute the callback with an error', function(done) { + var error = new Error('Error.'); + var apiResponse = { a: 'b', c: 'd' }; + + document.makeReq_ = function(method, path, query, body, callback) { + callback(error, apiResponse); + }; + + document.getMetadata(function(err, doc, apiResponse_) { + assert.deepEqual(err, error); + assert.deepEqual(apiResponse, apiResponse_); + done(); + }); + }); + + it('should reset the localized fields', function(done) { + document.makeReq_ = function(method, path, query, body, callback) { + callback(null, {}); + }; + + document.fields = { a: 'b', c: 'd' }; + + document.getMetadata(function(err) { + assert.ifError(err); + + assert.strictEqual(Object.keys(document.fields).length, 0); + + done(); + }); + }); + + it('should create and localize Field instances', function(done) { + document.makeReq_ = function(method, path, query, body, callback) { + callback(null, DOCUMENT_JSON); + }; + + document.addField = function(fieldName) { + assert(DOCUMENT_JSON.fields[fieldName]); + this.fields[fieldName] = {}; + return this.fields[fieldName]; + }; + + document.getMetadata(function(err) { + assert.ifError(err); + + var numOriginalFields = Object.keys(DOCUMENT_JSON.fields).length; + var numNewFields = Object.keys(document.fields).length; + + assert.strictEqual(numOriginalFields, numNewFields); + + for (var fieldName in document.fields) { + var originalField = DOCUMENT_JSON.fields[fieldName]; + var newField = document.fields[fieldName]; + + assert.deepEqual(originalField, newField); + } + + done(); + }); + }); + + it('should reset the localized rank', function(done) { + document.makeReq_ = function(method, path, query, body, callback) { + callback(null, {}); + }; + + document.rank = Date.now(); + + document.getMetadata(function(err) { + assert.ifError(err); + assert.strictEqual(document.rank, undefined); + done(); + }); + }); + + it('should localize a new rank', function(done) { + document.makeReq_ = function(method, path, query, body, callback) { + callback(null, DOCUMENT_JSON); + }; + + document.getMetadata(function(err) { + assert.ifError(err); + assert.strictEqual(document.rank, DOCUMENT_JSON.rank); + done(); + }); + }); + + it('should execute the callback with Document & api resp', function(done) { + document.makeReq_ = function(method, path, query, body, callback) { + callback(null, DOCUMENT_JSON); + }; + + document.getMetadata(function(err, document_, apiResponse) { + assert.ifError(err); + + assert.deepEqual(document_, document); + assert.deepEqual(apiResponse, DOCUMENT_JSON); + + done(); + }); + }); + }); + + describe('setRank', function() { + it('should throw if the given rank is not a number', function() { + assert.throws(function() { + document.setRank(); + }, /rank should be a positive integer/); + + assert.throws(function() { + document.setRank(true); + }, /rank should be a positive integer/); + + assert.throws(function() { + document.setRank(function() {}); + }, /rank should be a positive integer/); + }); + + it('should throw if the given rank is a negative number', function() { + assert.throws(function() { + document.setRank(-3); + }, /rank should be a positive integer/); + }); + + it('should localize the new rank', function() { + document.setRank(8); + assert.equal(document.rank, 8); + }); + }); + + describe('toJSON', function() { + it('should return an object with a fields property', function() { + document.fields = DOCUMENT_JSON.fields; + var documentJson = document.toJSON(); + assert.deepEqual(documentJson.fields, DOCUMENT_JSON.fields); + }); + + it('should return an object with a docId property', function() { + var documentJson = document.toJSON(); + assert.equal(documentJson.docId, ID); + }); + + it('should return an object with a rank property', function() { + document.rank = 8; + var documentJson = document.toJSON(); + assert.equal(documentJson.rank, 8); + }); + }); + + describe('makeReq_', function() { + it('should call index instance makeReq_', function(done) { + var method = 'POST'; + var path = '/'; + var query = 'query'; + var body = 'body'; + var callback = 'callback'; + + document.index_.makeReq_ = function(m, p, q, b, c) { + assert.equal(m, method); + assert.equal(p, '/documents/' + ID + path); + assert.equal(q, query); + assert.equal(b, body); + assert.equal(c, callback); + done(); + }; + + document.makeReq_(method, path, query, body, callback); + }); + }); +}); diff --git a/test/search/field.js b/test/search/field.js new file mode 100644 index 00000000000..3609820b5e5 --- /dev/null +++ b/test/search/field.js @@ -0,0 +1,149 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); + +var Field = require('../../lib/search/field.js'); + +describe('Field', function() { + var field; + + function fieldHasValue(field, type, value) { + return field.values + .filter(function(value) { return type in value; }) + .some(function(value_) { return value_[type] === value; }); + } + + beforeEach(function() { + field = new Field(); + }); + + describe('instantiation', function() { + it('should localize an empty values array', function() { + assert.deepEqual(field.values, []); + }); + }); + + describe('addAtomValue', function() { + it('should pass value to addTextValue', function(done) { + var atom = 'text-value'; + + field.addTextValue = function(text, options) { + assert.equal(text, atom); + assert.equal(options.format, 'ATOM'); + done(); + }; + + field.addAtomValue(atom); + }); + }); + + describe('addGeoValue', function() { + it('should push geo value to values', function() { + var geoValue = '+lat, -long'; + + field.addGeoValue(geoValue); + + assert(fieldHasValue(field, 'geoValue', geoValue)); + }); + + it('should parse an object', function() { + var geoValue = { latitude: 42.3314, longitude: 83.0458 }; + var expectedGeoValue = geoValue.latitude + ', ' + geoValue.longitude; + + field.addGeoValue(geoValue); + + assert(fieldHasValue(field, 'geoValue', expectedGeoValue)); + }); + }); + + describe('addHtmlValue', function() { + it('should pass value to addTextValue', function(done) { + var html = ''; + + field.addTextValue = function(text, options) { + assert.equal(text, html); + assert.equal(options.format, 'HTML'); + done(); + }; + + field.addHtmlValue(html); + }); + }); + + describe('addNumberValue', function() { + it('should throw if a number is not provided', function() { + assert.throws(function() { + field.addNumberValue(); + }, /number must be a number/); + + assert.throws(function() { + field.addNumberValue(true); + }, /number must be a number/); + + assert.throws(function() { + field.addNumberValue(function() {}); + }, /number must be a number/); + }); + + it('should push number value to values', function() { + field.addNumberValue(8); + + assert(fieldHasValue(field, 'numberValue', 8)); + }); + }); + + describe('addTextValue', function() { + it('should push formatted value to values', function() { + var text = 'HELLO'; + + field.addTextValue(text); + + assert(fieldHasValue(field, 'stringValue', text.toLowerCase())); + assert(fieldHasValue(field, 'stringFormat', 'TEXT')); + }); + + it('should allow customization of text value', function() { + var text = 'HELLO'; + var format = 'html'; + + field.addTextValue(text, { format: format }); + + assert(fieldHasValue(field, 'stringValue', text.toLowerCase())); + assert(fieldHasValue(field, 'stringFormat', format.toUpperCase())); + }); + }); + + describe('addTimestampValue', function() { + it('should push timestamp value to values', function() { + var timestampValue = 'timestamp-value'; + + field.addTimestampValue(timestampValue); + + assert(fieldHasValue(field, 'timestampValue', timestampValue)); + }); + + it('should parse a date object', function() { + var date = new Date(); + + field.addTimestampValue(date); + + assert(fieldHasValue(field, 'timestampValue', date.toJSON())); + }); + }); +}); diff --git a/test/search/index-class.js b/test/search/index-class.js new file mode 100644 index 00000000000..4f9a5c5bbd0 --- /dev/null +++ b/test/search/index-class.js @@ -0,0 +1,426 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var extend = require('extend'); +var mockery = require('mockery'); +var util = require('../../lib/common/util.js'); + +function FakeDocument() { + this.calledWith_ = util.toArray(arguments); +} +FakeDocument.prototype.toJSON = util.noop; + +var extended = false; +var fakeStreamRouter = { + extend: function(Class, methods) { + extended = true; + methods = util.arrayize(methods); + assert.equal(Class.name, 'Index'); + assert.deepEqual(methods, ['getDocuments', 'search']); + } +}; + +describe('Index', function() { + var Index; + var index; + + var SEARCH_INSTANCE = { + projectId: 'project-id' + }; + + var ID = 'index-id'; + + before(function() { + mockery.registerMock('./document.js', FakeDocument); + mockery.registerMock('../common/stream-router.js', fakeStreamRouter); + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + + Index = require('../../lib/search/index-class.js'); + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + beforeEach(function() { + index = new Index(SEARCH_INSTANCE, ID); + }); + + describe('instantiation', function() { + it('should extend the correct methods', function() { + assert(extended); // See `fakeStreamRouter.extend` + }); + + it('should localize the Search instance', function() { + assert.deepEqual(index.search_, SEARCH_INSTANCE); + }); + + it('should localize the id', function() { + assert.equal(index.id, ID); + }); + + it('should throw if an ID is not provided', function() { + assert.throws(function() { + new Index(SEARCH_INSTANCE); + }, /An ID is needed/); + }); + }); + + describe('createDocument', function() { + beforeEach(function() { + index.makeReq_ = util.noop; + }); + + it('should accept a Document object', function(done) { + var document = new FakeDocument(); + document.toJSON = done; + + index.createDocument(document, assert.ifError); + }); + + it('should create a Document from a JSON object', function(done) { + var documentObject = { a: 'b', c: 'd' }; + + index.documentFromObject_ = function(documentObject_) { + assert.deepEqual(documentObject_, documentObject); + done(); + }; + + index.createDocument(documentObject, assert.ifError); + }); + + it('should post document to the API', function(done) { + var expectedDocumentJson = { a: 'b', c: 'd' }; + + var document = new FakeDocument(); + document.toJSON = function() { return expectedDocumentJson; }; + + index.makeReq_ = function(method, path, query, body) { + assert.equal(method, 'POST'); + assert.equal(path, '/documents'); + assert.strictEqual(query, null); + assert.deepEqual(body, expectedDocumentJson); + done(); + }; + + index.createDocument(document, assert.ifError); + }); + + it('should execute callback with error & API response', function(done) { + var error = new Error('Error.'); + var apiResponse = { a: 'b', c: 'd' }; + + index.makeReq_ = function(method, path, query, body, callback) { + callback(error, apiResponse); + }; + + var document = new FakeDocument(); + + index.createDocument(document, function(err, document, apiResp) { + assert.deepEqual(err, error); + assert.strictEqual(document, null); + assert.deepEqual(apiResp, apiResponse); + done(); + }); + }); + + it('should execute callback with Document object', function(done) { + var apiResponse = { a: 'b', c: 'd' }; + + index.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + var document = new FakeDocument(); + + index.createDocument(document, function(err, document_, apiResp) { + assert.ifError(err); + assert.deepEqual(document_, document); + assert.deepEqual(apiResp, apiResponse); + done(); + }); + }); + }); + + describe('document', function() { + it('should return a new Document object', function() { + var docId = 'doc-id'; + var document = index.document(docId); + + assert.deepEqual(document.calledWith_, [index, docId]); + }); + }); + + describe('getDocuments', function() { + it('should get document from the API', function(done) { + var query = { a: 'b', c: 'd' }; + + index.makeReq_ = function(method, path, q, body) { + assert.equal(method, 'GET'); + assert.equal(path, '/documents'); + assert.deepEqual(q, query); + assert.strictEqual(body, null); + done(); + }; + + index.getDocuments(query, assert.ifError); + }); + + it('should send empty query if only a callback is given', function(done) { + index.makeReq_ = function(method, path, query) { + assert.deepEqual(query, {}); + done(); + }; + + index.getDocuments(assert.ifError); + }); + + it('should execute callback with error & API response', function(done) { + var error = new Error('Error.'); + var apiResponse = { a: 'b', c: 'd' }; + + index.makeReq_ = function(method, path, query, body, callback) { + callback(error, apiResponse); + }; + + index.getDocuments({}, function(err, documents, nextQuery, apiResp) { + assert.deepEqual(err, error); + assert.strictEqual(documents, null); + assert.strictEqual(nextQuery, null); + assert.deepEqual(apiResp, apiResponse); + done(); + }); + }); + + it('should execute callback with document objects', function(done) { + var documentObjects = [{ a: 'b' }, { c: 'd' }, { e: 'f' }]; + var apiResponse = { documents: documentObjects }; + + index.documentFromObject_ = function(documentObject) { + assert(documentObjects.indexOf(documentObject) > -1); + return true; // Used in the test callback to assure the value returned + // to the callback is what's returned from this method. + }; + + index.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + index.getDocuments({}, function(err, documents, nextQuery, apiResp) { + assert.ifError(err); + + assert.strictEqual(documents.length, documentObjects.length); + assert(documents.every(function (document) { return document; })); + + assert.strictEqual(nextQuery, null); + assert.deepEqual(apiResp, apiResponse); + + done(); + }); + }); + + it('should provide nextQuery to callback', function(done) { + var apiResponse = { nextPageToken: 'page-token' }; + + var query = { a: 'b', c: 'd' }; + var originalQuery = extend({}, query); + var expectedNextQuery = extend({}, query, { + pageToken: apiResponse.nextPageToken + }); + + index.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + index.getDocuments(query, function(err, documents, nextQuery) { + assert.ifError(err); + assert.deepEqual(query, originalQuery); + assert.deepEqual(nextQuery, expectedNextQuery); + done(); + }); + }); + }); + + describe('search', function() { + it('should throw if a query string or object is not provided', function() { + assert.throws(function() { + index.search(); + }, /A query must be either a string or object/); + + assert.throws(function() { + index.search(util.noop); + }, /A query must be either a string or object/); + }); + + it('should send a query to the API', function(done) { + var query = { + query: 'completed=true' + }; + + index.makeReq_ = function(method, path, q, body) { + assert.equal(method, 'GET'); + assert.equal(path, '/search'); + assert.deepEqual(q, query); + assert.strictEqual(body, null); + done(); + }; + + index.search(query, assert.ifError); + }); + + it('should build a query object from a string', function(done) { + var query = 'completed=true'; + + index.makeReq_ = function(method, path, q) { + assert.deepEqual(q, { query: query }); + done(); + }; + + index.search(query, assert.ifError); + }); + + it('should execute callback with error & API response', function(done) { + var apiResponse = { a: 'b', c: 'd' }; + var error = new Error('Error.'); + + index.makeReq_ = function(method, path, query, body, callback) { + callback(error, apiResponse); + }; + + index.search({}, function(err, documents, nextQuery, apiResp) { + assert.deepEqual(err, error); + assert.strictEqual(nextQuery, null); + assert.deepEqual(apiResp, apiResponse); + done(); + }); + }); + + it('should execute callback with document objects', function(done) { + var documentObjects = [{ a: 'b' }, { c: 'd' }, { e: 'f' }]; + var apiResponse = { results: documentObjects }; + + index.documentFromObject_ = function(documentObject) { + assert(documentObjects.indexOf(documentObject) > -1); + return true; // Used in the test callback to assure the value returned + // to the callback is what's returned from this method. + }; + + index.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + index.search({}, function(err, documents, nextQuery, apiResp) { + assert.ifError(err); + + assert.strictEqual(documents.length, documentObjects.length); + assert(documents.every(function (document) { return document; })); + + assert.strictEqual(nextQuery, null); + assert.deepEqual(apiResp, apiResponse); + + done(); + }); + }); + + it('should provide nextQuery to callback', function(done) { + var apiResponse = { nextPageToken: 'page-token' }; + + var query = { a: 'b', c: 'd' }; + var originalQuery = extend({}, query); + var expectedNextQuery = extend({}, query, { + pageToken: apiResponse.nextPageToken + }); + + index.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + index.search(query, function(err, documents, nextQuery) { + assert.ifError(err); + assert.deepEqual(query, originalQuery); + assert.deepEqual(nextQuery, expectedNextQuery); + done(); + }); + }); + }); + + describe('documentFromObject_', function() { + beforeEach(function() { + index.document = function() { + return {}; + }; + }); + + it('should create a document object', function(done) { + var documentObject = { docId: 'doc-id' }; + + index.document = function(docId) { + assert.equal(docId, documentObject.docId); + done(); + }; + + index.documentFromObject_(documentObject); + }); + + it('should return an object with a fields property', function() { + var documentObject = { + docId: 'doc-id', + fields: { a: 'b', c: 'd' } + }; + + var document = index.documentFromObject_(documentObject); + assert.deepEqual(document.fields, documentObject.fields); + }); + + it('should return an object with a rank property', function() { + var documentObject = { + docId: 'doc-id', + rank: 8 + }; + + var document = index.documentFromObject_(documentObject); + assert.equal(document.rank, documentObject.rank); + }); + }); + + describe('makeReq_', function() { + it('should call search instance makeReq_', function(done) { + var method = 'POST'; + var path = '/'; + var query = 'query'; + var body = 'body'; + var callback = 'callback'; + + index.search_.makeReq_ = function(m, p, q, b, c) { + assert.equal(m, method); + assert.equal(p, '/indexes/' + ID + path); + assert.equal(q, query); + assert.equal(b, body); + assert.equal(c, callback); + done(); + }; + + index.makeReq_(method, path, query, body, callback); + }); + }); +}); diff --git a/test/search/index.js b/test/search/index.js new file mode 100644 index 00000000000..dcb98fcbd8f --- /dev/null +++ b/test/search/index.js @@ -0,0 +1,281 @@ +/*! + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var extend = require('extend'); +var mockery = require('mockery'); +var util = require('../../lib/common/util.js'); + +function FakeIndex() { + this.calledWith_ = util.toArray(arguments); +} + +var extended = false; +var fakeStreamRouter = { + extend: function(Class, methods) { + if (Class.name !== 'Search') { + return; + } + + methods = util.arrayize(methods); + assert.equal(Class.name, 'Search'); + assert.deepEqual(methods, ['getIndexes']); + extended = true; + } +}; + +var makeAuthorizedRequestFactory_Override; +var fakeUtil = extend({}, util, { + makeAuthorizedRequestFactory: function() { + if (makeAuthorizedRequestFactory_Override) { + return makeAuthorizedRequestFactory_Override.apply(null, arguments); + } else { + return util.makeAuthorizedRequestFactory.apply(null, arguments); + } + } +}); + +describe('Search', function() { + var Search; + var search; + + var PROJECT_ID = 'project-id'; + + before(function() { + mockery.registerMock('./index-class.js', FakeIndex); + mockery.registerMock('../common/stream-router.js', fakeStreamRouter); + mockery.registerMock('../common/util.js', fakeUtil); + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + + Search = require('../../lib/search/index.js'); + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + beforeEach(function() { + makeAuthorizedRequestFactory_Override = null; + + search = new Search({ + projectId: PROJECT_ID + }); + }); + + describe('instantiation', function() { + it('should extend the correct methods', function() { + assert(extended); // See `fakeStreamRouter.extend` + }); + + it('should throw if a projectId is not specified', function() { + assert.throws(function() { + new Search(); + }, /Sorry, we cannot connect/); + }); + + it('should create an authorized request function', function(done) { + var options = { + projectId: 'projectId', + credentials: 'credentials', + email: 'email', + keyFilename: 'keyFile' + }; + + makeAuthorizedRequestFactory_Override = function(options_) { + assert.deepEqual(options_, { + credentials: options.credentials, + email: options.email, + keyFile: options.keyFilename, + scopes: [ + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/cloudsearch', + 'https://www.googleapis.com/auth/userinfo.email' + ] + }); + return done; + }; + + var search = new Search(options); + search.makeAuthorizedRequest_(); + }); + + it('should localize the projectId', function() { + assert.equal(search.projectId_, PROJECT_ID); + }); + }); + + describe('getIndexes', function() { + it('should get indexes from the API', function(done) { + var query = { a: 'b', c: 'd' }; + + search.makeReq_ = function(method, path, q, body) { + assert.equal(method, 'GET'); + assert.equal(path, '/indexes'); + assert.deepEqual(q, query); + assert.strictEqual(body, null); + done(); + }; + + search.getIndexes(query, assert.ifError); + }); + + it('should rename query.prefix to indexNamePrefix', function(done) { + var query = { prefix: 'prefix' }; + + search.makeReq_ = function(method, path, q) { + assert.equal(typeof q.prefix, 'undefined'); + assert.equal(q.indexNamePrefix, query.prefix); + done(); + }; + + search.getIndexes(query, assert.ifError); + }); + + it('should send empty query if only a callback is given', function(done) { + search.makeReq_ = function(method, path, query) { + assert.deepEqual(query, {}); + done(); + }; + + search.getIndexes(assert.ifError); + }); + + it('should execute callback with error & API response', function(done) { + var error = new Error('Error.'); + var apiResponse = { a: 'b', c: 'd' }; + + search.makeReq_ = function(method, path, query, body, callback) { + callback(error, apiResponse); + }; + + search.getIndexes({}, function(err, indexes, nextQuery, apiResp) { + assert.deepEqual(err, error); + assert.strictEqual(indexes, null); + assert.strictEqual(nextQuery, null); + assert.deepEqual(apiResp, apiResponse); + done(); + }); + }); + + it('should execute callback with index objects', function(done) { + var indexObjects = [{ indexId: 'a' }, { indexId: 'b' }, { indexId: 'c' }]; + var indexIds = indexObjects.map(util.prop('indexId')); + + var apiResponse = { indexes: indexObjects }; + + search.index = function(indexId) { + assert(indexIds.indexOf(indexId) > -1); + return true; // Used in the test callback to assure the value returned + // to the callback is what's returned from this method. + }; + + search.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + search.getIndexes({}, function(err, indexes, nextQuery, apiResp) { + assert.ifError(err); + + assert.strictEqual(indexes.length, indexObjects.length); + assert(indexes.every(function (index) { return index; })); + + assert.strictEqual(nextQuery, null); + assert.deepEqual(apiResp, apiResponse); + + done(); + }); + }); + + it('should assign fields property on new Index', function(done) { + var indexObjects = [{ indexId: 'a', indexedField: 'b' }]; + + var apiResponse = { indexes: indexObjects }; + + search.index = function() { + return {}; + }; + + search.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + search.getIndexes({}, function(err, indexes) { + assert.ifError(err); + assert.equal(indexes[0].fields, indexObjects.indexedField); + done(); + }); + }); + + it('should provide nextQuery to callback', function(done) { + var apiResponse = { nextPageToken: 'page-token' }; + + var query = { a: 'b', c: 'd' }; + var originalQuery = extend({}, query); + var expectedNextQuery = extend({}, query, { + pageToken: apiResponse.nextPageToken + }); + + search.makeReq_ = function(method, path, query, body, callback) { + callback(null, apiResponse); + }; + + search.getIndexes(query, function(err, indexes, nextQuery) { + assert.ifError(err); + assert.deepEqual(query, originalQuery); + assert.deepEqual(nextQuery, expectedNextQuery); + done(); + }); + }); + }); + + describe('index', function() { + it('should return a new Index object', function() { + var indexId = 'index-id'; + var index = search.index(indexId); + + assert.deepEqual(index.calledWith_, [search, indexId]); + }); + }); + + describe('makeReq_', function() { + it('should make correct authorized request', function(done) { + var method = 'POST'; + var path = '/'; + var query = 'query'; + var body = 'body'; + + search.makeAuthorizedRequest_ = function(reqOpts, callback) { + assert.equal(reqOpts.method, method); + assert.equal(reqOpts.qs, query); + + var baseUri = 'https://cloudsearch.googleapis.com/v1/'; + assert.equal(reqOpts.uri, baseUri + 'projects/' + PROJECT_ID + path); + + assert.equal(reqOpts.json, body); + + callback(); + }; + + search.makeReq_(method, path, query, body, done); + }); + }); +}); diff --git a/test/testdata/search-document.json b/test/testdata/search-document.json new file mode 100644 index 00000000000..82742d8ccf4 --- /dev/null +++ b/test/testdata/search-document.json @@ -0,0 +1,31 @@ +{ + "docId": "doc1", + "fields": { + "field2": { + "values": [ + { + "stringValue": "helloworld", + "stringFormat": "HTML" + } + ] + }, + "field1": { + "values": [ + { + "stringValue": "helloworld", + "stringFormat": "TEXT" + }, + { + "timestampValue": "2014-08-18T21:19:55.000Z" + }, + { + "numberValue": 10 + }, + { + "geoValue": "40.6894, -74.0447" + } + ] + } + }, + "rank": 8 +} From abca6f186c2046fac99bbcfa2459822a5f3832f3 Mon Sep 17 00:00:00 2001 From: Stephen Sawchuk Date: Thu, 18 Jun 2015 15:55:54 -0400 Subject: [PATCH 2/2] stream-router: support premature stream ending --- lib/common/stream-router.js | 65 +++++++++++++++++++++++------------- system-test/search.js | 4 ++- test/common/stream-router.js | 52 +++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 24 deletions(-) diff --git a/lib/common/stream-router.js b/lib/common/stream-router.js index 12490d2288d..a2ae6d15c74 100644 --- a/lib/common/stream-router.js +++ b/lib/common/stream-router.js @@ -84,35 +84,54 @@ streamRouter.router_ = function(args, originalMethod) { var callback = args[args.length - 1]; var isStreamMode = !util.is(callback, 'function'); - if (isStreamMode) { - var stream = streamEvents(through.obj()); - - var onResultSet = function(err, results, nextQuery) { - if (err) { - stream.emit('error', err); - stream.end(); - return; - } + if (!isStreamMode) { + originalMethod.apply(null, args); + return; + } - results.forEach(function(result) { + var stream = streamEvents(through.obj()); + + // Results from the API are split apart for the user. If 50 results are + // returned, we emit 50 data events. While the user is consuming these, they + // might choose to end the stream early by calling ".end()". We keep track of + // this state to prevent pushing more results to the stream, ending it again, + // or making unnecessary API calls. + var streamEnded = false; + var _end = stream.end; + stream.end = function() { + streamEnded = true; + _end.apply(this, arguments); + }; + + function onResultSet(err, results, nextQuery) { + if (err) { + stream.emit('error', err); + stream.end(); + return; + } + + results.forEach(function(result) { + if (!streamEnded) { stream.push(result); - }); - - if (nextQuery) { - originalMethod(nextQuery, onResultSet); - } else { - stream.end(); } - }; - - stream.once('reading', function() { - originalMethod.apply(null, args.concat(onResultSet)); }); - return stream; - } else { - originalMethod.apply(null, args); + if (streamEnded) { + return; + } + + if (nextQuery) { + originalMethod(nextQuery, onResultSet); + } else { + stream.end(); + } } + + stream.once('reading', function() { + originalMethod.apply(null, args.concat(onResultSet)); + }); + + return stream; }; module.exports = streamRouter; diff --git a/system-test/search.js b/system-test/search.js index 4d059633e1c..c5e517530aa 100644 --- a/system-test/search.js +++ b/system-test/search.js @@ -146,7 +146,9 @@ describe('Search', function() { search.getIndexes() .on('error', done) - .on('data', function() { resultsMatched++; }) + .on('data', function() { + resultsMatched++; + }) .on('end', function() { assert(resultsMatched > 0); done(); diff --git a/test/common/stream-router.js b/test/common/stream-router.js index b25986ace1c..c4de9f9ff0e 100644 --- a/test/common/stream-router.js +++ b/test/common/stream-router.js @@ -208,6 +208,58 @@ describe('streamRouter', function() { var rs = streamRouter.router_(ARGS_WITHOUT_CALLBACK, originalMethod); rs.on('data', util.noop); // Trigger the underlying `_read` event. }); + + it('should not push more results if stream ends early', function(done) { + var results = ['a', 'b', 'c']; + + function originalMethod() { + var callback = [].slice.call(arguments).pop(); + setImmediate(function() { + callback(null, results); + }); + } + + var rs = streamRouter.router_(ARGS_WITHOUT_CALLBACK, originalMethod); + rs.on('data', function(result) { + if (result === 'b') { + // Pre-maturely end the stream. + this.end(); + } + + assert.notEqual(result, 'c'); + }); + rs.on('end', function() { + done(); + }); + }); + + it('should not get more results if stream ends early', function(done) { + var results = ['a', 'b', 'c']; + + var originalMethodCalledCount = 0; + + function originalMethod() { + originalMethodCalledCount++; + + var callback = [].slice.call(arguments).pop(); + + setImmediate(function() { + callback(null, results, {}); + }); + } + + var rs = streamRouter.router_(ARGS_WITHOUT_CALLBACK, originalMethod); + rs.on('data', function(result) { + if (result === 'b') { + // Pre-maturely end the stream. + this.end(); + } + }); + rs.on('end', function() { + assert.equal(originalMethodCalledCount, 1); + done(); + }); + }); }); describe('callback mode', function() {