From 141012f225b955df043acc1a66e1b3385cf247da Mon Sep 17 00:00:00 2001 From: Stephen Sawchuk Date: Mon, 13 Jul 2015 23:55:58 -0400 Subject: [PATCH] stream-router: default `autoPaginate: true` --- lib/bigquery/index.js | 75 ++++++++------- lib/bigquery/job.js | 2 +- lib/bigquery/table.js | 26 +++-- lib/common/stream-router.js | 76 +++++++++------ lib/common/util.js | 30 +++++- lib/datastore/query.js | 2 +- lib/datastore/request.js | 25 ++--- lib/pubsub/index.js | 28 +++--- lib/search/index-class.js | 82 +++++++--------- lib/search/index.js | 29 +++--- lib/storage/bucket.js | 51 +++++----- lib/storage/index.js | 36 ++++--- system-test/bigquery.js | 26 +++-- system-test/datastore.js | 15 +-- system-test/pubsub.js | 8 +- system-test/search.js | 52 ++-------- system-test/storage.js | 33 ++----- test/common/stream-router.js | 178 ++++++++++++++++------------------- test/common/util.js | 83 +++++++++++++--- test/datastore/query.js | 10 +- 20 files changed, 439 insertions(+), 428 deletions(-) diff --git a/lib/bigquery/index.js b/lib/bigquery/index.js index 462552d91f67..3a128efd090b 100644 --- a/lib/bigquery/index.js +++ b/lib/bigquery/index.js @@ -163,13 +163,23 @@ BigQuery.prototype.dataset = function(id) { * @param {object=} query - Configuration object. * @param {boolean} query.all - List all datasets, including hidden ones. * @param {boolean} query.autoPaginate - Have pagination handled automatically. - * Default: false. + * Default: true. * @param {number} query.maxResults - Maximum number of results to return. * @param {string} query.pageToken - Token returned from a previous call, to * request the next page of results. * @param {function} callback - The callback function. * * @example + * bigquery.getDatasets(function(err, datasets) { + * if (!err) { + * // datasets is an array of Dataset objects. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- * var callback = function(err, datasets, nextQuery, apiResponse) { * if (nextQuery) { * // More results exist. @@ -177,17 +187,9 @@ BigQuery.prototype.dataset = function(id) { * } * }; * - * bigquery.getDatasets(callback); - * - * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. - * //- * bigquery.getDatasets({ - * autoPaginate: true - * }, function(err, datasets) { - * // Called after all datasets have been retrieved. - * }); + * autoPaginate: false + * }, callback); * * //- * // Get the datasets from your project as a readable object stream. @@ -251,7 +253,7 @@ BigQuery.prototype.getDatasets = function(query, callback) { * @param {boolean=} options.allUsers - Display jobs owned by all users in the * project. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {number=} options.maxResults - Maximum number of results to return. * @param {string=} options.pageToken - Token returned from a previous call, to * request the next page of results. @@ -263,19 +265,27 @@ BigQuery.prototype.getDatasets = function(query, callback) { * @param {function} callback - The callback function. * * @example - * bigquery.getJobs(function(err, jobs, nextQuery, apiResponse) { - * // If `nextQuery` is non-null, there are more results to fetch. + * bigquery.getJobs(function(err, jobs) { + * if (!err) { + * // jobs is an array of Job objects. + * } * }); * * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. * //- + * var callback = function(err, jobs, nextQuery, apiRespose) { + * if (nextQuery) { + * // More results exist. + * bigquery.getJobs(nextQuery, callback); + * } + * }; + * * bigquery.getJobs({ - * autoPaginate: true - * }, function(err, jobs) { - * // Called after all jobs have been retrieved. - * }); + * autoPaginate: false + * }, callback); + * * //- * // Get the jobs from your project as a readable object stream. * //- @@ -360,7 +370,7 @@ BigQuery.prototype.job = function(id) { * * @param {string|object} options - A string SQL query or configuration object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {number} options.maxResults - Maximum number of results to read. * @param {string} options.query - A query string, following the BigQuery query * syntax, of the query to execute. @@ -373,29 +383,26 @@ BigQuery.prototype.job = function(id) { * @example * var query = 'SELECT url FROM [publicdata:samples.github_nested] LIMIT 100'; * + * bigquery.query(query, function(err, rows) { + * if (!err) { + * // Handle results here. + * } + * }); + * * //- - * // You can run a query against your data in a serial manner. + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. * //- * var callback = function(err, rows, nextQuery, apiResponse) { - * // Handle results here. - * * if (nextQuery) { * bigquery.query(nextQuery, callback); * } * }; * - * bigquery.query(query, callback); - * - * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. - * //- * bigquery.query({ * query: query, - * autoPaginate: true - * }, function(err, rows) { - * // Called after all rows have been retrieved. - * }); + * autoPaginate: false + * }, callback); * * //- * // You can also use the `query` method as a readable object stream by diff --git a/lib/bigquery/job.js b/lib/bigquery/job.js index 2d27c8964bd0..881f786951bf 100644 --- a/lib/bigquery/job.js +++ b/lib/bigquery/job.js @@ -92,7 +92,7 @@ Job.prototype.getMetadata = function(callback) { * * @param {object=} options - Configuration object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {number} options.maxResults - Maximum number of results to read. * @param {string} options.pageToken - Page token, returned by a previous call, * to request the next page of results. Note: This is automatically added to diff --git a/lib/bigquery/table.js b/lib/bigquery/table.js index 94a53923391a..7e400c8cd1e6 100644 --- a/lib/bigquery/table.js +++ b/lib/bigquery/table.js @@ -484,15 +484,21 @@ Table.prototype.getMetadata = function(callback) { * * @param {object=} options - The configuration object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {number} options.maxResults - Maximum number of results to return. * @param {function} callback - The callback function. * * @example - * var options = { - * maxResults: 100 - * }; + * table.getRows(function(err, rows) { + * if (!err) { + * // Handle results here. + * } + * }); * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- * var callback = function(err, rows, nextQuery, apiResponse) { * if (nextQuery) { * // More results exist. @@ -500,17 +506,9 @@ Table.prototype.getMetadata = function(callback) { * } * }; * - * table.getRows(options, callback); - * - * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. - * //- * table.getRows({ - * autoPaginate: true - * }, function(err, rows) { - * // Called after all rows have been retrieved. - * }); + * autoPaginate: false + * }, callback); * * //- * // Get the rows as a readable object stream. diff --git a/lib/common/stream-router.js b/lib/common/stream-router.js index 51dde04b9f32..f4d2708fbde4 100644 --- a/lib/common/stream-router.js +++ b/lib/common/stream-router.js @@ -74,22 +74,51 @@ streamRouter.extend = function(Class, methodNames) { * method received. */ streamRouter.parseArguments_ = function(args) { - var parsedArguments = {}; + var query; + var callback; + var maxResults = -1; + var autoPaginate = true; var firstArgument = args[0]; var lastArgument = args[args.length - 1]; if (util.is(firstArgument, 'function')) { - parsedArguments.callback = firstArgument; + callback = firstArgument; } else { - parsedArguments.query = firstArgument; + query = firstArgument; } if (util.is(lastArgument, 'function')) { - parsedArguments.callback = lastArgument; + callback = lastArgument; } - return parsedArguments; + if (util.is(query, 'object')) { + // Check if the user only asked for a certain amount of results. + if (util.is(query.maxResults, 'number')) { + // `maxResults` is used API-wide. + maxResults = query.maxResults; + } else if (util.is(query.limitVal, 'number')) { + // `limitVal` is part of a Datastore query. + maxResults = query.limitVal; + } else if (util.is(query.pageSize, 'number')) { + // `pageSize` is Pub/Sub's `maxResults`. + maxResults = query.pageSize; + } + + if (callback && + (maxResults !== -1 || // The user specified a limit. + query.autoPaginate === false || + query.autoPaginateVal === false)) { + autoPaginate = false; + } + } + + return { + query: query || {}, + callback: callback, + maxResults: maxResults, + autoPaginate: autoPaginate + }; }; /** @@ -103,20 +132,20 @@ streamRouter.parseArguments_ = function(args) { * commonly an object, but to make the API more simple, it can also be a * string in some places. * @param {function=} parsedArguments.callback - Callback function. + * @param {boolean} parsedArguments.autoPaginate - Auto-pagination enabled. + * @param {number} parsedArguments.maxResults - Maximum results to return. * @param {function} originalMethod - The cached method that accepts a callback * and returns `nextQuery` to receive more results. * @return {undefined|stream} */ streamRouter.router_ = function(parsedArguments, originalMethod) { - var query = parsedArguments.query || {}; + var query = parsedArguments.query; var callback = parsedArguments.callback; + var autoPaginate = parsedArguments.autoPaginate; if (callback) { - if (query.autoPaginate === true || query.autoPaginateVal === true) { - delete query.autoPaginate; - delete query.autoPaginateVal; - - this.runAsStream_(query, originalMethod) + if (autoPaginate) { + this.runAsStream_(parsedArguments, originalMethod) .on('error', callback) .pipe(concat(function(results) { callback(null, results); @@ -125,7 +154,7 @@ streamRouter.router_ = function(parsedArguments, originalMethod) { originalMethod(query, callback); } } else { - return this.runAsStream_(query, originalMethod); + return this.runAsStream_(parsedArguments, originalMethod); } }; @@ -136,26 +165,19 @@ streamRouter.router_ = function(parsedArguments, originalMethod) { * `maxResults` and `limitVal` (from Datastore) will act as a cap for how many * results are fetched and emitted to the stream. * - * @param {object=|string=} query - Query object. This is most + * @param {object=|string=} parsedArguments.query - Query object. This is most * commonly an object, but to make the API more simple, it can also be a * string in some places. + * @param {function=} parsedArguments.callback - Callback function. + * @param {boolean} parsedArguments.autoPaginate - Auto-pagination enabled. + * @param {number} parsedArguments.maxResults - Maximum results to return. * @param {function} originalMethod - The cached method that accepts a callback * and returns `nextQuery` to receive more results. * @return {stream} - Readable object stream. */ -streamRouter.runAsStream_ = function(query, originalMethod) { - query = query || {}; - - var resultsToSend = -1; - - // Check if the user only asked for a certain amount of results. - if (util.is(query.maxResults, 'number')) { - // `maxResults` is used API-wide. - resultsToSend = query.maxResults; - } else if (util.is(query.limitVal, 'number')) { - // `limitVal` is part of a Datastore query. - resultsToSend = query.limitVal; - } +streamRouter.runAsStream_ = function(parsedArguments, originalMethod) { + var query = parsedArguments.query; + var resultsToSend = parsedArguments.maxResults; var stream = streamEvents(through.obj()); @@ -201,7 +223,7 @@ streamRouter.runAsStream_ = function(query, originalMethod) { } stream.once('reading', function() { - originalMethod.call(null, query, onResultSet); + originalMethod(query, onResultSet); }); return stream; diff --git a/lib/common/util.js b/lib/common/util.js index e58e61137583..6c89bea50f7d 100644 --- a/lib/common/util.js +++ b/lib/common/util.js @@ -674,13 +674,12 @@ function makeRequest(reqOpts, config, callback) { config = config || {}; + reqOpts = util.decorateRequest(reqOpts); + var MAX_RETRIES = config.maxRetries || 3; var autoRetry = config.autoRetry !== false ? true : false; var attemptedRetries = 0; - reqOpts.headers = reqOpts.headers || {}; - reqOpts.headers['User-Agent'] = USER_AGENT; - function shouldRetry(err) { return autoRetry && MAX_RETRIES > attemptedRetries && @@ -704,3 +703,28 @@ function makeRequest(reqOpts, config, callback) { } util.makeRequest = makeRequest; + +/** + * Decorate the options about to be made in a request. + * + * @param {object} reqOpts - The options to be passed to `request`. + * @return {object} reqOpts - The decorated reqOpts. + */ +function decorateRequest(reqOpts) { + reqOpts.headers = reqOpts.headers || {}; + reqOpts.headers['User-Agent'] = USER_AGENT; + + if (util.is(reqOpts.qs, 'object')) { + delete reqOpts.qs.autoPaginate; + delete reqOpts.qs.autoPaginateVal; + } + + if (util.is(reqOpts.json, 'object')) { + delete reqOpts.json.autoPaginate; + delete reqOpts.json.autoPaginateVal; + } + + return reqOpts; +} + +util.decorateRequest = decorateRequest; diff --git a/lib/datastore/query.js b/lib/datastore/query.js index 8c502c2dee3c..0015be8770d1 100644 --- a/lib/datastore/query.js +++ b/lib/datastore/query.js @@ -73,7 +73,7 @@ function Query(namespace, kinds) { this.selectVal = []; // pagination - this.autoPaginateVal = false; + this.autoPaginateVal = true; this.startVal = null; this.endVal = null; this.limitVal = -1; diff --git a/lib/datastore/request.js b/lib/datastore/request.js index 929568973a72..84d1d4e98a79 100644 --- a/lib/datastore/request.js +++ b/lib/datastore/request.js @@ -493,6 +493,18 @@ DatastoreRequest.prototype.delete = function(keys, callback) { * //- * var query = dataset.createQuery('Lion'); * + * transaction.runQuery(query, function(err, entities) { + * if (!err) { + * // Handle entities here. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, call `autoPaginate(false)` on your query. + * //- + * var manualPageQuery = dataset.createQuery('Lion').autoPaginate(false); + * * var callback = function(err, entities, nextQuery, apiResponse) { * if (nextQuery) { * // More results might exist. @@ -500,18 +512,7 @@ DatastoreRequest.prototype.delete = function(keys, callback) { * } * }; * - * transaction.runQuery(query, callback); - * - * //- - * // To have pagination handled for you, call `autoPaginate()`. Note the - * // changed callback parameters. - * //- - * - * var queryWithAutoPagination = dataset.createQuery('Lion').autoPaginate(); - * - * transaction.runQuery(queryWithAutoPagination, function(err, entities) { - * // Called after all entities have been retrieved. - * }); + * transaction.runQuery(manualPageQuery, callback); * * //- * // If you omit the callback, runQuery will automatically call subsequent diff --git a/lib/pubsub/index.js b/lib/pubsub/index.js index 75b4e526fb0e..70ad17f8ed54 100644 --- a/lib/pubsub/index.js +++ b/lib/pubsub/index.js @@ -107,13 +107,13 @@ function PubSub(options) { * * @param {object=} query - Query object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {string=} query.pageToken - Page token. * @param {number=} query.pageSize - Max number of results to return. * @param {function} callback - The callback function. * @param {?error} callback.err - An error from the API call, may be null. * @param {module:pubsub/topic[]} callback.topics - The list of topics returned. - * @param {object} callback.nextQuery - A query object representing the next + * @param {?object} callback.nextQuery - A query object representing the next * page of topics. * @param {object} callback.apiResponse - The full API response from the * service. @@ -420,7 +420,7 @@ PubSub.prototype.topic = function(name, options) { * * @param {object=} options - Configuration object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {string|module:pubsub/topic} options.topic - The name of the topic to * list subscriptions from. * @param {number} options.pageSize - Maximum number of results to return. @@ -429,12 +429,22 @@ PubSub.prototype.topic = function(name, options) { * @param {?error} callback.err - An error from the API call, may be null. * @param {module:pubsub/subscription[]} callback.subscriptions - The list of * subscriptions returned. - * @param {object} callback.nextQuery - A query object representing the next + * @param {?object} callback.nextQuery - A query object representing the next * page of topics. * @param {object} callback.apiResponse - The full API response from the * service. * * @example + * pubsub.getSubscriptions(function(err, subscriptions) { + * if (!err) { + * // subscriptions is an array of Subscription objects. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- * var callback = function(err, subscriptions, nextQuery, apiResponse) { * if (nextQuery) { * // More results exist. @@ -442,16 +452,8 @@ PubSub.prototype.topic = function(name, options) { * } * }; * - * //- - * // Get all subscriptions for this project. - * //- - * pubsub.getSubscriptions(callback); - * - * //- - * // Customize the query. - * //- * pubsub.getSubscriptions({ - * pageSize: 3 + * autoPaginate: false, * }, callback); * * //- diff --git a/lib/search/index-class.js b/lib/search/index-class.js index b261d0c70fbd..e5366d7b6832 100644 --- a/lib/search/index-class.js +++ b/lib/search/index-class.js @@ -149,7 +149,7 @@ Index.prototype.document = function(id) { * * @param {object=} query - Query object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {string} query.pageSize - The maximum number of documents to return * per page. If not specified, 100 documents are returned per page. * @param {string} query.pageToken - A previously-returned page token @@ -159,39 +159,34 @@ Index.prototype.document = function(id) { * @param {function} callback - The callback function. * * @example + * index.getDocuments(function(err, documents) { + * if (!err) { + * // documents is an array of Document objects. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- * function onApiResponse(err, documents, nextQuery, apiResponse) { * if (err) { * console.error(err); * return; * } * - * // `documents` is an array of Document objects in this index. + * // `documents` is an array of Document objects. * * if (nextQuery) { * index.getDocuments(nextQuery, onApiResponse); * } * } * - * index.getDocuments(onApiResponse); - * - * //- - * // Customize the request using a query object. - * //- * index.getDocuments({ - * pageSize: 10 + * autoPaginate: false * }, onApiResponse); * * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. - * //- - * index.getDocuments({ - * autoPaginate: true - * }, function(err, documents) { - * // Called after all documents have been retrieved. - * }); - * - * //- * // Get the documents as a readable object stream. * //- * index.getDocuments() @@ -251,7 +246,7 @@ Index.prototype.getDocuments = function(query, callback) { * * @param {string|object} query - A query object or simply a string query. * @param {boolean} query.autoPaginate - Have pagination handled automatically. - * Default: false. + * Default: true. * @param {string} query.pageSize - The maximum number of documents to return * per page. If not specified, 100 documents are returned per page. * @param {string} query.pageToken - A previously-returned page token @@ -261,53 +256,40 @@ Index.prototype.getDocuments = function(query, callback) { * @param {function} callback - The callback function. * * @example + * var query = 'person:stephen'; + * + * index.search(query, function(err, documents) { + * if (!err) { + * // `documents` is an array of Document objects. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- * function onApiResponse(err, documents, nextQuery, apiResponse) { * if (err) { * console.error(err); * return; * } * - * // `documents` is an array of Document objects that matched your query. + * // `documents` is an array of Document objects. * * if (nextQuery) { * index.search(nextQuery, onApiResponse); * } * } * - * //- - * // Run a simple query against all documents. - * //- - * var query = 'person:stephen'; - * - * index.search(query, onApiResponse); - * - * //- - * // Configure the query. - * //- - * var query = { - * query: 'person:stephen', - * pageSize: 10 - * }; - * - * index.search(query, onApiResponse); - * - * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. - * //- - * var query = { - * query: 'person:stephen', - * autoPaginate: true - * }; - * - * index.search(query, function(err, indexes) { - * // Called after all indexes have been retrieved. - * }); + * index.search({ + * autoPaginate: false, + * query: query + * }, onApiResponse); * * //- * // Get the documents that match your query as a readable object stream. * //- - * index.search('person:stephen') + * index.search(query) * .on('error', console.error) * .on('data', function(document) { * // document is a Document object. diff --git a/lib/search/index.js b/lib/search/index.js index 56ea03107f23..75f109bceb40 100644 --- a/lib/search/index.js +++ b/lib/search/index.js @@ -97,7 +97,7 @@ function Search(options) { * * @param {object=} query - Query object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {string} query.pageSize - The maximum number of indexes to return per * page. If not specified, 100 indexes are returned per page. * @param {string} query.pageToken - A previously-returned page token @@ -109,37 +109,32 @@ function Search(options) { * @param {function} callback - The callback function. * * @example + * search.getIndexes(function(err, indexes) { + * // indexes is an array of Index objects. + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- * function onApiResponse(err, indexes, nextQuery, apiResponse) { * if (err) { * console.error(err); * return; * } * + * // indexes is an array of Index objects. + * * if (nextQuery) { * search.getIndexes(nextQuery, onApiResponse); * } * } * - * search.getIndexes(onApiResponse); - * - * //- - * // Customize the request using a query object. - * //- * search.getIndexes({ - * pageSize: 10 + * autoPaginate: false * }, onApiResponse); * * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. - * //- - * search.getIndexes({ - * autoPaginate: true - * }, function(err, indexes) { - * // Called after all indexes have been retrieved. - * }); - * - * //- * // Get the indexes as a readable object stream. * //- * search.getIndexes() diff --git a/lib/storage/bucket.js b/lib/storage/bucket.js index 823e20da6513..d8e872496666 100644 --- a/lib/storage/bucket.js +++ b/lib/storage/bucket.js @@ -456,7 +456,7 @@ Bucket.prototype.file = function(name, options) { * * @param {object=} query - Query object. * @param {boolean} query.autoPaginate - Have pagination handled automatically. - * Default: false. + * Default: true. * @param {string} query.delimiter - Results will contain only objects whose * names, aside from the prefix, do not contain delimiter. Objects whose * names, aside from the prefix, contain delimiter will have their name @@ -473,47 +473,44 @@ Bucket.prototype.file = function(name, options) { * @param {function} callback - The callback function. * * @example - * bucket.getFiles(function(err, files, nextQuery, apiResponse) { - * if (nextQuery) { - * // nextQuery will be non-null if there are more results. - * bucket.getFiles(nextQuery, function(err, files, nextQ, apiResponse) {}); + * bucket.getFiles(function(err, files) { + * if (!err) { + * // files is an array of File objects. * } - * - * // The `metadata` property is populated for you with the metadata at the - * // time of fetching. - * files[0].metadata; - * - * // However, in cases where you are concerned the metadata could have - * // changed, use the `getMetadata` method. - * files[0].getMetadata(function(err, metadata) {}); * }); * * //- - * // Fetch using a query. - * //- - * bucket.getFiles({ - * maxResults: 5 - * }, function(err, files, nextQuery, apiResponse) {}); - * - * //- * // If your bucket has versioning enabled, you can get all of your files * // scoped to their generation. * //- * bucket.getFiles({ * versions: true - * }, function(err, files, nextQuery, apiResponse) { + * }, function(err, files) { * // Each file is scoped to its generation. * }); * * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. * //- + * var callback = function(err, files, nextQuery, apiResponse) { + * if (nextQuery) { + * // More results exist. + * bucket.getFiles(nextQuery, callback); + * } + * + * // The `metadata` property is populated for you with the metadata at the + * // time of fetching. + * files[0].metadata; + * + * // However, in cases where you are concerned the metadata could have + * // changed, use the `getMetadata` method. + * files[0].getMetadata(function(err, metadata) {}); + * }; + * * bucket.getFiles({ - * autoPaginate: true - * }, function(err, files) { - * // Called after all files have been retrieved. - * }); + * autoPaginate: false + * }, callback); * * //- * // Get the files from your bucket as a readable object stream. diff --git a/lib/storage/index.js b/lib/storage/index.js index 6ba1bdc12460..3c9dee7f287a 100644 --- a/lib/storage/index.js +++ b/lib/storage/index.js @@ -250,7 +250,7 @@ Storage.prototype.createBucket = function(name, metadata, callback) { * * @param {object=} query - Query object. * @param {boolean} options.autoPaginate - Have pagination handled - * automatically. Default: false. + * automatically. Default: true. * @param {number} query.maxResults - Maximum number of items plus prefixes to * return. * @param {string} query.pageToken - A previously-returned page token @@ -258,10 +258,19 @@ Storage.prototype.createBucket = function(name, metadata, callback) { * @param {function} callback - The callback function. * * @example - * gcs.getBuckets(function(err, buckets, nextQuery) { + * gcs.getBuckets(function(err, buckets) { + * if (!err) { + * // buckets is an array of Bucket objects. + * } + * }); + * + * //- + * // To control how many API requests are made and page through the results + * // manually, set `autoPaginate` to `false`. + * //- + * var callback = function(err, buckets, nextQuery) { * if (nextQuery) { - * // nextQuery will be non-null if there are more results. - * var callback = function(err, buckets, nextQuery, apiResponse){}; + * // More results exist. * gcs.getBuckets(nextQuery, callback); * } * @@ -272,24 +281,11 @@ Storage.prototype.createBucket = function(name, metadata, callback) { * // However, in cases where you are concerned the metadata could have * // changed, use the `getMetadata` method. * buckets[0].getMetadata(function(err, metadata, apiResponse) {}); - * }); - * - * //- - * // Fetch using a query. - * //- - * gcs.getBuckets({ - * maxResults: 5 - * }, function(err, buckets, nextQuery, apiResponse) {}); + * }; * - * //- - * // To have pagination handled for you, set `autoPaginate`. Note the changed - * // callback parameters. - * //- * gcs.getBuckets({ - * autoPaginate: true - * }, function(err, buckets) { - * // Called after all buckets have been retrieved. - * }); + * autoPaginate: false + * }, callback); * * //- * // Get the buckets from your project as a readable object stream. diff --git a/system-test/bigquery.js b/system-test/bigquery.js index 3b70c1725962..dbb7c001a812 100644 --- a/system-test/bigquery.js +++ b/system-test/bigquery.js @@ -139,7 +139,7 @@ describe('BigQuery', function() { }); it('should list datasets with autoPaginate', function(done) { - bigquery.getDatasets({ autoPaginate: true }, function(err, datasets) { + bigquery.getDatasets(function(err, datasets) { assert(datasets.length > 0); assert(datasets[0] instanceof Dataset); done(); @@ -166,6 +166,7 @@ describe('BigQuery', function() { assert(job instanceof Job); job.getQueryResults(function(err, rows) { + assert.ifError(err); assert.equal(rows.length, 100); assert.equal(typeof rows[0].url, 'string'); done(); @@ -207,25 +208,22 @@ describe('BigQuery', function() { }); }); - it('should allow querying in series', function(done) { - bigquery.query({ - query: query, - maxResults: 10 - }, function(err, rows, nextQuery) { + it('should query', function(done) { + bigquery.query(query, function(err, rows) { assert.ifError(err); - assert.equal(rows.length, 10); - assert.equal(typeof nextQuery.pageToken, 'string'); + assert.equal(rows.length, 100); done(); }); }); - it('should query with autoPaginate', function(done) { + it('should allow querying in series', function(done) { bigquery.query({ query: query, - autoPaginate: true - }, function(err, rows) { + maxResults: 10 + }, function(err, rows, nextQuery) { assert.ifError(err); - assert.equal(rows.length, 100); + assert.equal(rows.length, 10); + assert.equal(typeof nextQuery.pageToken, 'string'); done(); }); }); @@ -239,7 +237,7 @@ describe('BigQuery', function() { }); it('should list jobs with autoPaginate', function(done) { - bigquery.getJobs({ autoPaginate: true }, function(err, jobs) { + bigquery.getJobs(function(err, jobs) { assert.ifError(err); assert(jobs[0] instanceof Job); done(); @@ -299,7 +297,7 @@ describe('BigQuery', function() { }); it('should get the rows in a table with autoPaginate', function(done) { - table.getRows({ autoPaginate: true }, function(err, rows) { + table.getRows(function(err, rows) { assert.ifError(err); assert(Array.isArray(rows)); done(); diff --git a/system-test/datastore.js b/system-test/datastore.js index 33b6a5c3ea1b..1fff93925ee4 100644 --- a/system-test/datastore.js +++ b/system-test/datastore.js @@ -280,7 +280,8 @@ describe('datastore', function() { }); it('should limit queries', function(done) { - var q = ds.createQuery('Character').hasAncestor(ancestor).limit(5); + var q = ds.createQuery('Character').hasAncestor(ancestor).limit(5) + .autoPaginate(false); ds.runQuery(q, function(err, firstEntities, secondQuery) { assert.ifError(err); @@ -301,8 +302,7 @@ describe('datastore', function() { it('should run query with autoPaginate', function(done) { var q = ds.createQuery('Character') - .hasAncestor(ancestor) - .autoPaginate(); + .hasAncestor(ancestor); ds.runQuery(q, function(err, results) { assert.ifError(err); @@ -316,8 +316,7 @@ describe('datastore', function() { var q = ds.createQuery('Character') .hasAncestor(ancestor) - .limit(limit) - .autoPaginate(); + .limit(limit); ds.runQuery(q, function(err, results) { assert.ifError(err); @@ -429,7 +428,8 @@ describe('datastore', function() { .hasAncestor(ancestor) .offset(2) .limit(3) - .order('appearances'); + .order('appearances') + .autoPaginate(false); ds.runQuery(q, function(err, entities, secondQuery) { assert.ifError(err); @@ -454,7 +454,8 @@ describe('datastore', function() { .hasAncestor(ancestor) .offset(2) .limit(2) - .order('appearances'); + .order('appearances') + .autoPaginate(false); ds.runQuery(q, function(err, entities, nextQuery) { assert.ifError(err); diff --git a/system-test/pubsub.js b/system-test/pubsub.js index 51221bcd5ccf..f772251acf6b 100644 --- a/system-test/pubsub.js +++ b/system-test/pubsub.js @@ -95,14 +95,14 @@ describe('pubsub', function() { }); }); - it('should return a nextQuery if there are more results', function(done) { + it('should allow manual paging', function(done) { pubsub.getTopics({ pageSize: TOPIC_NAMES.length - 1 - }, function(err, topics, next) { + }, function(err, topics, nextQuery) { assert.ifError(err); assert(topics.length, TOPIC_NAMES.length - 1); - assert(next.pageSize, TOPIC_NAMES.length - 1); - assert(!!next.pageToken, true); + assert(nextQuery.pageSize, TOPIC_NAMES.length - 1); + assert(!!nextQuery.pageToken, true); done(); }); }); diff --git a/system-test/search.js b/system-test/search.js index 6b8035b84647..1083e09a9147 100644 --- a/system-test/search.js +++ b/system-test/search.js @@ -34,53 +34,25 @@ function deleteDocument(document, callback) { } function deleteIndexContents(index, callback) { - function handleResp(err, documents, nextQuery) { + index.getDocuments(function(err, documents) { if (err) { callback(err); return; } - async.eachLimit(documents, MAX_PARALLEL, deleteDocument, function(err) { - if (err) { - callback(err); - return; - } - - if (nextQuery) { - index.getDocuments(nextQuery, handleResp); - return; - } - - callback(); - }); - } - - index.getDocuments(handleResp); + async.eachLimit(documents, MAX_PARALLEL, deleteDocument, callback); + }); } function deleteAllDocuments(callback) { - function handleResp(err, indexes, nextQuery) { + search.getIndexes(function(err, indexes) { if (err) { callback(err); return; } - async.eachLimit(indexes, MAX_PARALLEL, deleteIndexContents, function(err) { - if (err) { - callback(err); - return; - } - - if (nextQuery) { - search.getIndexes(nextQuery, handleResp); - return; - } - - callback(); - }); - } - - search.getIndexes(handleResp); + async.eachLimit(indexes, MAX_PARALLEL, deleteIndexContents, callback); + }); } function generateIndexName() { @@ -301,18 +273,6 @@ describe('Search', function() { }); }); - it('should search document with autoPaginate', function(done) { - index.search({ - query: query, - autoPaginate: true - }, function(err, results) { - assert.ifError(err); - assert.equal(results.length, 1); - assert.equal(results[0].id, DOCUMENT_NAME); - done(); - }); - }); - it('should search document in stream mode', function(done) { var results = []; diff --git a/system-test/storage.js b/system-test/storage.js index c46720bd05c9..0e07d9d50a5f 100644 --- a/system-test/storage.js +++ b/system-test/storage.js @@ -395,31 +395,14 @@ describe('storage', function() { }); it('should get buckets', function(done) { - storage.getBuckets(getBucketsHandler); - - var createdBuckets = []; - var retries = 0; - var MAX_RETRIES = 2; - - function getBucketsHandler(err, buckets, nextQuery) { - buckets.forEach(function(bucket) { - if (bucketsToCreate.indexOf(bucket.name) > -1) { - createdBuckets.push(bucket); - } + storage.getBuckets(function(err, buckets) { + var createdBuckets = buckets.filter(function(bucket) { + return bucketsToCreate.indexOf(bucket.name) > -1; }); - if (createdBuckets.length < bucketsToCreate.length && nextQuery) { - retries++; - - if (retries <= MAX_RETRIES) { - storage.getBuckets(nextQuery, getBucketsHandler); - return; - } - } - assert.equal(createdBuckets.length, bucketsToCreate.length); done(); - } + }); }); it('should get buckets with autoPaginate', function(done) { @@ -719,10 +702,9 @@ describe('storage', function() { }); it('should get files', function(done) { - bucket.getFiles(function(err, files, nextQuery) { + bucket.getFiles(function(err, files) { assert.ifError(err); assert.equal(files.length, filenames.length); - assert.equal(nextQuery, null); done(); }); }); @@ -751,7 +733,10 @@ describe('storage', function() { }); it('should paginate the list', function(done) { - var query = { maxResults: filenames.length - 1 }; + var query = { + maxResults: filenames.length - 1 + }; + bucket.getFiles(query, function(err, files, nextQuery) { assert.ifError(err); assert.equal(files.length, filenames.length - 1); diff --git a/test/common/stream-router.js b/test/common/stream-router.js index 003537f07450..deff15c88619 100644 --- a/test/common/stream-router.js +++ b/test/common/stream-router.js @@ -145,6 +145,15 @@ describe('streamRouter', function() { }); describe('parseArguments_', function() { + it('should set defaults', function() { + var parsedArguments = streamRouter.parseArguments_([]); + + assert.strictEqual(Object.keys(parsedArguments.query).length, 0); + assert.strictEqual(parsedArguments.callback, undefined); + assert.strictEqual(parsedArguments.maxResults, -1); + assert.strictEqual(parsedArguments.autoPaginate, true); + }); + it('should detect a callback if first argument is a function', function() { var args = [ util.noop ]; var parsedArguments = streamRouter.parseArguments_(args); @@ -159,6 +168,13 @@ describe('streamRouter', function() { assert.strictEqual(parsedArguments.query, args[0]); }); + it('should not make an undefined value the query', function() { + var args = [ undefined, util.noop ]; + var parsedArguments = streamRouter.parseArguments_(args); + + assert.deepEqual(parsedArguments.query, {}); + }); + it('should detect a callback if last argument is a function', function() { var args = [ 'string', util.noop ]; var parsedArguments = streamRouter.parseArguments_(args); @@ -172,91 +188,65 @@ describe('streamRouter', function() { assert.strictEqual(parsedArguments.callback, undefined); }); - }); - describe('router_', function() { - beforeEach(function() { - streamRouterOverrides.runAsStream_ = util.noop; - }); + it('should set maxResults from query.maxResults', function() { + var args = [ { maxResults: 10 } ]; + var parsedArguments = streamRouter.parseArguments_(args); - describe('callback mode', function() { - describe('autoPaginate', function() { - it('should recognize autoPaginate', function(done) { - var parsedArguments = { - query: { - autoPaginate: true - }, - callback: util.noop - }; + assert.strictEqual(parsedArguments.maxResults, args[0].maxResults); + }); - streamRouterOverrides.runAsStream_ = function() { - done(); - return through(); - }; + it('should set maxResults from query.limitVal', function() { + var args = [ { limitVal: 10 } ]; + var parsedArguments = streamRouter.parseArguments_(args); - streamRouter.router_(parsedArguments, util.noop); - }); + assert.strictEqual(parsedArguments.maxResults, args[0].limitVal); + }); - it('should recognize autoPaginateVal', function(done) { - var parsedArguments = { - query: { - autoPaginateVal: true - }, - callback: util.noop - }; + it('should set maxResults from query.pageSize', function() { + var args = [ { pageSize: 10 } ]; + var parsedArguments = streamRouter.parseArguments_(args); - streamRouterOverrides.runAsStream_ = function() { - done(); - return through(); - }; + assert.strictEqual(parsedArguments.maxResults, args[0].pageSize); + }); - streamRouter.router_(parsedArguments, util.noop); - }); + it('should set autoPaginate: false if there is a maxResults', function() { + var args = [ { maxResults: 10 }, util.noop ]; + var parsedArguments = streamRouter.parseArguments_(args); - it('should delete the autoPaginate property', function(done) { - var parsedArguments = { - query: { - autoPaginate: true - }, - callback: util.noop - }; + assert.strictEqual(parsedArguments.autoPaginate, false); + }); - streamRouterOverrides.runAsStream_ = function(query) { - assert.strictEqual(query.autoPaginate, undefined); - done(); - return through(); - }; + it('should set autoPaginate: false query.autoPaginate', function() { + var args = [ { autoPaginate: false }, util.noop ]; + var parsedArguments = streamRouter.parseArguments_(args); - streamRouter.router_(parsedArguments, util.noop); - }); + assert.strictEqual(parsedArguments.autoPaginate, false); + }); - it('should delete the autoPaginateVal property', function(done) { - var parsedArguments = { - query: { - autoPaginateVal: true - }, - callback: util.noop - }; + it('should set autoPaginate: false with query.autoPaginateVal', function() { + var args = [ { autoPaginateVal: false }, util.noop ]; + var parsedArguments = streamRouter.parseArguments_(args); - streamRouterOverrides.runAsStream_ = function(query) { - assert.strictEqual(query.autoPaginateVal, undefined); - done(); - return through(); - }; + assert.strictEqual(parsedArguments.autoPaginate, false); + }); + }); - streamRouter.router_(parsedArguments, util.noop); - }); + describe('router_', function() { + beforeEach(function() { + streamRouterOverrides.runAsStream_ = util.noop; + }); - it('should runAsStream', function(done) { + describe('callback mode', function() { + describe('autoPaginate', function() { + it('should call runAsStream_ when autoPaginate:true', function(done) { var parsedArguments = { - query: { - autoPaginate: true - }, + autoPaginate: true, callback: util.noop }; - streamRouterOverrides.runAsStream_ = function(query, originalMethod) { - assert.deepEqual(query, {}); + streamRouterOverrides.runAsStream_ = function(args, originalMethod) { + assert.strictEqual(args, parsedArguments); originalMethod(); return through(); }; @@ -268,9 +258,7 @@ describe('streamRouter', function() { var error = new Error('Error.'); var parsedArguments = { - query: { - autoPaginate: true - }, + autoPaginate: true, callback: function(err) { assert.strictEqual(err, error); done(); @@ -292,9 +280,7 @@ describe('streamRouter', function() { var results = ['a', 'b', 'c']; var parsedArguments = { - query: { - autoPaginate: true - }, + autoPaginate: true, callback: function(err, results_) { assert.deepEqual(results_.toString().split(''), results); done(); @@ -320,14 +306,19 @@ describe('streamRouter', function() { }); describe('manual pagination', function() { - it('should call original method', function(done) { + it('should recoginze autoPaginate: false', function(done) { var parsedArguments = { - query: { a: 'b', c: 'd' }, + autoPaginate: false, + query: { + a: 'b', + c: 'd' + }, callback: done }; streamRouter.router_(parsedArguments, function(query, callback) { assert.deepEqual(query, parsedArguments.query); + callback(); }); }); @@ -340,8 +331,8 @@ describe('streamRouter', function() { query: { a: 'b', c: 'd' } }; - streamRouterOverrides.runAsStream_ = function(query, originalMethod) { - assert.deepEqual(query, parsedArguments.query); + streamRouterOverrides.runAsStream_ = function(args, originalMethod) { + assert.deepEqual(args, parsedArguments); originalMethod(); }; @@ -367,15 +358,19 @@ describe('streamRouter', function() { describe('runAsStream_', function() { describe('stream mode', function() { - var QUERY = { a: 'b', c: 'd' }; + var PARSED_ARGUMENTS = { + query: { + a: 'b', c: 'd' + } + }; it('should call original method when stream opens', function(done) { function originalMethod(query) { - assert.strictEqual(query, QUERY); + assert.strictEqual(query, PARSED_ARGUMENTS.query); done(); } - var rs = streamRouter.runAsStream_(QUERY, originalMethod); + var rs = streamRouter.runAsStream_(PARSED_ARGUMENTS, originalMethod); rs.on('data', util.noop); // Trigger the underlying `_read` event. }); @@ -388,7 +383,7 @@ describe('streamRouter', function() { }); } - var rs = streamRouter.runAsStream_(QUERY, originalMethod); + var rs = streamRouter.runAsStream_(PARSED_ARGUMENTS, originalMethod); rs.on('data', util.noop); // Trigger the underlying `_read` event. rs.on('error', function(err) { assert.deepEqual(err, error); @@ -406,7 +401,7 @@ describe('streamRouter', function() { }); } - var rs = streamRouter.runAsStream_(QUERY, originalMethod); + var rs = streamRouter.runAsStream_(PARSED_ARGUMENTS, originalMethod); rs.on('data', function(result) { resultsReceived.push(result); }); @@ -425,7 +420,7 @@ describe('streamRouter', function() { }); } - it('should respect query.maxResults', function(done) { + it('should respect maxResults', function(done) { var numResultsReceived = 0; streamRouter.runAsStream_({ maxResults: limit }, originalMethod) @@ -435,17 +430,6 @@ describe('streamRouter', function() { done(); }); }); - - it('should respect query.limitVal', function(done) { - var numResultsReceived = 0; - - streamRouter.runAsStream_({ limitVal: limit }, originalMethod) - .on('data', function() { numResultsReceived++; }) - .on('end', function() { - assert.strictEqual(numResultsReceived, limit); - done(); - }); - }); }); it('should get more results if nextQuery exists', function(done) { @@ -465,7 +449,7 @@ describe('streamRouter', function() { }); } - var rs = streamRouter.runAsStream_(QUERY, originalMethod); + var rs = streamRouter.runAsStream_(PARSED_ARGUMENTS, originalMethod); rs.on('data', util.noop); // Trigger the underlying `_read` event. }); @@ -478,7 +462,7 @@ describe('streamRouter', function() { }); } - var rs = streamRouter.runAsStream_(QUERY, originalMethod); + var rs = streamRouter.runAsStream_(PARSED_ARGUMENTS, originalMethod); rs.on('data', function(result) { if (result === 'b') { // Pre-maturely end the stream. @@ -505,7 +489,7 @@ describe('streamRouter', function() { }); } - var rs = streamRouter.runAsStream_(QUERY, originalMethod); + var rs = streamRouter.runAsStream_(PARSED_ARGUMENTS, originalMethod); rs.on('data', function(result) { if (result === 'b') { // Pre-maturely end the stream. diff --git a/test/common/util.js b/test/common/util.js index b447c64a0cc8..cf039ba67e64 100644 --- a/test/common/util.js +++ b/test/common/util.js @@ -851,30 +851,25 @@ describe('common/util', function() { }); describe('makeRequest', function() { - var PKG = require('../../package.json'); - var USER_AGENT = 'gcloud-node/' + PKG.version; - var reqOpts = { a: 'b', c: 'd' }; - var expectedReqOpts = extend(true, {}, reqOpts, { - headers: { - 'User-Agent': USER_AGENT - } - }); + it('should decorate the request', function(done) { + var reqOpts = { a: 'b', c: 'd' }; - it('should make a request', function(done) { - request_Override = function() { + request_Override = util.noop; + + utilOverrides.decorateRequest = function(reqOpts_) { + assert.strictEqual(reqOpts_, reqOpts); done(); }; - util.makeRequest({}, assert.ifError, {}); + util.makeRequest(reqOpts, {}, assert.ifError); }); - it('should add the user agent', function(done) { - request_Override = function(rOpts) { - assert.deepEqual(rOpts, expectedReqOpts); + it('should make a request', function(done) { + request_Override = function() { done(); }; - util.makeRequest(reqOpts, assert.ifError, {}); + util.makeRequest({}, assert.ifError, {}); }); it('should let handleResp handle the response', function(done) { @@ -1006,4 +1001,62 @@ describe('common/util', function() { }); }); }); + + describe('decorateRequest', function() { + it('should add the user agent', function() { + var PKG = require('../../package.json'); + var USER_AGENT = 'gcloud-node/' + PKG.version; + + var reqOpts = { a: 'b', c: 'd' }; + + var expectedReqOpts = extend({}, reqOpts, { + headers: { + 'User-Agent': USER_AGENT + } + }); + + var decoratedReqOpts = util.decorateRequest(reqOpts); + assert.deepEqual(decoratedReqOpts, expectedReqOpts); + }); + + it('should delete qs.autoPaginate', function() { + var decoratedReqOpts = util.decorateRequest({ + qs: { + autoPaginate: true + } + }); + + assert.strictEqual(decoratedReqOpts.autoPaginate, undefined); + }); + + it('should delete qs.autoPaginateVal', function() { + var decoratedReqOpts = util.decorateRequest({ + qs: { + autoPaginateVal: true + } + }); + + assert.strictEqual(decoratedReqOpts.autoPaginate, undefined); + }); + + it('should delete json.autoPaginate', function() { + var decoratedReqOpts = util.decorateRequest({ + json: { + autoPaginate: true + } + }); + + assert.strictEqual(decoratedReqOpts.autoPaginate, undefined); + }); + + it('should delete json.autoPaginateVal', function() { + var decoratedReqOpts = util.decorateRequest({ + json: { + autoPaginateVal: true + } + }); + + assert.strictEqual(decoratedReqOpts.autoPaginate, undefined); + }); + }); }); diff --git a/test/datastore/query.js b/test/datastore/query.js index a4d8ddd51d92..5098f6e94a09 100644 --- a/test/datastore/query.js +++ b/test/datastore/query.js @@ -45,15 +45,21 @@ describe('Query', function() { it('should default autoPaginate to false', function() { var query = new Query(['kind1']); - assert.strictEqual(query.autoPaginateVal, false); + assert.strictEqual(query.autoPaginateVal, true); }); - it('should support setting autoPagination', function() { + it('should default autoPaginate() to true', function() { var query = new Query(['kind1']) .autoPaginate(); assert.strictEqual(query.autoPaginateVal, true); }); + it('should support setting autoPagination to false', function() { + var query = new Query(['kind1']) + .autoPaginate(false); + assert.strictEqual(query.autoPaginateVal, false); + }); + it('should support field selection by field name', function() { var query = new Query(['kind1']) .select(['name', 'title']);