BigQuery samples.

README.md

@@ -8,6 +8,7 @@ This repository holds Node.js samples used throughout [cloud.google.com]().
 ## Table of Contents
 
 * [Google App Engine](#google-app-engine)
+* [Google BigQuery](#google-bigquery)
 * [Google Cloud Functions](#google-cloud-functions)
 * [Google Cloud Logging](#google-cloud-logging)
 * [Google Cloud Pub/Sub](#google-cloud-pubsub)
@@ -69,6 +70,13 @@ __Other Examples__
 - Reading/writing from/to disk - [Source code][aedisk_1]
 - Serving static files - [Source code][aestaticfiles_1]
 
+## Google BigQuery
+
+- Query sample - [Source code][bigquery_query_1] | [Documentation][bigquery_query_2]
+- Dataset size sample - [Source code][bigquery_size_1]
+- Load from file sample - [Source code][bigquery_file_1] | [Documentation][bigquery_file_2]
+- Load from GCS sample - [Source code][bigquery_gcs_1] | [Documentation][bigquery_gcs_2]
+
 ## Google Cloud Datastore
 
 - Tasks sample - [Source code][datastore_1] | [Documentation][datastore_2]
@@ -124,6 +132,8 @@ See [CONTRIBUTING.md](https://github.com/GoogleCloudPlatform/nodejs-docs-samples
 1. Start Redis
 1. Start Memcached
 1. Set the `GCLOUD_PROJECT` environment variable to id of your project
+1. Set the `TEST_BUCKET_NAME` environment variable to the name of a test Google
+Cloud Storage bucket.
 1. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to
 a service account file. You can download one from your Google project's
 "permissions" page.
@@ -253,6 +263,17 @@ See [LICENSE](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/ma
 [aeextending_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/appengine/extending-runtime
 [aestaticfiles_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/appengine/static-files
 
+[bigquery_query_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/getting_started.js
+[bigquery_query_2]: https://cloud.google.com/datastore/docs/concepts/overview
+
+[bigquery_size_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/dataset_size.js
+
+[bigquery_file_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/load_data_from_csv.js
+[bigquery_file_2]: https://cloud.google.com/bigquery/loading-data-into-bigquery#loaddatapostrequest
+
+[bigquery_gcs_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/load_data_from_gcs.js
+[bigquery_gcs_2]: https://cloud.google.com/bigquery/loading-data-into-bigquery#loaddatagcs
+
 [datastore_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/datastore/tasks.js
 [datastore_2]: https://cloud.google.com/datastore/docs/concepts/overview
 

bigquery/README.md

@@ -0,0 +1,41 @@
+## BigQuery Samples
+
+These samples require two environment variables to be set:
+
+- `GOOGLE_APPLICATION_CREDENTIALS` - Path to a service account file. You can
+download one from your Google project's "permissions" page.
+- `GCLOUD_PROJECT` - Id of your Google project.
+
+## Run the samples
+
+Install dependencies:
+
+    npm install
+
+### getting_started.js
+
+    npm run getting_started
+
+### dataset_size.js
+
+Usage: `npm run dataset_size -- <projectId> <datasetId>`
+
+Example:
+
+    npm run dataset_size -- bigquery-public-data hacker_news
+
+### load_data_from_csv.js
+
+Usage: `npm run load_data_from_csv -- <path-to-file> <dataset-id> <table-name>`
+
+Example:
+
+    npm run load_data_from_csv -- data.csv my-dataset my-table
+
+### load_data_from_gcs.js
+
+Usage: `npm run load_data_from_gcs -- <bucket-name> <filename> <dataset-id> <table-name>`
+
+Example:
+
+    npm run load_data_from_gcs -- my-bucket data.csv my-dataset my-table

bigquery/dataset_size.js

@@ -0,0 +1,159 @@
+// Copyright 2016, Google, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+var async = require('async');
+
+// [START auth]
+// You must set the GOOGLE_APPLICATION_CREDENTIALS and GCLOUD_PROJECT
+// environment variables to run this sample
+var projectId = process.env.GCLOUD_PROJECT;
+
+// Initialize gcloud
+var gcloud = require('gcloud')({
+  projectId: projectId
+});
+
+// Get a reference to the bigquery component
+var bigquery = gcloud.bigquery();
+// [END auth]
+
+// not going to use this bigquery instance
+bigquery = undefined;
+
+// [START list_tables]
+/**
+ * Retrieve all tables for the specified dataset.
+ *
+ * @param {Object} bigquery gcloud-node bigquery client.
+ * @param {string} datasetId Dataset of the tables to retrieve.
+ * @param {string} [pageToken] Page to retrieve.
+ * @param {Function} callback Callback function.
+ */
+function getAllTablesExample(bigquery, datasetId, pageToken, callback) {
+  if (typeof pageToken === 'function') {
+    callback = pageToken;
+    pageToken = undefined;
+  }
+  var dataset = bigquery.dataset(datasetId);
+  var options = {};
+  if (pageToken) {
+    options.pageToken = pageToken;
+  }
+
+  // Grab paginated tables
+  dataset.getTables(options, function (err, tables, nextQuery) {
+    // Quit on error
+    if (err) {
+      return callback(err);
+    }
+
+    // There is another page of tables
+    if (nextQuery) {
+      // Grab the remaining pages of tables recursively
+      return getAllTablesExample(
+        datasetId,
+        nextQuery.token,
+        function (err, _tables) {
+          if (err) {
+            return callback(err);
+          }
+          callback(null, tables.concat(_tables));
+        }
+      );
+    }
+    // Last page of tables
+    return callback(null, tables);
+  });
+}
+// [END list_tables]
+
+// [START get_size]
+/**
+ * Retrieve the size of the specified dataset.
+ *
+ * @param {string} projectId The project, .e.g. "bigquery-public-data"
+ * @param {string} datasetId The dataset, e.g. "hacker_news"
+ * @param {Function} callback Callback function.
+ */
+function getSizeExample(projectId, datasetId, callback) {
+  if (!projectId) {
+    return callback(new Error('projectId is required!'));
+  }
+  if (!datasetId) {
+    return callback(new Error('datasetId is require!'));
+  }
+
+  var gcloud = require('gcloud')({
+    projectId: projectId || process.env.GCLOUD_PROJECT
+  });
+  var bigquery = gcloud.bigquery();
+
+  // Fetch all tables in the dataset
+  getAllTablesExample(bigquery, datasetId, function (err, tables) {
+    return async.parallel(tables.map(function (table) {
+      return function (cb) {
+        // Fetch more detailed info for each table
+        table.get(function (err, tableInfo) {
+          if (err) {
+            return cb(err);
+          }
+          // Return numBytes converted to Megabytes
+          var numBytes = tableInfo.metadata.numBytes;
+          return cb(null, (parseInt(numBytes, 10) / 1000) / 1000);
+        });
+      };
+    }), function (err, sizes) {
+      if (err) {
+        return callback(err);
+      }
+      var sum = sizes.reduce(function (cur, prev) {
+        return cur + prev;
+      }, 0);
+      return callback(null, sum);
+    });
+  });
+}
+// [END get_size]
+
+// Run the examples
+exports.main = function (projectId, datasetId, cb) {
+  getSizeExample(projectId, datasetId, function (err, sum) {
+    if (err) {
+      return cb(err);
+    }
+    var size = 'MB';
+    if (sum > 1000) {
+      sum = sum / 1000;
+      size = 'GB';
+    }
+    if (sum > 1000) {
+      sum = sum / 1000;
+      size = 'TB';
+    }
+    cb(null, '' + sum.toPrecision(5) + ' ' + size);
+  });
+};
+
+if (module === require.main) {
+  var args = process.argv.slice(2);
+  if (args.length !== 2) {
+    throw new Error('Usage: node dataset_size.js <projectId> <datasetId>');
+  }
+  exports.main(
+    args[0],
+    args[1],
+    console.log
+  );
+}

bigquery/getting_started.js

@@ -0,0 +1,79 @@
+// Copyright 2016, Google, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START complete]
+'use strict';
+
+// [START auth]
+// You must set the GOOGLE_APPLICATION_CREDENTIALS and GCLOUD_PROJECT
+// environment variables to run this sample
+var projectId = process.env.GCLOUD_PROJECT;
+
+// Initialize gcloud
+var gcloud = require('gcloud')({
+  projectId: projectId
+});
+
+// Get a reference to the bigquery component
+var bigquery = gcloud.bigquery();
+// [END auth]
+
+// [START print]
+function printExample(rows) {
+  console.log('Query Results:');
+  rows.forEach(function (row) {
+    var str = '';
+    for (var key in row) {
+      if (str) {
+        str += '\t';
+      }
+      str += key + ': ' + row[key];
+    }
+    console.log(str);
+  });
+}
+// [END print]
+
+// [START query]
+/**
+ * Run an example query.
+ *
+ * @param {Function} callback Callback function.
+ */
+function queryExample(callback) {
+  var query = 'SELECT TOP(corpus, 10) as title, COUNT(*) as unique_words\n' +
+    'FROM [publicdata:samples.shakespeare];';
+
+  bigquery.query(query, function(err, rows) {
+    if (err) {
+      return callback(err);
+    }
+
+    printExample(rows);
+    callback(null, rows);
+  });
+}
+// [END query]
+
+// [END complete]
+
+// Run the examples
+exports.main = function (cb) {
+  queryExample(cb);
+};
+
+if (module === require.main) {
+  exports.main(
+    console.log
+  );
+}