diff --git a/dlp/README.md b/dlp/README.md new file mode 100644 index 0000000000..91ae4d9e27 --- /dev/null +++ b/dlp/README.md @@ -0,0 +1,120 @@ +Google Cloud Platform logo + +# Data Loss Prevention API Node.js Samples + +The [Data Loss Prevention][dlp_docs] (DLP) API provides programmatic access to a powerful detection engine for personally identifiable information and other privacy-sensitive data in unstructured data streams. + +This code provides a demonstration of the DLP API's functionality via REST in Node.js. It is intended for developers who want to be early adopters of the API. + +[gRPC](https://grpc.io)-based samples/client libraries for [several languages](https://cloud.google.com/docs/) are under active development, and will be released shortly. + +## Table of Contents + +* [Setup](#setup) +* [Samples](#samples) + * [Getting started with the Data Loss Prevention API](#getting-started-with-data-loss-prevention-api) + +## Setup + +1. Read [Prerequisites][prereq] and [How to run a sample][run] first. +1. Install dependencies: + + npm install + +[prereq]: ../README.md#prerequisities +[run]: ../README.md#how-to-run-a-sample + +## Samples + +### Getting started with the Data Loss Prevention API + +View the [DLP documentation][dlp_docs] or the [samples][dlp_samples]. + +__Run the samples:__ + +```sh +node inspect.js --help +``` + +``` +Commands: + Commands: + string Inspect a string using the Data Loss Prevention API. + file Inspects a local text, PNG, or JPEG file using the Data Loss Prevention API. + gcsFile Inspects a text file stored on Google Cloud Storage using the Data Loss Prevention + API. + datastore Inspect a Datastore instance using the Data Loss Prevention API. + +Options: + --help Show help [boolean] + -m, --minLikelihood + [string] [choices: "LIKELIHOOD_UNSPECIFIED", "VERY_UNLIKELY", "UNLIKELY", "POSSIBLE", "LIKELY", "VERY_LIKELY"] + [default: "LIKELIHOOD_UNSPECIFIED"] + -f, --maxFindings [default: 0] + -q, --includeQuote [boolean] [default: true] + -a, --authToken [string] [default: + "ab97.XXX..."] + -t, --infoTypes [array] [default: []] + +Examples: + node inspect.js string "My phone number is (123) 456-7890 + and my email address is me@somedomain.com" + node inspect.js file resources/test.txt + node inspect.js gcsFile my-bucket my-file.txt + +For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at +https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig +``` + +```sh +node metadata.js --help +``` + +``` +Commands: + infoTypes List types of sensitive information within a category. + categories List root categories of sensitive information. + +Options: + --help Show help [boolean] + -a, --authToken [string] [default: + "ab97.XXX..."] + oz0146E86Lk"] + +Examples: + node metadata.js infoTypes GOVERNMENT + node metadata.js categories + +For more information, see https://cloud.google.com/dlp/docs +``` + +```sh +node redact.js --help +``` + +``` +Commands: + string Redact sensitive data from a string using the Data Loss Prevention API. + +Options: + --help Show help [boolean] + -t, --infoTypes [array] [required] + -m, --minLikelihood + [string] [choices: "LIKELIHOOD_UNSPECIFIED", "VERY_UNLIKELY", "UNLIKELY", "POSSIBLE", "LIKELY", "VERY_LIKELY"] + [default: "LIKELIHOOD_UNSPECIFIED"] + -a, --authToken [string] [default: + "ab97.XXX..."] + htPfG_eIy04"] + +Examples: + node redact.js string "My name is Gary" "REDACTED" -t + US_MALE_NAME + +For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at +https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig +``` + +For more information, see [the docs][dlp_docs]. + +[dlp_samples]: ../dlp +[dlp_docs]: https://cloud.google.com/dlp/docs/ \ No newline at end of file diff --git a/dlp/inspect.js b/dlp/inspect.js new file mode 100644 index 0000000000..267d63bc18 --- /dev/null +++ b/dlp/inspect.js @@ -0,0 +1,401 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const API_URL = 'https://dlp.googleapis.com/v2beta1'; +const fs = require('fs'); +const requestPromise = require('request-promise'); +const mime = require('mime'); + +// Helper function to poll the rest API using exponential backoff +function pollJob (body, initialTimeout, tries, authToken) { + const jobName = body.name.split('/')[2]; + + // Construct polling function + const doPoll = (timeout, tries, resolve, reject) => { + // Construct REST request for polling an inspect job + const options = { + url: `${API_URL}/inspect/operations/${jobName}`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: true + }; + + // Poll the inspect job + setTimeout(() => { + requestPromise.get(options) + .then((body) => { + if (tries <= 0) { + reject('polling timed out'); + } + + // Job not finished - try again if possible + if (!(body && body.done)) { + return doPoll(timeout * 2, tries - 1, resolve, reject); + } + + // Job finished successfully! + return resolve(jobName); + }) + .catch((err) => { + reject(err); + }); + }, timeout); + }; + + // Return job-polling REST request as a Promise + return new Promise((resolve, reject) => { + doPoll(initialTimeout, tries, resolve, reject); + }); +} + +// Helper function to get results of a long-running (polling-required) job +function getJobResults (authToken, jobName) { + // Construct REST request to get results of finished inspect job + const options = { + url: `${API_URL}/inspect/results/${jobName}/findings`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: true + }; + + // Run job-results-fetching REST request + return requestPromise.get(options); +} + +function inspectString (authToken, string, inspectConfig) { + // [START inspect_string] + // Your gcloud auth token + // const authToken = 'YOUR_AUTH_TOKEN'; + + // The string to inspect + // const string = 'My name is Gary and my email is gary@example.com'; + + // Construct items to inspect + const items = [{ type: 'text/plain', value: string }]; + + // Construct REST request body + const requestBody = { + inspectConfig: { + infoTypes: inspectConfig.infoTypes, + minLikelihood: inspectConfig.minLikelihood, + maxFindings: inspectConfig.maxFindings, + includeQuote: inspectConfig.includeQuote + }, + items: items + }; + + // Construct REST request + const options = { + url: `${API_URL}/content:inspect`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: requestBody + }; + + // Run REST request + requestPromise.post(options) + .then((body) => { + const results = body.results[0].findings; + console.log(JSON.stringify(results, null, 2)); + }) + .catch((err) => { + console.log('Error in inspectString:', err); + }); + // [END inspect_string] +} + +function inspectFile (authToken, filepath, inspectConfig) { + // [START inspect_file] + // Your gcloud auth token. + // const authToken = 'YOUR_AUTH_TOKEN'; + + // The path to a local file to inspect. Can be a text, JPG, or PNG file. + // const fileName = 'path/to/image.png'; + + // Construct file data to inspect + const fileItems = [{ + type: mime.lookup(filepath) || 'application/octet-stream', + data: new Buffer(fs.readFileSync(filepath)).toString('base64') + }]; + + // Construct REST request body + const requestBody = { + inspectConfig: { + infoTypes: inspectConfig.infoTypes, + minLikelihood: inspectConfig.minLikelihood, + maxFindings: inspectConfig.maxFindings, + includeQuote: inspectConfig.includeQuote + }, + items: fileItems + }; + + // Construct REST request + const options = { + url: `${API_URL}/content:inspect`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: requestBody + }; + + // Run REST request + requestPromise.post(options) + .then((body) => { + const results = body.results[0].findings; + console.log(JSON.stringify(results, null, 2)); + }) + .catch((err) => { + console.log('Error in inspectFile:', err); + }); + // [END inspect_file] +} + +function inspectGCSFile (authToken, bucketName, fileName, inspectConfig) { + // [START inspect_gcs_file] + // Your gcloud auth token. + // const authToken = 'YOUR_AUTH_TOKEN'; + + // The name of the bucket where the file resides. + // const bucketName = 'YOUR-BUCKET'; + + // The path to the file within the bucket to inspect. + // Can contain wildcards, e.g. "my-image.*" + // const fileName = 'my-image.png'; + + // Get reference to the file to be inspected + const storageItems = { + cloudStorageOptions: { + fileSet: { url: `gs://${bucketName}/${fileName}` } + } + }; + + // Construct REST request body for creating an inspect job + const requestBody = { + inspectConfig: { + infoTypes: inspectConfig.infoTypes, + minLikelihood: inspectConfig.minLikelihood, + maxFindings: inspectConfig.maxFindings + }, + storageConfig: storageItems + }; + + // Construct REST request for creating an inspect job + let options = { + url: `${API_URL}/inspect/operations`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: requestBody + }; + + // Run inspect-job creation REST request + requestPromise.post(options) + .then((createBody) => pollJob(createBody, inspectConfig.initialTimeout, inspectConfig.tries, authToken)) + .then((jobName) => getJobResults(authToken, jobName)) + .then((findingsBody) => { + const findings = findingsBody.result.findings; + console.log(JSON.stringify(findings, null, 2)); + }) + .catch((err) => { + console.log('Error in inspectGCSFile:', err); + }); + // [END inspect_gcs_file] +} + +function inspectDatastore (authToken, namespaceId, kind, inspectConfig) { + // [START inspect_datastore] + // Your gcloud auth token + // const authToken = 'YOUR_AUTH_TOKEN'; + + // (Optional) The ID namespace of the Datastore document to inspect. + // To ignore Datastore namespaces, set this to an empty string ('') + // const namespace = ''; + + // The kind of the Datastore entity to inspect. + // const kind = 'Person'; + + // Get reference to the file to be inspected + const storageItems = { + datastoreOptions: { + partitionId: { + projectId: inspectConfig.projectId, + namespaceId: namespaceId + }, + kind: { + name: kind + } + } + }; + + // Construct REST request body for creating an inspect job + const requestBody = { + inspectConfig: { + infoTypes: inspectConfig.infoTypes, + minLikelihood: inspectConfig.minLikelihood, + maxFindings: inspectConfig.maxFindings + }, + storageConfig: storageItems + }; + + // Construct REST request for creating an inspect job + let options = { + url: `${API_URL}/inspect/operations`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: requestBody + }; + + // Run inspect-job creation REST request + requestPromise.post(options) + .then((createBody) => pollJob(createBody, inspectConfig.initialTimeout, inspectConfig.tries, authToken)) + .then((jobName) => getJobResults(authToken, jobName)) + .then((findingsBody) => { + const findings = findingsBody.result.findings; + console.log(JSON.stringify(findings, null, 2)); + }) + .catch((err) => { + console.log('Error in inspectDatastore:', err); + }); + // [END inspect_datastore] +} + +if (module === require.main) { + const auth = require('google-auto-auth')({ + keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, + scopes: ['https://www.googleapis.com/auth/cloud-platform'] + }); + auth.getToken((err, token) => { + if (err) { + console.err('Error fetching auth token:', err); + process.exit(1); + } + + const cli = require(`yargs`) + .demand(1) + .command( + `string `, + `Inspect a string using the Data Loss Prevention API.`, + {}, + (opts) => inspectString(opts.authToken, opts.string, opts) + ) + .command( + `file `, + `Inspects a local text, PNG, or JPEG file using the Data Loss Prevention API.`, + {}, + (opts) => inspectFile(opts.authToken, opts.filepath, opts) + ) + .command( + `gcsFile `, + `Inspects a text file stored on Google Cloud Storage using the Data Loss Prevention API.`, + { + initialTimeout: { + type: 'integer', + alias: '-i', + default: 5000 + }, + tries: { + type: 'integer', + default: 5 + } + }, + (opts) => inspectGCSFile(opts.authToken, opts.bucketName, opts.fileName, opts) + ) + .command( + `datastore `, + `Inspect a Datastore instance using the Data Loss Prevention API.`, + { + projectId: { + type: 'string', + default: process.env.GCLOUD_PROJECT + }, + namespaceId: { + type: 'string', + default: '' + }, + initialTimeout: { + type: 'integer', + alias: '-i', + default: 5000 + }, + tries: { + type: 'integer', + default: 5 + } + }, + (opts) => inspectDatastore(opts.authToken, opts.namespaceId, opts.kind, opts) + ) + .option('m', { + alias: 'minLikelihood', + default: 'LIKELIHOOD_UNSPECIFIED', + type: 'string', + choices: [ + 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', + 'UNLIKELY', + 'POSSIBLE', + 'LIKELY', + 'VERY_LIKELY' + ], + global: true + }) + .option('f', { + alias: 'maxFindings', + default: 0, + type: 'integer', + global: true + }) + .option('q', { + alias: 'includeQuote', + default: true, + type: 'boolean', + global: true + }) + .option('a', { + alias: 'authToken', + default: token, + type: 'string', + global: true + }) + .option('t', { + alias: 'infoTypes', + default: [], + type: 'array', + global: true, + coerce: (infoTypes) => infoTypes.map((type) => { + return { name: type }; + }) + }) + .example(`node $0 string "My phone number is (123) 456-7890 and my email address is me@somedomain.com"`) + .example(`node $0 file resources/test.txt`) + .example(`node $0 gcsFile my-bucket my-file.txt`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig`); + + cli.help().strict().argv; + }); +} diff --git a/dlp/metadata.js b/dlp/metadata.js new file mode 100644 index 0000000000..377344072c --- /dev/null +++ b/dlp/metadata.js @@ -0,0 +1,116 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const API_URL = 'https://dlp.googleapis.com/v2beta1'; +const requestPromise = require('request-promise'); + +function listInfoTypes (authToken, category) { + // [START list_info_types] + // Your gcloud auth token. + // const authToken = 'YOUR_AUTH_TOKEN'; + + // The category of info types to list. + // const category = 'CATEGORY_TO_LIST'; + + // Construct REST request + const options = { + url: `${API_URL}/rootCategories/${category}/infoTypes`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: true + }; + + // Run REST request + requestPromise.get(options) + .then((body) => { + console.log(body); + }) + .catch((err) => { + console.log('Error in listInfoTypes:', err); + }); + // [END list_info_types] +} + +function listCategories (authToken) { + // [START list_categories] + // Your gcloud auth token. + // const authToken = 'YOUR_AUTH_TOKEN'; + + // Construct REST request + const options = { + url: `${API_URL}/rootCategories`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: true + }; + + // Run REST request + requestPromise.get(options) + .then((body) => { + const categories = body.categories; + console.log(categories); + }) + .catch((err) => { + console.log('Error in listCategories:', err); + }); + // [END list_categories] +} + +if (module === require.main) { + const auth = require('google-auto-auth')({ + keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, + scopes: ['https://www.googleapis.com/auth/cloud-platform'] + }); + auth.getToken((err, token) => { + if (err) { + console.err('Error fetching auth token:', err); + process.exit(1); + } + + const cli = require(`yargs`) + .demand(1) + .command( + `infoTypes `, + `List types of sensitive information within a category.`, + {}, + (opts) => listInfoTypes(opts.authToken, opts.category) + ) + .command( + `categories`, + `List root categories of sensitive information.`, + {}, + (opts) => listCategories(opts.authToken) + ) + .option('a', { + alias: 'authToken', + default: token, + type: 'string', + global: true + }) + .example(`node $0 infoTypes GOVERNMENT`) + .example(`node $0 categories`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/dlp/docs`); + + cli.help().strict().argv; + }); +} diff --git a/dlp/package.json b/dlp/package.json new file mode 100644 index 0000000000..5e27385a2d --- /dev/null +++ b/dlp/package.json @@ -0,0 +1,40 @@ +{ + "name": "dlp-cli", + "description": "Command-line interface for Google Cloud Platform's Data Loss Prevention API", + "version": "0.0.1", + "private": true, + "license": "Apache Version 2.0", + "author": "Google Inc.", + "contributors": [ + { + "name": "Ace Nassri", + "email": "anassri@google.com" + }, + { + "name": "Jason Dobry", + "email": "jason.dobry@gmail.com" + }, + { + "name": "Jon Wayne Parrott", + "email": "jonwayne@google.com" + } + ], + "repository": { + "type": "git", + "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git" + }, + "scripts": { + "test": "ava system-test/*.test.js -c 20 -T 240s" + }, + "engines": { + "node": ">=4.3.2" + }, + "dependencies": { + "google-auth-library": "^0.10.0", + "google-auto-auth": "^0.5.2", + "mime": "1.3.4", + "request": "2.79.0", + "request-promise": "4.1.1", + "yargs": "6.6.0" + } +} diff --git a/dlp/redact.js b/dlp/redact.js new file mode 100644 index 0000000000..02ea1dfa54 --- /dev/null +++ b/dlp/redact.js @@ -0,0 +1,130 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const API_URL = 'https://dlp.googleapis.com/v2beta1'; +const requestPromise = require('request-promise'); + +function redactString (authToken, string, replaceString, inspectConfig) { + // [START redact_string] + // Your gcloud auth token + // const authToken = 'YOUR_AUTH_TOKEN'; + + // The string to inspect + // const string = 'My name is Gary and my email is gary@example.com'; + + // The string to replace sensitive data with + // const replaceString = 'REDACTED'; + + // Construct items to inspect + const items = [{ type: 'text/plain', value: string }]; + + // Construct info types + replacement configs + const replaceConfigs = inspectConfig.infoTypes.map((infoType) => { + return { + infoType: infoType, + replaceWith: replaceString + }; + }); + + // Construct REST request body + const requestBody = { + inspectConfig: { + infoTypes: inspectConfig.infoTypes, + minLikelihood: inspectConfig.minLikelihood + }, + items: items, + replaceConfigs: replaceConfigs + }; + + // Construct REST request + const options = { + url: `${API_URL}/content:redact`, + headers: { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + json: requestBody + }; + + // Run REST request + requestPromise.post(options) + .then((body) => { + const results = body.items[0].value; + console.log(results); + }) + .catch((err) => { + console.log('Error in redactString:', err); + }); + // [END redact_string] +} + +if (module === require.main) { + const auth = require('google-auto-auth')({ + keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, + scopes: ['https://www.googleapis.com/auth/cloud-platform'] + }); + auth.getToken((err, token) => { + if (err) { + console.err('Error fetching auth token:', err); + process.exit(1); + } + + const cli = require(`yargs`) + .demand(1) + .command( + `string `, + `Redact sensitive data from a string using the Data Loss Prevention API.`, + {}, + (opts) => redactString(opts.authToken, opts.string, opts.replaceString, opts) + ) + .option('m', { + alias: 'minLikelihood', + default: 'LIKELIHOOD_UNSPECIFIED', + type: 'string', + choices: [ + 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', + 'UNLIKELY', + 'POSSIBLE', + 'LIKELY', + 'VERY_LIKELY' + ], + global: true + }) + .option('a', { + alias: 'authToken', + default: token, + type: 'string', + global: true + }) + .option('t', { + alias: 'infoTypes', + required: true, + type: 'array', + global: true, + coerce: (infoTypes) => infoTypes.map((type) => { + return { name: type }; + }) + }) + .example(`node $0 string "My name is Gary" "REDACTED" -t US_MALE_NAME`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig`); + + cli.help().strict().argv; + }); +} diff --git a/dlp/resources/accounts.txt b/dlp/resources/accounts.txt new file mode 100644 index 0000000000..2763cd0ab8 --- /dev/null +++ b/dlp/resources/accounts.txt @@ -0,0 +1 @@ +My credit card number is 1234 5678 9012 3456, and my CVV is 789. \ No newline at end of file diff --git a/dlp/resources/harmless.txt b/dlp/resources/harmless.txt new file mode 100644 index 0000000000..5666de37ab --- /dev/null +++ b/dlp/resources/harmless.txt @@ -0,0 +1 @@ +This file is mostly harmless. diff --git a/dlp/resources/test.png b/dlp/resources/test.png new file mode 100644 index 0000000000..8f32c82588 Binary files /dev/null and b/dlp/resources/test.png differ diff --git a/dlp/resources/test.txt b/dlp/resources/test.txt new file mode 100644 index 0000000000..c2ee3815bc --- /dev/null +++ b/dlp/resources/test.txt @@ -0,0 +1 @@ +My phone number is (223) 456-7890 and my email address is gary@somedomain.com. \ No newline at end of file diff --git a/dlp/system-test/inspect.test.js b/dlp/system-test/inspect.test.js new file mode 100644 index 0000000000..9dd76c8af2 --- /dev/null +++ b/dlp/system-test/inspect.test.js @@ -0,0 +1,166 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +require(`../../system-test/_setup`); + +const cmd = 'node inspect'; + +// inspect_string +test(`should inspect a string`, async (t) => { + const output = await runAsync(`${cmd} string "I'm Gary and my email is gary@example.com"`); + t.regex(output, /"name": "EMAIL_ADDRESS"/); +}); + +test(`should handle a string with no sensitive data`, async (t) => { + const output = await runAsync(`${cmd} string "foo"`); + t.is(output, 'undefined'); +}); + +test(`should report string inspection handling errors`, async (t) => { + const output = await runAsync(`${cmd} string "I'm Gary and my email is gary@example.com" -a foo`); + t.regex(output, /Error in inspectString/); +}); + +// inspect_file +test(`should inspect a local text file`, async (t) => { + const output = await runAsync(`${cmd} file resources/test.txt`); + t.regex(output, /"name": "PHONE_NUMBER"/); + t.regex(output, /"name": "EMAIL_ADDRESS"/); +}); + +test(`should inspect a local image file`, async (t) => { + const output = await runAsync(`${cmd} file resources/test.png`); + t.regex(output, /"name": "PHONE_NUMBER"/); +}); + +test(`should handle a local file with no sensitive data`, async (t) => { + const output = await runAsync(`${cmd} file resources/harmless.txt`); + t.is(output, 'undefined'); +}); + +test(`should report local file handling errors`, async (t) => { + const output = await runAsync(`${cmd} file resources/harmless.txt -a foo`); + t.regex(output, /Error in inspectFile/); +}); + +// inspect_gcs_file +test.serial(`should inspect a GCS text file`, async (t) => { + const output = await runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp test.txt`); + t.regex(output, /"name": "PHONE_NUMBER"/); + t.regex(output, /"name": "EMAIL_ADDRESS"/); +}); + +test.serial(`should inspect multiple GCS text files`, async (t) => { + const output = await runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp *.txt`); + t.regex(output, /"name": "PHONE_NUMBER"/); + t.regex(output, /"name": "EMAIL_ADDRESS"/); + t.regex(output, /"name": "CREDIT_CARD_NUMBER"/); +}); + +test.serial(`should accept try limits for inspecting GCS files`, async (t) => { + const output = await runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp test.txt --tries 0`); + t.regex(output, /polling timed out/); +}); + +test.serial(`should handle a GCS file with no sensitive data`, async (t) => { + const output = await runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp harmless.txt`); + t.is(output, 'undefined'); +}); + +test.serial(`should report GCS file handling errors`, async (t) => { + const output = await runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp harmless.txt -a foo`); + t.regex(output, /Error in inspectGCSFile/); +}); + +// inspect_datastore +test.serial(`should inspect Datastore`, async (t) => { + const output = await runAsync(`${cmd} datastore Person --namespaceId DLP`); + t.regex(output, /"name": "PHONE_NUMBER"/); + t.regex(output, /"name": "EMAIL_ADDRESS"/); +}); + +test.serial(`should accept try limits for inspecting Datastore`, async (t) => { + const output = await runAsync(`${cmd} datastore Person --namespaceId DLP --tries 0`); + t.regex(output, /polling timed out/); +}); + +test.serial(`should handle Datastore with no sensitive data`, async (t) => { + const output = await runAsync(`${cmd} datastore Harmless --namespaceId DLP`); + t.is(output, 'undefined'); +}); + +test.serial(`should report Datastore file handling errors`, async (t) => { + const output = await runAsync(`${cmd} datastore Harmless --namespaceId DLP -a foo`); + t.regex(output, /Error in inspectDatastore/); +}); + +// CLI options +test(`should have a minLikelihood option`, async (t) => { + const promiseA = runAsync(`${cmd} string "My phone number is (123) 456-7890." -m POSSIBLE`); + const promiseB = runAsync(`${cmd} string "My phone number is (123) 456-7890." -m UNLIKELY`); + + const outputA = await promiseA; + t.truthy(outputA); + t.notRegex(outputA, /PHONE_NUMBER/); + + const outputB = await promiseB; + t.regex(outputB, /PHONE_NUMBER/); +}); + +test(`should have a maxFindings option`, async (t) => { + const promiseA = runAsync(`${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -f 1`); + const promiseB = runAsync(`${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -f 2`); + + const outputA = await promiseA; + t.not(outputA.includes('PHONE_NUMBER'), outputA.includes('EMAIL_ADDRESS')); // Exactly one of these should be included + + const outputB = await promiseB; + t.regex(outputB, /PHONE_NUMBER/); + t.regex(outputB, /EMAIL_ADDRESS/); +}); + +test(`should have an option to include quotes`, async (t) => { + const promiseA = runAsync(`${cmd} string "My phone number is (223) 456-7890." -q false`); + const promiseB = runAsync(`${cmd} string "My phone number is (223) 456-7890."`); + + const outputA = await promiseA; + t.truthy(outputA); + t.notRegex(outputA, /\(223\) 456-7890/); + + const outputB = await promiseB; + t.regex(outputB, /\(223\) 456-7890/); +}); + +test(`should have an option to filter results by infoType`, async (t) => { + const promiseA = runAsync(`${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890."`); + const promiseB = runAsync(`${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -t PHONE_NUMBER`); + + const outputA = await promiseA; + t.regex(outputA, /EMAIL_ADDRESS/); + t.regex(outputA, /PHONE_NUMBER/); + + const outputB = await promiseB; + t.notRegex(outputB, /EMAIL_ADDRESS/); + t.regex(outputB, /PHONE_NUMBER/); +}); + +test(`should have an option for custom auth tokens`, async (t) => { + const output = await runAsync(`${cmd} string "My name is Gary and my phone number is (223) 456-7890." -a foo`); + t.regex(output, /Error in inspectString/); + t.regex(output, /invalid authentication/); +}); + diff --git a/dlp/system-test/metadata.test.js b/dlp/system-test/metadata.test.js new file mode 100644 index 0000000000..967630d675 --- /dev/null +++ b/dlp/system-test/metadata.test.js @@ -0,0 +1,47 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +require(`../../system-test/_setup`); + +const cmd = `node metadata`; + +test(`should list info types for a given category`, async (t) => { + const output = await runAsync(`${cmd} infoTypes GOVERNMENT`); + t.regex(output, /name: 'US_DRIVERS_LICENSE_NUMBER'/); +}); + +test(`should inspect categories`, async (t) => { + const output = await runAsync(`${cmd} categories`); + t.regex(output, /name: 'FINANCE'/); +}); + +test(`should have an option for custom auth tokens`, async (t) => { + const output = await runAsync(`${cmd} categories -a foo`); + t.regex(output, /Error in listCategories/); + t.regex(output, /invalid authentication/); +}); + +// Error handling +test(`should report info type listing handling errors`, async (t) => { + const output = await runAsync(`${cmd} infoTypes GOVERNMENT -a foo`); + t.regex(output, /Error in listInfoTypes/); +}); + +test(`should report category listing handling errors`, async (t) => { + const output = await runAsync(`${cmd} categories -a foo`); + t.regex(output, /Error in listCategories/); +}); diff --git a/dlp/system-test/redact.test.js b/dlp/system-test/redact.test.js new file mode 100644 index 0000000000..d55c0a6f0f --- /dev/null +++ b/dlp/system-test/redact.test.js @@ -0,0 +1,54 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +require(`../../system-test/_setup`); + +const cmd = 'node redact'; + +// redact_string +test(`should redact sensitive data from a string`, async (t) => { + const output = await runAsync(`${cmd} string "I am Gary and my phone number is (123) 456-7890." REDACTED -t US_MALE_NAME PHONE_NUMBER`); + t.is(output, 'I am REDACTED and my phone number is REDACTED.'); +}); + +test(`should ignore unspecified type names when redacting from a string`, async (t) => { + const output = await runAsync(`${cmd} string "I am Gary and my phone number is (123) 456-7890." REDACTED -t PHONE_NUMBER`); + t.is(output, 'I am Gary and my phone number is REDACTED.'); +}); + +test(`should report string redaction handling errors`, async (t) => { + const output = await runAsync(`${cmd} string "My name is Gary and my phone number is (123) 456-7890." REDACTED -t PHONE_NUMBER -a foo`); + t.regex(output, /Error in redactString/); +}); + +// CLI options +test(`should have a minLikelihood option`, async (t) => { + const promiseA = runAsync(`${cmd} string "My phone number is (123) 456-7890." REDACTED -t PHONE_NUMBER -m VERY_LIKELY`); + const promiseB = runAsync(`${cmd} string "My phone number is (123) 456-7890." REDACTED -t PHONE_NUMBER -m UNLIKELY`); + + const outputA = await promiseA; + t.is(outputA, 'My phone number is (123) 456-7890.'); + + const outputB = await promiseB; + t.is(outputB, 'My phone number is REDACTED.'); +}); + +test(`should have an option for custom auth tokens`, async (t) => { + const output = await runAsync(`${cmd} string "My name is Gary and my phone number is (123) 456-7890." REDACTED -t PHONE_NUMBER -a foo`); + t.regex(output, /Error in redactString/); + t.regex(output, /invalid authentication/); +});