diff --git a/packages/google-cloud-datacatalog/package.json b/packages/google-cloud-datacatalog/package.json index 7e8cb8abcc0..a439dbfdbaf 100644 --- a/packages/google-cloud-datacatalog/package.json +++ b/packages/google-cloud-datacatalog/package.json @@ -52,7 +52,7 @@ "test": "c8 mocha build/test/*.js", "predocs-test": "npm run docs", "docs-test": "linkinator docs", - "samples-test": "c8 mocha samples/test/*.js", + "samples-test": "cd samples/ && npm link ../ && npm test", "system-test": "c8 mocha --timeout=5000 build/system-test/*.js", "compile": "tsc -p . && cp -r protos build/", "prepare": "npm run compile", diff --git a/packages/google-cloud-datacatalog/samples/README.md b/packages/google-cloud-datacatalog/samples/README.md new file mode 100644 index 00000000000..5208d51bb62 --- /dev/null +++ b/packages/google-cloud-datacatalog/samples/README.md @@ -0,0 +1,122 @@ +[//]: # "This README.md file is auto-generated, all changes to this file will be lost." +[//]: # "To regenerate it, use `python -m synthtool`." +Google Cloud Platform logo + +# [Data Catalog: Node.js Samples](https://github.com/googleapis/nodejs-datacatalog) + +[![Open in Cloud Shell][shell_img]][shell_link] + + + +## Table of Contents + +* [Before you begin](#before-you-begin) +* [Samples](#samples) + * [Create Custom Entry](#create-custom-entry) + * [Create Fileset](#create-fileset) + * [Grant Tag Template User Role](#grant-tag-template-user-role) + * [Quickstart](#quickstart) + * [Search Assets](#search-assets) + +## Before you begin + +Before running the samples, make sure you've followed the steps outlined in +[Using the client library](https://github.com/googleapis/nodejs-datacatalog#using-the-client-library). + +`cd samples` + +`npm install` + +`cd ..` + +## Samples + + + +### Create Custom Entry + +View the [source code](https://github.com/googleapis/nodejs-datacatalog/blob/master/samples/createCustomEntry.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-datacatalog&page=editor&open_in_editor=samples/createCustomEntry.js,samples/README.md) + +__Usage:__ + + +`node samples/createCustomEntry.js` + + +----- + + + + +### Create Fileset + +View the [source code](https://github.com/googleapis/nodejs-datacatalog/blob/master/samples/createFileset.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-datacatalog&page=editor&open_in_editor=samples/createFileset.js,samples/README.md) + +__Usage:__ + + +`node samples/createFileset.js` + + +----- + + + + +### Grant Tag Template User Role + +View the [source code](https://github.com/googleapis/nodejs-datacatalog/blob/master/samples/grantTagTemplateUserRole.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-datacatalog&page=editor&open_in_editor=samples/grantTagTemplateUserRole.js,samples/README.md) + +__Usage:__ + + +`node samples/grantTagTemplateUserRole.js` + + +----- + + + + +### Quickstart + +View the [source code](https://github.com/googleapis/nodejs-datacatalog/blob/master/samples/quickstart.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-datacatalog&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) + +__Usage:__ + + +`node samples/quickstart.js` + + +----- + + + + +### Search Assets + +View the [source code](https://github.com/googleapis/nodejs-datacatalog/blob/master/samples/searchAssets.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-datacatalog&page=editor&open_in_editor=samples/searchAssets.js,samples/README.md) + +__Usage:__ + + +`node samples/searchAssets.js` + + + + + + +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-datacatalog&page=editor&open_in_editor=samples/README.md +[product-docs]: https://cloud.google.com/data-catalog/ diff --git a/packages/google-cloud-datacatalog/samples/package.json b/packages/google-cloud-datacatalog/samples/package.json index 1d5d11d3bc0..91bf98acb2a 100644 --- a/packages/google-cloud-datacatalog/samples/package.json +++ b/packages/google-cloud-datacatalog/samples/package.json @@ -4,7 +4,7 @@ "license": "Apache-2.0", "author": "Google LLC", "engines": { - "node": ">=8" + "node": ">=10" }, "repository": "googleapis/nodejs-datacatalog", "private": true, @@ -15,7 +15,9 @@ "test": "mocha --timeout 600000" }, "dependencies": { - "@google-cloud/datacatalog": "^2.1.3" + "@google-cloud/bigquery": "^5.5.0", + "@google-cloud/datacatalog": "^2.1.3", + "uuid": "^8.3.1" }, "devDependencies": { "chai": "^4.2.0", diff --git a/packages/google-cloud-datacatalog/samples/quickstart.js b/packages/google-cloud-datacatalog/samples/quickstart.js new file mode 100644 index 00000000000..9e8ef194635 --- /dev/null +++ b/packages/google-cloud-datacatalog/samples/quickstart.js @@ -0,0 +1,170 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +async function main(projectId, datasetId, tableId) { + // [START data_catalog_quickstart] + // Import the Google Cloud client library and create a client. + const {DataCatalogClient} = require('@google-cloud/datacatalog').v1; + const datacatalog = new DataCatalogClient(); + + async function quickstart() { + // Common fields. + let request; + let responses; + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my_project'; // Google Cloud Platform project + // const datasetId = 'demo_dataset'; + // const tableId = 'trips'; + + // Currently, Data Catalog stores metadata in the + // us-central1 region. + const location = 'us-central1'; + + // Create Fields. + const fieldSource = { + displayName: 'Source of data asset', + type: { + primitiveType: 'STRING', + }, + }; + + const fieldNumRows = { + displayName: 'Number of rows in data asset', + type: { + primitiveType: 'DOUBLE', + }, + }; + + const fieldHasPII = { + displayName: 'Has PII', + type: { + primitiveType: 'BOOL', + }, + }; + + const fieldPIIType = { + displayName: 'PII type', + type: { + enumType: { + allowedValues: [ + { + displayName: 'EMAIL', + }, + { + displayName: 'SOCIAL SECURITY NUMBER', + }, + { + displayName: 'NONE', + }, + ], + }, + }, + }; + + // Create Tag Template. + const tagTemplateId = 'demo_tag_template'; + + const tagTemplate = { + displayName: 'Demo Tag Template', + fields: { + source: fieldSource, + num_rows: fieldNumRows, + has_pii: fieldHasPII, + pii_type: fieldPIIType, + }, + }; + + const tagTemplatePath = datacatalog.tagTemplatePath( + projectId, + location, + tagTemplateId + ); + + // Delete any pre-existing Template with the same name. + try { + request = { + name: tagTemplatePath, + force: true, + }; + await datacatalog.deleteTagTemplate(request); + console.log(`Deleted template: ${tagTemplatePath}`); + } catch (error) { + console.log(`Cannot delete template: ${tagTemplatePath}`); + } + + // Create the Tag Template request. + const locationPath = datacatalog.locationPath(projectId, location); + + request = { + parent: locationPath, + tagTemplateId: tagTemplateId, + tagTemplate: tagTemplate, + }; + + // Execute the request. + responses = await datacatalog.createTagTemplate(request); + const createdTagTemplate = responses[0]; + console.log(`Created template: ${createdTagTemplate.name}`); + + // Lookup Data Catalog's Entry referring to the table. + responses = await datacatalog.lookupEntry({ + linkedResource: + '//bigquery.googleapis.com/projects/' + + `${projectId}/datasets/${datasetId}/tables/${tableId}`, + }); + const entry = responses[0]; + console.log(`Entry name: ${entry.name}`); + console.log(`Entry type: ${entry.type}`); + console.log(`Linked resource: ${entry.linkedResource}`); + + // Attach a Tag to the table. + const tag = { + name: entry.name, + template: createdTagTemplate.name, + fields: { + source: { + stringValue: 'copied from tlc_yellow_trips_2017', + }, + num_rows: { + doubleValue: 113496874, + }, + has_pii: { + boolValue: false, + }, + pii_type: { + enumValue: { + displayName: 'NONE', + }, + }, + }, + }; + + request = { + parent: entry.name, + tag: tag, + }; + + // Create the Tag. + await datacatalog.createTag(request); + console.log(`Tag created for entry: ${entry.name}`); + } + quickstart(); + // [END data_catalog_quickstart] +} +main(...process.argv.slice(2)); diff --git a/packages/google-cloud-datacatalog/samples/test/quickstart.test.js b/packages/google-cloud-datacatalog/samples/test/quickstart.test.js new file mode 100644 index 00000000000..61efbdaf550 --- /dev/null +++ b/packages/google-cloud-datacatalog/samples/test/quickstart.test.js @@ -0,0 +1,88 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {describe, it, before} = require('mocha'); +const uuid = require('uuid'); +const cp = require('child_process'); +const {BigQuery} = require('@google-cloud/bigquery'); + +const bigquery = new BigQuery(); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const GCLOUD_TESTS_PREFIX = 'nodejs_data_catalog_samples'; +const generateUuid = () => + `${GCLOUD_TESTS_PREFIX}_${uuid.v4()}`.replace(/-/gi, '_'); + +describe('Quickstart', async () => { + const projectId = process.env.GCLOUD_PROJECT; + let datasetId; + let tableId; + + before(async () => { + // Delete any stale datasets from samples tests + await deleteDatasets(); + + // Create BigQuery dataset & table. + datasetId = generateUuid(); + tableId = generateUuid(); + await bigquery.createDataset(datasetId); + await bigquery.dataset(datasetId).createTable(tableId); + }); + + it('quickstart should attach tag to BigQuery table', async () => { + const output = execSync( + `node quickstart ${projectId} ${datasetId} ${tableId}` + ); + assert.include(output, 'Created template:'); + assert.include( + output, + `Linked resource: //bigquery.googleapis.com/projects/${projectId}/datasets/${datasetId}/tables/${tableId}` + ); + assert.include(output, 'Tag created for entry:'); + }); + + // Only delete a resource if it is older than 24 hours. That will prevent + // collisions with parallel CI test runs. + function isResourceStale(creationTime) { + const oneDayMs = 86400000; + const now = new Date(); + const created = new Date(creationTime); + return now.getTime() - created.getTime() >= oneDayMs; + } + + async function deleteDatasets() { + let [datasets] = await bigquery.getDatasets(); + datasets = datasets.filter(dataset => + dataset.id.includes(GCLOUD_TESTS_PREFIX) + ); + + for (const dataset of datasets) { + const [metadata] = await dataset.getMetadata(); + const creationTime = Number(metadata.creationTime); + + if (isResourceStale(creationTime)) { + try { + await dataset.delete({force: true}); + } catch (e) { + console.log(`dataset(${dataset.id}).delete() failed`); + console.log(e); + } + } + } + } +}); diff --git a/packages/google-cloud-datacatalog/src/v1beta1/data_catalog_client.ts b/packages/google-cloud-datacatalog/src/v1beta1/data_catalog_client.ts index 231a1043e12..e64cc421b56 100644 --- a/packages/google-cloud-datacatalog/src/v1beta1/data_catalog_client.ts +++ b/packages/google-cloud-datacatalog/src/v1beta1/data_catalog_client.ts @@ -3086,9 +3086,7 @@ export class DataCatalogClient { searchCatalogAsync( request?: protos.google.cloud.datacatalog.v1beta1.ISearchCatalogRequest, options?: gax.CallOptions - ): AsyncIterable< - protos.google.cloud.datacatalog.v1beta1.ISearchCatalogResult - > { + ): AsyncIterable { request = request || {}; options = options || {}; options = options || {}; @@ -3098,9 +3096,7 @@ export class DataCatalogClient { this.innerApiCalls['searchCatalog'] as GaxCall, (request as unknown) as RequestType, callSettings - ) as AsyncIterable< - protos.google.cloud.datacatalog.v1beta1.ISearchCatalogResult - >; + ) as AsyncIterable; } listEntryGroups( request: protos.google.cloud.datacatalog.v1beta1.IListEntryGroupsRequest,