From 16bef77f8808890276d459b82a8667a0c0f3b8fc Mon Sep 17 00:00:00 2001 From: dinesh-crest <127284636+dinesh-crest@users.noreply.github.com> Date: Tue, 25 Jul 2023 01:25:02 +0530 Subject: [PATCH] DLP: Added sample for update trigger and inspect data to hybrid job trigger (#3352) Added sample for same Co-authored-by: Patti Shin --- dlp/inspectDataToHybridJobTrigger.js | 94 ++++++++++++++++++ dlp/system-test/metadata.test.js | 2 +- dlp/system-test/triggers.test.js | 143 ++++++++++++++++++++++++++- dlp/updateTrigger.js | 74 ++++++++++++++ 4 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 dlp/inspectDataToHybridJobTrigger.js create mode 100644 dlp/updateTrigger.js diff --git a/dlp/inspectDataToHybridJobTrigger.js b/dlp/inspectDataToHybridJobTrigger.js new file mode 100644 index 0000000000..e5006fdcdb --- /dev/null +++ b/dlp/inspectDataToHybridJobTrigger.js @@ -0,0 +1,94 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// sample-metadata: +// title: Send data to the hybrid job trigger. +// description: Demonstrate an example of a hybridInspect request sent to Cloud DLP for processing by a hybrid job trigger. +// usage: node inspectDataToHybridJobTrigger.js my-project string jobTriggerId + +function main(projectId, string, jobTriggerId) { + // [START dlp_inspect_send_data_to_hybrid_job_trigger] + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlpClient = new DLP.DlpServiceClient(); + + // The project ID to run the API call under. + // const projectId = "your-project-id"; + + // The string to de-identify + // const string = 'My email is test@example.org'; + + // Job Trigger ID + // const jobTriggerId = 'your-job-trigger-id'; + + async function inspectDataToHybridJobTrigger() { + // Contains metadata to associate with the content. + const container = { + full_path: '10.0.0.2:logs1:app1', + relative_path: 'app1', + root_path: '10.0.0.2:logs1', + type: 'logging_sys', + version: '1.2', + }; + + const labels = {env: 'prod', 'appointment-bookings-comments': ''}; + + // Build the hybrid content item. + const hybridContentItem = { + item: {value: string}, + findingDetails: { + containerDetails: container, + labels, + }, + }; + + // Activate the job trigger. + try { + await dlpClient.activateJobTrigger({ + name: `projects/${projectId}/jobTriggers/${jobTriggerId}`, + }); + } catch (err) { + // Ignore error related to job trigger already active + if (err.code !== 3) { + console.log(err.message); + return; + } + } + + // Build the hybrid inspect request. + const request = { + name: `projects/${projectId}/jobTriggers/${jobTriggerId}`, + hybridItem: hybridContentItem, + }; + + // Send the hybrid inspect request. + const [response] = await dlpClient.hybridInspectJobTrigger(request); + + // Print the results + console.log(response); + } + inspectDataToHybridJobTrigger(); + // [END dlp_inspect_send_data_to_hybrid_job_trigger] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/dlp/system-test/metadata.test.js b/dlp/system-test/metadata.test.js index d2c8ceb89e..31d48b96ca 100644 --- a/dlp/system-test/metadata.test.js +++ b/dlp/system-test/metadata.test.js @@ -27,8 +27,8 @@ const tableId = 'github_nested'; const fieldId = 'url'; const storage = new Storage(); -const bucketName = `test-bucket-${uuid.v4()}`; const testFile = 'resources/test.txt'; +const bucketName = `test-dlp-metadata-bucket-${uuid.v4()}`; const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); diff --git a/dlp/system-test/triggers.test.js b/dlp/system-test/triggers.test.js index e19898947c..104f442db4 100644 --- a/dlp/system-test/triggers.test.js +++ b/dlp/system-test/triggers.test.js @@ -33,13 +33,103 @@ describe('triggers', () => { const infoType = 'PERSON_NAME'; const minLikelihood = 'VERY_LIKELY'; const maxFindings = 5; - const bucketName = process.env.BUCKET_NAME || 'long-door-651'; + const bucketName = process.env.BUCKET_NAME; + let tempTriggerName = ''; + + async function createTempTrigger() { + const triggerId = `trigger-test-${uuid.v4()}`; + await client.createJobTrigger({ + parent: `projects/${projectId}/locations/global`, + jobTrigger: { + inspectJob: { + inspectConfig: { + infoTypes: [{name: 'EMAIL_ADDRESS'}], + minLikelihood: 'LIKELIHOOD_UNSPECIFIED', + limits: { + maxFindingsPerRequest: 0, + }, + }, + storageConfig: { + cloudStorageOptions: { + fileSet: {url: `gs://${bucketName}/*`}, + }, + timeSpanConfig: { + enableAutoPopulationOfTimespanConfig: true, + }, + }, + }, + displayName: triggerDisplayName, + description: triggerDescription, + triggers: [ + { + schedule: { + recurrencePeriodDuration: { + seconds: 1 * 60 * 60 * 24, // Trigger the scan daily + }, + }, + }, + ], + status: 'HEALTHY', + }, + triggerId: triggerId, + }); + + return triggerId; + } + + async function createHybridTrigger() { + const hybridJobTriggerId = `hybrid-job-trigger-${uuid.v4()}`; + await client.createJobTrigger({ + parent: `projects/${projectId}/locations/global`, + triggerId: hybridJobTriggerId, + jobTrigger: { + triggers: [ + { + manual: {}, + }, + ], + inspectJob: { + actions: [{jobNotificationEmails: {}}, {publishToStackdriver: {}}], + inspectConfig: { + infoTypes: [{name: 'EMAIL_ADDRESS'}], + minLikelihood: 'POSSIBLE', + includeQuote: true, + }, + storageConfig: { + hybridOptions: { + description: + 'Hybrid job for data from the comments field of a table that contains customer appointment bookings', + requiredFindingLabelKeys: ['appointment-bookings-comments'], + labels: {env: 'prod'}, + tableOptions: {identifyingFields: [{name: 'booking_id'}]}, + }, + }, + }, + }, + }); + return hybridJobTriggerId; + } before(async () => { projectId = await client.getProjectId(); fullTriggerName = `projects/${projectId}/locations/global/jobTriggers/${triggerName}`; }); + // Delete triggers created in the snippets. + afterEach(async () => { + try { + if (tempTriggerName) { + const deleteJobTriggerRequest = { + name: `projects/${projectId}/locations/global/jobTriggers/${tempTriggerName}`, + }; + await client.deleteJobTrigger(deleteJobTriggerRequest); + tempTriggerName = ''; + } + } catch (err) { + throw `Error in deleting job/job trigger: ${err.message || err}`; + } + }); + it('should create a trigger', () => { const output = execSync( `node createTrigger.js ${projectId} ${triggerName} "${triggerDisplayName}" "${triggerDescription}" ${bucketName} true '1' ${infoType} ${minLikelihood} ${maxFindings}` @@ -88,4 +178,55 @@ describe('triggers', () => { } assert.include(output, 'fail'); }); + + // dlp_update_trigger + it('should update trigger', async () => { + let output = ''; + try { + tempTriggerName = await createTempTrigger(); + output = execSync( + `node updateTrigger.js ${projectId} ${tempTriggerName}` + ); + } catch (err) { + output = err.message; + } + assert.match(output, /Updated Trigger:/); + }); + + it('should handle errors while updating trigger', () => { + let output; + try { + output = execSync(`node updateTrigger.js ${projectId} BAD_TRIGGER_NAME`); + } catch (err) { + output = err.message; + } + assert.include(output, 'NOT_FOUND: Unknown trigger'); + }); + + // dlp_inspect_data_to_hybrid_job_trigger + it('should send data for inspection to hybrid job trigger', async () => { + let output = ''; + try { + tempTriggerName = await createHybridTrigger(); + output = execSync( + `node inspectDataToHybridJobTrigger.js ${projectId} "My email is test@example.org" ${tempTriggerName}` + ); + } catch (err) { + console.log(err); + output = err.message; + } + assert.match(output, /{}/); + }); + + it('should handle errors while sending data to hybrid job trigger', () => { + let output; + try { + output = execSync( + `node inspectDataToHybridJobTrigger.js ${projectId} "My email is test@example.org" BAD_HYBRID_TRIGGER_KEY` + ); + } catch (err) { + output = err.message; + } + assert.include(output, 'NOT_FOUND: Unknown trigger'); + }); }); diff --git a/dlp/updateTrigger.js b/dlp/updateTrigger.js new file mode 100644 index 0000000000..2d2a9b0c5d --- /dev/null +++ b/dlp/updateTrigger.js @@ -0,0 +1,74 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// sample-metadata: +// title: Update job trigger with suitable configuration and mask. +// description: Uses the Data Loss Prevention API to update job trigger with suitable configuration and mask. +// usage: node updateTrigger.js projectId, jobTriggerName +function main(projectId, jobTriggerName) { + // [START dlp_update_trigger] + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlpClient = new DLP.DlpServiceClient(); + + // The project ID to run the API call under + // const projectId = 'my-project'; + + // The job trigger ID to run the API call under + // const jobTriggerName = 'your-job-trigger-name'; + + async function updateTrigger() { + // Construct inspect configuration to match PERSON_NAME infotype + const inspectConfig = { + infoTypes: [{name: 'PERSON_NAME'}], + minLikelihood: 'LIKELY', + }; + + // Configure the job trigger we want to update. + const jobTrigger = {inspectJob: {inspectConfig}}; + + const updateMask = { + paths: [ + 'inspect_job.inspect_config.info_types', + 'inspect_job.inspect_config.min_likelihood', + ], + }; + + // Combine configurations into a request for the service. + const request = { + name: `projects/${projectId}/jobTriggers/${jobTriggerName}`, + jobTrigger, + updateMask, + }; + + // Send the request and receive response from the service + const [updatedJobTrigger] = await dlpClient.updateJobTrigger(request); + + // Print the results + console.log(`Updated Trigger: ${JSON.stringify(updatedJobTrigger)}`); + } + updateTrigger(projectId, jobTriggerName); + // [END dlp_update_trigger] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2));