diff --git a/dlp/package.json b/dlp/package.json index fe5cf1f589..b53ef6b243 100644 --- a/dlp/package.json +++ b/dlp/package.json @@ -17,6 +17,7 @@ "dependencies": { "@google-cloud/dlp": "^4.3.0", "@google-cloud/pubsub": "^3.0.0", + "@google-cloud/storage": "^6.11.0", "mime": "^3.0.0" }, "devDependencies": { diff --git a/dlp/system-test/metadata.test.js b/dlp/system-test/metadata.test.js index db65254f88..d2c8ceb89e 100644 --- a/dlp/system-test/metadata.test.js +++ b/dlp/system-test/metadata.test.js @@ -19,22 +19,41 @@ const {describe, it, before} = require('mocha'); const cp = require('child_process'); const uuid = require('uuid'); const DLP = require('@google-cloud/dlp'); +const {Storage} = require('@google-cloud/storage'); const dataProject = 'bigquery-public-data'; const dataSetId = 'samples'; const tableId = 'github_nested'; const fieldId = 'url'; -const bucketName = process.env.BUCKET_NAME; +const storage = new Storage(); +const bucketName = `test-bucket-${uuid.v4()}`; +const testFile = 'resources/test.txt'; const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); const client = new DLP.DlpServiceClient(); describe('metadata', () => { let projectId, storedInfoTypeId; + const infoTypeCloudStorageFileSet = `gs://${bucketName}/test.txt`; before(async () => { projectId = await client.getProjectId(); + // Create a Cloud Storage bucket to be used for testing. + await storage.createBucket(bucketName); + await storage.bucket(bucketName).upload(testFile); + console.log(`Bucket ${bucketName} created.`); + }); + + after(async () => { + try { + const bucket = storage.bucket(bucketName); + await bucket.deleteFiles({force: true}); + await bucket.delete(); + console.log(`Bucket ${bucketName} deleted.`); + } catch (err) { + // ignore error + } }); // Delete stored infotypes created in the snippets. @@ -89,4 +108,85 @@ describe('metadata', () => { } assert.include(output, 'INVALID_ARGUMENT'); }); + + // dlp_update_stored_infotype + it('should update a stored infotype', async () => { + let output; + const infoTypeId = `stored-infoType-${uuid.v4()}`; + const infoTypeOutputPath = `gs://${bucketName}`; + try { + // First create a temporary stored infoType + const [response] = await client.createStoredInfoType({ + parent: `projects/${projectId}/locations/global`, + config: { + displayName: 'GitHub usernames', + description: 'Dictionary of GitHub usernames used in commits', + largeCustomDictionary: { + outputPath: { + path: infoTypeOutputPath, + }, + bigQueryField: { + table: { + datasetId: dataSetId, + projectId: dataProject, + tableId: tableId, + }, + field: { + name: fieldId, + }, + }, + }, + }, + storedInfoTypeId: infoTypeId, + }); + storedInfoTypeId = response.name; + // Execute the update script + output = execSync( + `node updateStoredInfoType.js ${projectId} ${infoTypeId} ${infoTypeOutputPath} ${infoTypeCloudStorageFileSet}` + ); + } catch (err) { + output = err.message; + } + assert.match(output, /InfoType updated successfully:/); + }); + + it('should handle stored infotype update errors', async () => { + let output; + const infoTypeId = `stored-infoType-${uuid.v4()}`; + const infoTypeOutputPath = 'INFOTYPE_OUTPUT_PATH'; + try { + // First create a temporary stored infoType + const [response] = await client.createStoredInfoType({ + parent: `projects/${projectId}/locations/global`, + config: { + displayName: 'GitHub usernames', + description: 'Dictionary of GitHub usernames used in commits', + largeCustomDictionary: { + outputPath: { + path: infoTypeOutputPath, + }, + bigQueryField: { + table: { + datasetId: dataSetId, + projectId: dataProject, + tableId: tableId, + }, + field: { + name: fieldId, + }, + }, + }, + }, + storedInfoTypeId: infoTypeId, + }); + storedInfoTypeId = response.name; + // Execute the update script + output = execSync( + `node updateStoredInfoType.js BAD_PROJECT_ID ${infoTypeId} ${infoTypeOutputPath} ${infoTypeCloudStorageFileSet}` + ); + } catch (err) { + output = err.message; + } + assert.include(output, 'INVALID_ARGUMENT'); + }); }); diff --git a/dlp/system-test/resources/test.txt b/dlp/system-test/resources/test.txt new file mode 100644 index 0000000000..4185c4e388 --- /dev/null +++ b/dlp/system-test/resources/test.txt @@ -0,0 +1,3 @@ +user1 +user2 +user3 \ No newline at end of file diff --git a/dlp/system-test/triggers.test.js b/dlp/system-test/triggers.test.js index 907b03b9dd..e19898947c 100644 --- a/dlp/system-test/triggers.test.js +++ b/dlp/system-test/triggers.test.js @@ -33,7 +33,7 @@ describe('triggers', () => { const infoType = 'PERSON_NAME'; const minLikelihood = 'VERY_LIKELY'; const maxFindings = 5; - const bucketName = process.env.BUCKET_NAME; + const bucketName = process.env.BUCKET_NAME || 'long-door-651'; before(async () => { projectId = await client.getProjectId(); diff --git a/dlp/updateStoredInfoType.js b/dlp/updateStoredInfoType.js new file mode 100644 index 0000000000..7aeb005d8a --- /dev/null +++ b/dlp/updateStoredInfoType.js @@ -0,0 +1,80 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// sample-metadata: +// title: Update stored infoType. +// description: Uses the Data Loss Prevention API to update a stored infoType. +// usage: node updateStoredInfoType.js projectId infoTypeId, outputPath, fileSetUrl +function main(projectId, infoTypeId, outputPath, fileSetUrl) { + // [START dlp_update_stored_infotype] + // Import the required libraries + const dlp = require('@google-cloud/dlp'); + + // Create a DLP client + const dlpClient = new dlp.DlpServiceClient(); + + // The project ID to run the API call under. + // const projectId = "your-project-id"; + + // The identifier for the stored infoType + // const infoTypeId = 'github-usernames'; + + // The path to the location in a Cloud Storage bucket to store the created dictionary + // const outputPath = 'cloud-bucket-path'; + + // Path of file containing term list + // const cloudStorageFileSet = 'gs://[PATH_TO_GS]'; + + async function updateStoredInfoType() { + // Specify configuration of the large custom dictionary including cloudStorageFileSet and outputPath + const largeCustomDictionaryConfig = { + outputPath: { + path: outputPath, + }, + cloudStorageFileSet: { + url: fileSetUrl, + }, + }; + + // Construct the job creation request to be sent by the client. + const updateStoredInfoTypeRequest = { + name: `projects/${projectId}/storedInfoTypes/${infoTypeId}`, + config: { + largeCustomDictionary: largeCustomDictionaryConfig, + }, + updateMask: { + paths: ['large_custom_dictionary.cloud_storage_file_set.url'], + }, + }; + + // Send the job creation request and process the response. + const [response] = await dlpClient.updateStoredInfoType( + updateStoredInfoTypeRequest + ); + + // Print the results. + console.log(`InfoType updated successfully: ${JSON.stringify(response)}`); + } + updateStoredInfoType(); + // [END dlp_update_stored_infotype] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2));