Skip to content

Commit

Permalink
Merge branch 'main' into garethgeorge/typed
Browse files Browse the repository at this point in the history
  • Loading branch information
kweinmeister authored Jul 17, 2023
2 parents 0c60330 + 547f45f commit a628bb8
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 2 deletions.
1 change: 1 addition & 0 deletions dlp/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"dependencies": {
"@google-cloud/dlp": "^4.3.0",
"@google-cloud/pubsub": "^3.0.0",
"@google-cloud/storage": "^6.11.0",
"mime": "^3.0.0"
},
"devDependencies": {
Expand Down
102 changes: 101 additions & 1 deletion dlp/system-test/metadata.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,41 @@ const {describe, it, before} = require('mocha');
const cp = require('child_process');
const uuid = require('uuid');
const DLP = require('@google-cloud/dlp');
const {Storage} = require('@google-cloud/storage');

const dataProject = 'bigquery-public-data';
const dataSetId = 'samples';
const tableId = 'github_nested';
const fieldId = 'url';

const bucketName = process.env.BUCKET_NAME;
const storage = new Storage();
const bucketName = `test-bucket-${uuid.v4()}`;
const testFile = 'resources/test.txt';

const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});

const client = new DLP.DlpServiceClient();
describe('metadata', () => {
let projectId, storedInfoTypeId;
const infoTypeCloudStorageFileSet = `gs://${bucketName}/test.txt`;

before(async () => {
projectId = await client.getProjectId();
// Create a Cloud Storage bucket to be used for testing.
await storage.createBucket(bucketName);
await storage.bucket(bucketName).upload(testFile);
console.log(`Bucket ${bucketName} created.`);
});

after(async () => {
try {
const bucket = storage.bucket(bucketName);
await bucket.deleteFiles({force: true});
await bucket.delete();
console.log(`Bucket ${bucketName} deleted.`);
} catch (err) {
// ignore error
}
});

// Delete stored infotypes created in the snippets.
Expand Down Expand Up @@ -89,4 +108,85 @@ describe('metadata', () => {
}
assert.include(output, 'INVALID_ARGUMENT');
});

// dlp_update_stored_infotype
it('should update a stored infotype', async () => {
let output;
const infoTypeId = `stored-infoType-${uuid.v4()}`;
const infoTypeOutputPath = `gs://${bucketName}`;
try {
// First create a temporary stored infoType
const [response] = await client.createStoredInfoType({
parent: `projects/${projectId}/locations/global`,
config: {
displayName: 'GitHub usernames',
description: 'Dictionary of GitHub usernames used in commits',
largeCustomDictionary: {
outputPath: {
path: infoTypeOutputPath,
},
bigQueryField: {
table: {
datasetId: dataSetId,
projectId: dataProject,
tableId: tableId,
},
field: {
name: fieldId,
},
},
},
},
storedInfoTypeId: infoTypeId,
});
storedInfoTypeId = response.name;
// Execute the update script
output = execSync(
`node updateStoredInfoType.js ${projectId} ${infoTypeId} ${infoTypeOutputPath} ${infoTypeCloudStorageFileSet}`
);
} catch (err) {
output = err.message;
}
assert.match(output, /InfoType updated successfully:/);
});

it('should handle stored infotype update errors', async () => {
let output;
const infoTypeId = `stored-infoType-${uuid.v4()}`;
const infoTypeOutputPath = 'INFOTYPE_OUTPUT_PATH';
try {
// First create a temporary stored infoType
const [response] = await client.createStoredInfoType({
parent: `projects/${projectId}/locations/global`,
config: {
displayName: 'GitHub usernames',
description: 'Dictionary of GitHub usernames used in commits',
largeCustomDictionary: {
outputPath: {
path: infoTypeOutputPath,
},
bigQueryField: {
table: {
datasetId: dataSetId,
projectId: dataProject,
tableId: tableId,
},
field: {
name: fieldId,
},
},
},
},
storedInfoTypeId: infoTypeId,
});
storedInfoTypeId = response.name;
// Execute the update script
output = execSync(
`node updateStoredInfoType.js BAD_PROJECT_ID ${infoTypeId} ${infoTypeOutputPath} ${infoTypeCloudStorageFileSet}`
);
} catch (err) {
output = err.message;
}
assert.include(output, 'INVALID_ARGUMENT');
});
});
3 changes: 3 additions & 0 deletions dlp/system-test/resources/test.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
user1
user2
user3
2 changes: 1 addition & 1 deletion dlp/system-test/triggers.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ describe('triggers', () => {
const infoType = 'PERSON_NAME';
const minLikelihood = 'VERY_LIKELY';
const maxFindings = 5;
const bucketName = process.env.BUCKET_NAME;
const bucketName = process.env.BUCKET_NAME || 'long-door-651';

before(async () => {
projectId = await client.getProjectId();
Expand Down
80 changes: 80 additions & 0 deletions dlp/updateStoredInfoType.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

'use strict';

// sample-metadata:
// title: Update stored infoType.
// description: Uses the Data Loss Prevention API to update a stored infoType.
// usage: node updateStoredInfoType.js projectId infoTypeId, outputPath, fileSetUrl
function main(projectId, infoTypeId, outputPath, fileSetUrl) {
// [START dlp_update_stored_infotype]
// Import the required libraries
const dlp = require('@google-cloud/dlp');

// Create a DLP client
const dlpClient = new dlp.DlpServiceClient();

// The project ID to run the API call under.
// const projectId = "your-project-id";

// The identifier for the stored infoType
// const infoTypeId = 'github-usernames';

// The path to the location in a Cloud Storage bucket to store the created dictionary
// const outputPath = 'cloud-bucket-path';

// Path of file containing term list
// const cloudStorageFileSet = 'gs://[PATH_TO_GS]';

async function updateStoredInfoType() {
// Specify configuration of the large custom dictionary including cloudStorageFileSet and outputPath
const largeCustomDictionaryConfig = {
outputPath: {
path: outputPath,
},
cloudStorageFileSet: {
url: fileSetUrl,
},
};

// Construct the job creation request to be sent by the client.
const updateStoredInfoTypeRequest = {
name: `projects/${projectId}/storedInfoTypes/${infoTypeId}`,
config: {
largeCustomDictionary: largeCustomDictionaryConfig,
},
updateMask: {
paths: ['large_custom_dictionary.cloud_storage_file_set.url'],
},
};

// Send the job creation request and process the response.
const [response] = await dlpClient.updateStoredInfoType(
updateStoredInfoTypeRequest
);

// Print the results.
console.log(`InfoType updated successfully: ${JSON.stringify(response)}`);
}
updateStoredInfoType();
// [END dlp_update_stored_infotype]
}

process.on('unhandledRejection', err => {
console.error(err.message);
process.exitCode = 1;
});

main(...process.argv.slice(2));

0 comments on commit a628bb8

Please sign in to comment.