From 51bc8a7445ab8b3d2239493b69d9c271c1086dde Mon Sep 17 00:00:00 2001 From: Htut Khine Htay Win Date: Sun, 20 Oct 2024 23:06:17 -0700 Subject: [PATCH] feat: allow multiple KMS keys to create CMEK database/backup (#2099) Add kms_key_names field to create database/create backup code snippets. --- README.md | 4 + samples/README.md | 72 ++++++++ .../backups-copy-with-multiple-kms-keys.js | 105 ++++++++++++ .../backups-create-with-multiple-kms-keys.js | 125 ++++++++++++++ .../backups-restore-with-multiple-kms-keys.js | 89 ++++++++++ .../database-create-with-multiple-kms-keys.js | 77 +++++++++ samples/system-test/spanner.test.js | 158 ++++++++++++++++-- 7 files changed, 618 insertions(+), 12 deletions(-) create mode 100644 samples/backups-copy-with-multiple-kms-keys.js create mode 100644 samples/backups-create-with-multiple-kms-keys.js create mode 100644 samples/backups-restore-with-multiple-kms-keys.js create mode 100644 samples/database-create-with-multiple-kms-keys.js diff --git a/README.md b/README.md index dd197448a..31c800712 100644 --- a/README.md +++ b/README.md @@ -91,14 +91,17 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-spanner/tre | --------------------------- | --------------------------------- | ------ | | Add and drop new database role | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/add-and-drop-new-database-role.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/add-and-drop-new-database-role.js,samples/README.md) | | Backups-cancel | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-cancel.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-cancel.js,samples/README.md) | +| Copies a source backup | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-copy-with-multiple-kms-keys.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-copy-with-multiple-kms-keys.js,samples/README.md) | | Copies a source backup | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-copy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-copy.js,samples/README.md) | | Backups-create-with-encryption-key | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-create-with-encryption-key.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-create-with-encryption-key.js,samples/README.md) | +| Backups-create-with-multiple-kms-keys | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-create-with-multiple-kms-keys.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-create-with-multiple-kms-keys.js,samples/README.md) | | Backups-create | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-create.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-create.js,samples/README.md) | | Backups-delete | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-delete.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-delete.js,samples/README.md) | | Backups-get-database-operations | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-get-database-operations.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-get-database-operations.js,samples/README.md) | | Backups-get-operations | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-get-operations.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-get-operations.js,samples/README.md) | | Backups-get | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-get.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-get.js,samples/README.md) | | Backups-restore-with-encryption-key | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-restore-with-encryption-key.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-restore-with-encryption-key.js,samples/README.md) | +| Backups-restore-with-multiple-kms-keys | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-restore-with-multiple-kms-keys.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-restore-with-multiple-kms-keys.js,samples/README.md) | | Backups-restore | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-restore.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-restore.js,samples/README.md) | | Backups-update | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-update.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-update.js,samples/README.md) | | Backups | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups.js,samples/README.md) | @@ -109,6 +112,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-spanner/tre | CRUD | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/crud.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/crud.js,samples/README.md) | | Creates a new database with a specific default leader | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-create-with-default-leader.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/database-create-with-default-leader.js,samples/README.md) | | Database-create-with-encryption-key | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-create-with-encryption-key.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/database-create-with-encryption-key.js,samples/README.md) | +| Database-create-with-multiple-kms-keys | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-create-with-multiple-kms-keys.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/database-create-with-multiple-kms-keys.js,samples/README.md) | | Database-create-with-version-retention-period | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-create-with-version-retention-period.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/database-create-with-version-retention-period.js,samples/README.md) | | Gets the schema definition of an existing database | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-get-ddl.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/database-get-ddl.js,samples/README.md) | | Gets the default leader option of an existing database | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-get-default-leader.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/database-get-default-leader.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index bdbd28844..9a2d55d5b 100644 --- a/samples/README.md +++ b/samples/README.md @@ -17,13 +17,16 @@ and automatic, synchronous replication for high availability. * [Add and drop new database role](#add-and-drop-new-database-role) * [Backups-cancel](#backups-cancel) * [Copies a source backup](#copies-a-source-backup) + * [Copies a source backup](#copies-a-source-backup) * [Backups-create-with-encryption-key](#backups-create-with-encryption-key) + * [Backups-create-with-multiple-kms-keys](#backups-create-with-multiple-kms-keys) * [Backups-create](#backups-create) * [Backups-delete](#backups-delete) * [Backups-get-database-operations](#backups-get-database-operations) * [Backups-get-operations](#backups-get-operations) * [Backups-get](#backups-get) * [Backups-restore-with-encryption-key](#backups-restore-with-encryption-key) + * [Backups-restore-with-multiple-kms-keys](#backups-restore-with-multiple-kms-keys) * [Backups-restore](#backups-restore) * [Backups-update](#backups-update) * [Backups](#backups) @@ -34,6 +37,7 @@ and automatic, synchronous replication for high availability. * [CRUD](#crud) * [Creates a new database with a specific default leader](#creates-a-new-database-with-a-specific-default-leader) * [Database-create-with-encryption-key](#database-create-with-encryption-key) + * [Database-create-with-multiple-kms-keys](#database-create-with-multiple-kms-keys) * [Database-create-with-version-retention-period](#database-create-with-version-retention-period) * [Gets the schema definition of an existing database](#gets-the-schema-definition-of-an-existing-database) * [Gets the default leader option of an existing database](#gets-the-default-leader-option-of-an-existing-database) @@ -176,6 +180,23 @@ __Usage:__ +### Copies a source backup + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-copy-with-multiple-kms-keys.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-copy-with-multiple-kms-keys.js,samples/README.md) + +__Usage:__ + + +`node spannerCopyBackup ` + + +----- + + + + ### Copies a source backup View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-copy.js). @@ -210,6 +231,23 @@ __Usage:__ +### Backups-create-with-multiple-kms-keys + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-create-with-multiple-kms-keys.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-create-with-multiple-kms-keys.js,samples/README.md) + +__Usage:__ + + +`node samples/backups-create-with-multiple-kms-keys.js` + + +----- + + + + ### Backups-create View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-create.js). @@ -312,6 +350,23 @@ __Usage:__ +### Backups-restore-with-multiple-kms-keys + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-restore-with-multiple-kms-keys.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/backups-restore-with-multiple-kms-keys.js,samples/README.md) + +__Usage:__ + + +`node samples/backups-restore-with-multiple-kms-keys.js` + + +----- + + + + ### Backups-restore View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/backups-restore.js). @@ -482,6 +537,23 @@ __Usage:__ +### Database-create-with-multiple-kms-keys + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-create-with-multiple-kms-keys.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/database-create-with-multiple-kms-keys.js,samples/README.md) + +__Usage:__ + + +`node samples/database-create-with-multiple-kms-keys.js` + + +----- + + + + ### Database-create-with-version-retention-period View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/database-create-with-version-retention-period.js). diff --git a/samples/backups-copy-with-multiple-kms-keys.js b/samples/backups-copy-with-multiple-kms-keys.js new file mode 100644 index 000000000..f1e2fb52c --- /dev/null +++ b/samples/backups-copy-with-multiple-kms-keys.js @@ -0,0 +1,105 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// sample-metadata: +// title: Copies a source backup +// usage: node spannerCopyBackup + +'use strict'; + +function main( + instanceId = 'my-instance', + backupId = 'my-backup', + sourceBackupPath = 'projects/my-project-id/instances/my-source-instance/backups/my-source-backup', + projectId = 'my-project-id', + kmsKeyNames = 'key1,key2' +) { + // [START spanner_copy_backup_with_MR_CMEK] + /** + * TODO(developer): Uncomment these variables before running the sample. + */ + // const instanceId = 'my-instance'; + // const backupId = 'my-backup', + // const sourceBackupPath = 'projects/my-project-id/instances/my-source-instance/backups/my-source-backup', + // const projectId = 'my-project-id'; + // const kmsKeyNames = + // 'projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key1, + // projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key2'; + + // Imports the Google Cloud Spanner client library + const {Spanner} = require('@google-cloud/spanner'); + const {PreciseDate} = require('@google-cloud/precise-date'); + + // Creates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + // Gets a reference to a Cloud Spanner Database Admin Client object + const databaseAdminClient = spanner.getDatabaseAdminClient(); + + async function spannerCopyBackupWithMultipleKmsKeys() { + // Expire copy backup 14 days in the future + const expireTime = Spanner.timestamp( + Date.now() + 1000 * 60 * 60 * 24 * 14 + ).toStruct(); + + // Copy the source backup + try { + console.log(`Creating copy of the source backup ${sourceBackupPath}.`); + const [operation] = await databaseAdminClient.copyBackup({ + parent: databaseAdminClient.instancePath(projectId, instanceId), + sourceBackup: sourceBackupPath, + backupId: backupId, + expireTime: expireTime, + kmsKeyNames: kmsKeyNames.split(','), + }); + + console.log( + `Waiting for backup copy ${databaseAdminClient.backupPath( + projectId, + instanceId, + backupId + )} to complete...` + ); + await operation.promise(); + + // Verify the copy backup is ready + const [copyBackup] = await databaseAdminClient.getBackup({ + name: databaseAdminClient.backupPath(projectId, instanceId, backupId), + }); + + if (copyBackup.state === 'READY') { + console.log( + `Backup copy ${copyBackup.name} of size ` + + `${copyBackup.sizeBytes} bytes was created at ` + + `${new PreciseDate(copyBackup.createTime).toISOString()} ` + + 'with version time ' + + `${new PreciseDate(copyBackup.versionTime).toISOString()}` + ); + } else { + console.error('ERROR: Copy of backup is not ready.'); + } + } catch (err) { + console.error('ERROR:', err); + } + } + spannerCopyBackupWithMultipleKmsKeys(); + // [END spanner_copy_backup_with_MR_CMEK] +} +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/backups-create-with-multiple-kms-keys.js b/samples/backups-create-with-multiple-kms-keys.js new file mode 100644 index 000000000..f59263c13 --- /dev/null +++ b/samples/backups-create-with-multiple-kms-keys.js @@ -0,0 +1,125 @@ +/** + * Copyright 2024 Google LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +function main( + instanceId = 'my-instance', + databaseId = 'my-database', + backupId = 'my-backup', + projectId = 'my-project-id', + kmsKeyNames = 'key1,key2' +) { + // [START spanner_create_backup_with_MR_CMEK] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my-project-id'; + // const instanceId = 'my-instance'; + // const databaseId = 'my-database'; + // const backupId = 'my-backup'; + // const kmsKeyNames = + // 'projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key1, + // 'projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key2'; + + // Imports the Google Cloud client library + const {Spanner, protos} = require('@google-cloud/spanner'); + const {PreciseDate} = require('@google-cloud/precise-date'); + + // Creates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + // Gets a reference to a Cloud Spanner Database Admin Client object + const databaseAdminClient = spanner.getDatabaseAdminClient(); + async function createBackupWithMultipleKmsKeys() { + // Creates a new backup of the database + try { + console.log( + `Creating backup of database ${databaseAdminClient.databasePath( + projectId, + instanceId, + databaseId + )}.` + ); + + // Expire backup 14 days in the future + const expireTime = Date.now() + 1000 * 60 * 60 * 24 * 14; + + // Create a backup of the state of the database at the current time. + const [operation] = await databaseAdminClient.createBackup({ + parent: databaseAdminClient.instancePath(projectId, instanceId), + backupId: backupId, + backup: (protos.google.spanner.admin.database.v1.Backup = { + database: databaseAdminClient.databasePath( + projectId, + instanceId, + databaseId + ), + expireTime: Spanner.timestamp(expireTime).toStruct(), + name: databaseAdminClient.backupPath(projectId, instanceId, backupId), + }), + encryptionConfig: { + encryptionType: 'CUSTOMER_MANAGED_ENCRYPTION', + kmsKeyNames: kmsKeyNames.split(','), + }, + }); + + console.log( + `Waiting for backup ${databaseAdminClient.backupPath( + projectId, + instanceId, + backupId + )} to complete...` + ); + await operation.promise(); + + // Verify backup is ready + const [backupInfo] = await databaseAdminClient.getBackup({ + name: databaseAdminClient.backupPath(projectId, instanceId, backupId), + }); + + const kmsKeyVersions = backupInfo.encryptionInformation + .map(encryptionInfo => encryptionInfo.kmsKeyVersion) + .join(', '); + + if (backupInfo.state === 'READY') { + console.log( + `Backup ${backupInfo.name} of size ` + + `${backupInfo.sizeBytes} bytes was created at ` + + `${new PreciseDate(backupInfo.createTime).toISOString()} ` + + `using encryption key ${kmsKeyVersions}` + ); + } else { + console.error('ERROR: Backup is not ready.'); + } + } catch (err) { + console.error('ERROR:', err); + } finally { + // Close the spanner client when finished. + // The databaseAdminClient does not require explicit closure. The closure of the Spanner client will automatically close the databaseAdminClient. + spanner.close(); + } + } + createBackupWithMultipleKmsKeys(); + // [END spanner_create_backup_with_MR_CMEK] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/backups-restore-with-multiple-kms-keys.js b/samples/backups-restore-with-multiple-kms-keys.js new file mode 100644 index 000000000..1bb4c5d6b --- /dev/null +++ b/samples/backups-restore-with-multiple-kms-keys.js @@ -0,0 +1,89 @@ +/** + * Copyright 2024 Google LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +function main( + instanceId = 'my-instance', + databaseId = 'my-database', + backupId = 'my-backup', + projectId = 'my-project', + kmsKeyNames = 'key1,key2' +) { + // [START spanner_restore_backup_with_MR_CMEK] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my-project-id'; + // const instanceId = 'my-instance'; + // const databaseId = 'my-database'; + // const backupId = 'my-backup'; + // const kmsKeyNames = + // 'projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key1, + // projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key2'; + + // Imports the Google Cloud client library and precise date library + const {Spanner} = require('@google-cloud/spanner'); + + // Creates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + // Gets a reference to a Cloud Spanner Database Admin Client object + const databaseAdminClient = spanner.getDatabaseAdminClient(); + + async function restoreBackupWithMultipleKmsKeys() { + // Restore the database + console.log( + `Restoring database ${databaseAdminClient.databasePath( + projectId, + instanceId, + databaseId + )} from backup ${backupId}.` + ); + const [restoreOperation] = await databaseAdminClient.restoreDatabase({ + parent: databaseAdminClient.instancePath(projectId, instanceId), + databaseId: databaseId, + backup: databaseAdminClient.backupPath(projectId, instanceId, backupId), + encryptionConfig: { + encryptionType: 'CUSTOMER_MANAGED_ENCRYPTION', + kmsKeyNames: kmsKeyNames.split(','), + }, + }); + + // Wait for restore to complete + console.log('Waiting for database restore to complete...'); + await restoreOperation.promise(); + + console.log('Database restored from backup.'); + const [metadata] = await databaseAdminClient.getDatabase({ + name: databaseAdminClient.databasePath(projectId, instanceId, databaseId), + }); + console.log( + `Database ${metadata.restoreInfo.backupInfo.sourceDatabase} was restored ` + + `to ${databaseId} from backup ${metadata.restoreInfo.backupInfo.backup} ` + + `using encryption key ${metadata.encryptionConfig.kmsKeyNames}.` + ); + } + restoreBackupWithMultipleKmsKeys(); + // [END spanner_restore_backup_with_MR_CMEK] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/database-create-with-multiple-kms-keys.js b/samples/database-create-with-multiple-kms-keys.js new file mode 100644 index 000000000..760a6ad67 --- /dev/null +++ b/samples/database-create-with-multiple-kms-keys.js @@ -0,0 +1,77 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main( + instanceId = 'my-instance', + databaseId = 'my-database', + projectId = 'my-project', + kmsKeyNames = 'key1,key2,key3' +) { + // [START spanner_create_database_with_MR_CMEK] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my-project-id'; + // const instanceId = 'my-instance'; + // const databaseId = 'my-database'; + // const kmsKeyNames = + // 'projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key1,projects/my-project-id/my-region/keyRings/my-key-ring/cryptoKeys/my-key2'; + + // Imports the Google Cloud client library + const {Spanner, protos} = require('@google-cloud/spanner'); + + // creates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + // Gets a reference to a Cloud Spanner Database Admin Client object + const databaseAdminClient = spanner.getDatabaseAdminClient(); + + async function createDatabaseWithMultipleKmsKeys() { + // Creates a database + const [operation] = await databaseAdminClient.createDatabase({ + createStatement: 'CREATE DATABASE `' + databaseId + '`', + parent: databaseAdminClient.instancePath(projectId, instanceId), + encryptionConfig: + (protos.google.spanner.admin.database.v1.EncryptionConfig = { + kmsKeyNames: kmsKeyNames.split(','), + }), + }); + + console.log(`Waiting for operation on ${databaseId} to complete...`); + await operation.promise(); + + console.log(`Created database ${databaseId} on instance ${instanceId}.`); + + // Get encryption key + const [metadata] = await databaseAdminClient.getDatabase({ + name: databaseAdminClient.databasePath(projectId, instanceId, databaseId), + }); + + console.log( + `Database encrypted with keys ${metadata.encryptionConfig.kmsKeyNames}.` + ); + } + createDatabaseWithMultipleKmsKeys(); + // [END spanner_create_database_with_MR_CMEK] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/system-test/spanner.test.js b/samples/system-test/spanner.test.js index d1354c8f2..227f96dee 100644 --- a/samples/system-test/spanner.test.js +++ b/samples/system-test/spanner.test.js @@ -56,6 +56,8 @@ const PROJECT_ID = process.env.GCLOUD_PROJECT; const PREFIX = 'test-instance'; const INSTANCE_ID = process.env.SPANNERTEST_INSTANCE || `${PREFIX}-${CURRENT_TIME}`; +const MULTI_REGION_INSTANCE_ID = + process.env.SPANNERTEST_MR_INSTANCE || `${PREFIX}-mr-${CURRENT_TIME}`; const SAMPLE_INSTANCE_ID = `${PREFIX}-my-sample-instance-${CURRENT_TIME}`; const SAMPLE_INSTANCE_CONFIG_ID = `custom-my-sample-instance-config-${CURRENT_TIME}`; const BASE_INSTANCE_CONFIG_ID = 'regional-us-central1'; @@ -74,8 +76,11 @@ const COPY_BACKUP_ID = `test-copy-backup-${CURRENT_TIME}`; const ENCRYPTED_BACKUP_ID = `test-backup-${CURRENT_TIME}-enc`; const CANCELLED_BACKUP_ID = `test-backup-${CURRENT_TIME}-c`; const LOCATION_ID = 'regional-us-central1'; +const MULTI_REGION_LOCATION_ID = 'nam3'; const PG_LOCATION_ID = 'regional-us-west2'; -const KEY_LOCATION_ID = 'us-central1'; +const KEY_LOCATION_ID1 = 'us-central1'; +const KEY_LOCATION_ID2 = 'us-east1'; +const KEY_LOCATION_ID3 = 'us-east4'; const KEY_RING_ID = 'test-key-ring-node'; const KEY_ID = 'test-key'; const DEFAULT_LEADER = 'us-central1'; @@ -139,18 +144,14 @@ async function deleteInstance(instance) { return instance.delete(GAX_OPTIONS); } -async function getCryptoKey() { +async function getCryptoKey(key_location) { const NOT_FOUND = 5; // Instantiates a client. const client = new KeyManagementServiceClient(); // Build the parent key ring name. - const keyRingName = client.keyRingPath( - PROJECT_ID, - KEY_LOCATION_ID, - KEY_RING_ID - ); + const keyRingName = client.keyRingPath(PROJECT_ID, key_location, KEY_RING_ID); // Get key ring. try { @@ -159,7 +160,7 @@ async function getCryptoKey() { // Create key ring if it doesn't exist. if (err.code === NOT_FOUND) { // Build the parent location name. - const locationName = client.locationPath(PROJECT_ID, KEY_LOCATION_ID); + const locationName = client.locationPath(PROJECT_ID, key_location); await client.createKeyRing({ parent: locationName, keyRingId: KEY_RING_ID, @@ -174,7 +175,7 @@ async function getCryptoKey() { // Build the key name const keyName = client.cryptoKeyPath( PROJECT_ID, - KEY_LOCATION_ID, + key_location, KEY_RING_ID, KEY_ID ); @@ -405,7 +406,7 @@ describe('Autogenerated Admin Clients', () => { // create_database_with_encryption_key it('should create a database with an encryption key', async () => { - const key = await getCryptoKey(); + const key = await getCryptoKey(KEY_LOCATION_ID1); const output = execSync( `${schemaCmd} createDatabaseWithEncryptionKey "${INSTANCE_ID}" "${ENCRYPTED_DATABASE_ID}" ${PROJECT_ID} "${key.name}"` @@ -1339,7 +1340,7 @@ describe('Autogenerated Admin Clients', () => { // create_backup_with_encryption_key it('should create an encrypted backup of the database', async () => { - const key = await getCryptoKey(); + const key = await getCryptoKey(KEY_LOCATION_ID1); const output = execSync( `${backupsCmd} createBackupWithEncryptionKey ${INSTANCE_ID} ${DATABASE_ID} ${ENCRYPTED_BACKUP_ID} ${PROJECT_ID} ${key.name}` @@ -1441,7 +1442,7 @@ describe('Autogenerated Admin Clients', () => { // Delay the start of the test, if this is a retry. await delay(this.test); - const key = await getCryptoKey(); + const key = await getCryptoKey(KEY_LOCATION_ID1); const output = execSync( `${backupsCmd} restoreBackupWithEncryptionKey ${INSTANCE_ID} ${ENCRYPTED_RESTORE_DATABASE_ID} ${ENCRYPTED_BACKUP_ID} ${PROJECT_ID} ${key.name}` @@ -2286,4 +2287,137 @@ describe('Autogenerated Admin Clients', () => { assert.match(output, new RegExp('SingerId: 2')); }); }); + + describe('encrypted database and backups with multiple KMS keys', () => { + const MR_CMEK_DB = `test-mr-${CURRENT_TIME}-db`; + const MR_CMEK_BACKUP = `test-mr-${CURRENT_TIME}-backup`; + const MR_CMEK_COPIED = `test-mr-${CURRENT_TIME}-copied`; + const MR_CMEK_RESTORED = `test-mr-${CURRENT_TIME}-restored`; + let instance_already_exists = false; + let key1, key2, key3; + before(async () => { + // Create multiple KMS keys covering `nam3`. + key1 = await getCryptoKey(KEY_LOCATION_ID1); + key2 = await getCryptoKey(KEY_LOCATION_ID2); + key3 = await getCryptoKey(KEY_LOCATION_ID3); + + const multi_region_instance = spanner.instance(MULTI_REGION_INSTANCE_ID); + [instance_already_exists] = await multi_region_instance.exists(); + if (!instance_already_exists) { + const [, operation] = await multi_region_instance.create({ + config: MULTI_REGION_LOCATION_ID, + nodes: 1, + labels: { + [LABEL]: 'true', + created: CURRENT_TIME, + }, + gaxOptions: GAX_OPTIONS, + }); + await operation.promise(); + console.log( + `Created temp instance, using + ${multi_region_instance.formattedName_}...` + ); + } else { + console.log( + `Not creating temp instance, using + ${multi_region_instance.formattedName_}...` + ); + } + }); + + after(async () => { + const instance = spanner.instance(MULTI_REGION_INSTANCE_ID); + const restored_db = instance.database(MR_CMEK_RESTORED); + function sleep(timeMillis) { + return new Promise(resolve => setTimeout(resolve, timeMillis)); + } + // Backup cannot be deleted when restored db is not in READY_OPTIMIZING state. + while ((await restored_db.getState()) === 'READY_OPTIMIZING') { + await sleep(1000); + } + await Promise.all([ + instance.database(MR_CMEK_DB).delete(GAX_OPTIONS), + instance.database(MR_CMEK_RESTORED).delete(), + instance.backup(MR_CMEK_BACKUP).delete(GAX_OPTIONS), + instance.backup(MR_CMEK_COPIED).delete(GAX_OPTIONS), + ]); + if (!instance_already_exists) { + await spanner.instance(MULTI_REGION_INSTANCE_ID).delete(GAX_OPTIONS); + } + }); + + it('should create a database with multiple KMS keys', async () => { + const output = execSync( + `node database-create-with-multiple-kms-keys.js \ + "${MULTI_REGION_INSTANCE_ID}" \ + "${MR_CMEK_DB}" \ + "${PROJECT_ID}" \ + "${key1.name},${key2.name},${key3.name}"` + ); + assert.match( + output, + new RegExp(`Waiting for operation on ${MR_CMEK_DB} to complete...`) + ); + assert.match( + output, + new RegExp( + `Created database ${MR_CMEK_DB} on instance ${MULTI_REGION_INSTANCE_ID}.` + ) + ); + assert.match(output, new RegExp('Database encrypted with keys')); + }); + + it('should create backup with multiple KMS keys', async () => { + const output = execSync( + `node backups-create-with-multiple-kms-keys.js \ + ${MULTI_REGION_INSTANCE_ID} \ + ${MR_CMEK_DB} \ + ${MR_CMEK_BACKUP} \ + ${PROJECT_ID} \ + "${key1.name},${key2.name},${key3.name}"` + ); + assert.match(output, new RegExp(`Backup (.+)${MR_CMEK_BACKUP} of size`)); + assert.include(output, 'using encryption key'); + }); + + it('should copy backup with multiple KMS keys', async () => { + const sourceBackupPath = `projects/${PROJECT_ID}/instances/${MULTI_REGION_INSTANCE_ID}/backups/${MR_CMEK_BACKUP}`; + const output = execSync( + `node backups-copy-with-multiple-kms-keys.js \ + ${MULTI_REGION_INSTANCE_ID} \ + ${MR_CMEK_COPIED} \ + ${sourceBackupPath} \ + ${PROJECT_ID} \ + "${key1.name},${key2.name},${key3.name}"` + ); + assert.match( + output, + new RegExp(`(.*)Backup copy(.*)${MR_CMEK_COPIED} of size(.*)`) + ); + }); + + it('should restore backup with multiple KMS keys', async function () { + // Restoring a backup can be a slow operation so the test may timeout and + // we'll have to retry. + this.retries(5); + // Delay the start of the test, if this is a retry. + await delay(this.test); + + const output = execSync( + `node backups-restore-with-multiple-kms-keys.js \ + ${MULTI_REGION_INSTANCE_ID} \ + ${MR_CMEK_RESTORED} \ + ${MR_CMEK_BACKUP} \ + ${PROJECT_ID} \ + "${key1.name},${key2.name},${key3.name}"` + ); + assert.match(output, /Database restored from backup./); + assert.match( + output, + new RegExp( + `Database (.+) was restored to ${MR_CMEK_RESTORED} from backup ` + + `(.+)${MR_CMEK_BACKUP}.` + ) + ); + }); + }); });