From 012df48e01d5343ab07d8bc422cffb097f4926d2 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 22 Mar 2023 12:48:46 +0000 Subject: [PATCH 01/44] addition: connection name --- packages/@aws-cdk/aws-glue/lib/table.ts | 15 ++++++++ packages/@aws-cdk/aws-glue/test/table.test.ts | 35 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 83efe3f86f772..773355e62b9c1 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -5,6 +5,7 @@ import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from '@aws-cdk import * as cr from '@aws-cdk/custom-resources'; import { AwsCustomResource } from '@aws-cdk/custom-resources'; import { Construct } from 'constructs'; +import { IConnection } from './connection'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; import { CfnTable } from './glue.generated'; @@ -183,6 +184,13 @@ export interface TableProps { * @default - The parameter is not defined */ readonly enablePartitionFiltering?: boolean; + + /** + * The connection the table will use when performing reads and writes. + * + * @default - No connection + */ + readonly connection?: IConnection; } /** @@ -275,6 +283,11 @@ export class Table extends Resource implements ITable { */ public readonly partitionIndexes?: PartitionIndex[]; + /** + * The connection this table is associated with. + */ + public readonly connection?: IConnection; + /** * Partition indexes must be created one at a time. To avoid * race conditions, we store the resource and add dependencies @@ -303,6 +316,7 @@ export class Table extends Resource implements ITable { this.bucket = bucket; this.encryption = encryption; this.encryptionKey = encryptionKey; + this.connection = props.connection; const tableResource = new CfnTable(this, 'Table', { catalogId: props.database.catalogId, @@ -319,6 +333,7 @@ export class Table extends Resource implements ITable { 'classification': props.dataFormat.classificationString?.value, 'has_encrypted_data': this.encryption !== TableEncryption.UNENCRYPTED, 'partition_filtering.enabled': props.enablePartitionFiltering, + 'connectionName': props.connection?.connectionName, }, storageDescriptor: { location: `s3://${this.bucket.bucketName}/${this.s3Prefix}`, diff --git a/packages/@aws-cdk/aws-glue/test/table.test.ts b/packages/@aws-cdk/aws-glue/test/table.test.ts index 520c6a8528f49..c666546f7f85a 100644 --- a/packages/@aws-cdk/aws-glue/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue/test/table.test.ts @@ -1,4 +1,4 @@ -import { Template, Match } from '@aws-cdk/assertions'; +import { Match, Template } from '@aws-cdk/assertions'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; @@ -1600,6 +1600,39 @@ test('can specify a physical name', () => { }); }); +test('can associate a connection with the glue table', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + new glue.Table(stack, 'Table', { + database, + tableName: 'my_table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + connection, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + Parameters: { + connectionName: { Ref: 'Connection89AD5CF5' }, + }, + }, + }); +}); + function createTable(props: Pick>): void { const stack = new cdk.Stack(); new glue.Table(stack, 'table', { From cfed556cacb05206eddabfff653e0550d44935ab Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 22 Mar 2023 17:40:44 +0000 Subject: [PATCH 02/44] addition: custom location of data --- packages/@aws-cdk/aws-glue/lib/table.ts | 48 +++++--- .../aws-cdk-glue.assets.json | 2 +- .../integ.partition-index.js.snapshot/cdk.out | 2 +- .../integ.json | 2 +- .../manifest.json | 2 +- .../tree.json | 2 +- .../aws-cdk-glue.assets.json | 6 +- .../aws-cdk-glue.template.json | 53 +++++++++ .../test/integ.table.js.snapshot/cdk.out | 2 +- .../test/integ.table.js.snapshot/integ.json | 2 +- .../integ.table.js.snapshot/manifest.json | 22 ++-- .../test/integ.table.js.snapshot/tree.json | 109 ++++++++++++++++-- .../@aws-cdk/aws-glue/test/integ.table.ts | 8 ++ packages/@aws-cdk/aws-glue/test/table.test.ts | 43 +++++-- 14 files changed, 252 insertions(+), 51 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 773355e62b9c1..0f8cf6333ebf2 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -191,6 +191,15 @@ export interface TableProps { * @default - No connection */ readonly connection?: IConnection; + + /** + * The data source location of the glue table, (e.g. `default_db.public.example` for Redshift). + * + * If this property is set, it will override both `bucket` and `s3Prefix`. + * + * @default - No outsourced data source location + */ + readonly externalDataLocation?: string; } /** @@ -236,7 +245,7 @@ export class Table extends Resource implements ITable { /** * The type of encryption enabled for the table. */ - public readonly encryption: TableEncryption; + public readonly encryption?: TableEncryption; /** * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`. @@ -246,12 +255,12 @@ export class Table extends Resource implements ITable { /** * S3 bucket in which the table's data resides. */ - public readonly bucket: s3.IBucket; + public readonly bucket?: s3.IBucket; /** * S3 Key Prefix under which this table's files are stored in S3. */ - public readonly s3Prefix: string; + public readonly s3Prefix?: string; /** * Name of this table. @@ -288,6 +297,11 @@ export class Table extends Resource implements ITable { */ public readonly connection?: IConnection; + /** + * The location of the tables' data. + */ + readonly location?: string; + /** * Partition indexes must be created one at a time. To avoid * race conditions, we store the resource and add dependencies @@ -305,18 +319,23 @@ export class Table extends Resource implements ITable { this.database = props.database; this.dataFormat = props.dataFormat; - this.s3Prefix = props.s3Prefix ?? ''; + this.connection = props.connection; validateSchema(props.columns, props.partitionKeys); this.columns = props.columns; this.partitionKeys = props.partitionKeys; this.compressed = props.compressed ?? false; - const { bucket, encryption, encryptionKey } = createBucket(this, props); - this.bucket = bucket; - this.encryption = encryption; - this.encryptionKey = encryptionKey; - this.connection = props.connection; + if (props.externalDataLocation) { + this.location = props.externalDataLocation; + } else { + this.s3Prefix = props.s3Prefix ?? ''; + const { bucket, encryption, encryptionKey } = createBucket(this, props); + this.bucket = bucket; + this.encryption = encryption; + this.encryptionKey = encryptionKey; + this.location = `s3://${bucket.bucketName}/${this.s3Prefix}`; + } const tableResource = new CfnTable(this, 'Table', { catalogId: props.database.catalogId, @@ -336,7 +355,7 @@ export class Table extends Resource implements ITable { 'connectionName': props.connection?.connectionName, }, storageDescriptor: { - location: `s3://${this.bucket.bucketName}/${this.s3Prefix}`, + location: this.location, compressed: this.compressed, storedAsSubDirectories: props.storedAsSubDirectories ?? false, columns: renderColumns(props.columns), @@ -438,7 +457,8 @@ export class Table extends Resource implements ITable { * * @param grantee the principal */ - public grantRead(grantee: iam.IGrantable): iam.Grant { + public grantRead(grantee: iam.IGrantable): iam.Grant | undefined { + if (!this.bucket) return; const ret = this.grant(grantee, readPermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); @@ -450,7 +470,8 @@ export class Table extends Resource implements ITable { * * @param grantee the principal */ - public grantWrite(grantee: iam.IGrantable): iam.Grant { + public grantWrite(grantee: iam.IGrantable): iam.Grant | undefined { + if (!this.bucket) return; const ret = this.grant(grantee, writePermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); @@ -462,7 +483,8 @@ export class Table extends Resource implements ITable { * * @param grantee the principal */ - public grantReadWrite(grantee: iam.IGrantable): iam.Grant { + public grantReadWrite(grantee: iam.IGrantable): iam.Grant | undefined { + if (!this.bucket) return; const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); diff --git a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/aws-cdk-glue.assets.json index c735542618561..512e8a543dd1b 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/aws-cdk-glue.assets.json @@ -1,5 +1,5 @@ { - "version": "22.0.0", + "version": "31.0.0", "files": { "a268caa53756f51bda8ad5f499be4ed8484a81b314811806fbb66f874837c476": { "source": { diff --git a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/cdk.out index 145739f539580..7925065efbcc4 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"22.0.0"} \ No newline at end of file +{"version":"31.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/integ.json index 270eec2a064d6..91108d72131de 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "22.0.0", + "version": "31.0.0", "testCases": { "integ.partition-index": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/manifest.json index 81098186327a7..e55b3084ded1a 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "22.0.0", + "version": "31.0.0", "artifacts": { "aws-cdk-glue.assets": { "type": "cdk:asset-manifest", diff --git a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/tree.json index b740b1f893927..56cbfc51f3d3f 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.partition-index.js.snapshot/tree.json @@ -743,7 +743,7 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.189" + "version": "10.1.264" } } }, diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 5381bea5e40ca..9dd350e7c830e 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { - "version": "20.0.0", + "version": "31.0.0", "files": { - "eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f": { + "8b5896a268ed7744b472a9268f050bba5aad447cbebc753a0de4bd9533f2d1c3": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f.json", + "objectKey": "8b5896a268ed7744b472a9268f050bba5aad447cbebc753a0de4bd9533f2d1c3.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json index 92dfc8e76e2d5..de91c5e153f56 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -493,6 +493,59 @@ } } }, + "MyTableWithCustomLocationTable43A19D42": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "custom_location_table generated by CDK", + "Name": "custom_location_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": "default_db.public.test", + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, "MyUserDC45028B": { "Type": "AWS::IAM::User" }, diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out index 588d7b269d34f..7925065efbcc4 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"20.0.0"} \ No newline at end of file +{"version":"31.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json index 1f604630bc610..46daab575ad89 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "20.0.0", + "version": "31.0.0", "testCases": { "integ.table": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json index 931c2276f10a3..5ea29e941a9bb 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json @@ -1,12 +1,6 @@ { - "version": "20.0.0", + "version": "31.0.0", "artifacts": { - "Tree": { - "type": "cdk:tree", - "properties": { - "file": "tree.json" - } - }, "aws-cdk-glue.assets": { "type": "cdk:asset-manifest", "properties": { @@ -23,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/8b5896a268ed7744b472a9268f050bba5aad447cbebc753a0de4bd9533f2d1c3.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -105,6 +99,12 @@ "data": "MyPartitionFilteredTable324BA27A" } ], + "/aws-cdk-glue/MyTableWithCustomLocation/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithCustomLocationTable43A19D42" + } + ], "/aws-cdk-glue/MyUser/Resource": [ { "type": "aws:cdk:logicalId", @@ -143,6 +143,12 @@ ] }, "displayName": "aws-cdk-glue" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json index 441cf0ac8901f..e2bacd1f45171 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json @@ -4,14 +4,6 @@ "id": "App", "path": "", "children": { - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.1.85" - } - }, "aws-cdk-glue": { "id": "aws-cdk-glue", "path": "aws-cdk-glue", @@ -701,6 +693,77 @@ "version": "0.0.0" } }, + "MyTableWithCustomLocation": { + "id": "MyTableWithCustomLocation", + "path": "aws-cdk-glue/MyTableWithCustomLocation", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyTableWithCustomLocation/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "custom_location_table", + "description": "custom_location_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "storageDescriptor": { + "location": "default_db.public.test", + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Table", + "version": "0.0.0" + } + }, "MyUser": { "id": "MyUser", "path": "aws-cdk-glue/MyUser", @@ -1095,17 +1158,41 @@ "fqn": "@aws-cdk/aws-iam.User", "version": "0.0.0" } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue/BootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue/CheckBootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnRule", + "version": "0.0.0" + } } }, + "constructInfo": { + "fqn": "@aws-cdk/core.Stack", + "version": "0.0.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.85" + "version": "10.1.264" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.1.85" + "fqn": "@aws-cdk/core.App", + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index d9d543a124d16..8ccb69340f344 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -95,6 +95,14 @@ new glue.Table(stack, 'MyPartitionFilteredTable', { enablePartitionFiltering: true, }); +new glue.Table(stack, 'MyTableWithCustomLocation', { + database, + tableName: 'custom_location_table', + columns, + dataFormat: glue.DataFormat.JSON, + externalDataLocation: 'default_db.public.test', +}); + const user = new iam.User(stack, 'MyUser'); csvTable.grantReadWrite(user); encryptedTable.grantReadWrite(user); diff --git a/packages/@aws-cdk/aws-glue/test/table.test.ts b/packages/@aws-cdk/aws-glue/test/table.test.ts index c666546f7f85a..4403e5655b968 100644 --- a/packages/@aws-cdk/aws-glue/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue/test/table.test.ts @@ -95,7 +95,8 @@ test('partitioned JSON table', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.UNENCRYPTED); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket.encryptionKey).toEqual(undefined); + expect(table.bucket).not.toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { @@ -163,7 +164,7 @@ test('compressed table', () => { dataFormat: glue.DataFormat.JSON, }); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { @@ -246,7 +247,7 @@ test('encrypted table: SSE-S3', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { @@ -320,7 +321,7 @@ test('encrypted table: SSE-KMS (implicitly created key)', () => { dataFormat: glue.DataFormat.JSON, }); expect(table.encryption).toEqual(glue.TableEncryption.KMS); - expect(table.encryptionKey).toEqual(table.bucket.encryptionKey); + expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { Description: 'Created by Default/Table/Bucket', @@ -408,7 +409,7 @@ test('encrypted table: SSE-KMS (explicitly created key)', () => { dataFormat: glue.DataFormat.JSON, }); expect(table.encryption).toEqual(glue.TableEncryption.KMS); - expect(table.encryptionKey).toEqual(table.bucket.encryptionKey); + expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey); expect(table.encryptionKey).not.toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { @@ -494,7 +495,7 @@ test('encrypted table: SSE-KMS_MANAGED', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.KMS_MANAGED); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::S3::Bucket', { BucketEncryption: { @@ -569,7 +570,7 @@ test('encrypted table: CSE-KMS (implicitly created key)', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); expect(table.encryptionKey).not.toEqual(undefined); - expect(table.bucket.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(stack).resourceCountIs('AWS::KMS::Key', 1); @@ -638,7 +639,7 @@ test('encrypted table: CSE-KMS (explicitly created key)', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); expect(table.encryptionKey).not.toEqual(undefined); - expect(table.bucket.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { Description: 'MyKey', @@ -711,7 +712,7 @@ test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); expect(table.encryptionKey).not.toEqual(undefined); - expect(table.bucket.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { Description: 'MyKey', @@ -1633,6 +1634,30 @@ test('can associate a connection with the glue table', () => { }); }); +test('can associate an external location with the glue table', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + new glue.Table(stack, 'Table', { + database, + tableName: 'my_table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + externalDataLocation: 'default_db.public.test', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + StorageDescriptor: { + Location: 'default_db.public.test', + }, + }, + }); +}); + function createTable(props: Pick>): void { const stack = new cdk.Stack(); new glue.Table(stack, 'table', { From 13dcaa810cbc242a7da9a80afe166dcc87cfcb4c Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 22 Mar 2023 18:17:19 +0000 Subject: [PATCH 03/44] adding connection integ --- .../aws-cdk-glue.assets.json | 4 +- .../aws-cdk-glue.template.json | 99 +++++++++++-- .../integ.table.js.snapshot/manifest.json | 25 +++- .../test/integ.table.js.snapshot/tree.json | 138 +++++++++++++++--- .../@aws-cdk/aws-glue/test/integ.table.ts | 29 +++- 5 files changed, 256 insertions(+), 39 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 9dd350e7c830e..45f4b31336101 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "31.0.0", "files": { - "8b5896a268ed7744b472a9268f050bba5aad447cbebc753a0de4bd9533f2d1c3": { + "5e918941c8c1948aee7eef54957c6e0f9e2b2f8b36e6378143ee00f89977fbe2": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "8b5896a268ed7744b472a9268f050bba5aad447cbebc753a0de4bd9533f2d1c3.json", + "objectKey": "5e918941c8c1948aee7eef54957c6e0f9e2b2f8b36e6378143ee00f89977fbe2.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json index de91c5e153f56..9d68a1ffca382 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -2,8 +2,25 @@ "Resources": { "DataBucketE3889A50": { "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "MyConnection5621880D": { + "Type": "AWS::Glue::Connection", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "ConnectionInput": { + "ConnectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "ConnectionType": "JDBC", + "Name": "my_connection" + } + } }, "MyDatabase1E2517DB": { "Type": "AWS::Glue::Database", @@ -328,8 +345,8 @@ "Version": "2012-10-17" } }, - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" }, "MyEncryptedTableBucket7B28486D": { "Type": "AWS::S3::Bucket", @@ -423,11 +440,6 @@ } } }, - "MyPartitionFilteredTableBucket6ACAA137": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" - }, "MyPartitionFilteredTable324BA27A": { "Type": "AWS::Glue::Table", "Properties": { @@ -477,7 +489,74 @@ [ "s3://", { - "Ref": "MyPartitionFilteredTableBucket6ACAA137" + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyTableWithConnectionTable4BCA8495": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "connection_table generated by CDK", + "Name": "connection_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": false, + "connectionName": { + "Ref": "MyConnection5621880D" + } + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" }, "/" ] diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json index 5ea29e941a9bb..99d96cb314815 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/8b5896a268ed7744b472a9268f050bba5aad447cbebc753a0de4bd9533f2d1c3.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5e918941c8c1948aee7eef54957c6e0f9e2b2f8b36e6378143ee00f89977fbe2.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -39,6 +39,12 @@ "data": "DataBucketE3889A50" } ], + "/aws-cdk-glue/MyConnection/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyConnection5621880D" + } + ], "/aws-cdk-glue/MyDatabase/Resource": [ { "type": "aws:cdk:logicalId", @@ -87,16 +93,16 @@ "data": "MyEncryptedTable981A88C6" } ], - "/aws-cdk-glue/MyPartitionFilteredTable/Bucket/Resource": [ + "/aws-cdk-glue/MyPartitionFilteredTable/Table": [ { "type": "aws:cdk:logicalId", - "data": "MyPartitionFilteredTableBucket6ACAA137" + "data": "MyPartitionFilteredTable324BA27A" } ], - "/aws-cdk-glue/MyPartitionFilteredTable/Table": [ + "/aws-cdk-glue/MyTableWithConnection/Table": [ { "type": "aws:cdk:logicalId", - "data": "MyPartitionFilteredTable324BA27A" + "data": "MyTableWithConnectionTable4BCA8495" } ], "/aws-cdk-glue/MyTableWithCustomLocation/Table": [ @@ -140,6 +146,15 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } + ], + "MyPartitionFilteredTableBucket6ACAA137": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPartitionFilteredTableBucket6ACAA137", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } ] }, "displayName": "aws-cdk-glue" diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json index e2bacd1f45171..a1729baa5068a 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json @@ -30,6 +30,41 @@ "version": "0.0.0" } }, + "MyConnection": { + "id": "MyConnection", + "path": "aws-cdk-glue/MyConnection", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue/MyConnection/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Connection", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "connectionInput": { + "connectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "connectionType": "JDBC", + "name": "my_connection" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnConnection", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Connection", + "version": "0.0.0" + } + }, "MyDatabase": { "id": "MyDatabase", "path": "aws-cdk-glue/MyDatabase", @@ -592,31 +627,92 @@ "id": "MyPartitionFilteredTable", "path": "aws-cdk-glue/MyPartitionFilteredTable", "children": { - "Bucket": { - "id": "Bucket", - "path": "aws-cdk-glue/MyPartitionFilteredTable/Bucket", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-cdk-glue/MyPartitionFilteredTable/Bucket/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::S3::Bucket", - "aws:cdk:cloudformation:props": {} + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyPartitionFilteredTable/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" }, - "constructInfo": { - "fqn": "@aws-cdk/aws-s3.CfnBucket", - "version": "0.0.0" + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "partition_filtered_table", + "description": "partition_filtered_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": false, + "partition_filtering.enabled": true + }, + "storageDescriptor": { + "location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.Bucket", + "fqn": "@aws-cdk/aws-glue.CfnTable", "version": "0.0.0" } - }, + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Table", + "version": "0.0.0" + } + }, + "MyTableWithConnection": { + "id": "MyTableWithConnection", + "path": "aws-cdk-glue/MyTableWithConnection", + "children": { "Table": { "id": "Table", - "path": "aws-cdk-glue/MyPartitionFilteredTable/Table", + "path": "aws-cdk-glue/MyTableWithConnection/Table", "attributes": { "aws:cdk:cloudformation:type": "AWS::Glue::Table", "aws:cdk:cloudformation:props": { @@ -627,12 +723,14 @@ "Ref": "MyDatabase1E2517DB" }, "tableInput": { - "name": "partition_filtered_table", - "description": "partition_filtered_table generated by CDK", + "name": "connection_table", + "description": "connection_table generated by CDK", "parameters": { "classification": "json", "has_encrypted_data": false, - "partition_filtering.enabled": true + "connectionName": { + "Ref": "MyConnection5621880D" + } }, "storageDescriptor": { "location": { @@ -641,7 +739,7 @@ [ "s3://", { - "Ref": "MyPartitionFilteredTableBucket6ACAA137" + "Ref": "DataBucketE3889A50" }, "/" ] diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index 8ccb69340f344..38488d22a293b 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -9,7 +9,19 @@ const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-cdk-glue'); -const bucket = new s3.Bucket(stack, 'DataBucket'); +const bucket = new s3.Bucket(stack, 'DataBucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, +}); + +const connection = new glue.Connection(stack, 'MyConnection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:mysql://mysql.example.com:3306', + USERNAME: 'username', + PASSWORD: 'password', + }, +}); const database = new glue.Database(stack, 'MyDatabase', { databaseName: 'my_database', @@ -84,19 +96,32 @@ const encryptedTable = new glue.Table(stack, 'MyEncryptedTable', { partitionKeys, dataFormat: glue.DataFormat.JSON, encryption: glue.TableEncryption.KMS, - encryptionKey: new kms.Key(stack, 'MyKey'), + encryptionKey: new kms.Key(stack, 'MyKey', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + }), }); new glue.Table(stack, 'MyPartitionFilteredTable', { database, + bucket, tableName: 'partition_filtered_table', columns, dataFormat: glue.DataFormat.JSON, enablePartitionFiltering: true, }); +new glue.Table(stack, 'MyTableWithConnection', { + database, + bucket, + tableName: 'connection_table', + columns, + dataFormat: glue.DataFormat.JSON, + connection, +}); + new glue.Table(stack, 'MyTableWithCustomLocation', { database, + bucket, tableName: 'custom_location_table', columns, dataFormat: glue.DataFormat.JSON, From 5be4ffca69c940f0544e06f033c44f06cbaf8079 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 23 Mar 2023 11:38:20 +0000 Subject: [PATCH 04/44] addition: README --- packages/@aws-cdk/aws-glue/README.md | 37 +++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 83cb4a1c82434..d4e970aeec1e6 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -225,7 +225,23 @@ new glue.Table(this, 'MyTable', { }); ``` -By default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`: +However, if your data will not reside in a S3 bucket, you can use the `externalDataLocation` property to point to the data location: + +```ts +declare const myDatabase: glue.Database; +new glue.Table(this, 'MyTable', { + externalDataLocation: 'default_db_public_example', // A table in Redshift + // ... + database: myDatabase, + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, +}); +``` + +Using `externalDataLocation` will override the `bucket` and `s3Prefix` properties. ### Partition Keys @@ -418,6 +434,25 @@ new glue.Table(this, 'MyTable', { *Note: you cannot provide a `Bucket` when creating the `Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*. +### Glue Connections + +Glue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog. The `connection` property of the `Table` class allows you to specify a Glue Connection to be associated with the table. + +```ts +declare const myConnection: glue.Connection; +declare const myDatabase: glue.Database; +new glue.Table(this, 'MyTable', { + connection: myConnection, + // ... + database: myDatabase, + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, +}); +``` + ## Types A table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types: From 6581bf4d29499b731a7706e9a80f2920ea8e4ca5 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 14 Apr 2023 14:19:58 +0000 Subject: [PATCH 05/44] integ --- .../aws-cdk-glue.assets.json | 6 +- .../aws-cdk-glue.template.json | 1864 +++++++++-------- .../integ.table.js.snapshot/manifest.json | 19 +- .../test/integ.table.js.snapshot/tree.json | 102 +- 4 files changed, 1015 insertions(+), 976 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 3ba20d1f6f2e9..68f4bdec2bed8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "31.0.0", "files": { - "5e918941c8c1948aee7eef54957c6e0f9e2b2f8b36e6378143ee00f89977fbe2": { + "d430893c9d9551ffbfaf5ef745ae853ce6c50ff4aae68844cceb265e18bf69b7": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,11 +9,11 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "5e918941c8c1948aee7eef54957c6e0f9e2b2f8b36e6378143ee00f89977fbe2.json", + "objectKey": "d430893c9d9551ffbfaf5ef745ae853ce6c50ff4aae68844cceb265e18bf69b7.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } } }, "dockerImages": {} -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index 9b7df3066d3e0..e4ec8aa14f084 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -1,954 +1,984 @@ { - "Resources": { - "DataBucketE3889A50": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" + "Resources": { + "DataBucketE3889A50": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "MyConnection5621880D": { + "Type": "AWS::Glue::Connection", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyConnection5621880D": { - "Type": "AWS::Glue::Connection", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "ConnectionInput": { - "ConnectionProperties": { - "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", - "USERNAME": "username", - "PASSWORD": "password" - }, - "ConnectionType": "JDBC", - "Name": "my_connection" - } - } + "ConnectionInput": { + "ConnectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "ConnectionType": "JDBC", + "Name": "my_connection" + } + } + }, + "MyDatabase1E2517DB": { + "Type": "AWS::Glue::Database", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyDatabase1E2517DB": { - "Type": "AWS::Glue::Database", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseInput": { - "Name": "my_database" - } - } + "DatabaseInput": { + "Name": "my_database" + } + } + }, + "AVROTable58646ABF": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "AVROTable58646ABF": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "avro_table generated by CDK", - "Name": "avro_table", - "Parameters": { - "classification": "avro", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.apache.hadoop.hive.serde2.avro.AvroSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "CSVTableE499CABA": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "csv_table generated by CDK", - "Name": "csv_table", - "Parameters": { - "classification": "csv", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.apache.hadoop.hive.serde2.OpenCSVSerde" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "TableInput": { + "Description": "avro_table generated by CDK", + "Name": "avro_table", + "Parameters": { + "classification": "avro", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.serde2.avro.AvroSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "CSVTableE499CABA": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "JSONTable00348F1D": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "json_table generated by CDK", - "Name": "json_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "ParquetTableE84E985F": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "parquet_table generated by CDK", - "Name": "parquet_table", - "Parameters": { - "classification": "parquet", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "TableInput": { + "Description": "csv_table generated by CDK", + "Name": "csv_table", + "Parameters": { + "classification": "csv", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" } - }, - "MyKey6AB29FA6": { - "Type": "AWS::KMS::Key", - "Properties": { - "KeyPolicy": { - "Statement": [ - { - "Action": "kms:*", - "Effect": "Allow", - "Principal": { - "AWS": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::", - { - "Ref": "AWS::AccountId" - }, - ":root" - ] - ] - } - }, - "Resource": "*" - } - ], - "Version": "2012-10-17" - } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] }, - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" - }, - "MyEncryptedTableBucket7B28486D": { - "Type": "AWS::S3::Bucket", - "Properties": { - "BucketEncryption": { - "ServerSideEncryptionConfiguration": [ - { - "ServerSideEncryptionByDefault": { - "KMSMasterKeyID": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] - }, - "SSEAlgorithm": "aws:kms" - } - } - ] - } + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.serde2.OpenCSVSerde" }, - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "JSONTable00348F1D": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyEncryptedTable981A88C6": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "my_encrypted_table generated by CDK", - "Name": "my_encrypted_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "MyEncryptedTableBucket7B28486D" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "json_table generated by CDK", + "Name": "json_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "ParquetTableE84E985F": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyPartitionFilteredTable324BA27A": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "partition_filtered_table generated by CDK", - "Name": "partition_filtered_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true, - "partition_filtering.enabled": true - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "parquet_table generated by CDK", + "Name": "parquet_table", + "Parameters": { + "classification": "parquet", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" + } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyKey6AB29FA6": { + "Type": "AWS::KMS::Key", + "Properties": { + "KeyPolicy": { + "Statement": [ + { + "Action": "kms:*", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] } + }, + "Resource": "*" } - }, - "MyTableWithConnectionTable4BCA8495": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" + ], + "Version": "2012-10-17" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "MyEncryptedTableBucket7B28486D": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "ServerSideEncryptionByDefault": { + "KMSMasterKeyID": { + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" + ] }, - "TableInput": { - "Description": "connection_table generated by CDK", - "Name": "connection_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": false, - "connectionName": { - "Ref": "MyConnection5621880D" - } - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "SSEAlgorithm": "aws:kms" + } } + ] + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "MyEncryptedTable981A88C6": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyTableWithCustomLocationTable43A19D42": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "custom_location_table generated by CDK", - "Name": "custom_location_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": "default_db.public.test", - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "my_encrypted_table generated by CDK", + "Name": "my_encrypted_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "MyEncryptedTableBucket7B28486D" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyPartitionFilteredTable324BA27A": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyUserDC45028B": { - "Type": "AWS::IAM::User" + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "MyUserDefaultPolicy7B897426": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "glue:BatchCreatePartition", - "glue:BatchDeletePartition", - "glue:BatchGetPartition", - "glue:CreatePartition", - "glue:DeletePartition", - "glue:GetPartition", - "glue:GetPartitions", - "glue:GetTable", - "glue:GetTableVersion", - "glue:GetTableVersions", - "glue:GetTables", - "glue:UpdatePartition" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "CSVTableE499CABA" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "MyEncryptedTable981A88C6" - } - ] - ] - } - ] - }, - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - { - "Fn::GetAtt": ["MyEncryptedTableBucket7B28486D", "Arn"] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": ["MyEncryptedTableBucket7B28486D", "Arn"] - }, - "/*" - ] - ] - } - ] - }, - { - "Action": [ - "kms:Decrypt", - "kms:DescribeKey", - "kms:Encrypt", - "kms:GenerateDataKey*", - "kms:ReEncrypt*" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] - } - } - ], - "Version": "2012-10-17" - }, - "PolicyName": "MyUserDefaultPolicy7B897426", - "Users": [ - { - "Ref": "MyUserDC45028B" - } + "TableInput": { + "Description": "partition_filtered_table generated by CDK", + "Name": "partition_filtered_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true, + "partition_filtering.enabled": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyTableWithConnectionTable4BCA8495": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "connection_table generated by CDK", + "Name": "connection_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true, + "connectionName": { + "Ref": "MyConnection5621880D" } + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyTableWithCustomLocationTable43A19D42": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "AnotherUser254B09E3": { - "Type": "AWS::IAM::User" + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "AnotherUserDefaultPolicyDBDB9923": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "glue:BatchCreatePartition", - "glue:BatchDeletePartition", - "glue:BatchGetPartition", - "glue:CreatePartition", - "glue:DeletePartition", - "glue:GetPartition", - "glue:GetPartitions", - "glue:GetTable", - "glue:GetTableVersion", - "glue:GetTableVersions", - "glue:GetTables", - "glue:UpdatePartition" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "AVROTable58646ABF" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "JSONTable00348F1D" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "ParquetTableE84E985F" - } - ] - ] - } - ] - }, - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - "/*" - ] - ] - } - ] - } - ], - "Version": "2012-10-17" + "TableInput": { + "Description": "custom_location_table generated by CDK", + "Name": "custom_location_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": "default_db.public.test", + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyUserDC45028B": { + "Type": "AWS::IAM::User" + }, + "MyUserDefaultPolicy7B897426": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "glue:BatchCreatePartition", + "glue:BatchDeletePartition", + "glue:BatchGetPartition", + "glue:CreatePartition", + "glue:DeletePartition", + "glue:GetPartition", + "glue:GetPartitions", + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + "glue:GetTables", + "glue:UpdatePartition" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "CSVTableE499CABA" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "MyEncryptedTable981A88C6" + } + ] + ] + } + ] + }, + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] }, - "PolicyName": "AnotherUserDefaultPolicyDBDB9923", - "Users": [ - { - "Ref": "AnotherUser254B09E3" - } + { + "Fn::GetAtt": [ + "MyEncryptedTableBucket7B28486D", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "MyEncryptedTableBucket7B28486D", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" ] + } } - } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyUserDefaultPolicy7B897426", + "Users": [ + { + "Ref": "MyUserDC45028B" + } + ] + } }, - "Parameters": { - "BootstrapVersion": { - "Type": "AWS::SSM::Parameter::Value", - "Default": "/cdk-bootstrap/hnb659fds/version", - "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" - } + "AnotherUser254B09E3": { + "Type": "AWS::IAM::User" }, - "Rules": { - "CheckBootstrapVersion": { - "Assertions": [ + "AnotherUserDefaultPolicyDBDB9923": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "glue:BatchCreatePartition", + "glue:BatchDeletePartition", + "glue:BatchGetPartition", + "glue:CreatePartition", + "glue:DeletePartition", + "glue:GetPartition", + "glue:GetPartitions", + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + "glue:GetTables", + "glue:UpdatePartition" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "AVROTable58646ABF" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "JSONTable00348F1D" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "ParquetTableE84E985F" + } + ] + ] + } + ] + }, + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ { - "Assert": { - "Fn::Not": [ - { - "Fn::Contains": [ - ["1", "2", "3", "4", "5"], - { - "Ref": "BootstrapVersion" - } - ] - } + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" ] - }, - "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + }, + "/*" + ] + ] } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "AnotherUserDefaultPolicyDBDB9923", + "Users": [ + { + "Ref": "AnotherUser254B09E3" + } + ] + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." } + ] } -} + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index e6f8dc93c6415..6ce9e0fb8fe10 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -17,17 +17,21 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5e918941c8c1948aee7eef54957c6e0f9e2b2f8b36e6378143ee00f89977fbe2.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/d430893c9d9551ffbfaf5ef745ae853ce6c50ff4aae68844cceb265e18bf69b7.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", - "additionalDependencies": ["aws-cdk-glue.assets"], + "additionalDependencies": [ + "aws-cdk-glue.assets" + ], "lookupRole": { "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", "requiresBootstrapStackVersion": 8, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" } }, - "dependencies": ["aws-cdk-glue.assets"], + "dependencies": [ + "aws-cdk-glue.assets" + ], "metadata": { "/aws-cdk-glue/DataBucket/Resource": [ { @@ -142,13 +146,6 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } - ], - "MyPartitionFilteredTableBucket6ACAA137": [ - { - "type": "aws:cdk:logicalId", - "data": "MyPartitionFilteredTableBucket6ACAA137", - "trace": ["!!DESTRUCTIVE_CHANGES: WILL_DESTROY"] - } ] }, "displayName": "aws-cdk-glue" @@ -160,4 +157,4 @@ } } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index ef89ab19492da..36a49864a5514 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -20,13 +20,13 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.CfnBucket", + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.Bucket", + "fqn": "aws-cdk-lib.aws_s3.Bucket", "version": "0.0.0" } }, @@ -55,13 +55,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnConnection", + "fqn": "aws-cdk-lib.aws_glue.CfnConnection", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Connection", + "fqn": "@aws-cdk/aws-glue-alpha.Connection", "version": "0.0.0" } }, @@ -84,13 +84,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnDatabase", + "fqn": "aws-cdk-lib.aws_glue.CfnDatabase", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Database", + "fqn": "@aws-cdk/aws-glue-alpha.Database", "version": "0.0.0" } }, @@ -172,13 +172,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -260,13 +260,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -348,13 +348,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -436,13 +436,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -487,13 +487,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-kms.CfnKey", + "fqn": "aws-cdk-lib.aws_kms.CfnKey", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-kms.Key", + "fqn": "aws-cdk-lib.aws_kms.Key", "version": "0.0.0" } }, @@ -517,7 +517,10 @@ "serverSideEncryptionByDefault": { "sseAlgorithm": "aws:kms", "kmsMasterKeyId": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" + ] } } } @@ -526,13 +529,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.CfnBucket", + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.Bucket", + "fqn": "aws-cdk-lib.aws_s3.Bucket", "version": "0.0.0" } }, @@ -610,13 +613,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -693,13 +696,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -724,7 +727,7 @@ "description": "connection_table generated by CDK", "parameters": { "classification": "json", - "has_encrypted_data": false, + "has_encrypted_data": true, "connectionName": { "Ref": "MyConnection5621880D" } @@ -778,13 +781,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -849,13 +852,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -871,7 +874,7 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnUser", + "fqn": "aws-cdk-lib.aws_iam.CfnUser", "version": "0.0.0" } }, @@ -976,7 +979,10 @@ "Effect": "Allow", "Resource": [ { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] }, { "Fn::GetAtt": [ @@ -1024,7 +1030,10 @@ ], "Effect": "Allow", "Resource": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" + ] } } ], @@ -1039,19 +1048,19 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.Policy", + "fqn": "aws-cdk-lib.aws_iam.Policy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.User", + "fqn": "aws-cdk-lib.aws_iam.User", "version": "0.0.0" } }, @@ -1067,7 +1076,7 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnUser", + "fqn": "aws-cdk-lib.aws_iam.CfnUser", "version": "0.0.0" } }, @@ -1199,7 +1208,10 @@ "Effect": "Allow", "Resource": [ { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] }, { "Fn::Join": [ @@ -1229,19 +1241,19 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.Policy", + "fqn": "aws-cdk-lib.aws_iam.Policy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.User", + "fqn": "aws-cdk-lib.aws_iam.User", "version": "0.0.0" } }, @@ -1249,7 +1261,7 @@ "id": "BootstrapVersion", "path": "aws-cdk-glue/BootstrapVersion", "constructInfo": { - "fqn": "@aws-cdk/core.CfnParameter", + "fqn": "aws-cdk-lib.CfnParameter", "version": "0.0.0" } }, @@ -1257,13 +1269,13 @@ "id": "CheckBootstrapVersion", "path": "aws-cdk-glue/CheckBootstrapVersion", "constructInfo": { - "fqn": "@aws-cdk/core.CfnRule", + "fqn": "aws-cdk-lib.CfnRule", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/core.Stack", + "fqn": "aws-cdk-lib.Stack", "version": "0.0.0" } }, @@ -1272,13 +1284,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.264" + "version": "10.1.270" } } }, "constructInfo": { - "fqn": "@aws-cdk/core.App", + "fqn": "aws-cdk-lib.App", "version": "0.0.0" } } -} +} \ No newline at end of file From 00593b7abe6cd4c39c169be3f33eb6c0b6102ffd Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 4 Jul 2023 10:34:45 +0000 Subject: [PATCH 06/44] updating README to combine 2 examples into one --- packages/@aws-cdk/aws-glue-alpha/README.md | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 6b54de13b47b8..68df4f8aeecbe 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -221,24 +221,6 @@ new glue.Table(this, 'MyTable', { }); ``` -However, if your data will not reside in a S3 bucket, you can use the `externalDataLocation` property to point to the data location: - -```ts -declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { - externalDataLocation: 'default_db_public_example', // A table in Redshift - // ... - database: myDatabase, - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - dataFormat: glue.DataFormat.JSON, -}); -``` - -Using `externalDataLocation` will override the `bucket` and `s3Prefix` properties. - ### Partition Keys To improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window: @@ -431,13 +413,14 @@ new glue.Table(this, 'MyTable', { ### Glue Connections -Glue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog. The `connection` property of the `Table` class allows you to specify a Glue Connection to be associated with the table. +Glue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog. The `connection` property of the `Table` class allows you to specify a Glue Connection to be associated with the table. Use this in combination with the `externalDataLocation` property to specify an internal destination that the connection will point to: ```ts declare const myConnection: glue.Connection; declare const myDatabase: glue.Database; new glue.Table(this, 'MyTable', { connection: myConnection, + externalDataLocation: 'default_db_public_example', // A table in Redshift // ... database: myDatabase, columns: [{ From 431fbca48c94e947384d2caa666abced5f7ca158 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 13 Jul 2023 10:16:25 +0000 Subject: [PATCH 07/44] grant behaviour incorrect --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 25 +++--- packages/aws-cdk-lib/cx-api/FEATURE_FLAGS.md | 80 ------------------- packages/aws-cdk/vendor/noctilucent | 1 + 3 files changed, 15 insertions(+), 91 deletions(-) create mode 160000 packages/aws-cdk/vendor/noctilucent diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index 4264ec17e6987..a6ab19649b7fc 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -1,3 +1,4 @@ +import { CfnTable } from 'aws-cdk-lib/aws-glue'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; @@ -8,7 +9,6 @@ import { Construct } from 'constructs'; import { IConnection } from './connection'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; -import { CfnTable } from 'aws-cdk-lib/aws-glue'; import { Column } from './schema'; /** @@ -191,7 +191,7 @@ export interface TableProps { readonly connection?: IConnection; /** - * The data source location of the glue table, (e.g. `default_db.public.example` for Redshift). + * The data source location of the glue table, (e.g. `default_db_public_example` for Redshift). * * If this property is set, it will override both `bucket` and `s3Prefix`. * @@ -456,10 +456,11 @@ export class Table extends Resource implements ITable { * @param grantee the principal */ public grantRead(grantee: iam.IGrantable): iam.Grant | undefined { - if (!this.bucket) return; const ret = this.grant(grantee, readPermissions); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } - this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); + if (this.bucket) { + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } + this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); + } return ret; } @@ -469,10 +470,11 @@ export class Table extends Resource implements ITable { * @param grantee the principal */ public grantWrite(grantee: iam.IGrantable): iam.Grant | undefined { - if (!this.bucket) return; const ret = this.grant(grantee, writePermissions); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } - this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); + if (this.bucket) { + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } + this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); + } return ret; } @@ -482,10 +484,11 @@ export class Table extends Resource implements ITable { * @param grantee the principal */ public grantReadWrite(grantee: iam.IGrantable): iam.Grant | undefined { - if (!this.bucket) return; const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } - this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); + if (this.bucket) { + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } + this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); + } return ret; } diff --git a/packages/aws-cdk-lib/cx-api/FEATURE_FLAGS.md b/packages/aws-cdk-lib/cx-api/FEATURE_FLAGS.md index 2acd23b556a2f..901042138a6ee 100644 --- a/packages/aws-cdk-lib/cx-api/FEATURE_FLAGS.md +++ b/packages/aws-cdk-lib/cx-api/FEATURE_FLAGS.md @@ -17,10 +17,6 @@ Flags come in three types: | Flag | Summary | Since | Type | | ----- | ----- | ----- | ----- | -| [@aws-cdk/aws-apigateway:requestValidatorUniqueId](#aws-cdkaws-apigatewayrequestvalidatoruniqueid) | Generate a unique id for each RequestValidator added to a method | V2·NEXT | (fix) | -| [@aws-cdk/aws-ec2:restrictDefaultSecurityGroup](#aws-cdkaws-ec2restrictdefaultsecuritygroup) | Restrict access to the VPC default security group | V2·NEXT | (default) | -| [@aws-cdk/aws-kms:aliasNameRef](#aws-cdkaws-kmsaliasnameref) | KMS Alias name and keyArn will have implicit reference to KMS Key | V2·NEXT | (fix) | -| [@aws-cdk/aws-route53-patters:useCertificate](#aws-cdkaws-route53-pattersusecertificate) | Use the official `Certificate` resource instead of `DnsValidatedCertificate` | V2·NEXT | (default) | | [@aws-cdk/core:newStyleStackSynthesis](#aws-cdkcorenewstylestacksynthesis) | Switch to new stack synthesis method which enables CI/CD | 2.0.0 | (fix) | | [@aws-cdk/core:stackRelativeExports](#aws-cdkcorestackrelativeexports) | Name exports based on the construct paths relative to the stack, rather than the global construct path | 2.0.0 | (fix) | | [@aws-cdk/aws-rds:lowercaseDbIdentifier](#aws-cdkaws-rdslowercasedbidentifier) | Force lowercasing of RDS Cluster names in CDK | 2.0.0 | (fix) | @@ -332,82 +328,6 @@ Encryption can also be configured explicitly using the `encrypted` property. **Compatibility with old behavior:** Pass the `encrypted: false` property to the `FileSystem` construct to disable encryption. -### @aws-cdk/aws-apigateway:requestValidatorUniqueId - -*Generate a unique id for each RequestValidator added to a method* (fix) - -This flag allows multiple RequestValidators to be added to a RestApi when -providing the `RequestValidatorOptions` in the `addMethod()` method. - -If the flag is not set then only a single RequestValidator can be added in this way. -Any additional RequestValidators have to be created directly with `new RequestValidator`. - - -| Since | Default | Recommended | -| ----- | ----- | ----- | -| (not in v1) | | | -| V2·NEXT | `false` | `true` | - - -### @aws-cdk/aws-ec2:restrictDefaultSecurityGroup - -*Restrict access to the VPC default security group* (default) - -Enable this feature flag to remove the default ingress/egress rules from the -VPC default security group. - -When a VPC is created, a default security group is created as well and this cannot -be deleted. The default security group is created with ingress/egress rules that allow -_all_ traffic. [AWS Security best practices recommend](https://docs.aws.amazon.com/securityhub/latest/userguide/ec2-controls.html#ec2-2) -removing these ingress/egress rules in order to restrict access to the default security group. - - -| Since | Default | Recommended | -| ----- | ----- | ----- | -| (not in v1) | | | -| V2·NEXT | `false` | `true` | - -**Compatibility with old behavior:** - To allow all ingress/egress traffic to the VPC default security group you - can set the `restrictDefaultSecurityGroup: false`. - - - -### @aws-cdk/aws-kms:aliasNameRef - -*KMS Alias name and keyArn will have implicit reference to KMS Key* (fix) - -This flag allows an implicit dependency to be created between KMS Alias and KMS Key -when referencing key.aliasName or key.keyArn. - -If the flag is not set then a raw string is passed as the Alias name and no -implicit dependencies will be set. - - -| Since | Default | Recommended | -| ----- | ----- | ----- | -| (not in v1) | | | -| V2·NEXT | `false` | `true` | - - -### @aws-cdk/aws-route53-patters:useCertificate - -*Use the official `Certificate` resource instead of `DnsValidatedCertificate`* (default) - -Enable this feature flag to use the official CloudFormation supported `Certificate` resource instead -of the deprecated `DnsValidatedCertificate` construct. If this flag is enabled and you are creating -the stack in a region other than us-east-1 then you must also set `crossRegionReferences=true` on the -stack. - - -| Since | Default | Recommended | -| ----- | ----- | ----- | -| (not in v1) | | | -| V2·NEXT | `false` | `true` | - -**Compatibility with old behavior:** Define a `DnsValidatedCertificate` explicitly and pass in the `certificate` property - - ### @aws-cdk/core:newStyleStackSynthesis *Switch to new stack synthesis method which enables CI/CD* (fix) diff --git a/packages/aws-cdk/vendor/noctilucent b/packages/aws-cdk/vendor/noctilucent new file mode 160000 index 0000000000000..6da7c9fade55f --- /dev/null +++ b/packages/aws-cdk/vendor/noctilucent @@ -0,0 +1 @@ +Subproject commit 6da7c9fade55f8443bba7b8fdfcd4ebfe5208fb1 From 3af090707c95d61c2a3cb55791f17537755f31c8 Mon Sep 17 00:00:00 2001 From: rizbir khan Date: Thu, 13 Jul 2023 11:45:11 +0100 Subject: [PATCH 08/44] adding package by accident --- packages/aws-cdk/vendor/noctilucent | 1 - 1 file changed, 1 deletion(-) delete mode 160000 packages/aws-cdk/vendor/noctilucent diff --git a/packages/aws-cdk/vendor/noctilucent b/packages/aws-cdk/vendor/noctilucent deleted file mode 160000 index 6da7c9fade55f..0000000000000 --- a/packages/aws-cdk/vendor/noctilucent +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6da7c9fade55f8443bba7b8fdfcd4ebfe5208fb1 From bac46a63d11acabdb4e32cd9204f5f3e98db55a6 Mon Sep 17 00:00:00 2001 From: rizbir khan Date: Thu, 13 Jul 2023 18:30:37 +0100 Subject: [PATCH 09/44] removing undefined --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index a6ab19649b7fc..1bc96da4d5801 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -455,7 +455,7 @@ export class Table extends Resource implements ITable { * * @param grantee the principal */ - public grantRead(grantee: iam.IGrantable): iam.Grant | undefined { + public grantRead(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, readPermissions); if (this.bucket) { if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } @@ -469,7 +469,7 @@ export class Table extends Resource implements ITable { * * @param grantee the principal */ - public grantWrite(grantee: iam.IGrantable): iam.Grant | undefined { + public grantWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, writePermissions); if (this.bucket) { if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } @@ -483,7 +483,7 @@ export class Table extends Resource implements ITable { * * @param grantee the principal */ - public grantReadWrite(grantee: iam.IGrantable): iam.Grant | undefined { + public grantReadWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); if (this.bucket) { if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } From 501249d08294eba64349e32f186974f9b55c674a Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 1 Aug 2023 10:28:36 +0000 Subject: [PATCH 10/44] integ test --- .../aws-cdk-glue.assets.json | 4 +- .../aws-cdk-glue.template.json | 137 +++++++++++++ .../integ.table.js.snapshot/manifest.json | 23 ++- .../test/integ.table.js.snapshot/tree.json | 191 ++++++++++++++++++ 4 files changed, 343 insertions(+), 12 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index d714c2d54aaba..0bdc49903d98a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "33.0.0", "files": { - "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754": { + "9b5d36ea7b36547b6402bd05ce79d201a7f7312456ad595aac8c4b86352f053a": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", + "objectKey": "9b5d36ea7b36547b6402bd05ce79d201a7f7312456ad595aac8c4b86352f053a.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index f4415b64a1333..afe72d3d57110 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -5,6 +5,23 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, + "MyConnection5621880D": { + "Type": "AWS::Glue::Connection", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "ConnectionInput": { + "ConnectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "ConnectionType": "JDBC", + "Name": "my_connection" + } + } + }, "MyDatabase1E2517DB": { "Type": "AWS::Glue::Database", "Properties": { @@ -488,6 +505,126 @@ } } }, + "MyTableWithConnectionTable4BCA8495": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "connection_table generated by CDK", + "Name": "connection_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true, + "connectionName": { + "Ref": "MyConnection5621880D" + } + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyTableWithCustomLocationTable43A19D42": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "custom_location_table generated by CDK", + "Name": "custom_location_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": "default_db.public.test", + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, "MyTableWithStorageDescriptorParametersTable1A347345": { "Type": "AWS::Glue::Table", "Properties": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index c4178bd4636d9..89cf755d601d6 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/9b5d36ea7b36547b6402bd05ce79d201a7f7312456ad595aac8c4b86352f053a.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -99,6 +99,18 @@ "data": "MyPartitionFilteredTable324BA27A" } ], + "/aws-cdk-glue/MyTableWithConnection/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithConnectionTable4BCA8495" + } + ], + "/aws-cdk-glue/MyTableWithCustomLocation/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithCustomLocationTable43A19D42" + } + ], "/aws-cdk-glue/MyTableWithStorageDescriptorParameters/Table": [ { "type": "aws:cdk:logicalId", @@ -140,15 +152,6 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } - ], - "MyPartitionFilteredTableBucket6ACAA137": [ - { - "type": "aws:cdk:logicalId", - "data": "MyPartitionFilteredTableBucket6ACAA137", - "trace": [ - "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" - ] - } ] }, "displayName": "aws-cdk-glue" diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index 71c545fff8226..43e2bb0354f86 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -30,6 +30,41 @@ "version": "0.0.0" } }, + "MyConnection": { + "id": "MyConnection", + "path": "aws-cdk-glue/MyConnection", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue/MyConnection/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Connection", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "connectionInput": { + "connectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "connectionType": "JDBC", + "name": "my_connection" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnConnection", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Connection", + "version": "0.0.0" + } + }, "MyDatabase": { "id": "MyDatabase", "path": "aws-cdk-glue/MyDatabase", @@ -671,6 +706,162 @@ "version": "0.0.0" } }, + "MyTableWithConnection": { + "id": "MyTableWithConnection", + "path": "aws-cdk-glue/MyTableWithConnection", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyTableWithConnection/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "connection_table", + "description": "connection_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": true, + "connectionName": { + "Ref": "MyConnection5621880D" + } + }, + "storageDescriptor": { + "location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Table", + "version": "0.0.0" + } + }, + "MyTableWithCustomLocation": { + "id": "MyTableWithCustomLocation", + "path": "aws-cdk-glue/MyTableWithCustomLocation", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyTableWithCustomLocation/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "custom_location_table", + "description": "custom_location_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "storageDescriptor": { + "location": "default_db.public.test", + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Table", + "version": "0.0.0" + } + }, "MyTableWithStorageDescriptorParameters": { "id": "MyTableWithStorageDescriptorParameters", "path": "aws-cdk-glue/MyTableWithStorageDescriptorParameters", From 3a43fe87411587d9c99975802bfd8cd92990efa3 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 11:48:51 +0000 Subject: [PATCH 11/44] deprecating table --- .../aws-cdk-glue.assets.json | 4 +- .../aws-cdk-glue.template.json | 137 ------------- .../integ.table.js.snapshot/manifest.json | 29 +-- .../test/integ.table.js.snapshot/tree.json | 191 ------------------ 4 files changed, 12 insertions(+), 349 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 0bdc49903d98a..d714c2d54aaba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "33.0.0", "files": { - "9b5d36ea7b36547b6402bd05ce79d201a7f7312456ad595aac8c4b86352f053a": { + "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "9b5d36ea7b36547b6402bd05ce79d201a7f7312456ad595aac8c4b86352f053a.json", + "objectKey": "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index afe72d3d57110..f4415b64a1333 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -5,23 +5,6 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "MyConnection5621880D": { - "Type": "AWS::Glue::Connection", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "ConnectionInput": { - "ConnectionProperties": { - "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", - "USERNAME": "username", - "PASSWORD": "password" - }, - "ConnectionType": "JDBC", - "Name": "my_connection" - } - } - }, "MyDatabase1E2517DB": { "Type": "AWS::Glue::Database", "Properties": { @@ -505,126 +488,6 @@ } } }, - "MyTableWithConnectionTable4BCA8495": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "connection_table generated by CDK", - "Name": "connection_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true, - "connectionName": { - "Ref": "MyConnection5621880D" - } - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } - } - }, - "MyTableWithCustomLocationTable43A19D42": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "custom_location_table generated by CDK", - "Name": "custom_location_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": "default_db.public.test", - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } - } - }, "MyTableWithStorageDescriptorParametersTable1A347345": { "Type": "AWS::Glue::Table", "Properties": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index 89cf755d601d6..02bab428fcf54 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/9b5d36ea7b36547b6402bd05ce79d201a7f7312456ad595aac8c4b86352f053a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -39,12 +39,6 @@ "data": "DataBucketE3889A50" } ], - "/aws-cdk-glue/MyConnection/Resource": [ - { - "type": "aws:cdk:logicalId", - "data": "MyConnection5621880D" - } - ], "/aws-cdk-glue/MyDatabase/Resource": [ { "type": "aws:cdk:logicalId", @@ -99,18 +93,6 @@ "data": "MyPartitionFilteredTable324BA27A" } ], - "/aws-cdk-glue/MyTableWithConnection/Table": [ - { - "type": "aws:cdk:logicalId", - "data": "MyTableWithConnectionTable4BCA8495" - } - ], - "/aws-cdk-glue/MyTableWithCustomLocation/Table": [ - { - "type": "aws:cdk:logicalId", - "data": "MyTableWithCustomLocationTable43A19D42" - } - ], "/aws-cdk-glue/MyTableWithStorageDescriptorParameters/Table": [ { "type": "aws:cdk:logicalId", @@ -152,6 +134,15 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } + ], + "MyPartitionFilteredTableBucket6ACAA137": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPartitionFilteredTableBucket6ACAA137", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } ] }, "displayName": "aws-cdk-glue" diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index 43e2bb0354f86..71c545fff8226 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -30,41 +30,6 @@ "version": "0.0.0" } }, - "MyConnection": { - "id": "MyConnection", - "path": "aws-cdk-glue/MyConnection", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-cdk-glue/MyConnection/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Connection", - "aws:cdk:cloudformation:props": { - "catalogId": { - "Ref": "AWS::AccountId" - }, - "connectionInput": { - "connectionProperties": { - "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", - "USERNAME": "username", - "PASSWORD": "password" - }, - "connectionType": "JDBC", - "name": "my_connection" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnConnection", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Connection", - "version": "0.0.0" - } - }, "MyDatabase": { "id": "MyDatabase", "path": "aws-cdk-glue/MyDatabase", @@ -706,162 +671,6 @@ "version": "0.0.0" } }, - "MyTableWithConnection": { - "id": "MyTableWithConnection", - "path": "aws-cdk-glue/MyTableWithConnection", - "children": { - "Table": { - "id": "Table", - "path": "aws-cdk-glue/MyTableWithConnection/Table", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Table", - "aws:cdk:cloudformation:props": { - "catalogId": { - "Ref": "AWS::AccountId" - }, - "databaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "tableInput": { - "name": "connection_table", - "description": "connection_table generated by CDK", - "parameters": { - "classification": "json", - "has_encrypted_data": true, - "connectionName": { - "Ref": "MyConnection5621880D" - } - }, - "storageDescriptor": { - "location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "compressed": false, - "storedAsSubDirectories": false, - "columns": [ - { - "name": "col1", - "type": "string" - }, - { - "name": "col2", - "type": "string", - "comment": "col2 comment" - }, - { - "name": "col3", - "type": "array" - }, - { - "name": "col4", - "type": "map" - }, - { - "name": "col5", - "type": "struct" - } - ], - "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "serdeInfo": { - "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - } - }, - "tableType": "EXTERNAL_TABLE" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnTable", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", - "version": "0.0.0" - } - }, - "MyTableWithCustomLocation": { - "id": "MyTableWithCustomLocation", - "path": "aws-cdk-glue/MyTableWithCustomLocation", - "children": { - "Table": { - "id": "Table", - "path": "aws-cdk-glue/MyTableWithCustomLocation/Table", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Table", - "aws:cdk:cloudformation:props": { - "catalogId": { - "Ref": "AWS::AccountId" - }, - "databaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "tableInput": { - "name": "custom_location_table", - "description": "custom_location_table generated by CDK", - "parameters": { - "classification": "json", - "has_encrypted_data": true - }, - "storageDescriptor": { - "location": "default_db.public.test", - "compressed": false, - "storedAsSubDirectories": false, - "columns": [ - { - "name": "col1", - "type": "string" - }, - { - "name": "col2", - "type": "string", - "comment": "col2 comment" - }, - { - "name": "col3", - "type": "array" - }, - { - "name": "col4", - "type": "map" - }, - { - "name": "col5", - "type": "struct" - } - ], - "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "serdeInfo": { - "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - } - }, - "tableType": "EXTERNAL_TABLE" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnTable", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", - "version": "0.0.0" - } - }, "MyTableWithStorageDescriptorParameters": { "id": "MyTableWithStorageDescriptorParameters", "path": "aws-cdk-glue/MyTableWithStorageDescriptorParameters", From e083cffd5d7d6ebb8b3f17421b295bc453667a71 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 11:54:19 +0000 Subject: [PATCH 12/44] deprecating table --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 37 ++++++------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index 114ddf6ae838a..8d048a0ecd7dc 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -9,6 +9,7 @@ import { Construct } from 'constructs'; import { IConnection } from './connection'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; +import { S3Table } from './s3-table'; import { Column } from './schema'; import { StorageParameter } from './storage-parameter'; @@ -191,15 +192,6 @@ export interface TableProps { */ readonly connection?: IConnection; - /** - * The data source location of the glue table, (e.g. `default_db_public_example` for Redshift). - * - * If this property is set, it will override both `bucket` and `s3Prefix`. - * - * @default - No outsourced data source location - */ - readonly externalDataLocation?: string; - /** * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. * @@ -238,6 +230,8 @@ export interface TableProps { /** * A Glue table. + * + * @deprecate Use {@link S3Table} instead. */ export class Table extends Resource implements ITable { @@ -326,11 +320,6 @@ export class Table extends Resource implements ITable { */ public readonly partitionIndexes?: PartitionIndex[]; - /** - * The connection this table is associated with. - */ - public readonly connection?: IConnection; - /** * The location of the tables' data. */ @@ -358,7 +347,6 @@ export class Table extends Resource implements ITable { this.database = props.database; this.dataFormat = props.dataFormat; - this.connection = props.connection; validateSchema(props.columns, props.partitionKeys); this.columns = props.columns; @@ -366,16 +354,13 @@ export class Table extends Resource implements ITable { this.storageParameters = props.storageParameters; this.compressed = props.compressed ?? false; - if (props.externalDataLocation) { - this.location = props.externalDataLocation; - } else { - this.s3Prefix = props.s3Prefix ?? ''; - const { bucket, encryption, encryptionKey } = createBucket(this, props); - this.bucket = bucket; - this.encryption = encryption; - this.encryptionKey = encryptionKey; - this.location = `s3://${bucket.bucketName}/${this.s3Prefix}`; - } + + this.s3Prefix = props.s3Prefix ?? ''; + const { bucket, encryption, encryptionKey } = createBucket(this, props); + this.bucket = bucket; + this.encryption = encryption; + this.encryptionKey = encryptionKey; + this.location = `s3://${bucket.bucketName}/${this.s3Prefix}`; const tableResource = new CfnTable(this, 'Table', { catalogId: props.database.catalogId, @@ -569,7 +554,7 @@ export class Table extends Resource implements ITable { }); } - private getS3PrefixForGrant() { + protected getS3PrefixForGrant() { return this.s3Prefix + '*'; } } From cc1a180bef01c3fb59a2cf8db3410e39f80e5cff Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 11:55:17 +0000 Subject: [PATCH 13/44] deprecating --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index 8d048a0ecd7dc..45a854c68b2e6 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -231,7 +231,7 @@ export interface TableProps { /** * A Glue table. * - * @deprecate Use {@link S3Table} instead. + * @deprecated Use {@link S3Table} instead. */ export class Table extends Resource implements ITable { From 6aa651d825662c85eee9d1c1e787b7d95b5d88c6 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 11:57:12 +0000 Subject: [PATCH 14/44] deprecating --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index 45a854c68b2e6..932a0cfcefd65 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -6,7 +6,6 @@ import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk- import * as cr from 'aws-cdk-lib/custom-resources'; import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; -import { IConnection } from './connection'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; import { S3Table } from './s3-table'; @@ -185,13 +184,6 @@ export interface TableProps { */ readonly enablePartitionFiltering?: boolean; - /** - * The connection the table will use when performing reads and writes. - * - * @default - No connection - */ - readonly connection?: IConnection; - /** * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. * @@ -377,7 +369,6 @@ export class Table extends Resource implements ITable { 'classification': props.dataFormat.classificationString?.value, 'has_encrypted_data': true, 'partition_filtering.enabled': props.enablePartitionFiltering, - 'connectionName': props.connection?.connectionName, }, storageDescriptor: { location: this.location, From 772649de03158088d0ab8df75aa0da62990d3d85 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 12:56:10 +0000 Subject: [PATCH 15/44] Table -> S3Table --- .../aws-glue-alpha/test/table.test.ts | 84 +++++++------------ 1 file changed, 30 insertions(+), 54 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts index beeda08bd01b8..a5aa30d52da84 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts @@ -5,7 +5,6 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as glue from '../lib'; -import { PartitionIndex } from '../lib'; test('unpartitioned JSON table', () => { const app = new cdk.App(); @@ -13,7 +12,7 @@ test('unpartitioned JSON table', () => { const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - const table = new glue.Table(tableStack, 'Table', { + const table = new glue.S3Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -81,7 +80,7 @@ test('partitioned JSON table', () => { const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - const table = new glue.Table(tableStack, 'Table', { + const table = new glue.S3Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -154,7 +153,7 @@ test('compressed table', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -218,7 +217,7 @@ test('table.node.defaultChild', () => { const database = new glue.Database(stack, 'Database'); // WHEN - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -236,7 +235,7 @@ test('encrypted table: SSE-S3', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -311,7 +310,7 @@ test('encrypted table: SSE-KMS (implicitly created key)', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -398,7 +397,7 @@ test('encrypted table: SSE-KMS (explicitly created key)', () => { description: 'OurKey', }); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -484,7 +483,7 @@ test('encrypted table: SSE-KMS_MANAGED', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -559,7 +558,7 @@ test('encrypted table: CSE-KMS (implicitly created key)', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -627,7 +626,7 @@ test('encrypted table: CSE-KMS (explicitly created key)', () => { description: 'MyKey', }); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -699,7 +698,7 @@ test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { description: 'MyKey', }); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -771,7 +770,7 @@ test('explicit s3 bucket and prefix', () => { const bucket = new s3.Bucket(stack, 'ExplicitBucket'); const database = new glue.Database(dbStack, 'Database'); - new glue.Table(stack, 'Table', { + new glue.S3Table(stack, 'Table', { database, bucket, s3Prefix: 'prefix/', @@ -835,7 +834,7 @@ test('explicit s3 bucket and with empty prefix', () => { const bucket = new s3.Bucket(stack, 'ExplicitBucket'); const database = new glue.Database(dbStack, 'Database'); - new glue.Table(stack, 'Table', { + new glue.S3Table(stack, 'Table', { database, bucket, s3Prefix: '', @@ -897,7 +896,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -916,7 +915,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -939,7 +938,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -962,7 +961,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const indexes: PartitionIndex[] = [{ + const indexes: glue.PartitionIndex[] = [{ indexName: 'ind1', keyNames: ['part'], }, { @@ -976,7 +975,7 @@ describe('add partition index', () => { keyNames: ['part'], }]; - expect(() => new glue.Table(stack, 'Table', { + expect(() => new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -998,7 +997,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1061,7 +1060,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1162,7 +1161,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1265,7 +1264,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.Table(stack, 'Table', { + const table = new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1485,7 +1484,7 @@ describe('validate', () => { const stack = new cdk.Stack(app, 'Stack'); const database = new glue.Database(stack, 'Database'); - expect(() => new glue.Table(stack, 'Table', { + expect(() => new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1523,7 +1522,7 @@ test.each([ const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - new glue.Table(tableStack, 'Table', { + new glue.S3Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -1565,7 +1564,7 @@ test('Partition filtering on table is not defined (default behavior)', () => { const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - new glue.Table(tableStack, 'Table', { + new glue.S3Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -1604,7 +1603,7 @@ test('can specify a physical name', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Stack'); const database = new glue.Database(stack, 'Database'); - new glue.Table(stack, 'Table', { + new glue.S3Table(stack, 'Table', { database, tableName: 'my_table', columns: [{ @@ -1626,7 +1625,7 @@ test('storage descriptor parameters', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Stack'); const database = new glue.Database(stack, 'Database'); - new glue.Table(stack, 'Table', { + new glue.S3Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1655,7 +1654,7 @@ test('storage descriptor parameters', () => { }); }); -test('can associate a connection with the glue table', () => { +test('can associate an external location with the glue table', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Stack'); const database = new glue.Database(stack, 'Database'); @@ -1668,33 +1667,10 @@ test('can associate a connection with the glue table', () => { PASSWORD: 'password', }, }); - new glue.Table(stack, 'Table', { + new glue.ExternalTable(stack, 'Table', { database, tableName: 'my_table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - dataFormat: glue.DataFormat.JSON, connection, - }); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - TableInput: { - Parameters: { - connectionName: { Ref: 'Connection89AD5CF5' }, - }, - }, - }); -}); - -test('can associate an external location with the glue table', () => { - const app = new cdk.App(); - const stack = new cdk.Stack(app, 'Stack'); - const database = new glue.Database(stack, 'Database'); - new glue.Table(stack, 'Table', { - database, - tableName: 'my_table', columns: [{ name: 'col', type: glue.Schema.STRING, @@ -1714,7 +1690,7 @@ test('can associate an external location with the glue table', () => { function createTable(props: Pick>): void { const stack = new cdk.Stack(); - new glue.Table(stack, 'table', { + new glue.S3Table(stack, 'table', { ...props, database: new glue.Database(stack, 'db'), dataFormat: glue.DataFormat.JSON, From 31c0d44a90d06c02b45037f79d3ec1017d897aac Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 12:56:44 +0000 Subject: [PATCH 16/44] Table -> S3Table --- .../aws-glue-alpha/test/integ.table.ts | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index f50e3ec2db0d1..73acc2fb26a32 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -54,7 +54,7 @@ const partitionKeys = [{ type: glue.Schema.SMALL_INT, }]; -const avroTable = new glue.Table(stack, 'AVROTable', { +const avroTable = new glue.S3Table(stack, 'AVROTable', { database, bucket, tableName: 'avro_table', @@ -63,7 +63,7 @@ const avroTable = new glue.Table(stack, 'AVROTable', { dataFormat: glue.DataFormat.AVRO, }); -const csvTable = new glue.Table(stack, 'CSVTable', { +const csvTable = new glue.S3Table(stack, 'CSVTable', { database, bucket, tableName: 'csv_table', @@ -72,7 +72,7 @@ const csvTable = new glue.Table(stack, 'CSVTable', { dataFormat: glue.DataFormat.CSV, }); -const jsonTable = new glue.Table(stack, 'JSONTable', { +const jsonTable = new glue.S3Table(stack, 'JSONTable', { database, bucket, tableName: 'json_table', @@ -81,7 +81,7 @@ const jsonTable = new glue.Table(stack, 'JSONTable', { dataFormat: glue.DataFormat.JSON, }); -const parquetTable = new glue.Table(stack, 'ParquetTable', { +const parquetTable = new glue.S3Table(stack, 'ParquetTable', { database, bucket, tableName: 'parquet_table', @@ -90,7 +90,7 @@ const parquetTable = new glue.Table(stack, 'ParquetTable', { dataFormat: glue.DataFormat.PARQUET, }); -const encryptedTable = new glue.Table(stack, 'MyEncryptedTable', { +const encryptedTable = new glue.S3Table(stack, 'MyEncryptedTable', { database, tableName: 'my_encrypted_table', columns, @@ -102,7 +102,7 @@ const encryptedTable = new glue.Table(stack, 'MyEncryptedTable', { }), }); -new glue.Table(stack, 'MyPartitionFilteredTable', { +new glue.S3Table(stack, 'MyPartitionFilteredTable', { database, bucket, tableName: 'partition_filtered_table', @@ -111,25 +111,23 @@ new glue.Table(stack, 'MyPartitionFilteredTable', { enablePartitionFiltering: true, }); -new glue.Table(stack, 'MyTableWithConnection', { +new glue.S3Table(stack, 'MyTableWithConnection', { database, bucket, tableName: 'connection_table', columns, dataFormat: glue.DataFormat.JSON, - connection, }); -new glue.Table(stack, 'MyTableWithCustomLocation', { +new glue.S3Table(stack, 'MyTableWithCustomLocation', { database, bucket, tableName: 'custom_location_table', columns, dataFormat: glue.DataFormat.JSON, - externalDataLocation: 'default_db.public.test', }); -new glue.Table(stack, 'MyTableWithStorageDescriptorParameters', { +new glue.S3Table(stack, 'MyTableWithStorageDescriptorParameters', { database, bucket, tableName: 'table_with_storage_descriptor_parameters', @@ -144,6 +142,15 @@ new glue.Table(stack, 'MyTableWithStorageDescriptorParameters', { ], }); +new glue.ExternalTable(stack, 'MyTableWithCustomLocation', { + database, + connection, + tableName: 'custom_location_table', + columns, + dataFormat: glue.DataFormat.JSON, + externalDataLocation: 'default_db.public.test', +}); + const user = new iam.User(stack, 'MyUser'); csvTable.grantReadWrite(user); encryptedTable.grantReadWrite(user); From 5a9b46f646153d754dd9e037948e5bbcdcc18080 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 12:57:00 +0000 Subject: [PATCH 17/44] deprecating Table constructu --- .../lib/{table.ts => table-deprecated.ts} | 213 +----------------- 1 file changed, 3 insertions(+), 210 deletions(-) rename packages/@aws-cdk/aws-glue-alpha/lib/{table.ts => table-deprecated.ts} (71%) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts similarity index 71% rename from packages/@aws-cdk/aws-glue-alpha/lib/table.ts rename to packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index 932a0cfcefd65..2aa7830a53821 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -2,7 +2,7 @@ import { CfnTable } from 'aws-cdk-lib/aws-glue'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; -import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-lib/core'; +import { ArnFormat, Fn, Lazy, Names, Resource, Stack } from 'aws-cdk-lib/core'; import * as cr from 'aws-cdk-lib/custom-resources'; import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; @@ -11,214 +11,7 @@ import { IDatabase } from './database'; import { S3Table } from './s3-table'; import { Column } from './schema'; import { StorageParameter } from './storage-parameter'; - -/** - * Properties of a Partition Index. - */ -export interface PartitionIndex { - /** - * The name of the partition index. - * - * @default - a name will be generated for you. - */ - readonly indexName?: string; - - /** - * The partition key names that comprise the partition - * index. The names must correspond to a name in the - * table's partition keys. - */ - readonly keyNames: string[]; -} -export interface ITable extends IResource { - /** - * @attribute - */ - readonly tableArn: string; - - /** - * @attribute - */ - readonly tableName: string; -} - -/** - * Encryption options for a Table. - * - * @see https://docs.aws.amazon.com/athena/latest/ug/encryption.html - */ -export enum TableEncryption { - /** - * Server side encryption (SSE) with an Amazon S3-managed key. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html - */ - S3_MANAGED = 'SSE-S3', - - /** - * Server-side encryption (SSE) with an AWS KMS key managed by the account owner. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html - */ - KMS = 'SSE-KMS', - - /** - * Server-side encryption (SSE) with an AWS KMS key managed by the KMS service. - */ - KMS_MANAGED = 'SSE-KMS-MANAGED', - - /** - * Client-side encryption (CSE) with an AWS KMS key managed by the account owner. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html - */ - CLIENT_SIDE_KMS = 'CSE-KMS' -} - -export interface TableAttributes { - readonly tableArn: string; - readonly tableName: string; -} - -export interface TableProps { - /** - * Name of the table. - * - * @default - generated by CDK. - */ - readonly tableName?: string; - - /** - * Description of the table. - * - * @default generated - */ - readonly description?: string; - - /** - * Database in which to store the table. - */ - readonly database: IDatabase; - - /** - * S3 bucket in which to store data. - * - * @default one is created for you - */ - readonly bucket?: s3.IBucket; - - /** - * S3 prefix under which table objects are stored. - * - * @default - No prefix. The data will be stored under the root of the bucket. - */ - readonly s3Prefix?: string; - - /** - * Columns of the table. - */ - readonly columns: Column[]; - - /** - * Partition columns of the table. - * - * @default table is not partitioned - */ - readonly partitionKeys?: Column[]; - - /** - * Partition indexes on the table. A maximum of 3 indexes - * are allowed on a table. Keys in the index must be part - * of the table's partition keys. - * - * @default table has no partition indexes - */ - readonly partitionIndexes?: PartitionIndex[]; - - /** - * Storage type of the table's data. - */ - readonly dataFormat: DataFormat; - - /** - * Indicates whether the table's data is compressed or not. - * - * @default false - */ - readonly compressed?: boolean; - - /** - * The kind of encryption to secure the data with. - * - * You can only provide this option if you are not explicitly passing in a bucket. - * - * If you choose `SSE-KMS`, you *can* provide an un-managed KMS key with `encryptionKey`. - * If you choose `CSE-KMS`, you *must* provide an un-managed KMS key with `encryptionKey`. - * - * @default BucketEncryption.S3_MANAGED - */ - readonly encryption?: TableEncryption; - - /** - * External KMS key to use for bucket encryption. - * - * The `encryption` property must be `SSE-KMS` or `CSE-KMS`. - * - * @default key is managed by KMS. - */ - readonly encryptionKey?: kms.IKey; - - /** - * Indicates whether the table data is stored in subdirectories. - * - * @default false - */ - readonly storedAsSubDirectories?: boolean; - - /** - * Enables partition filtering. - * - * @see https://docs.aws.amazon.com/athena/latest/ug/glue-best-practices.html#glue-best-practices-partition-index - * - * @default - The parameter is not defined - */ - readonly enablePartitionFiltering?: boolean; - - /** - * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. - * - * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. - * - * Some keys will be auto-populated by glue crawlers, however, you can override them by specifying the key and value in this property. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/table-properties-crawler.html - * - * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ - * - * @example - * - * declare const glueDatabase: glue.IDatabase; - * const table = new glue.Table(this, 'Table', { - * storageParameters: [ - * glue.StorageParameter.skipHeaderLineCount(1), - * glue.StorageParameter.compressionType(glue.CompressionType.GZIP), - * glue.StorageParameter.custom('foo', 'bar'), // Will have no effect - * glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data - * glue.StorageParameter.custom(glue.StorageParameters.WRITE_PARALLEL, 'off'), - * ], - * // ... - * database: glueDatabase, - * columns: [{ - * name: 'col1', - * type: glue.Schema.STRING, - * }], - * dataFormat: glue.DataFormat.CSV, - * }); - * - * @default - The parameter is not defined - */ - readonly storageParameters?: StorageParameter[]; -} +import { ITable, PartitionIndex, TableAttributes, TableBase, TableEncryption, TableProps } from './table-base'; /** * A Glue table. @@ -230,7 +23,7 @@ export class Table extends Resource implements ITable { public static fromTableArn(scope: Construct, id: string, tableArn: string): ITable { const tableName = Fn.select(1, Fn.split('/', Stack.of(scope).splitArn(tableArn, ArnFormat.SLASH_RESOURCE_NAME).resourceName!)); - return Table.fromTableAttributes(scope, id, { + return TableBase.fromTableAttributes(scope, id, { tableArn, tableName, }); From 07cb52d117830b48ffd320024f428cfce57c6dc4 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 14:56:48 +0000 Subject: [PATCH 18/44] using S3 table --- packages/@aws-cdk/aws-glue-alpha/test/table.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts index a5aa30d52da84..3e455d9f163c5 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts @@ -1506,7 +1506,7 @@ test('Table.fromTableArn', () => { const stack = new cdk.Stack(); // WHEN - const table = glue.Table.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + const table = glue.S3Table.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); // THEN expect(table.tableArn).toEqual('arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); @@ -1688,7 +1688,7 @@ test('can associate an external location with the glue table', () => { }); }); -function createTable(props: Pick>): void { +function createTable(props: Pick>): void { const stack = new cdk.Stack(); new glue.S3Table(stack, 'table', { ...props, From d7ea017d73c15392e3a0e4e958759cfa68c4e60f Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 16:07:14 +0000 Subject: [PATCH 19/44] new table --- .../aws-glue-alpha/lib/external-table.ts | 180 ++++++++ packages/@aws-cdk/aws-glue-alpha/lib/index.ts | 5 +- .../@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 252 +++++++++++ .../@aws-cdk/aws-glue-alpha/lib/table-base.ts | 423 ++++++++++++++++++ 4 files changed, 859 insertions(+), 1 deletion(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts new file mode 100644 index 0000000000000..9da5fa01d1472 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts @@ -0,0 +1,180 @@ +import { CfnTable } from 'aws-cdk-lib/aws-glue'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import { Construct } from 'constructs'; +import { IConnection } from './connection'; +import { Column } from './schema'; +import { PartitionIndex, TableBase, TableEncryption, TableProps } from './table-base'; + +export interface ExternalTableProps extends TableProps { + /** + * The connection the table will use when performing reads and writes. + * + * @default - No connection + */ + readonly connection: IConnection; + + /** + * The data source location of the glue table, (e.g. `default_db_public_example` for Redshift). + * + * If this property is set, it will override both `bucket` and `s3Prefix`. + * + * @default - No outsourced data source location + */ + readonly externalDataLocation: string; +} + +/** + * A Glue table that targets an external data location (e.g. A table in a Redshift Cluster). + */ +export class ExternalTable extends TableBase { + /** + * Name of this table. + */ + public readonly tableName: string; + + /** + * ARN of this table. + */ + public readonly tableArn: string; + + /** + * The connection associated to this table + */ + public readonly connection: IConnection; + + /** + * The location of the tables' data. + */ + public readonly location: string; + + /** + * This table's partition indexes. + */ + public readonly partitionIndexes?: PartitionIndex[]; + + protected readonly tableResource: CfnTable; + + constructor(scope: Construct, id: string, props: ExternalTableProps) { + super(scope, id, props); + this.location = props.externalDataLocation; + this.connection = props.connection; + this.tableResource = new CfnTable(this, 'Table', { + catalogId: props.database.catalogId, + + databaseName: props.database.databaseName, + + tableInput: { + name: this.physicalName, + description: props.description || `${this.physicalName} generated by CDK`, + + partitionKeys: renderColumns(props.partitionKeys), + + parameters: { + 'classification': props.dataFormat.classificationString?.value, + 'has_encrypted_data': true, + 'partition_filtering.enabled': props.enablePartitionFiltering, + 'connectionName': props.connection.connectionName, + }, + storageDescriptor: { + location: this.location, + compressed: this.compressed, + storedAsSubDirectories: props.storedAsSubDirectories ?? false, + columns: renderColumns(props.columns), + inputFormat: props.dataFormat.inputFormat.className, + outputFormat: props.dataFormat.outputFormat.className, + serdeInfo: { + serializationLibrary: props.dataFormat.serializationLibrary.className, + }, + parameters: props.storageParameters ? props.storageParameters.reduce((acc, param) => { + if (param.key in acc) { + throw new Error(`Duplicate storage parameter key: ${param.key}`); + } + const key = param.key; + acc[key] = param.value; + return acc; + }, {} as { [key: string]: string }) : undefined, + }, + + tableType: 'EXTERNAL_TABLE', + }, + }); + + this.tableName = this.getResourceNameAttribute(this.tableResource.ref); + this.tableArn = this.stack.formatArn({ + service: 'glue', + resource: 'table', + resourceName: `${this.database.databaseName}/${this.tableName}`, + }); + this.node.defaultChild = this.tableResource; + + // Partition index creation relies on created table. + if (props.partitionIndexes) { + this.partitionIndexes = props.partitionIndexes; + this.partitionIndexes.forEach((index) => this.addPartitionIndex(index)); + } + } + + /** + * Grant read permissions to the table and the underlying data stored in S3 to an IAM principal. + * + * @param grantee the principal + */ + public grantRead(grantee: iam.IGrantable): iam.Grant { + const ret = this.grant(grantee, readPermissions); + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } + return ret; + } + + /** + * Grant write permissions to the table and the underlying data stored in S3 to an IAM principal. + * + * @param grantee the principal + */ + public grantWrite(grantee: iam.IGrantable): iam.Grant { + const ret = this.grant(grantee, writePermissions); + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } + return ret; + } + + /** + * Grant read and write permissions to the table and the underlying data stored in S3 to an IAM principal. + * + * @param grantee the principal + */ + public grantReadWrite(grantee: iam.IGrantable): iam.Grant { + const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } + return ret; + } +} + +const readPermissions = [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', +]; + +const writePermissions = [ + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', +]; + +function renderColumns(columns?: Array) { + if (columns === undefined) { + return undefined; + } + return columns.map(column => { + return { + name: column.name, + type: column.type.inputString, + comment: column.comment, + }; + }); +} diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts index c6a611242c925..1b9514c14625e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts @@ -5,9 +5,12 @@ export * from './connection'; export * from './data-format'; export * from './data-quality-ruleset'; export * from './database'; +export * from './external-table'; export * from './job'; export * from './job-executable'; +export * from './s3-table'; export * from './schema'; export * from './security-configuration'; export * from './storage-parameter'; -export * from './table'; +export * from './table-base'; +export * from './table-deprecated'; diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts new file mode 100644 index 0000000000000..20ac57cb5f218 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -0,0 +1,252 @@ +import { CfnTable } from 'aws-cdk-lib/aws-glue'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import { Construct } from 'constructs'; +import { Column } from './schema'; +import { PartitionIndex, TableBase, TableEncryption, TableProps } from './table-base'; + +export interface S3TableProps extends TableProps { + /** + * S3 bucket in which to store data. + * + * @default one is created for you + */ + readonly bucket?: s3.IBucket; + + /** + * S3 prefix under which table objects are stored. + * + * @default - No prefix. The data will be stored under the root of the bucket. + */ + readonly s3Prefix?: string; +} + +/** + * A Glue table that targets an S3 dataset. + */ +export class S3Table extends TableBase { + /** + * Name of this table. + */ + public readonly tableName: string; + + /** + * ARN of this table. + */ + public readonly tableArn: string; + + /** + * S3 bucket in which the table's data resides. + */ + public readonly bucket: s3.IBucket; + + /** + * S3 Key Prefix under which this table's files are stored in S3. + */ + public readonly s3Prefix: string; + + /** + * The type of encryption enabled for the table. + */ + public readonly encryption?: TableEncryption; + + /** + * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`. + */ + public readonly encryptionKey?: kms.IKey; + + /** + * The location of the tables' data. + */ + public readonly location: string; + + /** + * This table's partition indexes. + */ + public readonly partitionIndexes?: PartitionIndex[]; + + protected readonly tableResource: CfnTable; + + constructor(scope: Construct, id: string, props: S3TableProps) { + super(scope, id, props); + this.s3Prefix = props.s3Prefix ?? ''; + const { bucket, encryption, encryptionKey } = createBucket(this, props); + this.bucket = bucket; + this.encryption = encryption; + this.encryptionKey = encryptionKey; + this.location = `s3://${bucket.bucketName}/${this.s3Prefix}`; + + this.tableResource = new CfnTable(this, 'Table', { + catalogId: props.database.catalogId, + + databaseName: props.database.databaseName, + + tableInput: { + name: this.physicalName, + description: props.description || `${this.physicalName} generated by CDK`, + + partitionKeys: renderColumns(props.partitionKeys), + + parameters: { + 'classification': props.dataFormat.classificationString?.value, + 'has_encrypted_data': true, + 'partition_filtering.enabled': props.enablePartitionFiltering, + }, + storageDescriptor: { + location: this.location, + compressed: this.compressed, + storedAsSubDirectories: props.storedAsSubDirectories ?? false, + columns: renderColumns(props.columns), + inputFormat: props.dataFormat.inputFormat.className, + outputFormat: props.dataFormat.outputFormat.className, + serdeInfo: { + serializationLibrary: props.dataFormat.serializationLibrary.className, + }, + parameters: props.storageParameters ? props.storageParameters.reduce((acc, param) => { + if (param.key in acc) { + throw new Error(`Duplicate storage parameter key: ${param.key}`); + } + const key = param.key; + acc[key] = param.value; + return acc; + }, {} as { [key: string]: string }) : undefined, + }, + + tableType: 'EXTERNAL_TABLE', + }, + }); + + this.tableName = this.getResourceNameAttribute(this.tableResource.ref); + this.tableArn = this.stack.formatArn({ + service: 'glue', + resource: 'table', + resourceName: `${this.database.databaseName}/${this.tableName}`, + }); + this.node.defaultChild = this.tableResource; + + // Partition index creation relies on created table. + if (props.partitionIndexes) { + this.partitionIndexes = props.partitionIndexes; + this.partitionIndexes.forEach((index) => this.addPartitionIndex(index)); + } + } + + /** + * Grant read permissions to the table and the underlying data stored in S3 to an IAM principal. + * + * @param grantee the principal + */ + public grantRead(grantee: iam.IGrantable): iam.Grant { + const ret = this.grant(grantee, readPermissions); + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } + this.bucket.grantRead(grantee, this.generateS3PrefixForGrant()); + return ret; + } + + /** + * Grant write permissions to the table and the underlying data stored in S3 to an IAM principal. + * + * @param grantee the principal + */ + public grantWrite(grantee: iam.IGrantable): iam.Grant { + const ret = this.grant(grantee, writePermissions); + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } + this.bucket.grantWrite(grantee, this.generateS3PrefixForGrant()); + return ret; + } + + /** + * Grant read and write permissions to the table and the underlying data stored in S3 to an IAM principal. + * + * @param grantee the principal + */ + public grantReadWrite(grantee: iam.IGrantable): iam.Grant { + const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } + this.bucket.grantReadWrite(grantee, this.generateS3PrefixForGrant()); + return ret; + } + + private generateS3PrefixForGrant() { + return this.s3Prefix + '*'; + } +} + +const readPermissions = [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', +]; + +const writePermissions = [ + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', +]; + +// map TableEncryption to bucket's SSE configuration (s3.BucketEncryption) +const encryptionMappings = { + [TableEncryption.S3_MANAGED]: s3.BucketEncryption.S3_MANAGED, + [TableEncryption.KMS_MANAGED]: s3.BucketEncryption.KMS_MANAGED, + [TableEncryption.KMS]: s3.BucketEncryption.KMS, + [TableEncryption.CLIENT_SIDE_KMS]: s3.BucketEncryption.S3_MANAGED, +}; + +// create the bucket to store a table's data depending on the `encryption` and `encryptionKey` properties. +function createBucket(table: TableBase, props: S3TableProps) { + let bucket = props.bucket; + + if (bucket && (props.encryption !== undefined && props.encryption !== TableEncryption.CLIENT_SIDE_KMS)) { + throw new Error('you can not specify encryption settings if you also provide a bucket'); + } + + const encryption = props.encryption || TableEncryption.S3_MANAGED; + + let encryptionKey: kms.IKey | undefined; + if (encryption === TableEncryption.CLIENT_SIDE_KMS && props.encryptionKey === undefined) { + // CSE-KMS should behave the same as SSE-KMS - use the provided key or create one automatically + // Since Bucket only knows about SSE, we repeat the logic for CSE-KMS at the Table level. + encryptionKey = new kms.Key(table, 'Key'); + } else { + encryptionKey = props.encryptionKey; + } + + // create the bucket if none was provided + if (!bucket) { + if (encryption === TableEncryption.CLIENT_SIDE_KMS) { + bucket = new s3.Bucket(table, 'Bucket'); + } else { + bucket = new s3.Bucket(table, 'Bucket', { + encryption: encryptionMappings[encryption], + encryptionKey, + }); + encryptionKey = bucket.encryptionKey; + } + } + + return { + bucket, + encryption, + encryptionKey, + }; +} + +function renderColumns(columns?: Array) { + if (columns === undefined) { + return undefined; + } + return columns.map(column => { + return { + name: column.name, + type: column.type.inputString, + comment: column.comment, + }; + }); +} diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts new file mode 100644 index 0000000000000..3be953edf706e --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts @@ -0,0 +1,423 @@ +import { CfnTable } from 'aws-cdk-lib/aws-glue'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-lib/core'; +import * as cr from 'aws-cdk-lib/custom-resources'; +import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; +import { Construct } from 'constructs'; +import { DataFormat } from './data-format'; +import { IDatabase } from './database'; +import { Column } from './schema'; +import { StorageParameter } from './storage-parameter'; + +/** + * Properties of a Partition Index. + */ +export interface PartitionIndex { + /** + * The name of the partition index. + * + * @default - a name will be generated for you. + */ + readonly indexName?: string; + + /** + * The partition key names that comprise the partition + * index. The names must correspond to a name in the + * table's partition keys. + */ + readonly keyNames: string[]; +} +export interface ITable extends IResource { + /** + * @attribute + */ + readonly tableArn: string; + + /** + * @attribute + */ + readonly tableName: string; +} + +/** + * Encryption options for a Table. + * + * @see https://docs.aws.amazon.com/athena/latest/ug/encryption.html + */ +export enum TableEncryption { + /** + * Server side encryption (SSE) with an Amazon S3-managed key. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html + */ + S3_MANAGED = 'SSE-S3', + + /** + * Server-side encryption (SSE) with an AWS KMS key managed by the account owner. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + */ + KMS = 'SSE-KMS', + + /** + * Server-side encryption (SSE) with an AWS KMS key managed by the KMS service. + */ + KMS_MANAGED = 'SSE-KMS-MANAGED', + + /** + * Client-side encryption (CSE) with an AWS KMS key managed by the account owner. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html + */ + CLIENT_SIDE_KMS = 'CSE-KMS' +} + +export interface TableAttributes { + readonly tableArn: string; + readonly tableName: string; +} + +export interface TableProps { + /** + * Name of the table. + * + * @default - generated by CDK. + */ + readonly tableName?: string; + + /** + * Description of the table. + * + * @default generated + */ + readonly description?: string; + + /** + * Database in which to store the table. + */ + readonly database: IDatabase; + + /** + * Columns of the table. + */ + readonly columns: Column[]; + + /** + * Partition columns of the table. + * + * @default table is not partitioned + */ + readonly partitionKeys?: Column[]; + + /** + * Partition indexes on the table. A maximum of 3 indexes + * are allowed on a table. Keys in the index must be part + * of the table's partition keys. + * + * @default table has no partition indexes + */ + readonly partitionIndexes?: PartitionIndex[]; + + /** + * Storage type of the table's data. + */ + readonly dataFormat: DataFormat; + + /** + * Indicates whether the table's data is compressed or not. + * + * @default false + */ + readonly compressed?: boolean; + + /** + * The kind of encryption to secure the data with. + * + * You can only provide this option if you are not explicitly passing in a bucket. + * + * If you choose `SSE-KMS`, you *can* provide an un-managed KMS key with `encryptionKey`. + * If you choose `CSE-KMS`, you *must* provide an un-managed KMS key with `encryptionKey`. + * + * @default BucketEncryption.S3_MANAGED + */ + readonly encryption?: TableEncryption; + + /** + * External KMS key to use for bucket encryption. + * + * The `encryption` property must be `SSE-KMS` or `CSE-KMS`. + * + * @default key is managed by KMS. + */ + readonly encryptionKey?: kms.IKey; + + /** + * Indicates whether the table data is stored in subdirectories. + * + * @default false + */ + readonly storedAsSubDirectories?: boolean; + + /** + * Enables partition filtering. + * + * @see https://docs.aws.amazon.com/athena/latest/ug/glue-best-practices.html#glue-best-practices-partition-index + * + * @default - The parameter is not defined + */ + readonly enablePartitionFiltering?: boolean; + + /** + * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. + * + * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. + * + * Some keys will be auto-populated by glue crawlers, however, you can override them by specifying the key and value in this property. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/table-properties-crawler.html + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ + * + * @example + * + * declare const glueDatabase: glue.IDatabase; + * const table = new glue.Table(this, 'Table', { + * storageParameters: [ + * glue.StorageParameter.skipHeaderLineCount(1), + * glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + * glue.StorageParameter.custom('foo', 'bar'), // Will have no effect + * glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data + * glue.StorageParameter.custom(glue.StorageParameters.WRITE_PARALLEL, 'off'), + * ], + * // ... + * database: glueDatabase, + * columns: [{ + * name: 'col1', + * type: glue.Schema.STRING, + * }], + * dataFormat: glue.DataFormat.CSV, + * }); + * + * @default - The parameter is not defined + */ + readonly storageParameters?: StorageParameter[]; +} + +/** + * A Glue table. + */ +export abstract class TableBase extends Resource implements ITable { + + public static fromTableArn(scope: Construct, id: string, tableArn: string): ITable { + const tableName = Fn.select(1, Fn.split('/', Stack.of(scope).splitArn(tableArn, ArnFormat.SLASH_RESOURCE_NAME).resourceName!)); + + return TableBase.fromTableAttributes(scope, id, { + tableArn, + tableName, + }); + } + + /** + * Creates a Table construct that represents an external table. + * + * @param scope The scope creating construct (usually `this`). + * @param id The construct's id. + * @param attrs Import attributes + */ + public static fromTableAttributes(scope: Construct, id: string, attrs: TableAttributes): ITable { + class Import extends Resource implements ITable { + public readonly tableArn = attrs.tableArn; + public readonly tableName = attrs.tableName; + } + + return new Import(scope, id); + } + + protected abstract readonly tableResource: CfnTable; + public abstract readonly tableName: string; + public abstract readonly tableArn: string; + public abstract readonly location: string; + public abstract readonly partitionIndexes?: PartitionIndex[]; + + /** + * Database this table belongs to. + */ + public readonly database: IDatabase; + + /** + * Indicates whether the table's data is compressed or not. + */ + public readonly compressed: boolean; + + /** + * The type of encryption enabled for the table. + */ + public readonly encryption?: TableEncryption; + + /** + * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`. + */ + public readonly encryptionKey?: kms.IKey; + + /** + * Format of this table's data files. + */ + public readonly dataFormat: DataFormat; + + /** + * This table's columns. + */ + public readonly columns: Column[]; + + /** + * This table's partition keys if the table is partitioned. + */ + public readonly partitionKeys?: Column[]; + + /** + * The tables' storage descriptor properties. + */ + public readonly storageParameters?: StorageParameter[]; + + /** + * Partition indexes must be created one at a time. To avoid + * race conditions, we store the resource and add dependencies + * each time a new partition index is created. + */ + private partitionIndexCustomResources: AwsCustomResource[] = []; + + constructor(scope: Construct, id: string, props: TableProps) { + super(scope, id, { + physicalName: props.tableName ?? + Lazy.string({ + produce: () => Names.uniqueResourceName(this, {}).toLowerCase(), + }), + }); + + this.database = props.database; + this.dataFormat = props.dataFormat; + + validateSchema(props.columns, props.partitionKeys); + this.columns = props.columns; + this.partitionKeys = props.partitionKeys; + this.storageParameters = props.storageParameters; + + this.compressed = props.compressed ?? false; + + this.encryption = props.encryption; + this.encryptionKey = props.encryptionKey; + } + + public abstract grantRead(grantee: iam.IGrantable): iam.Grant; + public abstract grantWrite(grantee: iam.IGrantable): iam.Grant; + public abstract grantReadWrite(grantee: iam.IGrantable): iam.Grant; + + /** + * Add a partition index to the table. You can have a maximum of 3 partition + * indexes to a table. Partition index keys must be a subset of the table's + * partition keys. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/partition-indexes.html + */ + public addPartitionIndex(index: PartitionIndex) { + const numPartitions = this.partitionIndexCustomResources.length; + if (numPartitions >= 3) { + throw new Error('Maximum number of partition indexes allowed is 3'); + } + this.validatePartitionIndex(index); + + const indexName = index.indexName ?? this.generateIndexName(index.keyNames); + const partitionIndexCustomResource = new cr.AwsCustomResource(this, `partition-index-${indexName}`, { + onCreate: { + service: 'Glue', + action: 'createPartitionIndex', + parameters: { + DatabaseName: this.database.databaseName, + TableName: this.tableName, + PartitionIndex: { + IndexName: indexName, + Keys: index.keyNames, + }, + }, + physicalResourceId: cr.PhysicalResourceId.of( + indexName, + ), + }, + policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ + resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE, + }), + // APIs are available in 2.1055.0 + installLatestAwsSdk: false, + }); + this.grantToUnderlyingResources(partitionIndexCustomResource, ['glue:UpdateTable']); + + // Depend on previous partition index if possible, to avoid race condition + if (numPartitions > 0) { + this.partitionIndexCustomResources[numPartitions-1].node.addDependency(partitionIndexCustomResource); + } + this.partitionIndexCustomResources.push(partitionIndexCustomResource); + } + + private generateIndexName(keys: string[]): string { + const prefix = keys.join('-') + '-'; + const uniqueId = Names.uniqueId(this); + const maxIndexLength = 80; // arbitrarily specified + const startIndex = Math.max(0, uniqueId.length - (maxIndexLength - prefix.length)); + return prefix + uniqueId.substring(startIndex); + } + + private validatePartitionIndex(index: PartitionIndex) { + if (index.indexName !== undefined && (index.indexName.length < 1 || index.indexName.length > 255)) { + throw new Error(`Index name must be between 1 and 255 characters, but got ${index.indexName.length}`); + } + if (!this.partitionKeys || this.partitionKeys.length === 0) { + throw new Error('The table must have partition keys to create a partition index'); + } + const keyNames = this.partitionKeys.map(pk => pk.name); + if (!index.keyNames.every(k => keyNames.includes(k))) { + throw new Error(`All index keys must also be partition keys. Got ${index.keyNames} but partition key names are ${keyNames}`); + } + } + + /** + * Grant the given identity custom permissions. + */ + public grant(grantee: iam.IGrantable, actions: string[]) { + return iam.Grant.addToPrincipal({ + grantee, + resourceArns: [this.tableArn], + actions, + }); + } + + /** + * Grant the given identity custom permissions to ALL underlying resources of the table. + * Permissions will be granted to the catalog, the database, and the table. + */ + public grantToUnderlyingResources(grantee: iam.IGrantable, actions: string[]) { + return iam.Grant.addToPrincipal({ + grantee, + resourceArns: [ + this.tableArn, + this.database.catalogArn, + this.database.databaseArn, + ], + actions, + }); + } +} + +function validateSchema(columns: Column[], partitionKeys?: Column[]): void { + if (columns.length === 0) { + throw new Error('you must specify at least one column for the table'); + } + // Check there is at least one column and no duplicated column names or partition keys. + const names = new Set(); + (columns.concat(partitionKeys || [])).forEach(column => { + if (names.has(column.name)) { + throw new Error(`column names and partition keys must be unique, but \'${column.name}\' is duplicated`); + } + names.add(column.name); + }); +} From ea081de9dc56ee63e99d7cf390303e2d358d7318 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 16:07:22 +0000 Subject: [PATCH 20/44] old table deprecated --- .../aws-glue-alpha/lib/table-deprecated.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index 2aa7830a53821..750293b39c0dd 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -8,10 +8,10 @@ import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; -import { S3Table } from './s3-table'; +import { S3Table, S3TableProps } from './s3-table'; import { Column } from './schema'; import { StorageParameter } from './storage-parameter'; -import { ITable, PartitionIndex, TableAttributes, TableBase, TableEncryption, TableProps } from './table-base'; +import { ITable, PartitionIndex, TableAttributes, TableBase, TableEncryption } from './table-base'; /** * A Glue table. @@ -122,7 +122,7 @@ export class Table extends Resource implements ITable { */ private partitionIndexCustomResources: AwsCustomResource[] = []; - constructor(scope: Construct, id: string, props: TableProps) { + constructor(scope: Construct, id: string, props: S3TableProps) { super(scope, id, { physicalName: props.tableName ?? Lazy.string({ @@ -278,7 +278,7 @@ export class Table extends Resource implements ITable { const ret = this.grant(grantee, readPermissions); if (this.bucket) { if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } - this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); + this.bucket.grantRead(grantee, this.generateS3PrefixForGrant()); } return ret; } @@ -292,7 +292,7 @@ export class Table extends Resource implements ITable { const ret = this.grant(grantee, writePermissions); if (this.bucket) { if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } - this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); + this.bucket.grantWrite(grantee, this.generateS3PrefixForGrant()); } return ret; } @@ -306,7 +306,7 @@ export class Table extends Resource implements ITable { const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); if (this.bucket) { if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } - this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); + this.bucket.grantReadWrite(grantee, this.generateS3PrefixForGrant()); } return ret; } @@ -338,7 +338,7 @@ export class Table extends Resource implements ITable { }); } - protected getS3PrefixForGrant() { + protected generateS3PrefixForGrant() { return this.s3Prefix + '*'; } } @@ -366,7 +366,7 @@ const encryptionMappings = { }; // create the bucket to store a table's data depending on the `encryption` and `encryptionKey` properties. -function createBucket(table: Table, props: TableProps) { +function createBucket(table: Table, props: S3TableProps) { let bucket = props.bucket; if (bucket && (props.encryption !== undefined && props.encryption !== TableEncryption.CLIENT_SIDE_KMS)) { From c4317b1858edb5e3e44eda4c7c8eb8859942bc39 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 23 Aug 2023 16:10:55 +0000 Subject: [PATCH 21/44] integ test --- .../test/integ.partition-index.ts | 6 +- .../aws-cdk-glue.assets.json | 4 +- .../aws-cdk-glue.template.json | 137 ++++++++++++ .../integ.table.js.snapshot/manifest.json | 29 ++- .../test/integ.table.js.snapshot/tree.json | 205 +++++++++++++++++- .../aws-glue-alpha/test/integ.table.ts | 8 - 6 files changed, 359 insertions(+), 30 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.partition-index.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.partition-index.ts index 0982b86fa1a68..5455b6eeab618 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.partition-index.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.partition-index.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node -import * as s3 from 'aws-cdk-lib/aws-s3'; import * as cdk from 'aws-cdk-lib'; +import * as s3 from 'aws-cdk-lib/aws-s3'; import * as glue from '../lib'; /** @@ -38,7 +38,7 @@ const partitionKeys = [{ type: glue.Schema.BIG_INT, }]; -const csvTable = new glue.Table(stack, 'CSVTable', { +const csvTable = new glue.S3Table(stack, 'CSVTable', { database, bucket, tableName: 'csv_table', @@ -56,7 +56,7 @@ csvTable.addPartitionIndex({ keyNames: ['month', 'year'], }); -const jsonTable = new glue.Table(stack, 'JSONTable', { +const jsonTable = new glue.S3Table(stack, 'JSONTable', { database, bucket, tableName: 'json_table', diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index d714c2d54aaba..188bdf753be37 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "33.0.0", "files": { - "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754": { + "5f9f9d1c02cf84a8d8e0742f90161a825f9d867c85ac45b9dbbc4e025a539b55": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", + "objectKey": "5f9f9d1c02cf84a8d8e0742f90161a825f9d867c85ac45b9dbbc4e025a539b55.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index f4415b64a1333..36ca24dfffa82 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -5,6 +5,23 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, + "MyConnection5621880D": { + "Type": "AWS::Glue::Connection", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "ConnectionInput": { + "ConnectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "ConnectionType": "JDBC", + "Name": "my_connection" + } + } + }, "MyDatabase1E2517DB": { "Type": "AWS::Glue::Database", "Properties": { @@ -488,6 +505,70 @@ } } }, + "MyTableWithConnectionTable4BCA8495": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "connection_table generated by CDK", + "Name": "connection_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, "MyTableWithStorageDescriptorParametersTable1A347345": { "Type": "AWS::Glue::Table", "Properties": { @@ -559,6 +640,62 @@ } } }, + "MyTableWithCustomLocationTable43A19D42": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "custom_location_table generated by CDK", + "Name": "custom_location_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true, + "connectionName": { + "Ref": "MyConnection5621880D" + } + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": "default_db.public.test", + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, "MyUserDC45028B": { "Type": "AWS::IAM::User" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index 02bab428fcf54..78b63511c19cb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5f9f9d1c02cf84a8d8e0742f90161a825f9d867c85ac45b9dbbc4e025a539b55.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -39,6 +39,12 @@ "data": "DataBucketE3889A50" } ], + "/aws-cdk-glue/MyConnection/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyConnection5621880D" + } + ], "/aws-cdk-glue/MyDatabase/Resource": [ { "type": "aws:cdk:logicalId", @@ -93,12 +99,24 @@ "data": "MyPartitionFilteredTable324BA27A" } ], + "/aws-cdk-glue/MyTableWithConnection/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithConnectionTable4BCA8495" + } + ], "/aws-cdk-glue/MyTableWithStorageDescriptorParameters/Table": [ { "type": "aws:cdk:logicalId", "data": "MyTableWithStorageDescriptorParametersTable1A347345" } ], + "/aws-cdk-glue/MyTableWithCustomLocation/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithCustomLocationTable43A19D42" + } + ], "/aws-cdk-glue/MyUser/Resource": [ { "type": "aws:cdk:logicalId", @@ -134,15 +152,6 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } - ], - "MyPartitionFilteredTableBucket6ACAA137": [ - { - "type": "aws:cdk:logicalId", - "data": "MyPartitionFilteredTableBucket6ACAA137", - "trace": [ - "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" - ] - } ] }, "displayName": "aws-cdk-glue" diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index 71c545fff8226..b4157c9205ff4 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -30,6 +30,41 @@ "version": "0.0.0" } }, + "MyConnection": { + "id": "MyConnection", + "path": "aws-cdk-glue/MyConnection", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue/MyConnection/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Connection", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "connectionInput": { + "connectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "connectionType": "JDBC", + "name": "my_connection" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnConnection", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Connection", + "version": "0.0.0" + } + }, "MyDatabase": { "id": "MyDatabase", "path": "aws-cdk-glue/MyDatabase", @@ -143,7 +178,7 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", "version": "0.0.0" } }, @@ -231,7 +266,7 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", "version": "0.0.0" } }, @@ -319,7 +354,7 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", "version": "0.0.0" } }, @@ -407,7 +442,7 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", "version": "0.0.0" } }, @@ -584,7 +619,7 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", "version": "0.0.0" } }, @@ -667,7 +702,89 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", + "version": "0.0.0" + } + }, + "MyTableWithConnection": { + "id": "MyTableWithConnection", + "path": "aws-cdk-glue/MyTableWithConnection", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyTableWithConnection/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "connection_table", + "description": "connection_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "storageDescriptor": { + "location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", "version": "0.0.0" } }, @@ -756,7 +873,81 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Table", + "fqn": "@aws-cdk/aws-glue-alpha.S3Table", + "version": "0.0.0" + } + }, + "MyTableWithCustomLocation": { + "id": "MyTableWithCustomLocation", + "path": "aws-cdk-glue/MyTableWithCustomLocation", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyTableWithCustomLocation/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "custom_location_table", + "description": "custom_location_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": true, + "connectionName": { + "Ref": "MyConnection5621880D" + } + }, + "storageDescriptor": { + "location": "default_db.public.test", + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.ExternalTable", "version": "0.0.0" } }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index 73acc2fb26a32..0dc3555e9f2eb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -119,14 +119,6 @@ new glue.S3Table(stack, 'MyTableWithConnection', { dataFormat: glue.DataFormat.JSON, }); -new glue.S3Table(stack, 'MyTableWithCustomLocation', { - database, - bucket, - tableName: 'custom_location_table', - columns, - dataFormat: glue.DataFormat.JSON, -}); - new glue.S3Table(stack, 'MyTableWithStorageDescriptorParameters', { database, bucket, From 130bd3c286ee71bdf134be5fb02982339c707a89 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 24 Aug 2023 10:13:20 +0000 Subject: [PATCH 22/44] bump From dc90f5889da29a2a14fec031e0834bda195d533a Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 25 Aug 2023 16:18:54 +0000 Subject: [PATCH 23/44] separating table tests --- .../aws-glue-alpha/lib/external-table.ts | 6 +- .../@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 2 +- .../test/external-table.test.ts | 1751 +++++++++++++++++ .../aws-glue-alpha/test/table.test.ts | 34 - 4 files changed, 1755 insertions(+), 38 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts index 9da5fa01d1472..8f7164e17d7d4 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts @@ -115,7 +115,7 @@ export class ExternalTable extends TableBase { } /** - * Grant read permissions to the table and the underlying data stored in S3 to an IAM principal. + * Grant read permissions to the table * * @param grantee the principal */ @@ -126,7 +126,7 @@ export class ExternalTable extends TableBase { } /** - * Grant write permissions to the table and the underlying data stored in S3 to an IAM principal. + * Grant write permissions to the table * * @param grantee the principal */ @@ -137,7 +137,7 @@ export class ExternalTable extends TableBase { } /** - * Grant read and write permissions to the table and the underlying data stored in S3 to an IAM principal. + * Grant read and write permissions to the table * * @param grantee the principal */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts index 20ac57cb5f218..52feec05c41a2 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -200,7 +200,7 @@ const encryptionMappings = { }; // create the bucket to store a table's data depending on the `encryption` and `encryptionKey` properties. -function createBucket(table: TableBase, props: S3TableProps) { +function createBucket(table: S3Table, props: S3TableProps) { let bucket = props.bucket; if (bucket && (props.encryption !== undefined && props.encryption !== TableEncryption.CLIENT_SIDE_KMS)) { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts new file mode 100644 index 0000000000000..06dcdc8bfb3c7 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts @@ -0,0 +1,1751 @@ +import * as cdk from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { CfnTable } from 'aws-cdk-lib/aws-glue'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as glue from '../lib'; + +const externalDataLocation = 'default_db.public.test'; +const readPermissions = [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', +]; +const writePermissions = [ + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', +]; + +test('unpartitioned JSON table', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + + const tableStack = new cdk.Stack(app, 'table'); + const connection = new glue.Connection(tableStack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + const table = new glue.ExternalTable(tableStack, 'Table', { + database, + connection, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.S3_MANAGED, + dataFormat: glue.DataFormat.JSON, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('partitioned JSON table', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + + const tableStack = new cdk.Stack(app, 'table'); + const connection = new glue.Connection(tableStack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + const table = new glue.ExternalTable(tableStack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], + encryption: glue.TableEncryption.S3_MANAGED, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); + expect(table.encryptionKey).toEqual(undefined); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + PartitionKeys: [ + { + Name: 'year', + Type: 'smallint', + }, + ], + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('compressed table', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: true, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('table.node.defaultChild', () => { + // GIVEN + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + // WHEN + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + // THEN + expect(table.node.defaultChild instanceof CfnTable).toEqual(true); +}); + +test('encrypted table: SSE-S3', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.S3_MANAGED, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); + expect(table.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: SSE-KMS (implicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.KMS); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'Created by Default/Table/Bucket', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: SSE-KMS (explicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const encryptionKey = new kms.Key(stack, 'MyKey', { + description: 'OurKey', + }); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.KMS); + expect(table.encryptionKey).not.toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'OurKey', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: SSE-KMS_MANAGED', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS_MANAGED, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.KMS_MANAGED); + expect(table.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: CSE-KMS (implicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); + expect(table.encryptionKey).not.toEqual(undefined); + + Template.fromStack(stack).resourceCountIs('AWS::KMS::Key', 1); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: CSE-KMS (explicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const encryptionKey = new kms.Key(stack, 'MyKey', { + description: 'MyKey', + }); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); + expect(table.encryptionKey).not.toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'MyKey', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const encryptionKey = new kms.Key(stack, 'MyKey', { + description: 'MyKey', + }); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); + expect(table.encryptionKey).not.toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'MyKey', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('explicit s3 bucket and prefix', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const stack = new cdk.Stack(app, 'app'); + const database = new glue.Database(dbStack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Description: 'apptablecb9c398f generated by CDK', + Name: 'apptablecb9c398f', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('explicit s3 bucket and with empty prefix', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const stack = new cdk.Stack(app, 'app'); + const database = new glue.Database(dbStack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Description: 'apptablecb9c398f generated by CDK', + Name: 'apptablecb9c398f', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: externalDataLocation, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +describe('add partition index', () => { + test('fails if no partition keys', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + expect(() => table.addPartitionIndex({ + indexName: 'my-part', + keyNames: ['part'], + })).toThrowError(/The table must have partition keys to create a partition index/); + }); + + test('fails if partition index does not match partition keys', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + expect(() => table.addPartitionIndex({ + indexName: 'my-part', + keyNames: ['not-part'], + })).toThrowError(/All index keys must also be partition keys/); + }); + + test('fails with index name < 1 character', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + expect(() => table.addPartitionIndex({ + indexName: '', + keyNames: ['part'], + })).toThrowError(/Index name must be between 1 and 255 characters, but got 0/); + }); + + test('fails with > 3 indexes', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const indexes: glue.PartitionIndex[] = [{ + indexName: 'ind1', + keyNames: ['part'], + }, { + indexName: 'ind2', + keyNames: ['part'], + }, { + indexName: 'ind3', + keyNames: ['part'], + }, { + indexName: 'ind4', + keyNames: ['part'], + }]; + + expect(() => new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + partitionIndexes: indexes, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + })).toThrowError('Maximum number of partition indexes allowed is 3'); + }); +}); + +describe('grants', () => { + test('custom permissions', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + table.grant(user, ['glue:UpdateTable']); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 'glue:UpdateTable', + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); + + describe('read only', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + test('no encryption key', () => { + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + table.grantRead(user); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: readPermissions, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); + + test('with encryption key', () => { + const encryptionKey = new kms.Key(stack, 'MyKey', { + description: 'OurKey', + }); + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + encryptionKey, + externalDataLocation, + }); + + table.grantRead(user); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: readPermissions, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 'kms:Decrypt', + 'kms:Encrypt', + 'kms:ReEncrypt*', + 'kms:GenerateDataKey*', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); + }); + + test('no encryption key', () => { + + }); + + test('read and write', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + table.grantReadWrite(user); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); +}); + +describe('validate', () => { + test('at least one', () => { + expect(() => { + createTable({ + columns: [], + }); + }).toThrowError('you must specify at least one column for the table'); + }); + + test('unique column names', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }, { + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + }); + + test('unique partition keys', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'p1', + type: glue.Schema.STRING, + }, { + name: 'p1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'p1' is duplicated"); + }); + + test('column names and partition keys are all unique', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + }); + + test('can not specify an explicit bucket and encryption', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: glue.TableEncryption.KMS, + }); + }).toThrowError('you can not specify encryption settings if you also provide a bucket'); + }); + + test('can explicitly pass bucket if Encryption undefined', () => { + expect(() => createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })).not.toThrow(); + }); + + test('can explicitly pass bucket if encryption is not set', () => { + expect(() => createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })).not.toThrow(); + }); + + test('can explicitly pass bucket if ClientSideKms', () => { + expect(() => createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + })).not.toThrow(); + }); + + test('unique storage descriptor parameters', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + expect(() => new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storageParameters: [ + glue.StorageParameter.skipHeaderLineCount(2), + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('foo', 'bar'), + glue.StorageParameter.custom(glue.StorageParameters.COMPRESSION_TYPE, 'true'), + ], + connection, + externalDataLocation, + })).toThrowError('Duplicate storage parameter key: compression_type'); + }); +}); + +test('Table.fromTableArn', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + const table = glue.ExternalTable.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + + // THEN + expect(table.tableArn).toEqual('arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + expect(table.tableName).toEqual('tbl1'); +}); + +test.each([ + ['enabled', true], + ['disabled', false], +])('Partition filtering on table %s', (_, enabled) => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + const connection = new glue.Connection(dbStack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const tableStack = new cdk.Stack(app, 'table'); + new glue.ExternalTable(tableStack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + enablePartitionFiltering: enabled, + connection, + externalDataLocation, + }); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + 'classification': 'json', + 'has_encrypted_data': true, + 'partition_filtering.enabled': enabled, + }, + PartitionKeys: Match.anyValue(), + StorageDescriptor: Match.anyValue(), + TableType: Match.anyValue(), + }, + }); +}); + +test('Partition filtering on table is not defined (default behavior)', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + const connection = new glue.Connection(dbStack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + + const tableStack = new cdk.Stack(app, 'table'); + new glue.ExternalTable(tableStack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + enablePartitionFiltering: undefined, + connection, + externalDataLocation, + }); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + PartitionKeys: Match.anyValue(), + StorageDescriptor: Match.anyValue(), + TableType: Match.anyValue(), + }, + }); +}); + +test('can specify a physical name', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + new glue.ExternalTable(stack, 'Table', { + database, + tableName: 'my_table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + Name: 'my_table', + Description: 'my_table generated by CDK', + }, + }); +}); + +test('can specify a description', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + new glue.ExternalTable(stack, 'Table', { + database, + tableName: 'my_table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + description: 'This is a test table.', + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + Name: 'my_table', + Description: 'my_table generated by CDK', + }, + }); +}); + +test('storage descriptor parameters', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storageParameters: [ + glue.StorageParameter.skipHeaderLineCount(2), + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('foo', 'bar'), + glue.StorageParameter.custom('separatorChar', ','), + ], + connection, + externalDataLocation, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + StorageDescriptor: { + Parameters: { + 'skip.header.line.count': '2', + 'separatorChar': ',', + 'foo': 'bar', + 'compression_type': 'gzip', + }, + }, + }, + }); +}); + +test('can associate an external location with the glue table', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + new glue.ExternalTable(stack, 'Table', { + database, + tableName: 'my_table', + connection, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + externalDataLocation, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + StorageDescriptor: { + Location: externalDataLocation, + }, + Parameters: { + connectionName: { + Ref: 'Connection89AD5CF5', + }, + }, + }, + }); +}); + +function createTable(props: Pick>): void { + const stack = new cdk.Stack(); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + new glue.ExternalTable(stack, 'table', { + ...props, + database: new glue.Database(stack, 'db'), + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); +} diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts index 3e455d9f163c5..e50821af96750 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts @@ -1654,40 +1654,6 @@ test('storage descriptor parameters', () => { }); }); -test('can associate an external location with the glue table', () => { - const app = new cdk.App(); - const stack = new cdk.Stack(app, 'Stack'); - const database = new glue.Database(stack, 'Database'); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - new glue.ExternalTable(stack, 'Table', { - database, - tableName: 'my_table', - connection, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - dataFormat: glue.DataFormat.JSON, - externalDataLocation: 'default_db.public.test', - }); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - TableInput: { - StorageDescriptor: { - Location: 'default_db.public.test', - }, - }, - }); -}); - function createTable(props: Pick>): void { const stack = new cdk.Stack(); new glue.S3Table(stack, 'table', { From b9ba3f29909ae2293c08eb40c7d2df31f7702860 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 29 Aug 2023 15:23:34 +0000 Subject: [PATCH 24/44] moving table encryption to s3 tables only --- .../aws-glue-alpha/lib/external-table.ts | 5 +- .../@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 56 ++++++++++++++- .../@aws-cdk/aws-glue-alpha/lib/table-base.ts | 68 ------------------- .../aws-glue-alpha/lib/table-deprecated.ts | 4 +- 4 files changed, 58 insertions(+), 75 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts index 8f7164e17d7d4..0c61e9df40511 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts @@ -3,7 +3,7 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import { Construct } from 'constructs'; import { IConnection } from './connection'; import { Column } from './schema'; -import { PartitionIndex, TableBase, TableEncryption, TableProps } from './table-base'; +import { PartitionIndex, TableBase, TableProps } from './table-base'; export interface ExternalTableProps extends TableProps { /** @@ -121,7 +121,6 @@ export class ExternalTable extends TableBase { */ public grantRead(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, readPermissions); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } return ret; } @@ -132,7 +131,6 @@ export class ExternalTable extends TableBase { */ public grantWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, writePermissions); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } return ret; } @@ -143,7 +141,6 @@ export class ExternalTable extends TableBase { */ public grantReadWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } return ret; } } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts index 52feec05c41a2..6151a8f2f7eb1 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -4,7 +4,40 @@ import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; import { Construct } from 'constructs'; import { Column } from './schema'; -import { PartitionIndex, TableBase, TableEncryption, TableProps } from './table-base'; +import { PartitionIndex, TableBase, TableProps } from './table-base'; + +/** + * Encryption options for a Table. + * + * @see https://docs.aws.amazon.com/athena/latest/ug/encryption.html + */ +export enum TableEncryption { + /** + * Server side encryption (SSE) with an Amazon S3-managed key. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html + */ + S3_MANAGED = 'SSE-S3', + + /** + * Server-side encryption (SSE) with an AWS KMS key managed by the account owner. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + */ + KMS = 'SSE-KMS', + + /** + * Server-side encryption (SSE) with an AWS KMS key managed by the KMS service. + */ + KMS_MANAGED = 'SSE-KMS-MANAGED', + + /** + * Client-side encryption (CSE) with an AWS KMS key managed by the account owner. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html + */ + CLIENT_SIDE_KMS = 'CSE-KMS' +} export interface S3TableProps extends TableProps { /** @@ -20,6 +53,27 @@ export interface S3TableProps extends TableProps { * @default - No prefix. The data will be stored under the root of the bucket. */ readonly s3Prefix?: string; + + /** + * The kind of encryption to secure the data with. + * + * You can only provide this option if you are not explicitly passing in a bucket. + * + * If you choose `SSE-KMS`, you *can* provide an un-managed KMS key with `encryptionKey`. + * If you choose `CSE-KMS`, you *must* provide an un-managed KMS key with `encryptionKey`. + * + * @default BucketEncryption.S3_MANAGED + */ + readonly encryption?: TableEncryption; + + /** + * External KMS key to use for bucket encryption. + * + * The `encryption` property must be `SSE-KMS` or `CSE-KMS`. + * + * @default key is managed by KMS. + */ + readonly encryptionKey?: kms.IKey; } /** diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts index 3be953edf706e..080dfc6af0ea0 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts @@ -1,6 +1,5 @@ import { CfnTable } from 'aws-cdk-lib/aws-glue'; import * as iam from 'aws-cdk-lib/aws-iam'; -import * as kms from 'aws-cdk-lib/aws-kms'; import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-lib/core'; import * as cr from 'aws-cdk-lib/custom-resources'; import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; @@ -40,39 +39,6 @@ export interface ITable extends IResource { readonly tableName: string; } -/** - * Encryption options for a Table. - * - * @see https://docs.aws.amazon.com/athena/latest/ug/encryption.html - */ -export enum TableEncryption { - /** - * Server side encryption (SSE) with an Amazon S3-managed key. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html - */ - S3_MANAGED = 'SSE-S3', - - /** - * Server-side encryption (SSE) with an AWS KMS key managed by the account owner. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html - */ - KMS = 'SSE-KMS', - - /** - * Server-side encryption (SSE) with an AWS KMS key managed by the KMS service. - */ - KMS_MANAGED = 'SSE-KMS-MANAGED', - - /** - * Client-side encryption (CSE) with an AWS KMS key managed by the account owner. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html - */ - CLIENT_SIDE_KMS = 'CSE-KMS' -} - export interface TableAttributes { readonly tableArn: string; readonly tableName: string; @@ -131,27 +97,6 @@ export interface TableProps { */ readonly compressed?: boolean; - /** - * The kind of encryption to secure the data with. - * - * You can only provide this option if you are not explicitly passing in a bucket. - * - * If you choose `SSE-KMS`, you *can* provide an un-managed KMS key with `encryptionKey`. - * If you choose `CSE-KMS`, you *must* provide an un-managed KMS key with `encryptionKey`. - * - * @default BucketEncryption.S3_MANAGED - */ - readonly encryption?: TableEncryption; - - /** - * External KMS key to use for bucket encryption. - * - * The `encryption` property must be `SSE-KMS` or `CSE-KMS`. - * - * @default key is managed by KMS. - */ - readonly encryptionKey?: kms.IKey; - /** * Indicates whether the table data is stored in subdirectories. * @@ -250,16 +195,6 @@ export abstract class TableBase extends Resource implements ITable { */ public readonly compressed: boolean; - /** - * The type of encryption enabled for the table. - */ - public readonly encryption?: TableEncryption; - - /** - * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`. - */ - public readonly encryptionKey?: kms.IKey; - /** * Format of this table's data files. */ @@ -304,9 +239,6 @@ export abstract class TableBase extends Resource implements ITable { this.storageParameters = props.storageParameters; this.compressed = props.compressed ?? false; - - this.encryption = props.encryption; - this.encryptionKey = props.encryptionKey; } public abstract grantRead(grantee: iam.IGrantable): iam.Grant; diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index 750293b39c0dd..cedb03015a800 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -8,10 +8,10 @@ import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; -import { S3Table, S3TableProps } from './s3-table'; +import { S3Table, S3TableProps, TableEncryption } from './s3-table'; import { Column } from './schema'; import { StorageParameter } from './storage-parameter'; -import { ITable, PartitionIndex, TableAttributes, TableBase, TableEncryption } from './table-base'; +import { ITable, PartitionIndex, TableAttributes, TableBase } from './table-base'; /** * A Glue table. From 43484006333c878f10927360bc6f219cb275df94 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 29 Aug 2023 15:26:20 +0000 Subject: [PATCH 25/44] bug with table description --- packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts index 06dcdc8bfb3c7..ded336cefd44b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts @@ -1642,7 +1642,7 @@ test('can specify a description', () => { Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { TableInput: { Name: 'my_table', - Description: 'my_table generated by CDK', + Description: 'This is a test table.', }, }); }); From 634f717bf9c39245e83197a907777086d8792fbc Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 29 Aug 2023 15:26:43 +0000 Subject: [PATCH 26/44] removing encryption test --- .../test/external-table.test.ts | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts index ded336cefd44b..6a7273e3baf17 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts @@ -1381,52 +1381,6 @@ describe('validate', () => { }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); }); - test('can not specify an explicit bucket and encryption', () => { - expect(() => { - createTable({ - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: glue.TableEncryption.KMS, - }); - }).toThrowError('you can not specify encryption settings if you also provide a bucket'); - }); - - test('can explicitly pass bucket if Encryption undefined', () => { - expect(() => createTable({ - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: undefined, - })).not.toThrow(); - }); - - test('can explicitly pass bucket if encryption is not set', () => { - expect(() => createTable({ - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: undefined, - })).not.toThrow(); - }); - - test('can explicitly pass bucket if ClientSideKms', () => { - expect(() => createTable({ - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - })).not.toThrow(); - }); - test('unique storage descriptor parameters', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Stack'); From e259fbd6b57365ab3544c846cab8f17991e3f2c5 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 30 Aug 2023 10:56:12 +0000 Subject: [PATCH 27/44] removing encryption tests --- .../test/external-table.test.ts | 954 +++--------------- 1 file changed, 163 insertions(+), 791 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts index 6a7273e3baf17..b863e3ce1c48f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts @@ -2,8 +2,6 @@ import * as cdk from 'aws-cdk-lib'; import { Match, Template } from 'aws-cdk-lib/assertions'; import { CfnTable } from 'aws-cdk-lib/aws-glue'; import * as iam from 'aws-cdk-lib/aws-iam'; -import * as kms from 'aws-cdk-lib/aws-kms'; -import * as s3 from 'aws-cdk-lib/aws-s3'; import * as glue from '../lib'; const externalDataLocation = 'default_db.public.test'; @@ -39,588 +37,27 @@ test('unpartitioned JSON table', () => { PASSWORD: 'password', }, }); - const table = new glue.ExternalTable(tableStack, 'Table', { - database, - connection, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.S3_MANAGED, - dataFormat: glue.DataFormat.JSON, - externalDataLocation, - }); - expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); - - Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', - }, - TableInput: { - Name: 'tabletable8fff2c2b', - Description: 'tabletable8fff2c2b generated by CDK', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('partitioned JSON table', () => { - const app = new cdk.App(); - const dbStack = new cdk.Stack(app, 'db'); - const database = new glue.Database(dbStack, 'Database'); - - const tableStack = new cdk.Stack(app, 'table'); - const connection = new glue.Connection(tableStack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - const table = new glue.ExternalTable(tableStack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - partitionKeys: [{ - name: 'year', - type: glue.Schema.SMALL_INT, - }], - encryption: glue.TableEncryption.S3_MANAGED, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); - expect(table.encryptionKey).toEqual(undefined); - - Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', - }, - TableInput: { - Name: 'tabletable8fff2c2b', - Description: 'tabletable8fff2c2b generated by CDK', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - PartitionKeys: [ - { - Name: 'year', - Type: 'smallint', - }, - ], - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('compressed table', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - expect(table.encryptionKey).toEqual(undefined); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: true, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('table.node.defaultChild', () => { - // GIVEN - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - - // WHEN - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - - // THEN - expect(table.node.defaultChild instanceof CfnTable).toEqual(true); -}); - -test('encrypted table: SSE-S3', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.S3_MANAGED, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); - expect(table.encryptionKey).toEqual(undefined); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('encrypted table: SSE-KMS (implicitly created key)', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.KMS, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - expect(table.encryption).toEqual(glue.TableEncryption.KMS); - - Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { - Description: 'Created by Default/Table/Bucket', - }); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('encrypted table: SSE-KMS (explicitly created key)', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const encryptionKey = new kms.Key(stack, 'MyKey', { - description: 'OurKey', - }); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.KMS, - encryptionKey, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - expect(table.encryption).toEqual(glue.TableEncryption.KMS); - expect(table.encryptionKey).not.toEqual(undefined); - - Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { - Description: 'OurKey', - }); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('encrypted table: SSE-KMS_MANAGED', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.KMS_MANAGED, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - expect(table.encryption).toEqual(glue.TableEncryption.KMS_MANAGED); - expect(table.encryptionKey).toEqual(undefined); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('encrypted table: CSE-KMS (implicitly created key)', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); - expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); - expect(table.encryptionKey).not.toEqual(undefined); - - Template.fromStack(stack).resourceCountIs('AWS::KMS::Key', 1); - - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); -}); - -test('encrypted table: CSE-KMS (explicitly created key)', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const encryptionKey = new kms.Key(stack, 'MyKey', { - description: 'MyKey', - }); - const connection = new glue.Connection(stack, 'Connection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', - USERNAME: 'username', - PASSWORD: 'password', - }, - }); - - const table = new glue.ExternalTable(stack, 'Table', { + new glue.ExternalTable(tableStack, 'Table', { database, + connection, columns: [{ name: 'col', type: glue.Schema.STRING, }], - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - encryptionKey, dataFormat: glue.DataFormat.JSON, - connection, externalDataLocation, }); - expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); - expect(table.encryptionKey).not.toEqual(undefined); - - Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { - Description: 'MyKey', - }); - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { Ref: 'AWS::AccountId', }, DatabaseName: { - Ref: 'DatabaseB269D8BB', + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', }, TableInput: { - Description: 'table generated by CDK', - Name: 'table', + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', Parameters: { classification: 'json', has_encrypted_data: true, @@ -646,13 +83,13 @@ test('encrypted table: CSE-KMS (explicitly created key)', () => { }); }); -test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database'); - const encryptionKey = new kms.Key(stack, 'MyKey', { - description: 'MyKey', - }); - const connection = new glue.Connection(stack, 'Connection', { +test('partitioned JSON table', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + + const tableStack = new cdk.Stack(app, 'table'); + const connection = new glue.Connection(tableStack, 'Connection', { connectionName: 'my_connection', type: glue.ConnectionType.JDBC, properties: { @@ -661,40 +98,41 @@ test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { PASSWORD: 'password', }, }); - - const table = new glue.ExternalTable(stack, 'Table', { + new glue.ExternalTable(tableStack, 'Table', { database, columns: [{ name: 'col', type: glue.Schema.STRING, }], - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - encryptionKey, + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], dataFormat: glue.DataFormat.JSON, connection, externalDataLocation, }); - expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); - expect(table.encryptionKey).not.toEqual(undefined); - - Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { - Description: 'MyKey', - }); - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { Ref: 'AWS::AccountId', }, DatabaseName: { - Ref: 'DatabaseB269D8BB', + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', }, TableInput: { - Description: 'table generated by CDK', - Name: 'table', + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', Parameters: { classification: 'json', has_encrypted_data: true, }, + PartitionKeys: [ + { + Name: 'year', + Type: 'smallint', + }, + ], StorageDescriptor: { Columns: [ { @@ -716,11 +154,10 @@ test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { }); }); -test('explicit s3 bucket and prefix', () => { - const app = new cdk.App(); - const dbStack = new cdk.Stack(app, 'db'); - const stack = new cdk.Stack(app, 'app'); - const database = new glue.Database(dbStack, 'Database'); +test('compressed table', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { connectionName: 'my_connection', type: glue.ConnectionType.JDBC, @@ -730,13 +167,13 @@ test('explicit s3 bucket and prefix', () => { PASSWORD: 'password', }, }); - new glue.ExternalTable(stack, 'Table', { database, columns: [{ name: 'col', type: glue.Schema.STRING, }], + compressed: true, dataFormat: glue.DataFormat.JSON, connection, externalDataLocation, @@ -747,11 +184,11 @@ test('explicit s3 bucket and prefix', () => { Ref: 'AWS::AccountId', }, DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + Ref: 'DatabaseB269D8BB', }, TableInput: { - Description: 'apptablecb9c398f generated by CDK', - Name: 'apptablecb9c398f', + Name: 'table', + Description: 'table generated by CDK', Parameters: { classification: 'json', has_encrypted_data: true, @@ -763,7 +200,7 @@ test('explicit s3 bucket and prefix', () => { Type: 'string', }, ], - Compressed: false, + Compressed: true, InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', Location: externalDataLocation, OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', @@ -777,11 +214,10 @@ test('explicit s3 bucket and prefix', () => { }); }); -test('explicit s3 bucket and with empty prefix', () => { - const app = new cdk.App(); - const dbStack = new cdk.Stack(app, 'db'); - const stack = new cdk.Stack(app, 'app'); - const database = new glue.Database(dbStack, 'Database'); +test('table.node.defaultChild', () => { + // GIVEN + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); const connection = new glue.Connection(stack, 'Connection', { connectionName: 'my_connection', type: glue.ConnectionType.JDBC, @@ -792,50 +228,21 @@ test('explicit s3 bucket and with empty prefix', () => { }, }); - new glue.ExternalTable(stack, 'Table', { + // WHEN + const table = new glue.ExternalTable(stack, 'Table', { database, columns: [{ name: 'col', type: glue.Schema.STRING, }], + compressed: true, dataFormat: glue.DataFormat.JSON, connection, externalDataLocation, }); - Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', - }, - TableInput: { - Description: 'apptablecb9c398f generated by CDK', - Name: 'apptablecb9c398f', - Parameters: { - classification: 'json', - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: externalDataLocation, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - }); + // THEN + expect(table.node.defaultChild instanceof CfnTable).toEqual(true); }); describe('add partition index', () => { @@ -1057,7 +464,7 @@ describe('grants', () => { }); }); - describe('read only', () => { + test('read only', () => { const stack = new cdk.Stack(); const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); @@ -1070,173 +477,139 @@ describe('grants', () => { PASSWORD: 'password', }, }); - test('no encryption key', () => { - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - connection, - externalDataLocation, - }); + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); - table.grantRead(user); + table.grantRead(user); - Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: readPermissions, - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: readPermissions, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, ], - }, + ], }, - ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', }, ], - }); + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], }); + }); - test('with encryption key', () => { - const encryptionKey = new kms.Key(stack, 'MyKey', { - description: 'OurKey', - }); - const table = new glue.ExternalTable(stack, 'Table', { - database, - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - connection, - encryptionKey, - externalDataLocation, - }); + test('write only', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + const connection = new glue.Connection(stack, 'Connection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection', + USERNAME: 'username', + PASSWORD: 'password', + }, + }); + const table = new glue.ExternalTable(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + connection, + externalDataLocation, + }); - table.grantRead(user); + table.grantWrite(user); - Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: readPermissions, - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], - ], - }, - }, - { - Action: [ - 'kms:Decrypt', - 'kms:Encrypt', - 'kms:ReEncrypt*', - 'kms:GenerateDataKey*', - ], - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: writePermissions, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, ], - }, + ], }, - ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', }, ], - }); + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], }); }); - test('no encryption key', () => { - - }); - test('read and write', () => { const stack = new cdk.Stack(); const user = new iam.User(stack, 'User'); @@ -1250,7 +623,6 @@ describe('grants', () => { PASSWORD: 'password', }, }); - const table = new glue.ExternalTable(stack, 'Table', { database, columns: [{ From 8fcc9344ae935eca04d0841b5bb351bf0dc1dfa2 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 30 Aug 2023 15:26:09 +0000 Subject: [PATCH 28/44] adding deprecated table tests --- .../test/external-table.test.ts | 26 +- .../aws-glue-alpha/test/s3-table.test.ts | 1105 +++++++++++++++++ .../aws-glue-alpha/test/table-base.test.ts | 684 ++++++++++ ...table.test.ts => table-deprecated.test.ts} | 62 +- 4 files changed, 1838 insertions(+), 39 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/s3-table.test.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/table-base.test.ts rename packages/@aws-cdk/aws-glue-alpha/test/{table.test.ts => table-deprecated.test.ts} (96%) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts index b863e3ce1c48f..012722e050ebc 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/external-table.test.ts @@ -786,16 +786,26 @@ describe('validate', () => { }); }); -test('Table.fromTableArn', () => { - // GIVEN - const stack = new cdk.Stack(); +describe('Table.fromTableArn', () => { + test('success', () => { + // GIVEN + const stack = new cdk.Stack(); - // WHEN - const table = glue.ExternalTable.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + // WHEN + const table = glue.ExternalTable.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); - // THEN - expect(table.tableArn).toEqual('arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); - expect(table.tableName).toEqual('tbl1'); + // THEN + expect(table.tableArn).toEqual('arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + expect(table.tableName).toEqual('tbl1'); + }); + + test('throws if no ARN is provided', () => { + // GIVEN + const stack = new cdk.Stack(); + + // THEN + expect(() => glue.ExternalTable.fromTableArn(stack, 'boom', '')).toThrowError(/ARNs must start with \"arn:\" and have at least 6 components: /); + }); }); test.each([ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/s3-table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/s3-table.test.ts new file mode 100644 index 0000000000000..c5e498ad61c41 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/s3-table.test.ts @@ -0,0 +1,1105 @@ +import * as cdk from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as glue from '../lib'; + +test('encrypted table: SSE-S3', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.S3_MANAGED, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); + expect(table.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + }, + }, + ], + }, + }); +}); + +test('encrypted table: SSE-KMS (implicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.KMS); + expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'Created by Default/Table/Bucket', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + KMSMasterKeyID: { + 'Fn::GetAtt': [ + 'TableBucketKey3E9F984A', + 'Arn', + ], + }, + SSEAlgorithm: 'aws:kms', + }, + }, + ], + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: SSE-KMS (explicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const encryptionKey = new kms.Key(stack, 'MyKey', { + description: 'OurKey', + }); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.KMS); + expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey); + expect(table.encryptionKey).not.toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'OurKey', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + KMSMasterKeyID: { + 'Fn::GetAtt': [ + 'MyKey6AB29FA6', + 'Arn', + ], + }, + SSEAlgorithm: 'aws:kms', + }, + }, + ], + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: SSE-KMS_MANAGED', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS_MANAGED, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.KMS_MANAGED); + expect(table.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + SSEAlgorithm: 'aws:kms', + }, + }, + ], + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: CSE-KMS (implicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); + expect(table.encryptionKey).not.toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).resourceCountIs('AWS::KMS::Key', 1); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: CSE-KMS (explicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const encryptionKey = new kms.Key(stack, 'MyKey', { + description: 'MyKey', + }); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); + expect(table.encryptionKey).not.toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'MyKey', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + const bucket = new s3.Bucket(stack, 'Bucket'); + const encryptionKey = new kms.Key(stack, 'MyKey', { + description: 'MyKey', + }); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + bucket, + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); + expect(table.encryptionKey).not.toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { + Description: 'MyKey', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'Bucket83908E77', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('explicit s3 bucket and prefix', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const stack = new cdk.Stack(app, 'app'); + const bucket = new s3.Bucket(stack, 'ExplicitBucket'); + const database = new glue.Database(dbStack, 'Database'); + + new glue.S3Table(stack, 'Table', { + database, + bucket, + s3Prefix: 'prefix/', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Description: 'apptablecb9c398f generated by CDK', + Name: 'apptablecb9c398f', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'ExplicitBucket0AA51A3F', + }, + '/prefix/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('explicit s3 bucket and with empty prefix', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const stack = new cdk.Stack(app, 'app'); + const bucket = new s3.Bucket(stack, 'ExplicitBucket'); + const database = new glue.Database(dbStack, 'Database'); + + new glue.S3Table(stack, 'Table', { + database, + bucket, + s3Prefix: '', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Description: 'apptablecb9c398f generated by CDK', + Name: 'apptablecb9c398f', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'ExplicitBucket0AA51A3F', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +describe('grants', () => { + test('custom permissions', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grant(user, ['glue:UpdateTable']); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 'glue:UpdateTable', + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); + + test('read only', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grantRead(user); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); + + test('write only', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grantWrite(user); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 's3:DeleteObject*', + 's3:PutObject', + 's3:PutObjectLegalHold', + 's3:PutObjectRetention', + 's3:PutObjectTagging', + 's3:PutObjectVersionTagging', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); + + test('read and write', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grantReadWrite(user); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject', + 's3:PutObjectLegalHold', + 's3:PutObjectRetention', + 's3:PutObjectTagging', + 's3:PutObjectVersionTagging', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); +}); + +describe('validate', () => { + test('can not specify an explicit bucket and encryption', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: glue.TableEncryption.KMS, + }); + }).toThrowError('you can not specify encryption settings if you also provide a bucket'); + }); + + test('can explicitly pass bucket if Encryption undefined', () => { + expect(() => createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })).not.toThrow(); + }); + + test('can explicitly pass bucket if encryption is not set', () => { + expect(() => createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })).not.toThrow(); + }); + + test('can explicitly pass bucket if ClientSideKms', () => { + expect(() => createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + })).not.toThrow(); + }); +}); + +function createTable(props: Pick>): void { + const stack = new cdk.Stack(); + new glue.S3Table(stack, 'table', { + ...props, + database: new glue.Database(stack, 'db'), + dataFormat: glue.DataFormat.JSON, + }); +} diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table-base.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table-base.test.ts new file mode 100644 index 0000000000000..d39345c288f24 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/table-base.test.ts @@ -0,0 +1,684 @@ +import * as cdk from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { CfnTable } from 'aws-cdk-lib/aws-glue'; +import * as glue from '../lib'; + +test('unpartitioned JSON table', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + + const tableStack = new cdk.Stack(app, 'table'); + const table = new glue.S3Table(tableStack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); + + Template.fromStack(tableStack).hasResource('AWS::S3::Bucket', { + Type: 'AWS::S3::Bucket', + DeletionPolicy: 'Retain', + UpdateReplacePolicy: 'Retain', + }); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('partitioned JSON table', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + + const tableStack = new cdk.Stack(app, 'table'); + const table = new glue.S3Table(tableStack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); + expect(table.encryptionKey).toEqual(undefined); + expect(table.bucket).not.toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + PartitionKeys: [ + { + Name: 'year', + Type: 'smallint', + }, + ], + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('compressed table', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + expect(table.encryptionKey).toEqual(undefined); + expect(table.bucket?.encryptionKey).toEqual(undefined); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: true, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + }); +}); + +test('table.node.defaultChild', () => { + // GIVEN + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + // WHEN + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + // THEN + expect(table.node.defaultChild instanceof CfnTable).toEqual(true); +}); + +describe('parition indexes', () => { + test('fails with > 3 indexes', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const indexes: glue.PartitionIndex[] = [{ + indexName: 'ind1', + keyNames: ['part'], + }, { + indexName: 'ind2', + keyNames: ['part'], + }, { + indexName: 'ind3', + keyNames: ['part'], + }, { + indexName: 'ind4', + keyNames: ['part'], + }]; + + expect(() => new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + partitionIndexes: indexes, + dataFormat: glue.DataFormat.JSON, + })).toThrowError('Maximum number of partition indexes allowed is 3'); + }); + + test('no indexName', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const indexes: glue.PartitionIndex[] = [{ + keyNames: ['part'], + }]; + + new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + partitionIndexes: indexes, + dataFormat: glue.DataFormat.JSON, + }); + }); + + describe('add partition index', () => { + test('fails if no partition keys', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + + expect(() => table.addPartitionIndex({ + indexName: 'my-part', + keyNames: ['part'], + })).toThrowError(/The table must have partition keys to create a partition index/); + }); + + test('fails if partition index does not match partition keys', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + }); + + expect(() => table.addPartitionIndex({ + indexName: 'my-part', + keyNames: ['not-part'], + })).toThrowError(/All index keys must also be partition keys/); + }); + + test('fails with index name < 1 character', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database'); + + const table = new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + }); + + expect(() => table.addPartitionIndex({ + indexName: '', + keyNames: ['part'], + })).toThrowError(/Index name must be between 1 and 255 characters, but got 0/); + }); + }); +}); + +describe('validate', () => { + test('at least one column', () => { + expect(() => { + createTable({ + columns: [], + }); + }).toThrowError('you must specify at least one column for the table'); + }); + + test('unique column names', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }, { + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + }); + + test('unique partition keys', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'p1', + type: glue.Schema.STRING, + }, { + name: 'p1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'p1' is duplicated"); + }); + + test('column names and partition keys are all unique', () => { + expect(() => { + createTable({ + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + }); + + test('unique storage descriptor parameters', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + + expect(() => new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storageParameters: [ + glue.StorageParameter.skipHeaderLineCount(2), + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('foo', 'bar'), + glue.StorageParameter.custom(glue.StorageParameters.COMPRESSION_TYPE, 'true'), + ], + })).toThrowError('Duplicate storage parameter key: compression_type'); + }); +}); + +describe('Table.fromTableArn', () => { + test('success', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + const table = glue.ExternalTable.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + + // THEN + expect(table.tableArn).toEqual('arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + expect(table.tableName).toEqual('tbl1'); + }); + + test('throws if no ARN is provided', () => { + // GIVEN + const stack = new cdk.Stack(); + + // THEN + expect(() => glue.ExternalTable.fromTableArn(stack, 'boom', '')).toThrowError(/ARNs must start with \"arn:\" and have at least 6 components: /); + }); +}); + +test.each([ + ['enabled', true], + ['disabled', false], +])('Partition filtering on table %s', (_, enabled) => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + + const tableStack = new cdk.Stack(app, 'table'); + new glue.S3Table(tableStack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + enablePartitionFiltering: enabled, + }); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + 'classification': 'json', + 'has_encrypted_data': true, + 'partition_filtering.enabled': enabled, + }, + PartitionKeys: Match.anyValue(), + StorageDescriptor: Match.anyValue(), + TableType: Match.anyValue(), + }, + }); +}); + +test('Partition filtering on table is not defined (default behavior)', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database'); + + const tableStack = new cdk.Stack(app, 'table'); + new glue.S3Table(tableStack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + enablePartitionFiltering: undefined, + }); + + Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'tabletable8fff2c2b', + Description: 'tabletable8fff2c2b generated by CDK', + Parameters: { + classification: 'json', + has_encrypted_data: true, + }, + PartitionKeys: Match.anyValue(), + StorageDescriptor: Match.anyValue(), + TableType: Match.anyValue(), + }, + }); +}); + +test('can specify a physical name', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + new glue.S3Table(stack, 'Table', { + database, + tableName: 'my_table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + Name: 'my_table', + Description: 'my_table generated by CDK', + }, + }); +}); + +test('can specify a description', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + new glue.S3Table(stack, 'Table', { + database, + tableName: 'my_table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + description: 'This is a test table.', + dataFormat: glue.DataFormat.JSON, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + Name: 'my_table', + Description: 'This is a test table.', + }, + }); +}); + +test('storage descriptor parameters', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storageParameters: [ + glue.StorageParameter.skipHeaderLineCount(2), + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('foo', 'bar'), + glue.StorageParameter.custom('separatorChar', ','), + ], + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + StorageDescriptor: { + Parameters: { + 'skip.header.line.count': '2', + 'separatorChar': ',', + 'foo': 'bar', + 'compression_type': 'gzip', + }, + }, + }, + }); +}); + +test('can specify there are subdirectories', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storedAsSubDirectories: true, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + StorageDescriptor: { + StoredAsSubDirectories: true, + }, + }, + }); +}); + +test('data format without classification string', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + const dataFormat = new glue.DataFormat({ + inputFormat: glue.InputFormat.TEXT, + outputFormat: glue.OutputFormat.HIVE_IGNORE_KEY_TEXT, + serializationLibrary: glue.SerializationLibrary.OPENX_JSON, + }); + new glue.S3Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + Parameters: { + classification: Match.absent(), + }, + StorageDescriptor: { + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + }, + }, + }); +}); + +function createTable(props: Pick>): void { + const stack = new cdk.Stack(); + new glue.S3Table(stack, 'table', { + ...props, + database: new glue.Database(stack, 'db'), + dataFormat: glue.DataFormat.JSON, + }); +} diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table-deprecated.test.ts similarity index 96% rename from packages/@aws-cdk/aws-glue-alpha/test/table.test.ts rename to packages/@aws-cdk/aws-glue-alpha/test/table-deprecated.test.ts index e50821af96750..6d72682053d66 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table-deprecated.test.ts @@ -5,6 +5,7 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as glue from '../lib'; +import { PartitionIndex } from '../lib'; test('unpartitioned JSON table', () => { const app = new cdk.App(); @@ -12,7 +13,7 @@ test('unpartitioned JSON table', () => { const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - const table = new glue.S3Table(tableStack, 'Table', { + const table = new glue.Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -80,7 +81,7 @@ test('partitioned JSON table', () => { const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - const table = new glue.S3Table(tableStack, 'Table', { + const table = new glue.Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -94,7 +95,6 @@ test('partitioned JSON table', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket).not.toEqual(undefined); expect(table.bucket?.encryptionKey).toEqual(undefined); Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { @@ -153,7 +153,7 @@ test('compressed table', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -217,7 +217,7 @@ test('table.node.defaultChild', () => { const database = new glue.Database(stack, 'Database'); // WHEN - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -235,7 +235,7 @@ test('encrypted table: SSE-S3', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -310,7 +310,7 @@ test('encrypted table: SSE-KMS (implicitly created key)', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -397,7 +397,7 @@ test('encrypted table: SSE-KMS (explicitly created key)', () => { description: 'OurKey', }); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -483,7 +483,7 @@ test('encrypted table: SSE-KMS_MANAGED', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -558,7 +558,7 @@ test('encrypted table: CSE-KMS (implicitly created key)', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -626,7 +626,7 @@ test('encrypted table: CSE-KMS (explicitly created key)', () => { description: 'MyKey', }); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -698,7 +698,7 @@ test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { description: 'MyKey', }); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -770,7 +770,7 @@ test('explicit s3 bucket and prefix', () => { const bucket = new s3.Bucket(stack, 'ExplicitBucket'); const database = new glue.Database(dbStack, 'Database'); - new glue.S3Table(stack, 'Table', { + new glue.Table(stack, 'Table', { database, bucket, s3Prefix: 'prefix/', @@ -834,7 +834,7 @@ test('explicit s3 bucket and with empty prefix', () => { const bucket = new s3.Bucket(stack, 'ExplicitBucket'); const database = new glue.Database(dbStack, 'Database'); - new glue.S3Table(stack, 'Table', { + new glue.Table(stack, 'Table', { database, bucket, s3Prefix: '', @@ -896,7 +896,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -915,7 +915,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -938,7 +938,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -961,7 +961,7 @@ describe('add partition index', () => { const stack = new cdk.Stack(); const database = new glue.Database(stack, 'Database'); - const indexes: glue.PartitionIndex[] = [{ + const indexes: PartitionIndex[] = [{ indexName: 'ind1', keyNames: ['part'], }, { @@ -975,7 +975,7 @@ describe('add partition index', () => { keyNames: ['part'], }]; - expect(() => new glue.S3Table(stack, 'Table', { + expect(() => new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -997,7 +997,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1060,7 +1060,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1161,7 +1161,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1264,7 +1264,7 @@ describe('grants', () => { const user = new iam.User(stack, 'User'); const database = new glue.Database(stack, 'Database'); - const table = new glue.S3Table(stack, 'Table', { + const table = new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1484,7 +1484,7 @@ describe('validate', () => { const stack = new cdk.Stack(app, 'Stack'); const database = new glue.Database(stack, 'Database'); - expect(() => new glue.S3Table(stack, 'Table', { + expect(() => new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1506,7 +1506,7 @@ test('Table.fromTableArn', () => { const stack = new cdk.Stack(); // WHEN - const table = glue.S3Table.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + const table = glue.Table.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); // THEN expect(table.tableArn).toEqual('arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); @@ -1522,7 +1522,7 @@ test.each([ const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - new glue.S3Table(tableStack, 'Table', { + new glue.Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -1564,7 +1564,7 @@ test('Partition filtering on table is not defined (default behavior)', () => { const database = new glue.Database(dbStack, 'Database'); const tableStack = new cdk.Stack(app, 'table'); - new glue.S3Table(tableStack, 'Table', { + new glue.Table(tableStack, 'Table', { database, columns: [{ name: 'col', @@ -1603,7 +1603,7 @@ test('can specify a physical name', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Stack'); const database = new glue.Database(stack, 'Database'); - new glue.S3Table(stack, 'Table', { + new glue.Table(stack, 'Table', { database, tableName: 'my_table', columns: [{ @@ -1625,7 +1625,7 @@ test('storage descriptor parameters', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Stack'); const database = new glue.Database(stack, 'Database'); - new glue.S3Table(stack, 'Table', { + new glue.Table(stack, 'Table', { database, columns: [{ name: 'col', @@ -1656,9 +1656,9 @@ test('storage descriptor parameters', () => { function createTable(props: Pick>): void { const stack = new cdk.Stack(); - new glue.S3Table(stack, 'table', { + new glue.Table(stack, 'table', { ...props, database: new glue.Database(stack, 'db'), dataFormat: glue.DataFormat.JSON, }); -} +} \ No newline at end of file From eabfa71085f594f5726d18cf59c92e7b3059e768 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 31 Aug 2023 10:09:52 +0000 Subject: [PATCH 29/44] updating README --- packages/@aws-cdk/aws-glue-alpha/README.md | 70 +++++++++++----------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index ac02692819c1e..589084bcd7093 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -211,7 +211,7 @@ A Glue table describes a table of data in S3: its structure (column names and ty ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { database: myDatabase, columns: [{ name: 'col1', @@ -230,7 +230,7 @@ By default, a S3 bucket will be created to store the table's data but you can ma ```ts declare const myBucket: s3.Bucket; declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { bucket: myBucket, s3Prefix: 'my-table/', // ... @@ -247,7 +247,7 @@ Glue tables can be configured to contain user-defined properties, to describe th ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { storageParameters: [ glue.StorageParameter.skipHeaderLineCount(1), glue.StorageParameter.compressionType(glue.CompressionType.GZIP), @@ -269,7 +269,7 @@ To improve query performance, a table can specify `partitionKeys` on which data ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { database: myDatabase, columns: [{ name: 'col1', @@ -300,7 +300,7 @@ property: ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { database: myDatabase, columns: [{ name: 'col1', @@ -337,7 +337,7 @@ If you have a table with a large number of partitions that grows over time, cons ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { database: myDatabase, columns: [{ name: 'col1', @@ -355,6 +355,28 @@ new glue.Table(this, 'MyTable', { }); ``` +### Glue Connections + +Glue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog. + +Whereas `S3Table` will point to (and if needed, create) a bucket to store the table's data, `ExternalTable` will point to an existing table in a data source. For example, to create a table in Glue that points to a table in Redshift: + +```ts +declare const myConnection: glue.Connection; +declare const myDatabase: glue.Database; +new glue.ExternalTable(this, 'MyTable', { + connection: myConnection, + externalDataLocation: 'default_db_public_example', // A table in Redshift + // ... + database: myDatabase, + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, +}); +``` + ## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html) You can enable encryption on a Table's data: @@ -363,7 +385,7 @@ You can enable encryption on a Table's data: ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { encryption: glue.TableEncryption.S3_MANAGED, // ... database: myDatabase, @@ -380,7 +402,7 @@ new glue.Table(this, 'MyTable', { ```ts declare const myDatabase: glue.Database; // KMS key is created automatically -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { encryption: glue.TableEncryption.KMS, // ... database: myDatabase, @@ -392,7 +414,7 @@ new glue.Table(this, 'MyTable', { }); // with an explicit KMS key -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { encryption: glue.TableEncryption.KMS, encryptionKey: new kms.Key(this, 'MyKey'), // ... @@ -409,7 +431,7 @@ new glue.Table(this, 'MyTable', { ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { encryption: glue.TableEncryption.KMS_MANAGED, // ... database: myDatabase, @@ -426,7 +448,7 @@ new glue.Table(this, 'MyTable', { ```ts declare const myDatabase: glue.Database; // KMS key is created automatically -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { encryption: glue.TableEncryption.CLIENT_SIDE_KMS, // ... database: myDatabase, @@ -438,7 +460,7 @@ new glue.Table(this, 'MyTable', { }); // with an explicit KMS key -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { encryption: glue.TableEncryption.CLIENT_SIDE_KMS, encryptionKey: new kms.Key(this, 'MyKey'), // ... @@ -451,27 +473,7 @@ new glue.Table(this, 'MyTable', { }); ``` -*Note: you cannot provide a `Bucket` when creating the `Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*. - -### Glue Connections - -Glue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog. The `connection` property of the `Table` class allows you to specify a Glue Connection to be associated with the table. Use this in combination with the `externalDataLocation` property to specify an internal destination that the connection will point to: - -```ts -declare const myConnection: glue.Connection; -declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { - connection: myConnection, - externalDataLocation: 'default_db_public_example', // A table in Redshift - // ... - database: myDatabase, - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - dataFormat: glue.DataFormat.JSON, -}); -``` +*Note: you cannot provide a `Bucket` when creating the `S3Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*. ## Types @@ -479,7 +481,7 @@ A table's schema is a collection of columns, each of which have a `name` and a ` ```ts declare const myDatabase: glue.Database; -new glue.Table(this, 'MyTable', { +new glue.S3Table(this, 'MyTable', { columns: [{ name: 'primitive_column', type: glue.Schema.STRING, From d5f762cf4d7b4504f6c28125716723fe66c726cf Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 1 Sep 2023 10:33:31 +0000 Subject: [PATCH 30/44] incorrect changes --- .../aws-glue-alpha/lib/table-deprecated.ts | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index cedb03015a800..0c03658ba9e65 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -58,7 +58,7 @@ export class Table extends Resource implements ITable { /** * The type of encryption enabled for the table. */ - public readonly encryption?: TableEncryption; + public readonly encryption: TableEncryption; /** * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`. @@ -68,12 +68,12 @@ export class Table extends Resource implements ITable { /** * S3 bucket in which the table's data resides. */ - public readonly bucket?: s3.IBucket; + public readonly bucket: s3.IBucket; /** * S3 Key Prefix under which this table's files are stored in S3. */ - public readonly s3Prefix?: string; + public readonly s3Prefix: string; /** * Name of this table. @@ -105,11 +105,6 @@ export class Table extends Resource implements ITable { */ public readonly partitionIndexes?: PartitionIndex[]; - /** - * The location of the tables' data. - */ - readonly location?: string; - /** * The tables' storage descriptor properties. */ @@ -145,7 +140,6 @@ export class Table extends Resource implements ITable { this.bucket = bucket; this.encryption = encryption; this.encryptionKey = encryptionKey; - this.location = `s3://${bucket.bucketName}/${this.s3Prefix}`; const tableResource = new CfnTable(this, 'Table', { catalogId: props.database.catalogId, @@ -164,7 +158,7 @@ export class Table extends Resource implements ITable { 'partition_filtering.enabled': props.enablePartitionFiltering, }, storageDescriptor: { - location: this.location, + location: `s3://${this.bucket.bucketName}/${this.s3Prefix}`, compressed: this.compressed, storedAsSubDirectories: props.storedAsSubDirectories ?? false, columns: renderColumns(props.columns), @@ -276,10 +270,8 @@ export class Table extends Resource implements ITable { */ public grantRead(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, readPermissions); - if (this.bucket) { - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } - this.bucket.grantRead(grantee, this.generateS3PrefixForGrant()); - } + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } + this.bucket.grantRead(grantee, this.generateS3PrefixForGrant()); return ret; } @@ -290,10 +282,8 @@ export class Table extends Resource implements ITable { */ public grantWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, writePermissions); - if (this.bucket) { - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } - this.bucket.grantWrite(grantee, this.generateS3PrefixForGrant()); - } + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } + this.bucket.grantWrite(grantee, this.generateS3PrefixForGrant()); return ret; } @@ -304,10 +294,8 @@ export class Table extends Resource implements ITable { */ public grantReadWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); - if (this.bucket) { - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } - this.bucket.grantReadWrite(grantee, this.generateS3PrefixForGrant()); - } + if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } + this.bucket.grantReadWrite(grantee, this.generateS3PrefixForGrant()); return ret; } @@ -338,7 +326,7 @@ export class Table extends Resource implements ITable { }); } - protected generateS3PrefixForGrant() { + private generateS3PrefixForGrant() { return this.s3Prefix + '*'; } } From dab9f2837178b642e37f14b687bce8c4683f85f0 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 1 Sep 2023 10:34:41 +0000 Subject: [PATCH 31/44] english... --- packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts index 6151a8f2f7eb1..ab3ec3cd2cbcb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -77,7 +77,7 @@ export interface S3TableProps extends TableProps { } /** - * A Glue table that targets an S3 dataset. + * A Glue table that targets a S3 dataset. */ export class S3Table extends TableBase { /** From 5abf2224e73de770b76527ff49c77708ca6b2693 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 1 Sep 2023 10:53:10 +0000 Subject: [PATCH 32/44] generate -> get --- packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 8 ++++---- packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts index ab3ec3cd2cbcb..01ab8f385ec66 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -194,7 +194,7 @@ export class S3Table extends TableBase { public grantRead(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, readPermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } - this.bucket.grantRead(grantee, this.generateS3PrefixForGrant()); + this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); return ret; } @@ -206,7 +206,7 @@ export class S3Table extends TableBase { public grantWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, writePermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } - this.bucket.grantWrite(grantee, this.generateS3PrefixForGrant()); + this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); return ret; } @@ -218,11 +218,11 @@ export class S3Table extends TableBase { public grantReadWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } - this.bucket.grantReadWrite(grantee, this.generateS3PrefixForGrant()); + this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); return ret; } - private generateS3PrefixForGrant() { + private getS3PrefixForGrant() { return this.s3Prefix + '*'; } } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index 0c03658ba9e65..02ac2d8f60765 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -271,7 +271,7 @@ export class Table extends Resource implements ITable { public grantRead(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, readPermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } - this.bucket.grantRead(grantee, this.generateS3PrefixForGrant()); + this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); return ret; } @@ -283,7 +283,7 @@ export class Table extends Resource implements ITable { public grantWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, writePermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } - this.bucket.grantWrite(grantee, this.generateS3PrefixForGrant()); + this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); return ret; } @@ -295,7 +295,7 @@ export class Table extends Resource implements ITable { public grantReadWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } - this.bucket.grantReadWrite(grantee, this.generateS3PrefixForGrant()); + this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); return ret; } @@ -326,7 +326,7 @@ export class Table extends Resource implements ITable { }); } - private generateS3PrefixForGrant() { + private getS3PrefixForGrant() { return this.s3Prefix + '*'; } } From 7d8a0057d358d6e2adfb6adf73ef700d71b3987b Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 1 Sep 2023 10:53:19 +0000 Subject: [PATCH 33/44] moving s3 declaration --- packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index 02ac2d8f60765..da2a44c2de88e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -127,6 +127,7 @@ export class Table extends Resource implements ITable { this.database = props.database; this.dataFormat = props.dataFormat; + this.s3Prefix = props.s3Prefix ?? ''; validateSchema(props.columns, props.partitionKeys); this.columns = props.columns; @@ -134,8 +135,6 @@ export class Table extends Resource implements ITable { this.storageParameters = props.storageParameters; this.compressed = props.compressed ?? false; - - this.s3Prefix = props.s3Prefix ?? ''; const { bucket, encryption, encryptionKey } = createBucket(this, props); this.bucket = bucket; this.encryption = encryption; From 05cd0ce7b494df1cefb7fc357b39d51a00d12ff8 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 1 Sep 2023 11:12:50 +0000 Subject: [PATCH 34/44] using Table.* --- packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index da2a44c2de88e..76621f4ad593d 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -11,7 +11,7 @@ import { IDatabase } from './database'; import { S3Table, S3TableProps, TableEncryption } from './s3-table'; import { Column } from './schema'; import { StorageParameter } from './storage-parameter'; -import { ITable, PartitionIndex, TableAttributes, TableBase } from './table-base'; +import { ITable, PartitionIndex, TableAttributes } from './table-base'; /** * A Glue table. @@ -23,7 +23,7 @@ export class Table extends Resource implements ITable { public static fromTableArn(scope: Construct, id: string, tableArn: string): ITable { const tableName = Fn.select(1, Fn.split('/', Stack.of(scope).splitArn(tableArn, ArnFormat.SLASH_RESOURCE_NAME).resourceName!)); - return TableBase.fromTableAttributes(scope, id, { + return Table.fromTableAttributes(scope, id, { tableArn, tableName, }); From 285de6c357b81425b1c26e8059485d48812e96ce Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 1 Sep 2023 11:40:03 +0000 Subject: [PATCH 35/44] keeping deprecated table the same --- .../aws-glue-alpha/lib/table-deprecated.ts | 217 +++++++++++++++++- 1 file changed, 212 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index 76621f4ad593d..b9d164352662c 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -2,16 +2,223 @@ import { CfnTable } from 'aws-cdk-lib/aws-glue'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; -import { ArnFormat, Fn, Lazy, Names, Resource, Stack } from 'aws-cdk-lib/core'; +import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-lib/core'; import * as cr from 'aws-cdk-lib/custom-resources'; import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; -import { S3Table, S3TableProps, TableEncryption } from './s3-table'; +import { S3Table } from './s3-table'; import { Column } from './schema'; import { StorageParameter } from './storage-parameter'; -import { ITable, PartitionIndex, TableAttributes } from './table-base'; + +/** + * Properties of a Partition Index. + */ +export interface PartitionIndex { + /** + * The name of the partition index. + * + * @default - a name will be generated for you. + */ + readonly indexName?: string; + + /** + * The partition key names that comprise the partition + * index. The names must correspond to a name in the + * table's partition keys. + */ + readonly keyNames: string[]; +} +export interface ITable extends IResource { + /** + * @attribute + */ + readonly tableArn: string; + + /** + * @attribute + */ + readonly tableName: string; +} + +/** + * Encryption options for a Table. + * + * @see https://docs.aws.amazon.com/athena/latest/ug/encryption.html + */ +export enum TableEncryption { + /** + * Server side encryption (SSE) with an Amazon S3-managed key. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html + */ + S3_MANAGED = 'SSE-S3', + + /** + * Server-side encryption (SSE) with an AWS KMS key managed by the account owner. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + */ + KMS = 'SSE-KMS', + + /** + * Server-side encryption (SSE) with an AWS KMS key managed by the KMS service. + */ + KMS_MANAGED = 'SSE-KMS-MANAGED', + + /** + * Client-side encryption (CSE) with an AWS KMS key managed by the account owner. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html + */ + CLIENT_SIDE_KMS = 'CSE-KMS' +} + +export interface TableAttributes { + readonly tableArn: string; + readonly tableName: string; +} + +export interface TableProps { + /** + * Name of the table. + * + * @default - generated by CDK. + */ + readonly tableName?: string; + + /** + * Description of the table. + * + * @default generated + */ + readonly description?: string; + + /** + * Database in which to store the table. + */ + readonly database: IDatabase; + + /** + * S3 bucket in which to store data. + * + * @default one is created for you + */ + readonly bucket?: s3.IBucket; + + /** + * S3 prefix under which table objects are stored. + * + * @default - No prefix. The data will be stored under the root of the bucket. + */ + readonly s3Prefix?: string; + + /** + * Columns of the table. + */ + readonly columns: Column[]; + + /** + * Partition columns of the table. + * + * @default table is not partitioned + */ + readonly partitionKeys?: Column[]; + + /** + * Partition indexes on the table. A maximum of 3 indexes + * are allowed on a table. Keys in the index must be part + * of the table's partition keys. + * + * @default table has no partition indexes + */ + readonly partitionIndexes?: PartitionIndex[]; + + /** + * Storage type of the table's data. + */ + readonly dataFormat: DataFormat; + + /** + * Indicates whether the table's data is compressed or not. + * + * @default false + */ + readonly compressed?: boolean; + + /** + * The kind of encryption to secure the data with. + * + * You can only provide this option if you are not explicitly passing in a bucket. + * + * If you choose `SSE-KMS`, you *can* provide an un-managed KMS key with `encryptionKey`. + * If you choose `CSE-KMS`, you *must* provide an un-managed KMS key with `encryptionKey`. + * + * @default BucketEncryption.S3_MANAGED + */ + readonly encryption?: TableEncryption; + + /** + * External KMS key to use for bucket encryption. + * + * The `encryption` property must be `SSE-KMS` or `CSE-KMS`. + * + * @default key is managed by KMS. + */ + readonly encryptionKey?: kms.IKey; + + /** + * Indicates whether the table data is stored in subdirectories. + * + * @default false + */ + readonly storedAsSubDirectories?: boolean; + + /** + * Enables partition filtering. + * + * @see https://docs.aws.amazon.com/athena/latest/ug/glue-best-practices.html#glue-best-practices-partition-index + * + * @default - The parameter is not defined + */ + readonly enablePartitionFiltering?: boolean; + + /** + * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. + * + * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. + * + * Some keys will be auto-populated by glue crawlers, however, you can override them by specifying the key and value in this property. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/table-properties-crawler.html + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ + * + * @example + * + * declare const glueDatabase: glue.IDatabase; + * const table = new glue.Table(this, 'Table', { + * storageParameters: [ + * glue.StorageParameter.skipHeaderLineCount(1), + * glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + * glue.StorageParameter.custom('foo', 'bar'), // Will have no effect + * glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data + * glue.StorageParameter.custom(glue.StorageParameters.WRITE_PARALLEL, 'off'), + * ], + * // ... + * database: glueDatabase, + * columns: [{ + * name: 'col1', + * type: glue.Schema.STRING, + * }], + * dataFormat: glue.DataFormat.CSV, + * }); + * + * @default - The parameter is not defined + */ + readonly storageParameters?: StorageParameter[]; +} /** * A Glue table. @@ -117,7 +324,7 @@ export class Table extends Resource implements ITable { */ private partitionIndexCustomResources: AwsCustomResource[] = []; - constructor(scope: Construct, id: string, props: S3TableProps) { + constructor(scope: Construct, id: string, props: TableProps) { super(scope, id, { physicalName: props.tableName ?? Lazy.string({ @@ -353,7 +560,7 @@ const encryptionMappings = { }; // create the bucket to store a table's data depending on the `encryption` and `encryptionKey` properties. -function createBucket(table: Table, props: S3TableProps) { +function createBucket(table: Table, props: TableProps) { let bucket = props.bucket; if (bucket && (props.encryption !== undefined && props.encryption !== TableEncryption.CLIENT_SIDE_KMS)) { From a8b494895d2076dc8ce84b6e75fc6b8a61d7397c Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 4 Sep 2023 10:36:52 +0000 Subject: [PATCH 36/44] Making a subclass of S3Table --- .../@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 8 +- .../aws-glue-alpha/lib/table-deprecated.ts | 624 +----------------- .../aws-cdk-glue.assets.json | 6 +- .../aws-cdk-glue.template.json | 64 ++ ...efaultTestDeployAssert8BFB5B70.assets.json | 2 +- .../test/integ.table.js.snapshot/cdk.out | 2 +- .../test/integ.table.js.snapshot/integ.json | 2 +- .../integ.table.js.snapshot/manifest.json | 10 +- .../test/integ.table.js.snapshot/tree.json | 82 +++ .../aws-glue-alpha/test/integ.table.ts | 8 + 10 files changed, 173 insertions(+), 635 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts index 01ab8f385ec66..b57d7cb40c0b7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -194,7 +194,7 @@ export class S3Table extends TableBase { public grantRead(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, readPermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } - this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); + this.bucket.grantRead(grantee, this.generateS3PrefixForGrant()); return ret; } @@ -206,7 +206,7 @@ export class S3Table extends TableBase { public grantWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, writePermissions); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } - this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); + this.bucket.grantWrite(grantee, this.generateS3PrefixForGrant()); return ret; } @@ -218,11 +218,11 @@ export class S3Table extends TableBase { public grantReadWrite(grantee: iam.IGrantable): iam.Grant { const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } - this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); + this.bucket.grantReadWrite(grantee, this.generateS3PrefixForGrant()); return ret; } - private getS3PrefixForGrant() { + protected generateS3PrefixForGrant() { return this.s3Prefix + '*'; } } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index b9d164352662c..095a0f39b6634 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -1,630 +1,8 @@ -import { CfnTable } from 'aws-cdk-lib/aws-glue'; -import * as iam from 'aws-cdk-lib/aws-iam'; -import * as kms from 'aws-cdk-lib/aws-kms'; -import * as s3 from 'aws-cdk-lib/aws-s3'; -import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-lib/core'; -import * as cr from 'aws-cdk-lib/custom-resources'; -import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; -import { Construct } from 'constructs'; -import { DataFormat } from './data-format'; -import { IDatabase } from './database'; import { S3Table } from './s3-table'; -import { Column } from './schema'; -import { StorageParameter } from './storage-parameter'; - -/** - * Properties of a Partition Index. - */ -export interface PartitionIndex { - /** - * The name of the partition index. - * - * @default - a name will be generated for you. - */ - readonly indexName?: string; - - /** - * The partition key names that comprise the partition - * index. The names must correspond to a name in the - * table's partition keys. - */ - readonly keyNames: string[]; -} -export interface ITable extends IResource { - /** - * @attribute - */ - readonly tableArn: string; - - /** - * @attribute - */ - readonly tableName: string; -} - -/** - * Encryption options for a Table. - * - * @see https://docs.aws.amazon.com/athena/latest/ug/encryption.html - */ -export enum TableEncryption { - /** - * Server side encryption (SSE) with an Amazon S3-managed key. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html - */ - S3_MANAGED = 'SSE-S3', - - /** - * Server-side encryption (SSE) with an AWS KMS key managed by the account owner. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html - */ - KMS = 'SSE-KMS', - - /** - * Server-side encryption (SSE) with an AWS KMS key managed by the KMS service. - */ - KMS_MANAGED = 'SSE-KMS-MANAGED', - - /** - * Client-side encryption (CSE) with an AWS KMS key managed by the account owner. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html - */ - CLIENT_SIDE_KMS = 'CSE-KMS' -} - -export interface TableAttributes { - readonly tableArn: string; - readonly tableName: string; -} - -export interface TableProps { - /** - * Name of the table. - * - * @default - generated by CDK. - */ - readonly tableName?: string; - - /** - * Description of the table. - * - * @default generated - */ - readonly description?: string; - - /** - * Database in which to store the table. - */ - readonly database: IDatabase; - - /** - * S3 bucket in which to store data. - * - * @default one is created for you - */ - readonly bucket?: s3.IBucket; - - /** - * S3 prefix under which table objects are stored. - * - * @default - No prefix. The data will be stored under the root of the bucket. - */ - readonly s3Prefix?: string; - - /** - * Columns of the table. - */ - readonly columns: Column[]; - - /** - * Partition columns of the table. - * - * @default table is not partitioned - */ - readonly partitionKeys?: Column[]; - - /** - * Partition indexes on the table. A maximum of 3 indexes - * are allowed on a table. Keys in the index must be part - * of the table's partition keys. - * - * @default table has no partition indexes - */ - readonly partitionIndexes?: PartitionIndex[]; - - /** - * Storage type of the table's data. - */ - readonly dataFormat: DataFormat; - - /** - * Indicates whether the table's data is compressed or not. - * - * @default false - */ - readonly compressed?: boolean; - - /** - * The kind of encryption to secure the data with. - * - * You can only provide this option if you are not explicitly passing in a bucket. - * - * If you choose `SSE-KMS`, you *can* provide an un-managed KMS key with `encryptionKey`. - * If you choose `CSE-KMS`, you *must* provide an un-managed KMS key with `encryptionKey`. - * - * @default BucketEncryption.S3_MANAGED - */ - readonly encryption?: TableEncryption; - - /** - * External KMS key to use for bucket encryption. - * - * The `encryption` property must be `SSE-KMS` or `CSE-KMS`. - * - * @default key is managed by KMS. - */ - readonly encryptionKey?: kms.IKey; - - /** - * Indicates whether the table data is stored in subdirectories. - * - * @default false - */ - readonly storedAsSubDirectories?: boolean; - - /** - * Enables partition filtering. - * - * @see https://docs.aws.amazon.com/athena/latest/ug/glue-best-practices.html#glue-best-practices-partition-index - * - * @default - The parameter is not defined - */ - readonly enablePartitionFiltering?: boolean; - - /** - * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. - * - * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. - * - * Some keys will be auto-populated by glue crawlers, however, you can override them by specifying the key and value in this property. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/table-properties-crawler.html - * - * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ - * - * @example - * - * declare const glueDatabase: glue.IDatabase; - * const table = new glue.Table(this, 'Table', { - * storageParameters: [ - * glue.StorageParameter.skipHeaderLineCount(1), - * glue.StorageParameter.compressionType(glue.CompressionType.GZIP), - * glue.StorageParameter.custom('foo', 'bar'), // Will have no effect - * glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data - * glue.StorageParameter.custom(glue.StorageParameters.WRITE_PARALLEL, 'off'), - * ], - * // ... - * database: glueDatabase, - * columns: [{ - * name: 'col1', - * type: glue.Schema.STRING, - * }], - * dataFormat: glue.DataFormat.CSV, - * }); - * - * @default - The parameter is not defined - */ - readonly storageParameters?: StorageParameter[]; -} /** * A Glue table. * * @deprecated Use {@link S3Table} instead. */ -export class Table extends Resource implements ITable { - - public static fromTableArn(scope: Construct, id: string, tableArn: string): ITable { - const tableName = Fn.select(1, Fn.split('/', Stack.of(scope).splitArn(tableArn, ArnFormat.SLASH_RESOURCE_NAME).resourceName!)); - - return Table.fromTableAttributes(scope, id, { - tableArn, - tableName, - }); - } - - /** - * Creates a Table construct that represents an external table. - * - * @param scope The scope creating construct (usually `this`). - * @param id The construct's id. - * @param attrs Import attributes - */ - public static fromTableAttributes(scope: Construct, id: string, attrs: TableAttributes): ITable { - class Import extends Resource implements ITable { - public readonly tableArn = attrs.tableArn; - public readonly tableName = attrs.tableName; - } - - return new Import(scope, id); - } - - /** - * Database this table belongs to. - */ - public readonly database: IDatabase; - - /** - * Indicates whether the table's data is compressed or not. - */ - public readonly compressed: boolean; - - /** - * The type of encryption enabled for the table. - */ - public readonly encryption: TableEncryption; - - /** - * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`. - */ - public readonly encryptionKey?: kms.IKey; - - /** - * S3 bucket in which the table's data resides. - */ - public readonly bucket: s3.IBucket; - - /** - * S3 Key Prefix under which this table's files are stored in S3. - */ - public readonly s3Prefix: string; - - /** - * Name of this table. - */ - public readonly tableName: string; - - /** - * ARN of this table. - */ - public readonly tableArn: string; - - /** - * Format of this table's data files. - */ - public readonly dataFormat: DataFormat; - - /** - * This table's columns. - */ - public readonly columns: Column[]; - - /** - * This table's partition keys if the table is partitioned. - */ - public readonly partitionKeys?: Column[]; - - /** - * This table's partition indexes. - */ - public readonly partitionIndexes?: PartitionIndex[]; - - /** - * The tables' storage descriptor properties. - */ - public readonly storageParameters?: StorageParameter[]; - - /** - * Partition indexes must be created one at a time. To avoid - * race conditions, we store the resource and add dependencies - * each time a new partition index is created. - */ - private partitionIndexCustomResources: AwsCustomResource[] = []; - - constructor(scope: Construct, id: string, props: TableProps) { - super(scope, id, { - physicalName: props.tableName ?? - Lazy.string({ - produce: () => Names.uniqueResourceName(this, {}).toLowerCase(), - }), - }); - - this.database = props.database; - this.dataFormat = props.dataFormat; - this.s3Prefix = props.s3Prefix ?? ''; - - validateSchema(props.columns, props.partitionKeys); - this.columns = props.columns; - this.partitionKeys = props.partitionKeys; - this.storageParameters = props.storageParameters; - - this.compressed = props.compressed ?? false; - const { bucket, encryption, encryptionKey } = createBucket(this, props); - this.bucket = bucket; - this.encryption = encryption; - this.encryptionKey = encryptionKey; - - const tableResource = new CfnTable(this, 'Table', { - catalogId: props.database.catalogId, - - databaseName: props.database.databaseName, - - tableInput: { - name: this.physicalName, - description: props.description || `${this.physicalName} generated by CDK`, - - partitionKeys: renderColumns(props.partitionKeys), - - parameters: { - 'classification': props.dataFormat.classificationString?.value, - 'has_encrypted_data': true, - 'partition_filtering.enabled': props.enablePartitionFiltering, - }, - storageDescriptor: { - location: `s3://${this.bucket.bucketName}/${this.s3Prefix}`, - compressed: this.compressed, - storedAsSubDirectories: props.storedAsSubDirectories ?? false, - columns: renderColumns(props.columns), - inputFormat: props.dataFormat.inputFormat.className, - outputFormat: props.dataFormat.outputFormat.className, - serdeInfo: { - serializationLibrary: props.dataFormat.serializationLibrary.className, - }, - parameters: props.storageParameters ? props.storageParameters.reduce((acc, param) => { - if (param.key in acc) { - throw new Error(`Duplicate storage parameter key: ${param.key}`); - } - const key = param.key; - acc[key] = param.value; - return acc; - }, {} as { [key: string]: string }) : undefined, - }, - - tableType: 'EXTERNAL_TABLE', - }, - }); - - this.tableName = this.getResourceNameAttribute(tableResource.ref); - this.tableArn = this.stack.formatArn({ - service: 'glue', - resource: 'table', - resourceName: `${this.database.databaseName}/${this.tableName}`, - }); - this.node.defaultChild = tableResource; - - // Partition index creation relies on created table. - if (props.partitionIndexes) { - this.partitionIndexes = props.partitionIndexes; - this.partitionIndexes.forEach((index) => this.addPartitionIndex(index)); - } - } - - /** - * Add a partition index to the table. You can have a maximum of 3 partition - * indexes to a table. Partition index keys must be a subset of the table's - * partition keys. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/partition-indexes.html - */ - public addPartitionIndex(index: PartitionIndex) { - const numPartitions = this.partitionIndexCustomResources.length; - if (numPartitions >= 3) { - throw new Error('Maximum number of partition indexes allowed is 3'); - } - this.validatePartitionIndex(index); - - const indexName = index.indexName ?? this.generateIndexName(index.keyNames); - const partitionIndexCustomResource = new cr.AwsCustomResource(this, `partition-index-${indexName}`, { - onCreate: { - service: 'Glue', - action: 'createPartitionIndex', - parameters: { - DatabaseName: this.database.databaseName, - TableName: this.tableName, - PartitionIndex: { - IndexName: indexName, - Keys: index.keyNames, - }, - }, - physicalResourceId: cr.PhysicalResourceId.of( - indexName, - ), - }, - policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ - resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE, - }), - // APIs are available in 2.1055.0 - installLatestAwsSdk: false, - }); - this.grantToUnderlyingResources(partitionIndexCustomResource, ['glue:UpdateTable']); - - // Depend on previous partition index if possible, to avoid race condition - if (numPartitions > 0) { - this.partitionIndexCustomResources[numPartitions-1].node.addDependency(partitionIndexCustomResource); - } - this.partitionIndexCustomResources.push(partitionIndexCustomResource); - } - - private generateIndexName(keys: string[]): string { - const prefix = keys.join('-') + '-'; - const uniqueId = Names.uniqueId(this); - const maxIndexLength = 80; // arbitrarily specified - const startIndex = Math.max(0, uniqueId.length - (maxIndexLength - prefix.length)); - return prefix + uniqueId.substring(startIndex); - } - - private validatePartitionIndex(index: PartitionIndex) { - if (index.indexName !== undefined && (index.indexName.length < 1 || index.indexName.length > 255)) { - throw new Error(`Index name must be between 1 and 255 characters, but got ${index.indexName.length}`); - } - if (!this.partitionKeys || this.partitionKeys.length === 0) { - throw new Error('The table must have partition keys to create a partition index'); - } - const keyNames = this.partitionKeys.map(pk => pk.name); - if (!index.keyNames.every(k => keyNames.includes(k))) { - throw new Error(`All index keys must also be partition keys. Got ${index.keyNames} but partition key names are ${keyNames}`); - } - } - - /** - * Grant read permissions to the table and the underlying data stored in S3 to an IAM principal. - * - * @param grantee the principal - */ - public grantRead(grantee: iam.IGrantable): iam.Grant { - const ret = this.grant(grantee, readPermissions); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); } - this.bucket.grantRead(grantee, this.getS3PrefixForGrant()); - return ret; - } - - /** - * Grant write permissions to the table and the underlying data stored in S3 to an IAM principal. - * - * @param grantee the principal - */ - public grantWrite(grantee: iam.IGrantable): iam.Grant { - const ret = this.grant(grantee, writePermissions); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); } - this.bucket.grantWrite(grantee, this.getS3PrefixForGrant()); - return ret; - } - - /** - * Grant read and write permissions to the table and the underlying data stored in S3 to an IAM principal. - * - * @param grantee the principal - */ - public grantReadWrite(grantee: iam.IGrantable): iam.Grant { - const ret = this.grant(grantee, [...readPermissions, ...writePermissions]); - if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); } - this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant()); - return ret; - } - - /** - * Grant the given identity custom permissions. - */ - public grant(grantee: iam.IGrantable, actions: string[]) { - return iam.Grant.addToPrincipal({ - grantee, - resourceArns: [this.tableArn], - actions, - }); - } - - /** - * Grant the given identity custom permissions to ALL underlying resources of the table. - * Permissions will be granted to the catalog, the database, and the table. - */ - public grantToUnderlyingResources(grantee: iam.IGrantable, actions: string[]) { - return iam.Grant.addToPrincipal({ - grantee, - resourceArns: [ - this.tableArn, - this.database.catalogArn, - this.database.databaseArn, - ], - actions, - }); - } - - private getS3PrefixForGrant() { - return this.s3Prefix + '*'; - } -} - -function validateSchema(columns: Column[], partitionKeys?: Column[]): void { - if (columns.length === 0) { - throw new Error('you must specify at least one column for the table'); - } - // Check there is at least one column and no duplicated column names or partition keys. - const names = new Set(); - (columns.concat(partitionKeys || [])).forEach(column => { - if (names.has(column.name)) { - throw new Error(`column names and partition keys must be unique, but \'${column.name}\' is duplicated`); - } - names.add(column.name); - }); -} - -// map TableEncryption to bucket's SSE configuration (s3.BucketEncryption) -const encryptionMappings = { - [TableEncryption.S3_MANAGED]: s3.BucketEncryption.S3_MANAGED, - [TableEncryption.KMS_MANAGED]: s3.BucketEncryption.KMS_MANAGED, - [TableEncryption.KMS]: s3.BucketEncryption.KMS, - [TableEncryption.CLIENT_SIDE_KMS]: s3.BucketEncryption.S3_MANAGED, -}; - -// create the bucket to store a table's data depending on the `encryption` and `encryptionKey` properties. -function createBucket(table: Table, props: TableProps) { - let bucket = props.bucket; - - if (bucket && (props.encryption !== undefined && props.encryption !== TableEncryption.CLIENT_SIDE_KMS)) { - throw new Error('you can not specify encryption settings if you also provide a bucket'); - } - - const encryption = props.encryption || TableEncryption.S3_MANAGED; - - let encryptionKey: kms.IKey | undefined; - if (encryption === TableEncryption.CLIENT_SIDE_KMS && props.encryptionKey === undefined) { - // CSE-KMS should behave the same as SSE-KMS - use the provided key or create one automatically - // Since Bucket only knows about SSE, we repeat the logic for CSE-KMS at the Table level. - encryptionKey = new kms.Key(table, 'Key'); - } else { - encryptionKey = props.encryptionKey; - } - - // create the bucket if none was provided - if (!bucket) { - if (encryption === TableEncryption.CLIENT_SIDE_KMS) { - bucket = new s3.Bucket(table, 'Bucket'); - } else { - bucket = new s3.Bucket(table, 'Bucket', { - encryption: encryptionMappings[encryption], - encryptionKey, - }); - encryptionKey = bucket.encryptionKey; - } - } - - return { - bucket, - encryption, - encryptionKey, - }; -} - -const readPermissions = [ - 'glue:BatchGetPartition', - 'glue:GetPartition', - 'glue:GetPartitions', - 'glue:GetTable', - 'glue:GetTables', - 'glue:GetTableVersion', - 'glue:GetTableVersions', -]; - -const writePermissions = [ - 'glue:BatchCreatePartition', - 'glue:BatchDeletePartition', - 'glue:CreatePartition', - 'glue:DeletePartition', - 'glue:UpdatePartition', -]; - -function renderColumns(columns?: Array) { - if (columns === undefined) { - return undefined; - } - return columns.map(column => { - return { - name: column.name, - type: column.type.inputString, - comment: column.comment, - }; - }); -} +export class Table extends S3Table {} diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 188bdf753be37..d878ca74b1eb7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { - "version": "33.0.0", + "version": "34.0.0", "files": { - "5f9f9d1c02cf84a8d8e0742f90161a825f9d867c85ac45b9dbbc4e025a539b55": { + "40e8a4e6d6f2beed80f0b9d7f9ea729ab78da8ccac9b479b28baa412b63fd2c8": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "5f9f9d1c02cf84a8d8e0742f90161a825f9d867c85ac45b9dbbc4e025a539b55.json", + "objectKey": "40e8a4e6d6f2beed80f0b9d7f9ea729ab78da8ccac9b479b28baa412b63fd2c8.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index 36ca24dfffa82..a9f4e24191c0f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -696,6 +696,70 @@ } } }, + "MyDeprecatedTableAA0364FD": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "custom_serde_table generated by CDK", + "Name": "custom_serde_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, "MyUserDC45028B": { "Type": "AWS::IAM::User" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json index 670371bb340e7..711c72d940d8a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json @@ -1,5 +1,5 @@ { - "version": "33.0.0", + "version": "34.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out index 560dae10d018f..2313ab5436501 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"33.0.0"} \ No newline at end of file +{"version":"34.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json index ab7e38c81b5c6..71d86f03a0888 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "33.0.0", + "version": "34.0.0", "testCases": { "aws-cdk-glue-table-integ/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index 78b63511c19cb..6968053d6d491 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "33.0.0", + "version": "34.0.0", "artifacts": { "aws-cdk-glue.assets": { "type": "cdk:asset-manifest", @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5f9f9d1c02cf84a8d8e0742f90161a825f9d867c85ac45b9dbbc4e025a539b55.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/40e8a4e6d6f2beed80f0b9d7f9ea729ab78da8ccac9b479b28baa412b63fd2c8.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -117,6 +117,12 @@ "data": "MyTableWithCustomLocationTable43A19D42" } ], + "/aws-cdk-glue/MyDeprecatedTable/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyDeprecatedTableAA0364FD" + } + ], "/aws-cdk-glue/MyUser/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index b4157c9205ff4..71b5c5d315659 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -951,6 +951,88 @@ "version": "0.0.0" } }, + "MyDeprecatedTable": { + "id": "MyDeprecatedTable", + "path": "aws-cdk-glue/MyDeprecatedTable", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyDeprecatedTable/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "custom_serde_table", + "description": "custom_serde_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "storageDescriptor": { + "location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Table", + "version": "0.0.0" + } + }, "MyUser": { "id": "MyUser", "path": "aws-cdk-glue/MyUser", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index 0dc3555e9f2eb..58c9387d81807 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -143,6 +143,14 @@ new glue.ExternalTable(stack, 'MyTableWithCustomLocation', { externalDataLocation: 'default_db.public.test', }); +new glue.Table(stack, 'MyDeprecatedTable', { + database, + bucket, + tableName: 'custom_serde_table', + columns, + dataFormat: glue.DataFormat.JSON, +}); + const user = new iam.User(stack, 'MyUser'); csvTable.grantReadWrite(user); encryptedTable.grantReadWrite(user); From 8062297991bc1c961b95a9785a0ca05ca6ab5760 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 4 Sep 2023 11:13:10 +0000 Subject: [PATCH 37/44] changing table name --- packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index 58c9387d81807..0ffb0e41620e6 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -146,7 +146,7 @@ new glue.ExternalTable(stack, 'MyTableWithCustomLocation', { new glue.Table(stack, 'MyDeprecatedTable', { database, bucket, - tableName: 'custom_serde_table', + tableName: 'deprecated_table', columns, dataFormat: glue.DataFormat.JSON, }); From da2f8782694565f2a3b29e1e33a0809515df72ba Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 4 Sep 2023 11:17:34 +0000 Subject: [PATCH 38/44] reverting test file --- .../test/table-deprecated.test.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table-deprecated.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table-deprecated.test.ts index 6d72682053d66..9803d72a36240 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table-deprecated.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table-deprecated.test.ts @@ -95,7 +95,7 @@ test('partitioned JSON table', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket?.encryptionKey).toEqual(undefined); + expect(table.bucket.encryptionKey).toEqual(undefined); Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { @@ -163,7 +163,7 @@ test('compressed table', () => { dataFormat: glue.DataFormat.JSON, }); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket?.encryptionKey).toEqual(undefined); + expect(table.bucket.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { @@ -246,7 +246,7 @@ test('encrypted table: SSE-S3', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket?.encryptionKey).toEqual(undefined); + expect(table.bucket.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { CatalogId: { @@ -320,7 +320,7 @@ test('encrypted table: SSE-KMS (implicitly created key)', () => { dataFormat: glue.DataFormat.JSON, }); expect(table.encryption).toEqual(glue.TableEncryption.KMS); - expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey); + expect(table.encryptionKey).toEqual(table.bucket.encryptionKey); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { Description: 'Created by Default/Table/Bucket', @@ -408,7 +408,7 @@ test('encrypted table: SSE-KMS (explicitly created key)', () => { dataFormat: glue.DataFormat.JSON, }); expect(table.encryption).toEqual(glue.TableEncryption.KMS); - expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey); + expect(table.encryptionKey).toEqual(table.bucket.encryptionKey); expect(table.encryptionKey).not.toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { @@ -494,7 +494,7 @@ test('encrypted table: SSE-KMS_MANAGED', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.KMS_MANAGED); expect(table.encryptionKey).toEqual(undefined); - expect(table.bucket?.encryptionKey).toEqual(undefined); + expect(table.bucket.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::S3::Bucket', { BucketEncryption: { @@ -569,7 +569,7 @@ test('encrypted table: CSE-KMS (implicitly created key)', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); expect(table.encryptionKey).not.toEqual(undefined); - expect(table.bucket?.encryptionKey).toEqual(undefined); + expect(table.bucket.encryptionKey).toEqual(undefined); Template.fromStack(stack).resourceCountIs('AWS::KMS::Key', 1); @@ -638,7 +638,7 @@ test('encrypted table: CSE-KMS (explicitly created key)', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); expect(table.encryptionKey).not.toEqual(undefined); - expect(table.bucket?.encryptionKey).toEqual(undefined); + expect(table.bucket.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { Description: 'MyKey', @@ -711,7 +711,7 @@ test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { }); expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS); expect(table.encryptionKey).not.toEqual(undefined); - expect(table.bucket?.encryptionKey).toEqual(undefined); + expect(table.bucket.encryptionKey).toEqual(undefined); Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', { Description: 'MyKey', @@ -1661,4 +1661,4 @@ function createTable(props: Pick Date: Mon, 4 Sep 2023 11:39:35 +0000 Subject: [PATCH 39/44] integ test --- .../test/integ.table.js.snapshot/aws-cdk-glue.assets.json | 4 ++-- .../test/integ.table.js.snapshot/aws-cdk-glue.template.json | 4 ++-- .../aws-glue-alpha/test/integ.table.js.snapshot/manifest.json | 2 +- .../aws-glue-alpha/test/integ.table.js.snapshot/tree.json | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index d878ca74b1eb7..ac741b3cb3447 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "34.0.0", "files": { - "40e8a4e6d6f2beed80f0b9d7f9ea729ab78da8ccac9b479b28baa412b63fd2c8": { + "5811f114b7ed52376bdd846d1c42ed0287d8adb0db529b9272151483a2f31ea3": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "40e8a4e6d6f2beed80f0b9d7f9ea729ab78da8ccac9b479b28baa412b63fd2c8.json", + "objectKey": "5811f114b7ed52376bdd846d1c42ed0287d8adb0db529b9272151483a2f31ea3.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index a9f4e24191c0f..7f6abb4c0051e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -706,8 +706,8 @@ "Ref": "MyDatabase1E2517DB" }, "TableInput": { - "Description": "custom_serde_table generated by CDK", - "Name": "custom_serde_table", + "Description": "deprecated_table generated by CDK", + "Name": "deprecated_table", "Parameters": { "classification": "json", "has_encrypted_data": true diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index 6968053d6d491..6692e1ac143af 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/40e8a4e6d6f2beed80f0b9d7f9ea729ab78da8ccac9b479b28baa412b63fd2c8.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5811f114b7ed52376bdd846d1c42ed0287d8adb0db529b9272151483a2f31ea3.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index 71b5c5d315659..ca212963be24a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -968,8 +968,8 @@ "Ref": "MyDatabase1E2517DB" }, "tableInput": { - "name": "custom_serde_table", - "description": "custom_serde_table generated by CDK", + "name": "deprecated_table", + "description": "deprecated_table generated by CDK", "parameters": { "classification": "json", "has_encrypted_data": true From 8e053a791dd3ec092a0233935d1fd685dc479b68 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 6 Sep 2023 10:17:37 +0000 Subject: [PATCH 40/44] removing location prop, keeping this for separate PR --- packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts | 8 +------- packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 8 +------- packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts | 1 - 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts index 0c61e9df40511..ee6376797c4c7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts @@ -42,11 +42,6 @@ export class ExternalTable extends TableBase { */ public readonly connection: IConnection; - /** - * The location of the tables' data. - */ - public readonly location: string; - /** * This table's partition indexes. */ @@ -56,7 +51,6 @@ export class ExternalTable extends TableBase { constructor(scope: Construct, id: string, props: ExternalTableProps) { super(scope, id, props); - this.location = props.externalDataLocation; this.connection = props.connection; this.tableResource = new CfnTable(this, 'Table', { catalogId: props.database.catalogId, @@ -76,7 +70,7 @@ export class ExternalTable extends TableBase { 'connectionName': props.connection.connectionName, }, storageDescriptor: { - location: this.location, + location: props.externalDataLocation, compressed: this.compressed, storedAsSubDirectories: props.storedAsSubDirectories ?? false, columns: renderColumns(props.columns), diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts index b57d7cb40c0b7..c476fa84660b7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -110,11 +110,6 @@ export class S3Table extends TableBase { */ public readonly encryptionKey?: kms.IKey; - /** - * The location of the tables' data. - */ - public readonly location: string; - /** * This table's partition indexes. */ @@ -129,7 +124,6 @@ export class S3Table extends TableBase { this.bucket = bucket; this.encryption = encryption; this.encryptionKey = encryptionKey; - this.location = `s3://${bucket.bucketName}/${this.s3Prefix}`; this.tableResource = new CfnTable(this, 'Table', { catalogId: props.database.catalogId, @@ -148,7 +142,7 @@ export class S3Table extends TableBase { 'partition_filtering.enabled': props.enablePartitionFiltering, }, storageDescriptor: { - location: this.location, + location: `s3://${this.bucket.bucketName}/${this.s3Prefix}`, compressed: this.compressed, storedAsSubDirectories: props.storedAsSubDirectories ?? false, columns: renderColumns(props.columns), diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts index 080dfc6af0ea0..b77165170ae93 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts @@ -182,7 +182,6 @@ export abstract class TableBase extends Resource implements ITable { protected abstract readonly tableResource: CfnTable; public abstract readonly tableName: string; public abstract readonly tableArn: string; - public abstract readonly location: string; public abstract readonly partitionIndexes?: PartitionIndex[]; /** From 4ebc18d1454b83b6e126f5a790d927d5a816428c Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 6 Sep 2023 10:29:30 +0000 Subject: [PATCH 41/44] the english language --- packages/@aws-cdk/aws-glue-alpha/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 589084bcd7093..cbbea0b1dbf84 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -359,7 +359,7 @@ new glue.S3Table(this, 'MyTable', { Glue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog. -Whereas `S3Table` will point to (and if needed, create) a bucket to store the table's data, `ExternalTable` will point to an existing table in a data source. For example, to create a table in Glue that points to a table in Redshift: +Whereas `S3Table` will point to (and if needed, create) a bucket to store the tables' data, `ExternalTable` will point to an existing table in a data source. For example, to create a table in Glue that points to a table in Redshift: ```ts declare const myConnection: glue.Connection; From f0c455b4983bfd0f06a4f2b9deba56d327264261 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 11 Sep 2023 11:26:11 +0000 Subject: [PATCH 42/44] splitting external table into own test file --- .../test/integ.external-table.ts | 57 +++++++++++++++++++ .../aws-glue-alpha/test/integ.table.ts | 19 ------- 2 files changed, 57 insertions(+), 19 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.ts diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.ts new file mode 100644 index 0000000000000..f451a671fe172 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.ts @@ -0,0 +1,57 @@ +import * as integ from '@aws-cdk/integ-tests-alpha'; +import * as cdk from 'aws-cdk-lib'; +import * as glue from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-glue'); + +const database = new glue.Database(stack, 'MyDatabase', { + databaseName: 'my_database', +}); + +const connection = new glue.Connection(stack, 'MyConnection', { + connectionName: 'my_connection', + type: glue.ConnectionType.JDBC, + properties: { + JDBC_CONNECTION_URL: 'jdbc:mysql://mysql.example.com:3306', + USERNAME: 'username', + PASSWORD: 'password', + }, +}); + +const columns = [{ + name: 'col1', + type: glue.Schema.STRING, +}, { + name: 'col2', + type: glue.Schema.STRING, + comment: 'col2 comment', +}, { + name: 'col3', + type: glue.Schema.array(glue.Schema.STRING), +}, { + name: 'col4', + type: glue.Schema.map(glue.Schema.STRING, glue.Schema.STRING), +}, { + name: 'col5', + type: glue.Schema.struct([{ + name: 'col1', + type: glue.Schema.STRING, + }]), +}]; + +new glue.ExternalTable(stack, 'MyTableWithCustomLocation', { + database, + connection, + tableName: 'custom_location_table', + columns, + dataFormat: glue.DataFormat.JSON, + externalDataLocation: 'default_db.public.test', +}); + +new integ.IntegTest(app, 'aws-cdk-glue-table-integ', { + testCases: [stack], +}); + +app.synth(); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index 0ffb0e41620e6..7c175b5bfa3bc 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -14,16 +14,6 @@ const bucket = new s3.Bucket(stack, 'DataBucket', { removalPolicy: cdk.RemovalPolicy.DESTROY, }); -const connection = new glue.Connection(stack, 'MyConnection', { - connectionName: 'my_connection', - type: glue.ConnectionType.JDBC, - properties: { - JDBC_CONNECTION_URL: 'jdbc:mysql://mysql.example.com:3306', - USERNAME: 'username', - PASSWORD: 'password', - }, -}); - const database = new glue.Database(stack, 'MyDatabase', { databaseName: 'my_database', }); @@ -134,15 +124,6 @@ new glue.S3Table(stack, 'MyTableWithStorageDescriptorParameters', { ], }); -new glue.ExternalTable(stack, 'MyTableWithCustomLocation', { - database, - connection, - tableName: 'custom_location_table', - columns, - dataFormat: glue.DataFormat.JSON, - externalDataLocation: 'default_db.public.test', -}); - new glue.Table(stack, 'MyDeprecatedTable', { database, bucket, From 45ea77c1ac07da898431790efde694f39170cd22 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 11 Sep 2023 11:51:01 +0000 Subject: [PATCH 43/44] compat issues, TableProps is now used for deprecated table --- packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts | 4 ++-- packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts | 6 +++--- packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts | 4 ++-- packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts | 4 +++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts index ee6376797c4c7..cca62739a9bcc 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/external-table.ts @@ -3,9 +3,9 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import { Construct } from 'constructs'; import { IConnection } from './connection'; import { Column } from './schema'; -import { PartitionIndex, TableBase, TableProps } from './table-base'; +import { PartitionIndex, TableBase, TableBaseProps } from './table-base'; -export interface ExternalTableProps extends TableProps { +export interface ExternalTableProps extends TableBaseProps { /** * The connection the table will use when performing reads and writes. * diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts index c476fa84660b7..ccb8c9d87bd6c 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/s3-table.ts @@ -4,7 +4,7 @@ import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; import { Construct } from 'constructs'; import { Column } from './schema'; -import { PartitionIndex, TableBase, TableProps } from './table-base'; +import { PartitionIndex, TableBase, TableBaseProps } from './table-base'; /** * Encryption options for a Table. @@ -39,7 +39,7 @@ export enum TableEncryption { CLIENT_SIDE_KMS = 'CSE-KMS' } -export interface S3TableProps extends TableProps { +export interface S3TableProps extends TableBaseProps { /** * S3 bucket in which to store data. * @@ -103,7 +103,7 @@ export class S3Table extends TableBase { /** * The type of encryption enabled for the table. */ - public readonly encryption?: TableEncryption; + public readonly encryption: TableEncryption; /** * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`. diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts index b77165170ae93..e77875c6c75e3 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-base.ts @@ -44,7 +44,7 @@ export interface TableAttributes { readonly tableName: string; } -export interface TableProps { +export interface TableBaseProps { /** * Name of the table. * @@ -221,7 +221,7 @@ export abstract class TableBase extends Resource implements ITable { */ private partitionIndexCustomResources: AwsCustomResource[] = []; - constructor(scope: Construct, id: string, props: TableProps) { + constructor(scope: Construct, id: string, props: TableBaseProps) { super(scope, id, { physicalName: props.tableName ?? Lazy.string({ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts index 095a0f39b6634..cf3ad58b492ae 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table-deprecated.ts @@ -1,4 +1,6 @@ -import { S3Table } from './s3-table'; +import { S3Table, S3TableProps } from './s3-table'; + +export interface TableProps extends S3TableProps {} /** * A Glue table. From 447acb9cec15abefa5fefb69759c0322c2608ec1 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 11 Sep 2023 11:58:48 +0000 Subject: [PATCH 44/44] integ test --- .../aws-cdk-glue.assets.json | 19 ++ .../aws-cdk-glue.template.json | 122 +++++++++ ...efaultTestDeployAssert8BFB5B70.assets.json | 19 ++ ...aultTestDeployAssert8BFB5B70.template.json | 36 +++ .../integ.external-table.js.snapshot/cdk.out | 1 + .../integ.json | 12 + .../manifest.json | 123 +++++++++ .../tree.json | 239 ++++++++++++++++++ .../aws-cdk-glue.assets.json | 4 +- .../aws-cdk-glue.template.json | 73 ------ .../integ.table.js.snapshot/manifest.json | 32 ++- .../test/integ.table.js.snapshot/tree.json | 113 +-------- 12 files changed, 594 insertions(+), 199 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.assets.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.template.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/cdk.out create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/integ.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/manifest.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/tree.json diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.assets.json new file mode 100644 index 0000000000000..61f1993482c17 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.assets.json @@ -0,0 +1,19 @@ +{ + "version": "34.0.0", + "files": { + "84e0753dbcfc1fd4c21499f0bad1d34eee7e6a23678af76661541de677da38e8": { + "source": { + "path": "aws-cdk-glue.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "84e0753dbcfc1fd4c21499f0bad1d34eee7e6a23678af76661541de677da38e8.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.template.json new file mode 100644 index 0000000000000..c6316d12d1455 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/aws-cdk-glue.template.json @@ -0,0 +1,122 @@ +{ + "Resources": { + "MyDatabase1E2517DB": { + "Type": "AWS::Glue::Database", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseInput": { + "Name": "my_database" + } + } + }, + "MyConnection5621880D": { + "Type": "AWS::Glue::Connection", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "ConnectionInput": { + "ConnectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "ConnectionType": "JDBC", + "Name": "my_connection" + } + } + }, + "MyTableWithCustomLocationTable43A19D42": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "custom_location_table generated by CDK", + "Name": "custom_location_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true, + "connectionName": { + "Ref": "MyConnection5621880D" + } + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": "default_db.public.test", + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json new file mode 100644 index 0000000000000..711c72d940d8a --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json @@ -0,0 +1,19 @@ +{ + "version": "34.0.0", + "files": { + "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { + "source": { + "path": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json new file mode 100644 index 0000000000000..ad9d0fb73d1dd --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json @@ -0,0 +1,36 @@ +{ + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/cdk.out new file mode 100644 index 0000000000000..2313ab5436501 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"34.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/integ.json new file mode 100644 index 0000000000000..71d86f03a0888 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/integ.json @@ -0,0 +1,12 @@ +{ + "version": "34.0.0", + "testCases": { + "aws-cdk-glue-table-integ/DefaultTest": { + "stacks": [ + "aws-cdk-glue" + ], + "assertionStack": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "assertionStackName": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/manifest.json new file mode 100644 index 0000000000000..20944a5cf59ef --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/manifest.json @@ -0,0 +1,123 @@ +{ + "version": "34.0.0", + "artifacts": { + "aws-cdk-glue.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "aws-cdk-glue.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "aws-cdk-glue": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "aws-cdk-glue.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/84e0753dbcfc1fd4c21499f0bad1d34eee7e6a23678af76661541de677da38e8.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "aws-cdk-glue.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "aws-cdk-glue.assets" + ], + "metadata": { + "/aws-cdk-glue/MyDatabase/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyDatabase1E2517DB" + } + ], + "/aws-cdk-glue/MyConnection/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyConnection5621880D" + } + ], + "/aws-cdk-glue/MyTableWithCustomLocation/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithCustomLocationTable43A19D42" + } + ], + "/aws-cdk-glue/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-cdk-glue/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-cdk-glue" + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "metadata": { + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/tree.json new file mode 100644 index 0000000000000..9b7fae3e761ec --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.external-table.js.snapshot/tree.json @@ -0,0 +1,239 @@ +{ + "version": "tree-0.1", + "tree": { + "id": "App", + "path": "", + "children": { + "aws-cdk-glue": { + "id": "aws-cdk-glue", + "path": "aws-cdk-glue", + "children": { + "MyDatabase": { + "id": "MyDatabase", + "path": "aws-cdk-glue/MyDatabase", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue/MyDatabase/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Database", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseInput": { + "name": "my_database" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnDatabase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Database", + "version": "0.0.0" + } + }, + "MyConnection": { + "id": "MyConnection", + "path": "aws-cdk-glue/MyConnection", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue/MyConnection/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Connection", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "connectionInput": { + "connectionProperties": { + "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", + "USERNAME": "username", + "PASSWORD": "password" + }, + "connectionType": "JDBC", + "name": "my_connection" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnConnection", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Connection", + "version": "0.0.0" + } + }, + "MyTableWithCustomLocation": { + "id": "MyTableWithCustomLocation", + "path": "aws-cdk-glue/MyTableWithCustomLocation", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyTableWithCustomLocation/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "custom_location_table", + "description": "custom_location_table generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": true, + "connectionName": { + "Ref": "MyConnection5621880D" + } + }, + "storageDescriptor": { + "location": "default_db.public.test", + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.ExternalTable", + "version": "0.0.0" + } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + }, + "aws-cdk-glue-table-integ": { + "id": "aws-cdk-glue-table-integ", + "path": "aws-cdk-glue-table-integ", + "children": { + "DefaultTest": { + "id": "DefaultTest", + "path": "aws-cdk-glue-table-integ/DefaultTest", + "children": { + "Default": { + "id": "Default", + "path": "aws-cdk-glue-table-integ/DefaultTest/Default", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.70" + } + }, + "DeployAssert": { + "id": "DeployAssert", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "children": { + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", + "version": "0.0.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.70" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index ac741b3cb3447..e08d0476521bf 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "34.0.0", "files": { - "5811f114b7ed52376bdd846d1c42ed0287d8adb0db529b9272151483a2f31ea3": { + "7a0cca4ed21fb1c6d9b050e5fd7c8d857b13af8ef7b8cead40cd08d2e25fc892": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "5811f114b7ed52376bdd846d1c42ed0287d8adb0db529b9272151483a2f31ea3.json", + "objectKey": "7a0cca4ed21fb1c6d9b050e5fd7c8d857b13af8ef7b8cead40cd08d2e25fc892.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index 7f6abb4c0051e..75020f0d007ad 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -5,23 +5,6 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "MyConnection5621880D": { - "Type": "AWS::Glue::Connection", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "ConnectionInput": { - "ConnectionProperties": { - "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", - "USERNAME": "username", - "PASSWORD": "password" - }, - "ConnectionType": "JDBC", - "Name": "my_connection" - } - } - }, "MyDatabase1E2517DB": { "Type": "AWS::Glue::Database", "Properties": { @@ -640,62 +623,6 @@ } } }, - "MyTableWithCustomLocationTable43A19D42": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "custom_location_table generated by CDK", - "Name": "custom_location_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true, - "connectionName": { - "Ref": "MyConnection5621880D" - } - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": "default_db.public.test", - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } - } - }, "MyDeprecatedTableAA0364FD": { "Type": "AWS::Glue::Table", "Properties": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index 6692e1ac143af..d511e7b0795fb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5811f114b7ed52376bdd846d1c42ed0287d8adb0db529b9272151483a2f31ea3.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/7a0cca4ed21fb1c6d9b050e5fd7c8d857b13af8ef7b8cead40cd08d2e25fc892.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -39,12 +39,6 @@ "data": "DataBucketE3889A50" } ], - "/aws-cdk-glue/MyConnection/Resource": [ - { - "type": "aws:cdk:logicalId", - "data": "MyConnection5621880D" - } - ], "/aws-cdk-glue/MyDatabase/Resource": [ { "type": "aws:cdk:logicalId", @@ -111,12 +105,6 @@ "data": "MyTableWithStorageDescriptorParametersTable1A347345" } ], - "/aws-cdk-glue/MyTableWithCustomLocation/Table": [ - { - "type": "aws:cdk:logicalId", - "data": "MyTableWithCustomLocationTable43A19D42" - } - ], "/aws-cdk-glue/MyDeprecatedTable/Table": [ { "type": "aws:cdk:logicalId", @@ -158,6 +146,24 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } + ], + "MyConnection5621880D": [ + { + "type": "aws:cdk:logicalId", + "data": "MyConnection5621880D", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "MyTableWithCustomLocationTable43A19D42": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithCustomLocationTable43A19D42", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } ] }, "displayName": "aws-cdk-glue" diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index ca212963be24a..ca372333c5ca8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -30,41 +30,6 @@ "version": "0.0.0" } }, - "MyConnection": { - "id": "MyConnection", - "path": "aws-cdk-glue/MyConnection", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-cdk-glue/MyConnection/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Connection", - "aws:cdk:cloudformation:props": { - "catalogId": { - "Ref": "AWS::AccountId" - }, - "connectionInput": { - "connectionProperties": { - "JDBC_CONNECTION_URL": "jdbc:mysql://mysql.example.com:3306", - "USERNAME": "username", - "PASSWORD": "password" - }, - "connectionType": "JDBC", - "name": "my_connection" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnConnection", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.Connection", - "version": "0.0.0" - } - }, "MyDatabase": { "id": "MyDatabase", "path": "aws-cdk-glue/MyDatabase", @@ -877,80 +842,6 @@ "version": "0.0.0" } }, - "MyTableWithCustomLocation": { - "id": "MyTableWithCustomLocation", - "path": "aws-cdk-glue/MyTableWithCustomLocation", - "children": { - "Table": { - "id": "Table", - "path": "aws-cdk-glue/MyTableWithCustomLocation/Table", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Table", - "aws:cdk:cloudformation:props": { - "catalogId": { - "Ref": "AWS::AccountId" - }, - "databaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "tableInput": { - "name": "custom_location_table", - "description": "custom_location_table generated by CDK", - "parameters": { - "classification": "json", - "has_encrypted_data": true, - "connectionName": { - "Ref": "MyConnection5621880D" - } - }, - "storageDescriptor": { - "location": "default_db.public.test", - "compressed": false, - "storedAsSubDirectories": false, - "columns": [ - { - "name": "col1", - "type": "string" - }, - { - "name": "col2", - "type": "string", - "comment": "col2 comment" - }, - { - "name": "col3", - "type": "array" - }, - { - "name": "col4", - "type": "map" - }, - { - "name": "col5", - "type": "struct" - } - ], - "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "serdeInfo": { - "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - } - }, - "tableType": "EXTERNAL_TABLE" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnTable", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.ExternalTable", - "version": "0.0.0" - } - }, "MyDeprecatedTable": { "id": "MyDeprecatedTable", "path": "aws-cdk-glue/MyDeprecatedTable", @@ -1463,7 +1354,7 @@ "path": "aws-cdk-glue-table-integ/DefaultTest/Default", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.69" + "version": "10.2.70" } }, "DeployAssert": { @@ -1509,7 +1400,7 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.69" + "version": "10.2.70" } } },