From ead3104a2010ca938c461040f4fa4c1e131bd4c0 Mon Sep 17 00:00:00 2001 From: Janardhan Molumuri Date: Wed, 9 Aug 2023 09:28:15 -0700 Subject: [PATCH] Skeleton code structure for Glue L2 construct --- .../aws-glue-alpha/lib/conditionaltrigger.ts | 7 +++++++ packages/@aws-cdk/aws-glue-alpha/lib/constants.ts | 4 ++++ packages/@aws-cdk/aws-glue-alpha/lib/flexjob.ts | 13 +++++++++++++ .../aws-glue-alpha/lib/notifyeventtrigger.ts | 10 ++++++++++ .../@aws-cdk/aws-glue-alpha/lib/ondemandtrigger.ts | 8 ++++++++ .../@aws-cdk/aws-glue-alpha/lib/pythonshelljob.ts | 8 ++++++++ packages/@aws-cdk/aws-glue-alpha/lib/rayjob.ts | 7 +++++++ .../@aws-cdk/aws-glue-alpha/lib/scheduledtrigger.ts | 12 ++++++++++++ packages/@aws-cdk/aws-glue-alpha/lib/sparketljob.ts | 10 ++++++++++ .../@aws-cdk/aws-glue-alpha/lib/streamingjob.ts | 13 +++++++++++++ packages/@aws-cdk/aws-glue-alpha/lib/trigger.ts | 7 +++++++ 11 files changed, 99 insertions(+) create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/conditionaltrigger.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/constants.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/flexjob.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/notifyeventtrigger.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/ondemandtrigger.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/pythonshelljob.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/rayjob.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/scheduledtrigger.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/sparketljob.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/streamingjob.ts create mode 100644 packages/@aws-cdk/aws-glue-alpha/lib/trigger.ts diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/conditionaltrigger.ts b/packages/@aws-cdk/aws-glue-alpha/lib/conditionaltrigger.ts new file mode 100644 index 0000000000000..f19c954f1c479 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/conditionaltrigger.ts @@ -0,0 +1,7 @@ +/** + * Conditional Trigger Class + * + * Conditional triggers have a predicate and actions associated with them. + * When the predicateCondition is true, the trigger actions will be executed. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/constants.ts b/packages/@aws-cdk/aws-glue-alpha/lib/constants.ts new file mode 100644 index 0000000000000..36ac8bef38776 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/constants.ts @@ -0,0 +1,4 @@ +/** + * Enums used by Job Types/Triggers + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/flexjob.ts b/packages/@aws-cdk/aws-glue-alpha/lib/flexjob.ts new file mode 100644 index 0000000000000..f9c25e74c64ff --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/flexjob.ts @@ -0,0 +1,13 @@ +/** + * Flex Jobs class + * + * Flex jobs supports Python and Scala language. + * The flexible execution class is appropriate for non-urgent jobs such as + * pre-production jobs, testing, and one-time data loads. + * Flexible job runs are supported for jobs using AWS Glue version 3.0 or later and G.1X or + * G.2X worker types but will default to the latest version of Glue (currently Glue 3.0.) + * + * Similar to ETL, we’ll enable these features: —enable-metrics, —enable-spark-ui, + * —enable-continuous-cloudwatch-log + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/notifyeventtrigger.ts b/packages/@aws-cdk/aws-glue-alpha/lib/notifyeventtrigger.ts new file mode 100644 index 0000000000000..5fa916cf4ed19 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/notifyeventtrigger.ts @@ -0,0 +1,10 @@ +/** + * Notify Event Trigger Class + * + * Workflows are mandatory for this trigger type. There are two types of notify event triggers, + * batching and non-batching trigger. + * For batching triggers, developers must specify BatchSize but for non-batching BatchSize will + * be set to 1. + * For both triggers, BatchWindow will be default to 900 seconds. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/ondemandtrigger.ts b/packages/@aws-cdk/aws-glue-alpha/lib/ondemandtrigger.ts new file mode 100644 index 0000000000000..b4209538a9a97 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/ondemandtrigger.ts @@ -0,0 +1,8 @@ +/** + * On Demand Trigger Class + * + * On demand triggers can start glue jobs or crawlers. + * The trigger method will take an optional description but abstract the requirement of an + * actions list using the job or crawler objects using conditional types. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/pythonshelljob.ts b/packages/@aws-cdk/aws-glue-alpha/lib/pythonshelljob.ts new file mode 100644 index 0000000000000..ff3d74a63ff9a --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/pythonshelljob.ts @@ -0,0 +1,8 @@ +/** + * Python Shell Jobs class + * + * A Python shell job runs Python scripts as a shell and supports a Python version that + * depends on the AWS Glue version you are using. + * This can be used to schedule and run tasks that don't require an Apache Spark environment. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/rayjob.ts b/packages/@aws-cdk/aws-glue-alpha/lib/rayjob.ts new file mode 100644 index 0000000000000..3bcff4328ba74 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/rayjob.ts @@ -0,0 +1,7 @@ +/** + * Ray Jobs class + * + * Glue ray only supports worker type Z.2X and Glue version 4.0. + * Runtime will default to Ray2.3 and min workers will default to 3. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/scheduledtrigger.ts b/packages/@aws-cdk/aws-glue-alpha/lib/scheduledtrigger.ts new file mode 100644 index 0000000000000..1abcaf030ecd7 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/scheduledtrigger.ts @@ -0,0 +1,12 @@ +/** + * Scheduled Trigger Base Class + * + * Schedule triggers are a way for developers to create jobs using cron expressions. + * We’ll provide daily, weekly, and monthly convenience functions, as well as a custom function + * that will allow developers to create their own custom timing using the existing + * event Schedule object without having to build their own cron expressions. + * + * The trigger method will take an optional description and list of Actions + * which can refer to Jobs or crawlers via conditional types. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/sparketljob.ts b/packages/@aws-cdk/aws-glue-alpha/lib/sparketljob.ts new file mode 100644 index 0000000000000..9f2d02e6d6677 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/sparketljob.ts @@ -0,0 +1,10 @@ +/** + * Spark ETL Jobs class + * + * ETL jobs supports Python and Scala language. + * ETL job type supports G1, G2, G4 and G8 worker type default as G2, which customer can override. + * It wil default to the best practice version of ETL 4.0, but allow developers to override to 3.0. + * We will also default to best practice enablement the following ETL features: + * —enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/streamingjob.ts b/packages/@aws-cdk/aws-glue-alpha/lib/streamingjob.ts new file mode 100644 index 0000000000000..5ff520e088eb4 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/streamingjob.ts @@ -0,0 +1,13 @@ +/** + * Streaming Jobs class + * + * A Streaming job is similar to an ETL job, except that it performs ETL on data streams + * using the Apache Spark Structured Streaming framework. + * These jobs will default to use Python 3.9. + * + * Similar to ETL jobs, streaming job supports Scala and Python languages. Similar to ETL, + * it supports G1 and G2 worker type and 2.0, 3.0 and 4.0 version. We’ll default to G2 worker + * and 4.0 version for streaming jobs which developers can override. + * We will enable —enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log. + * + */ diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/trigger.ts b/packages/@aws-cdk/aws-glue-alpha/lib/trigger.ts new file mode 100644 index 0000000000000..d2dd5e22c02b3 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/trigger.ts @@ -0,0 +1,7 @@ +/** + * Workflow Trigger Base Class + * + * In AWS Glue, developers can use workflows to create and visualize complex extract, + * transform, and load (ETL) activities involving multiple crawlers, jobs, and triggers. + * + */