From b6a5f340c63317ffccc7bb7d21a0e16758e592a1 Mon Sep 17 00:00:00 2001 From: chie8842 Date: Wed, 10 Aug 2022 06:49:31 +0000 Subject: [PATCH 01/13] add execution_class option at glue/job.go --- internal/service/glue/job.go | 5 ++++ internal/service/glue/job_test.go | 50 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/internal/service/glue/job.go b/internal/service/glue/job.go index 465364e137cc..77367b1d92b8 100644 --- a/internal/service/glue/job.go +++ b/internal/service/glue/job.go @@ -153,6 +153,11 @@ func ResourceJob() *schema.Resource { ConflictsWith: []string{"max_capacity"}, ValidateFunc: validation.IntAtLeast(2), }, + "execution_class": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice([]string{"FLEX", "STANDARD"}, true), + }, "non_overridable_arguments": { Type: schema.TypeMap, Optional: true, diff --git a/internal/service/glue/job_test.go b/internal/service/glue/job_test.go index 4eef24bcf5d1..1596164e1619 100644 --- a/internal/service/glue/job_test.go +++ b/internal/service/glue/job_test.go @@ -282,6 +282,40 @@ func TestAccGlueJob_glueVersion(t *testing.T) { }) } +func TestAccGlueJob_executionClass(t *testing.T) { + var job glue.Job + + rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + resourceName := "aws_glue_job.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(t) }, + ErrorCheck: acctest.ErrorCheck(t, glue.EndpointsID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckJobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccJobConfig_executionClass(rName, "FLEX"), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "execution_class", "FLEX"), + ), + }, + { + Config: testAccJobConfig_executionClass(rName, "STANDARD"), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "execution_class", "STANDARD"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} func TestAccGlueJob_executionProperty(t *testing.T) { var job glue.Job @@ -933,6 +967,22 @@ resource "aws_glue_job" "test" { `, testAccJobConfig_Base(rName), glueVersion, rName) } +func testAccJobConfig_executionClass(rName, executionClass string) string { + return fmt.Sprintf(` +%s +resource "aws_glue_job" "test" { + execution_class = "%s" + max_capacity = 10 + name = "%s" + role_arn = aws_iam_role.test.arn + command { + script_location = "testscriptlocation" + } + depends_on = [aws_iam_role_policy_attachment.test] +} +`, testAccJobConfig_Base(rName), executionClass, rName) +} + func testAccJobConfig_executionProperty(rName string, maxConcurrentRuns int) string { return fmt.Sprintf(` %s From c772d80470d9be59ffe6319be6e1d53984741146 Mon Sep 17 00:00:00 2001 From: chie8842 Date: Tue, 9 Aug 2022 13:56:12 +0000 Subject: [PATCH 02/13] add documentation about execution_class --- website/docs/r/glue_job.html.markdown | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/r/glue_job.html.markdown b/website/docs/r/glue_job.html.markdown index cf7c7365e5cd..5b8a82b773eb 100644 --- a/website/docs/r/glue_job.html.markdown +++ b/website/docs/r/glue_job.html.markdown @@ -90,6 +90,7 @@ The following arguments are supported: * `description` – (Optional) Description of the job. * `execution_property` – (Optional) Execution property of the job. Defined below. * `glue_version` - (Optional) The version of glue to use, for example "1.0". For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html). +* `execution_class` - (Optional) Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. * `max_capacity` – (Optional) The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `number_of_workers` and `worker_type` arguments instead with `glue_version` `2.0` and above. * `max_retries` – (Optional) The maximum number of times to retry this job if it fails. * `name` – (Required) The name you assign to this job. It must be unique in your account. From 60e57b8b5d07771c3c38cf75abd0bbd0eed186bc Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 09:42:52 -0400 Subject: [PATCH 03/13] Fix terrafmt error. --- internal/service/glue/job_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/service/glue/job_test.go b/internal/service/glue/job_test.go index 1596164e1619..8fa7719bf57b 100644 --- a/internal/service/glue/job_test.go +++ b/internal/service/glue/job_test.go @@ -968,19 +968,20 @@ resource "aws_glue_job" "test" { } func testAccJobConfig_executionClass(rName, executionClass string) string { - return fmt.Sprintf(` -%s + return acctest.ConfigCompose(testAccJobConfig_Base(rName), fmt.Sprintf(` resource "aws_glue_job" "test" { - execution_class = "%s" - max_capacity = 10 - name = "%s" - role_arn = aws_iam_role.test.arn + execution_class = %[2]q + max_capacity = 10 + name = %[1]q + role_arn = aws_iam_role.test.arn + command { script_location = "testscriptlocation" } + depends_on = [aws_iam_role_policy_attachment.test] } -`, testAccJobConfig_Base(rName), executionClass, rName) +`, rName, executionClass)) } func testAccJobConfig_executionProperty(rName string, maxConcurrentRuns int) string { From bf198d4504e80517db35e481df53cc2a87b16893 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 09:45:11 -0400 Subject: [PATCH 04/13] Add CHANGELOG entry. --- .changelog/26188.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/26188.txt diff --git a/.changelog/26188.txt b/.changelog/26188.txt new file mode 100644 index 000000000000..4dd8abe3d8db --- /dev/null +++ b/.changelog/26188.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +resource/aws_glue_job: Add `execution_class` argument +``` \ No newline at end of file From 70f9eee555dd53a8dbda8dff2beb66f727d4bacd Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 09:47:55 -0400 Subject: [PATCH 05/13] r/aws_glue_job: Alphabetize attributes. --- internal/service/glue/job.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/service/glue/job.go b/internal/service/glue/job.go index 77367b1d92b8..147488962a62 100644 --- a/internal/service/glue/job.go +++ b/internal/service/glue/job.go @@ -76,6 +76,11 @@ func ResourceJob() *schema.Resource { Optional: true, Computed: true, }, + "execution_class": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice(glue.ExecutionClass_Values(), true), + }, "execution_property": { Type: schema.TypeList, Optional: true, @@ -109,6 +114,11 @@ func ResourceJob() *schema.Resource { ForceNew: true, ValidateFunc: validation.NoZeroValues, }, + "non_overridable_arguments": { + Type: schema.TypeMap, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, "notification_property": { Type: schema.TypeList, Optional: true, @@ -124,6 +134,12 @@ func ResourceJob() *schema.Resource { }, }, }, + "number_of_workers": { + Type: schema.TypeInt, + Optional: true, + ConflictsWith: []string{"max_capacity"}, + ValidateFunc: validation.IntAtLeast(2), + }, "role_arn": { Type: schema.TypeString, Required: true, @@ -147,22 +163,6 @@ func ResourceJob() *schema.Resource { ConflictsWith: []string{"max_capacity"}, ValidateFunc: validation.StringInSlice(glue.WorkerType_Values(), false), }, - "number_of_workers": { - Type: schema.TypeInt, - Optional: true, - ConflictsWith: []string{"max_capacity"}, - ValidateFunc: validation.IntAtLeast(2), - }, - "execution_class": { - Type: schema.TypeString, - Optional: true, - ValidateFunc: validation.StringInSlice([]string{"FLEX", "STANDARD"}, true), - }, - "non_overridable_arguments": { - Type: schema.TypeMap, - Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, - }, }, } } @@ -171,8 +171,8 @@ func resourceJobCreate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*conns.AWSClient).GlueConn defaultTagsConfig := meta.(*conns.AWSClient).DefaultTagsConfig tags := defaultTagsConfig.MergeTags(tftags.New(d.Get("tags").(map[string]interface{}))) - name := d.Get("name").(string) + name := d.Get("name").(string) input := &glue.CreateJobInput{ Command: expandJobCommand(d.Get("command").([]interface{})), Name: aws.String(name), From 950c92a3187635eaf3ce322dcb9f2e235d89aa7d Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 09:49:37 -0400 Subject: [PATCH 06/13] r/aws_glue_job: Tweak documentation. --- website/docs/r/glue_job.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/r/glue_job.html.markdown b/website/docs/r/glue_job.html.markdown index 5b8a82b773eb..8723c2a9f817 100644 --- a/website/docs/r/glue_job.html.markdown +++ b/website/docs/r/glue_job.html.markdown @@ -90,7 +90,7 @@ The following arguments are supported: * `description` – (Optional) Description of the job. * `execution_property` – (Optional) Execution property of the job. Defined below. * `glue_version` - (Optional) The version of glue to use, for example "1.0". For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html). -* `execution_class` - (Optional) Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. +* `execution_class` - (Optional) Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`. * `max_capacity` – (Optional) The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `number_of_workers` and `worker_type` arguments instead with `glue_version` `2.0` and above. * `max_retries` – (Optional) The maximum number of times to retry this job if it fails. * `name` – (Required) The name you assign to this job. It must be unique in your account. From d34e60317e5dca700175c9e108a8072be8d11be3 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 10:01:33 -0400 Subject: [PATCH 07/13] r/aws_glue_job: Set ExecutionClass during resource Create. --- internal/service/glue/job.go | 79 ++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/internal/service/glue/job.go b/internal/service/glue/job.go index 147488962a62..33487d870117 100644 --- a/internal/service/glue/job.go +++ b/internal/service/glue/job.go @@ -180,67 +180,72 @@ func resourceJobCreate(d *schema.ResourceData, meta interface{}) error { Tags: Tags(tags.IgnoreAWS()), } - if v, ok := d.GetOk("timeout"); ok { - input.Timeout = aws.Int64(int64(v.(int))) - } - - if v, ok := d.GetOk("max_capacity"); ok { - input.MaxCapacity = aws.Float64(v.(float64)) - } - if v, ok := d.GetOk("connections"); ok { input.Connections = &glue.ConnectionsList{ Connections: flex.ExpandStringList(v.([]interface{})), } } - if kv, ok := d.GetOk("default_arguments"); ok { - input.DefaultArguments = flex.ExpandStringMap(kv.(map[string]interface{})) - } - - if kv, ok := d.GetOk("non_overridable_arguments"); ok { - input.NonOverridableArguments = flex.ExpandStringMap(kv.(map[string]interface{})) + if v, ok := d.GetOk("default_arguments"); ok { + input.DefaultArguments = flex.ExpandStringMap(v.(map[string]interface{})) } if v, ok := d.GetOk("description"); ok { input.Description = aws.String(v.(string)) } - if v, ok := d.GetOk("glue_version"); ok { - input.GlueVersion = aws.String(v.(string)) + if v, ok := d.GetOk("execution_class"); ok { + input.ExecutionClass = aws.String(v.(string)) } if v, ok := d.GetOk("execution_property"); ok { input.ExecutionProperty = expandExecutionProperty(v.([]interface{})) } + if v, ok := d.GetOk("glue_version"); ok { + input.GlueVersion = aws.String(v.(string)) + } + + if v, ok := d.GetOk("max_capacity"); ok { + input.MaxCapacity = aws.Float64(v.(float64)) + } + if v, ok := d.GetOk("max_retries"); ok { input.MaxRetries = aws.Int64(int64(v.(int))) } + if v, ok := d.GetOk("non_overridable_arguments"); ok { + input.NonOverridableArguments = flex.ExpandStringMap(v.(map[string]interface{})) + } + if v, ok := d.GetOk("notification_property"); ok { input.NotificationProperty = expandNotificationProperty(v.([]interface{})) } + if v, ok := d.GetOk("number_of_workers"); ok { + input.NumberOfWorkers = aws.Int64(int64(v.(int))) + } + if v, ok := d.GetOk("security_configuration"); ok { input.SecurityConfiguration = aws.String(v.(string)) } - if v, ok := d.GetOk("worker_type"); ok { - input.WorkerType = aws.String(v.(string)) + if v, ok := d.GetOk("timeout"); ok { + input.Timeout = aws.Int64(int64(v.(int))) } - if v, ok := d.GetOk("number_of_workers"); ok { - input.NumberOfWorkers = aws.Int64(int64(v.(int))) + if v, ok := d.GetOk("worker_type"); ok { + input.WorkerType = aws.String(v.(string)) } log.Printf("[DEBUG] Creating Glue Job: %s", input) - _, err := conn.CreateJob(input) + output, err := conn.CreateJob(input) + if err != nil { - return fmt.Errorf("error creating Glue Job (%s): %s", name, err) + return fmt.Errorf("creating Glue Job (%s): %w", name, err) } - d.SetId(name) + d.SetId(aws.StringValue(output.Name)) return resourceJobRead(d, meta) } @@ -254,7 +259,6 @@ func resourceJobRead(d *schema.ResourceData, meta interface{}) error { JobName: aws.String(d.Id()), } - log.Printf("[DEBUG] Reading Glue Job: %s", input) output, err := conn.GetJob(input) if err != nil { if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { @@ -337,9 +341,7 @@ func resourceJobRead(d *schema.ResourceData, meta interface{}) error { func resourceJobUpdate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*conns.AWSClient).GlueConn - if d.HasChanges("command", "connections", "default_arguments", "description", - "execution_property", "glue_version", "max_capacity", "max_retries", "notification_property", "number_of_workers", - "role_arn", "security_configuration", "timeout", "worker_type", "non_overridable_arguments") { + if d.HasChangesExcept("tags", "tags_all") { jobUpdate := &glue.JobUpdate{ Command: expandJobCommand(d.Get("command").([]interface{})), Role: aws.String(d.Get("role_arn").(string)), @@ -414,7 +416,7 @@ func resourceJobUpdate(d *schema.ResourceData, meta interface{}) error { if d.HasChange("tags_all") { o, n := d.GetChange("tags_all") if err := UpdateTags(conn, d.Get("arn").(string), o, n); err != nil { - return fmt.Errorf("error updating tags: %s", err) + return fmt.Errorf("updating tags: %w", err) } } @@ -425,25 +427,16 @@ func resourceJobDelete(d *schema.ResourceData, meta interface{}) error { conn := meta.(*conns.AWSClient).GlueConn log.Printf("[DEBUG] Deleting Glue Job: %s", d.Id()) - err := DeleteJob(conn, d.Id()) - if err != nil { - return fmt.Errorf("error deleting Glue Job (%s): %s", d.Id(), err) - } - - return nil -} + _, err := conn.DeleteJob(&glue.DeleteJobInput{ + JobName: aws.String(d.Id()), + }) -func DeleteJob(conn *glue.Glue, jobName string) error { - input := &glue.DeleteJobInput{ - JobName: aws.String(jobName), + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + return nil } - _, err := conn.DeleteJob(input) if err != nil { - if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { - return nil - } - return err + return fmt.Errorf("deleting Glue Job (%s): %w", d.Id(), err) } return nil From 8752adb4cb2996a99bf00d1c247a91b65a80b460 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 10:11:50 -0400 Subject: [PATCH 08/13] r/aws_glue_job: Add and use 'FindJobByName'. --- internal/service/glue/find.go | 26 ++++++++++++- internal/service/glue/job.go | 62 ++++++++++++------------------- internal/service/glue/job_test.go | 43 +++++++-------------- 3 files changed, 61 insertions(+), 70 deletions(-) diff --git a/internal/service/glue/find.go b/internal/service/glue/find.go index 3cd89049695b..d570a454f24b 100644 --- a/internal/service/glue/find.go +++ b/internal/service/glue/find.go @@ -27,13 +27,35 @@ func FindDevEndpointByName(conn *glue.Glue, name string) (*glue.DevEndpoint, err } if output == nil || output.DevEndpoint == nil { + return nil, tfresource.NewEmptyResultError(input) + } + + return output.DevEndpoint, nil +} + +func FindJobByName(conn *glue.Glue, name string) (*glue.Job, error) { + input := &glue.GetJobInput{ + JobName: aws.String(name), + } + + output, err := conn.GetJob(input) + + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { return nil, &resource.NotFoundError{ - Message: "Empty result", + LastError: err, LastRequest: input, } } - return output.DevEndpoint, nil + if err != nil { + return nil, err + } + + if output == nil || output.Job == nil { + return nil, tfresource.NewEmptyResultError(input) + } + + return output.Job, nil } // FindTableByName returns the Table corresponding to the specified name. diff --git a/internal/service/glue/job.go b/internal/service/glue/job.go index 33487d870117..c7f19f48b5b3 100644 --- a/internal/service/glue/job.go +++ b/internal/service/glue/job.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/terraform-provider-aws/internal/conns" "github.com/hashicorp/terraform-provider-aws/internal/flex" tftags "github.com/hashicorp/terraform-provider-aws/internal/tags" + "github.com/hashicorp/terraform-provider-aws/internal/tfresource" "github.com/hashicorp/terraform-provider-aws/internal/verify" ) @@ -255,27 +256,18 @@ func resourceJobRead(d *schema.ResourceData, meta interface{}) error { defaultTagsConfig := meta.(*conns.AWSClient).DefaultTagsConfig ignoreTagsConfig := meta.(*conns.AWSClient).IgnoreTagsConfig - input := &glue.GetJobInput{ - JobName: aws.String(d.Id()), - } + job, err := FindJobByName(conn, d.Id()) - output, err := conn.GetJob(input) - if err != nil { - if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { - log.Printf("[WARN] Glue Job (%s) not found, removing from state", d.Id()) - d.SetId("") - return nil - } - return fmt.Errorf("error reading Glue Job (%s): %s", d.Id(), err) - } - - job := output.Job - if job == nil { + if !d.IsNewResource() && tfresource.NotFound(err) { log.Printf("[WARN] Glue Job (%s) not found, removing from state", d.Id()) d.SetId("") return nil } + if err != nil { + return fmt.Errorf("reading Glue Job (%s): %w", d.Id(), err) + } + jobARN := arn.ARN{ Partition: meta.(*conns.AWSClient).Partition, Service: "glue", @@ -284,57 +276,49 @@ func resourceJobRead(d *schema.ResourceData, meta interface{}) error { Resource: fmt.Sprintf("job/%s", d.Id()), }.String() d.Set("arn", jobARN) - if err := d.Set("command", flattenJobCommand(job.Command)); err != nil { - return fmt.Errorf("error setting command: %s", err) + return fmt.Errorf("setting command: %w", err) } if err := d.Set("connections", flattenConnectionsList(job.Connections)); err != nil { - return fmt.Errorf("error setting connections: %s", err) - } - if err := d.Set("default_arguments", aws.StringValueMap(job.DefaultArguments)); err != nil { - return fmt.Errorf("error setting default_arguments: %s", err) - } - if err := d.Set("non_overridable_arguments", aws.StringValueMap(job.NonOverridableArguments)); err != nil { - return fmt.Errorf("error setting non_overridable_arguments: %w", err) + return fmt.Errorf("setting connections: %w", err) } + d.Set("default_arguments", aws.StringValueMap(job.DefaultArguments)) d.Set("description", job.Description) - d.Set("glue_version", job.GlueVersion) + d.Set("execution_class", job.ExecutionClass) if err := d.Set("execution_property", flattenExecutionProperty(job.ExecutionProperty)); err != nil { - return fmt.Errorf("error setting execution_property: %s", err) + return fmt.Errorf("setting execution_property: %w", err) } + d.Set("glue_version", job.GlueVersion) d.Set("max_capacity", job.MaxCapacity) d.Set("max_retries", job.MaxRetries) + d.Set("name", job.Name) + d.Set("non_overridable_arguments", aws.StringValueMap(job.NonOverridableArguments)) if err := d.Set("notification_property", flattenNotificationProperty(job.NotificationProperty)); err != nil { - return fmt.Errorf("error setting notification_property: #{err}") + return fmt.Errorf("setting notification_property: %w", err) } - d.Set("name", job.Name) + d.Set("number_of_workers", job.NumberOfWorkers) d.Set("role_arn", job.Role) + d.Set("security_configuration", job.SecurityConfiguration) + d.Set("timeout", job.Timeout) + d.Set("worker_type", job.WorkerType) tags, err := ListTags(conn, jobARN) if err != nil { - return fmt.Errorf("error listing tags for Glue Job (%s): %s", jobARN, err) + return fmt.Errorf("listing tags for Glue Job (%s): %w", jobARN, err) } tags = tags.IgnoreAWS().IgnoreConfig(ignoreTagsConfig) //lintignore:AWSR002 if err := d.Set("tags", tags.RemoveDefaultConfig(defaultTagsConfig).Map()); err != nil { - return fmt.Errorf("error setting tags: %w", err) + return fmt.Errorf("setting tags: %w", err) } if err := d.Set("tags_all", tags.Map()); err != nil { - return fmt.Errorf("error setting tags_all: %w", err) + return fmt.Errorf("setting tags_all: %w", err) } - d.Set("timeout", job.Timeout) - if err := d.Set("security_configuration", job.SecurityConfiguration); err != nil { - return fmt.Errorf("error setting security_configuration: %s", err) - } - - d.Set("worker_type", job.WorkerType) - d.Set("number_of_workers", job.NumberOfWorkers) - return nil } diff --git a/internal/service/glue/job_test.go b/internal/service/glue/job_test.go index 8fa7719bf57b..5fcf178d8f7b 100644 --- a/internal/service/glue/job_test.go +++ b/internal/service/glue/job_test.go @@ -5,15 +5,14 @@ import ( "regexp" "testing" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" - "github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2/tfawserr" sdkacctest "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/hashicorp/terraform-provider-aws/internal/acctest" "github.com/hashicorp/terraform-provider-aws/internal/conns" tfglue "github.com/hashicorp/terraform-provider-aws/internal/service/glue" + "github.com/hashicorp/terraform-provider-aws/internal/tfresource" ) func TestAccGlueJob_basic(t *testing.T) { @@ -747,11 +746,11 @@ func TestAccGlueJob_disappears(t *testing.T) { }) } -func testAccCheckJobExists(resourceName string, job *glue.Job) resource.TestCheckFunc { +func testAccCheckJobExists(n string, v *glue.Job) resource.TestCheckFunc { return func(s *terraform.State) error { - rs, ok := s.RootModule().Resources[resourceName] + rs, ok := s.RootModule().Resources[n] if !ok { - return fmt.Errorf("Not found: %s", resourceName) + return fmt.Errorf("Not found: %s", n) } if rs.Primary.ID == "" { @@ -760,23 +759,15 @@ func testAccCheckJobExists(resourceName string, job *glue.Job) resource.TestChec conn := acctest.Provider.Meta().(*conns.AWSClient).GlueConn - output, err := conn.GetJob(&glue.GetJobInput{ - JobName: aws.String(rs.Primary.ID), - }) + output, err := tfglue.FindJobByName(conn, rs.Primary.ID) + if err != nil { return err } - if output.Job == nil { - return fmt.Errorf("Glue Job (%s) not found", rs.Primary.ID) - } + *v = *output - if aws.StringValue(output.Job.Name) == rs.Primary.ID { - *job = *output.Job - return nil - } - - return fmt.Errorf("Glue Job (%s) not found", rs.Primary.ID) + return nil } } @@ -788,23 +779,17 @@ func testAccCheckJobDestroy(s *terraform.State) error { conn := acctest.Provider.Meta().(*conns.AWSClient).GlueConn - output, err := conn.GetJob(&glue.GetJobInput{ - JobName: aws.String(rs.Primary.ID), - }) - - if err != nil { - if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { - return nil - } + _, err := tfglue.FindJobByName(conn, rs.Primary.ID) + if tfresource.NotFound(err) { + continue } - job := output.Job - if job != nil && aws.StringValue(job.Name) == rs.Primary.ID { - return fmt.Errorf("Glue Job %s still exists", rs.Primary.ID) + if err != nil { + return err } - return err + return fmt.Errorf("Glue Job %s still exists", rs.Primary.ID) } return nil From e40e01fbda2e6e1b100aaf67c961e601434e3f99 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 10:16:09 -0400 Subject: [PATCH 09/13] r/aws_glue_job: Set ExecutionClass during resource Update. --- internal/service/glue/job.go | 43 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/internal/service/glue/job.go b/internal/service/glue/job.go index c7f19f48b5b3..4db137fd8eab 100644 --- a/internal/service/glue/job.go +++ b/internal/service/glue/job.go @@ -331,18 +331,6 @@ func resourceJobUpdate(d *schema.ResourceData, meta interface{}) error { Role: aws.String(d.Get("role_arn").(string)), } - if v, ok := d.GetOk("timeout"); ok { - jobUpdate.Timeout = aws.Int64(int64(v.(int))) - } - - if v, ok := d.GetOk("number_of_workers"); ok { - jobUpdate.NumberOfWorkers = aws.Int64(int64(v.(int))) - } else { - if v, ok := d.GetOk("max_capacity"); ok { - jobUpdate.MaxCapacity = aws.Float64(v.(float64)) - } - } - if v, ok := d.GetOk("connections"); ok { jobUpdate.Connections = &glue.ConnectionsList{ Connections: flex.ExpandStringList(v.([]interface{})), @@ -353,34 +341,50 @@ func resourceJobUpdate(d *schema.ResourceData, meta interface{}) error { jobUpdate.DefaultArguments = flex.ExpandStringMap(kv.(map[string]interface{})) } - if kv, ok := d.GetOk("non_overridable_arguments"); ok { - jobUpdate.NonOverridableArguments = flex.ExpandStringMap(kv.(map[string]interface{})) - } - if v, ok := d.GetOk("description"); ok { jobUpdate.Description = aws.String(v.(string)) } - if v, ok := d.GetOk("glue_version"); ok { - jobUpdate.GlueVersion = aws.String(v.(string)) + if v, ok := d.GetOk("execution_class"); ok { + jobUpdate.ExecutionClass = aws.String(v.(string)) } if v, ok := d.GetOk("execution_property"); ok { jobUpdate.ExecutionProperty = expandExecutionProperty(v.([]interface{})) } + if v, ok := d.GetOk("glue_version"); ok { + jobUpdate.GlueVersion = aws.String(v.(string)) + } + if v, ok := d.GetOk("max_retries"); ok { jobUpdate.MaxRetries = aws.Int64(int64(v.(int))) } + if kv, ok := d.GetOk("non_overridable_arguments"); ok { + jobUpdate.NonOverridableArguments = flex.ExpandStringMap(kv.(map[string]interface{})) + } + if v, ok := d.GetOk("notification_property"); ok { jobUpdate.NotificationProperty = expandNotificationProperty(v.([]interface{})) } + if v, ok := d.GetOk("number_of_workers"); ok { + jobUpdate.NumberOfWorkers = aws.Int64(int64(v.(int))) + } else { + if v, ok := d.GetOk("max_capacity"); ok { + jobUpdate.MaxCapacity = aws.Float64(v.(float64)) + } + } + if v, ok := d.GetOk("security_configuration"); ok { jobUpdate.SecurityConfiguration = aws.String(v.(string)) } + if v, ok := d.GetOk("timeout"); ok { + jobUpdate.Timeout = aws.Int64(int64(v.(int))) + } + if v, ok := d.GetOk("worker_type"); ok { jobUpdate.WorkerType = aws.String(v.(string)) } @@ -392,8 +396,9 @@ func resourceJobUpdate(d *schema.ResourceData, meta interface{}) error { log.Printf("[DEBUG] Updating Glue Job: %s", input) _, err := conn.UpdateJob(input) + if err != nil { - return fmt.Errorf("error updating Glue Job (%s): %s", d.Id(), err) + return fmt.Errorf("updating Glue Job (%s): %w", d.Id(), err) } } From e72baf66b89cd64d073c7efd21c4cf5c682e9867 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 10:28:10 -0400 Subject: [PATCH 10/13] r/aws_glue_job: Tidy up acceptance tests. --- internal/service/glue/dev_endpoint_test.go | 4 +- internal/service/glue/job_test.go | 294 +++++++++------------ 2 files changed, 123 insertions(+), 175 deletions(-) diff --git a/internal/service/glue/dev_endpoint_test.go b/internal/service/glue/dev_endpoint_test.go index 40e7ffc0311c..b2ae1fff3231 100644 --- a/internal/service/glue/dev_endpoint_test.go +++ b/internal/service/glue/dev_endpoint_test.go @@ -866,7 +866,7 @@ resource "aws_security_group" "test" { } func testAccDevEndpointConfig_tags1(rName, tagKey1, tagValue1 string) string { - return testAccJobConfig_Base(rName) + fmt.Sprintf(` + return testAccJobConfig_base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn @@ -879,7 +879,7 @@ resource "aws_glue_dev_endpoint" "test" { } func testAccDevEndpointConfig_tags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string { - return testAccJobConfig_Base(rName) + fmt.Sprintf(` + return testAccJobConfig_base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn diff --git a/internal/service/glue/job_test.go b/internal/service/glue/job_test.go index 5fcf178d8f7b..c8d5ea5a0f8f 100644 --- a/internal/service/glue/job_test.go +++ b/internal/service/glue/job_test.go @@ -17,8 +17,7 @@ import ( func TestAccGlueJob_basic(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" roleResourceName := "aws_iam_role.test" @@ -36,8 +35,9 @@ func TestAccGlueJob_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "command.#", "1"), resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"), resource.TestCheckResourceAttr(resourceName, "default_arguments.%", "0"), - resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.%", "0"), + resource.TestCheckResourceAttr(resourceName, "execution_class", ""), resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.%", "0"), resource.TestCheckResourceAttrPair(resourceName, "role_arn", roleResourceName, "arn"), resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), resource.TestCheckResourceAttr(resourceName, "timeout", "2880"), @@ -52,10 +52,32 @@ func TestAccGlueJob_basic(t *testing.T) { }) } -func TestAccGlueJob_basicStreaming(t *testing.T) { +func TestAccGlueJob_disappears(t *testing.T) { var job glue.Job + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_glue_job.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(t) }, + ErrorCheck: acctest.ErrorCheck(t, glue.EndpointsID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckJobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccJobConfig_required(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobExists(resourceName, &job), + acctest.CheckResourceDisappears(acctest.Provider, tfglue.ResourceJob(), resourceName), + ), + ExpectNonEmptyPlan: true, + }, + }, + }) +} - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) +func TestAccGlueJob_basicStreaming(t *testing.T) { + var job glue.Job + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" roleResourceName := "aws_iam_role.test" @@ -91,8 +113,7 @@ func TestAccGlueJob_basicStreaming(t *testing.T) { } func TestAccGlueJob_command(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -128,8 +149,7 @@ func TestAccGlueJob_command(t *testing.T) { func TestAccGlueJob_defaultArguments(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -167,8 +187,7 @@ func TestAccGlueJob_defaultArguments(t *testing.T) { func TestAccGlueJob_nonOverridableArguments(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -206,8 +225,7 @@ func TestAccGlueJob_nonOverridableArguments(t *testing.T) { func TestAccGlueJob_description(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -241,8 +259,7 @@ func TestAccGlueJob_description(t *testing.T) { func TestAccGlueJob_glueVersion(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -283,8 +300,7 @@ func TestAccGlueJob_glueVersion(t *testing.T) { func TestAccGlueJob_executionClass(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -317,8 +333,7 @@ func TestAccGlueJob_executionClass(t *testing.T) { } func TestAccGlueJob_executionProperty(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -358,8 +373,7 @@ func TestAccGlueJob_executionProperty(t *testing.T) { func TestAccGlueJob_maxRetries(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -397,8 +411,7 @@ func TestAccGlueJob_maxRetries(t *testing.T) { func TestAccGlueJob_notificationProperty(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -437,9 +450,8 @@ func TestAccGlueJob_notificationProperty(t *testing.T) { } func TestAccGlueJob_tags(t *testing.T) { - var job1, job2, job3 glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + var job glue.Job + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -451,7 +463,7 @@ func TestAccGlueJob_tags(t *testing.T) { { Config: testAccJobConfig_tags1(rName, "key1", "value1"), Check: resource.ComposeTestCheckFunc( - testAccCheckJobExists(resourceName, &job1), + testAccCheckJobExists(resourceName, &job), resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"), ), @@ -464,7 +476,7 @@ func TestAccGlueJob_tags(t *testing.T) { { Config: testAccJobConfig_tags2(rName, "key1", "value1updated", "key2", "value2"), Check: resource.ComposeTestCheckFunc( - testAccCheckJobExists(resourceName, &job2), + testAccCheckJobExists(resourceName, &job), resource.TestCheckResourceAttr(resourceName, "tags.%", "2"), resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"), resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), @@ -473,7 +485,7 @@ func TestAccGlueJob_tags(t *testing.T) { { Config: testAccJobConfig_tags1(rName, "key2", "value2"), Check: resource.ComposeTestCheckFunc( - testAccCheckJobExists(resourceName, &job3), + testAccCheckJobExists(resourceName, &job), resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), ), @@ -484,8 +496,7 @@ func TestAccGlueJob_tags(t *testing.T) { func TestAccGlueJob_streamingTimeout(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -518,8 +529,7 @@ func TestAccGlueJob_streamingTimeout(t *testing.T) { } func TestAccGlueJob_timeout(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -553,8 +563,7 @@ func TestAccGlueJob_timeout(t *testing.T) { func TestAccGlueJob_security(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -588,8 +597,7 @@ func TestAccGlueJob_security(t *testing.T) { func TestAccGlueJob_workerType(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -630,8 +638,7 @@ func TestAccGlueJob_workerType(t *testing.T) { func TestAccGlueJob_pythonShell(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -686,8 +693,7 @@ func TestAccGlueJob_pythonShell(t *testing.T) { func TestAccGlueJob_maxCapacity(t *testing.T) { var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_glue_job.test" resource.ParallelTest(t, resource.TestCase{ @@ -722,30 +728,6 @@ func TestAccGlueJob_maxCapacity(t *testing.T) { }) } -func TestAccGlueJob_disappears(t *testing.T) { - var job glue.Job - - rName := fmt.Sprintf("tf-acc-test-%s", sdkacctest.RandString(5)) - resourceName := "aws_glue_job.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(t) }, - ErrorCheck: acctest.ErrorCheck(t, glue.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckJobDestroy, - Steps: []resource.TestStep{ - { - Config: testAccJobConfig_required(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckJobExists(resourceName, &job), - acctest.CheckResourceDisappears(acctest.Provider, tfglue.ResourceJob(), resourceName), - ), - ExpectNonEmptyPlan: true, - }, - }, - }) -} - func testAccCheckJobExists(n string, v *glue.Job) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[n] @@ -795,7 +777,7 @@ func testAccCheckJobDestroy(s *terraform.State) error { return nil } -func testAccJobConfig_Base(rName string) string { +func testAccJobConfig_base(rName string) string { return fmt.Sprintf(` data "aws_partition" "current" {} @@ -804,7 +786,7 @@ data "aws_iam_policy" "AWSGlueServiceRole" { } resource "aws_iam_role" "test" { - name = "%s" + name = %[1]q assume_role_policy = < Date: Wed, 10 Aug 2022 10:59:58 -0400 Subject: [PATCH 11/13] r/aws_glue_dev_endpoint: Tidy up acceptance tests. --- internal/service/glue/dev_endpoint_test.go | 102 +++++++++++---------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/internal/service/glue/dev_endpoint_test.go b/internal/service/glue/dev_endpoint_test.go index b2ae1fff3231..5775858cb557 100644 --- a/internal/service/glue/dev_endpoint_test.go +++ b/internal/service/glue/dev_endpoint_test.go @@ -561,15 +561,15 @@ func TestAccGlueDevEndpoint_disappears(t *testing.T) { }) } -func testAccCheckDevEndpointExists(resourceName string, v *glue.DevEndpoint) resource.TestCheckFunc { +func testAccCheckDevEndpointExists(n string, v *glue.DevEndpoint) resource.TestCheckFunc { return func(s *terraform.State) error { - rs, ok := s.RootModule().Resources[resourceName] + rs, ok := s.RootModule().Resources[n] if !ok { - return fmt.Errorf("not found: %s", resourceName) + return fmt.Errorf("Not found: %s", n) } if rs.Primary.ID == "" { - return fmt.Errorf("no Glue Dev Endpoint ID is set") + return fmt.Errorf("No Glue Dev Endpoint ID is set") } conn := acctest.Provider.Meta().(*conns.AWSClient).GlueConn @@ -638,16 +638,16 @@ data "aws_partition" "current" {} } func testAccDevEndpointConfig_basic(rName string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn } -`, rName) +`, rName)) } func testAccDevEndpointConfig_arguments(rName, argKey, argValue string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn @@ -655,11 +655,11 @@ resource "aws_glue_dev_endpoint" "test" { %[2]q = %[3]q } } -`, rName, argKey, argValue) +`, rName, argKey, argValue)) } func testAccDevEndpointConfig_arguments2(rName, argKey1, argValue1, argKey2, argValue2 string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn @@ -668,102 +668,102 @@ resource "aws_glue_dev_endpoint" "test" { %[4]q = %[5]q } } -`, rName, argKey1, argValue1, argKey2, argValue2) +`, rName, argKey1, argValue1, argKey2, argValue2)) } func testAccDevEndpointConfig_extraJarsS3Path(rName string, extraJarsS3Path string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn - extra_jars_s3_path = %q + extra_jars_s3_path = %[2]q } -`, rName, extraJarsS3Path) +`, rName, extraJarsS3Path)) } func testAccDevEndpointConfig_extraPythonLibsS3Path(rName string, extraPythonLibsS3Path string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn - extra_python_libs_s3_path = %q + extra_python_libs_s3_path = %[2]q } -`, rName, extraPythonLibsS3Path) +`, rName, extraPythonLibsS3Path)) } func testAccDevEndpointConfig_version(rName string, glueVersion string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn glue_version = %[2]q } -`, rName, glueVersion) +`, rName, glueVersion)) } func testAccDevEndpointConfig_numberOfNodes(rName string, numberOfNodes int) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn - number_of_nodes = %d + number_of_nodes = %[2]d } -`, rName, numberOfNodes) +`, rName, numberOfNodes)) } func testAccDevEndpointConfig_numberOfWorkers(rName string, numberOfWorkers int) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn worker_type = "G.1X" - number_of_workers = %d + number_of_workers = %[2]d } -`, rName, numberOfWorkers) +`, rName, numberOfWorkers)) } func testAccDevEndpointConfig_publicKey(rName string, publicKey string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn - public_key = "%s" + public_key = %[2]q } -`, rName, publicKey) +`, rName, publicKey)) } func testAccDevEndpointConfig_publicKeys2(rName string, publicKey1 string, publicKey2 string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q] } -`, rName, publicKey1, publicKey2) +`, rName, publicKey1, publicKey2)) } func testAccDevEndpointConfig_publicKeys3(rName string, publicKey1 string, publicKey2 string, publicKey3 string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q, %[4]q] } -`, rName, publicKey1, publicKey2, publicKey3) +`, rName, publicKey1, publicKey2, publicKey3)) } func testAccDevEndpointConfig_publicKeys4(rName string, publicKey1 string, publicKey2 string, publicKey3 string, publicKey4 string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q, %[4]q, %[5]q] } -`, rName, publicKey1, publicKey2, publicKey3, publicKey4) +`, rName, publicKey1, publicKey2, publicKey3, publicKey4)) } func testAccDevEndpointConfig_securityConfiguration(rName string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn @@ -787,7 +787,7 @@ resource "aws_glue_security_configuration" "test" { } } } -`, rName) +`, rName)) } func testAccDevEndpointConfig_subnetIDSecurityGroupIDs(rName string) string { @@ -843,6 +843,10 @@ resource "aws_security_group" "test" { name = %[1]q vpc_id = aws_vpc.test.id + tags = { + Name = %[1]q + } + ingress { from_port = 0 to_port = 0 @@ -866,7 +870,7 @@ resource "aws_security_group" "test" { } func testAccDevEndpointConfig_tags1(rName, tagKey1, tagValue1 string) string { - return testAccJobConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn @@ -875,11 +879,11 @@ resource "aws_glue_dev_endpoint" "test" { %[2]q = %[3]q } } -`, rName, tagKey1, tagValue1) +`, rName, tagKey1, tagValue1)) } func testAccDevEndpointConfig_tags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string { - return testAccJobConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn @@ -889,27 +893,27 @@ resource "aws_glue_dev_endpoint" "test" { %[4]q = %[5]q } } -`, rName, tagKey1, tagValue1, tagKey2, tagValue2) +`, rName, tagKey1, tagValue1, tagKey2, tagValue2)) } func testAccDevEndpointConfig_workerType(rName, workerType string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn worker_type = %[2]q number_of_workers = 2 } -`, rName, workerType) +`, rName, workerType)) } func testAccDevEndpointConfig_workerTypeStandard(rName string) string { - return testAccDevEndpointConfig_base(rName) + fmt.Sprintf(` + return acctest.ConfigCompose(testAccDevEndpointConfig_base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn worker_type = "Standard" number_of_workers = 2 } -`, rName) +`, rName)) } From c92c18c9464f638227e16e4071db72767f156a6e Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 12:03:43 -0400 Subject: [PATCH 12/13] Tidy up 'sweepDevEndpoints' and 'sweepJobs'. --- internal/service/glue/sweep.go | 88 ++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/internal/service/glue/sweep.go b/internal/service/glue/sweep.go index d85f95a85766..33f4976df57a 100644 --- a/internal/service/glue/sweep.go +++ b/internal/service/glue/sweep.go @@ -40,7 +40,7 @@ func init() { resource.AddTestSweepers("aws_glue_dev_endpoint", &resource.Sweeper{ Name: "aws_glue_dev_endpoint", - F: sweepDevEndpoint, + F: sweepDevEndpoints, }) resource.AddTestSweepers("aws_glue_job", &resource.Sweeper{ @@ -249,42 +249,50 @@ func sweepCrawlers(region string) error { return nil } -func sweepDevEndpoint(region string) error { +func sweepDevEndpoints(region string) error { client, err := sweep.SharedRegionalSweepClient(region) if err != nil { return fmt.Errorf("error getting client: %s", err) } + input := &glue.GetDevEndpointsInput{} conn := client.(*conns.AWSClient).GlueConn + sweepResources := make([]*sweep.SweepResource, 0) - input := &glue.GetDevEndpointsInput{} err = conn.GetDevEndpointsPages(input, func(page *glue.GetDevEndpointsOutput, lastPage bool) bool { - if len(page.DevEndpoints) == 0 { - log.Printf("[INFO] No Glue Dev Endpoints to sweep") - return false + if page == nil { + return !lastPage } - for _, endpoint := range page.DevEndpoints { - name := aws.StringValue(endpoint.EndpointName) + + for _, v := range page.DevEndpoints { + name := aws.StringValue(v.EndpointName) if !strings.HasPrefix(name, sweep.ResourcePrefix) { log.Printf("[INFO] Skipping Glue Dev Endpoint: %s", name) continue } - log.Printf("[INFO] Deleting Glue Dev Endpoint: %s", name) - _, err := conn.DeleteDevEndpoint(&glue.DeleteDevEndpointInput{ - EndpointName: aws.String(name), - }) - if err != nil { - log.Printf("[ERROR] Failed to delete Glue Dev Endpoint %s: %s", name, err) - } + r := ResourceDevEndpoint() + d := r.Data(nil) + d.SetId(name) + + sweepResources = append(sweepResources, sweep.NewSweepResource(r, d, client)) } + return !lastPage }) + + if sweep.SkipSweepError(err) { + log.Printf("[WARN] Skipping Glue Dev Endpoint sweep for %s: %s", region, err) + return nil + } + if err != nil { - if sweep.SkipSweepError(err) { - log.Printf("[WARN] Skipping Glue Dev Endpoint sweep for %s: %s", region, err) - return nil - } - return fmt.Errorf("error retrieving Glue Dev Endpoint: %s", err) + return fmt.Errorf("error listing Glue Dev Endpoints (%s): %w", region, err) + } + + err = sweep.SweepOrchestrator(sweepResources) + + if err != nil { + return fmt.Errorf("error sweeping Glue Dev Endpoints (%s): %w", region, err) } return nil @@ -295,31 +303,39 @@ func sweepJobs(region string) error { if err != nil { return fmt.Errorf("error getting client: %s", err) } + input := &glue.GetJobsInput{} conn := client.(*conns.AWSClient).GlueConn + sweepResources := make([]*sweep.SweepResource, 0) - input := &glue.GetJobsInput{} err = conn.GetJobsPages(input, func(page *glue.GetJobsOutput, lastPage bool) bool { - if len(page.Jobs) == 0 { - log.Printf("[INFO] No Glue Jobs to sweep") - return false + if page == nil { + return !lastPage } - for _, job := range page.Jobs { - name := aws.StringValue(job.Name) - log.Printf("[INFO] Deleting Glue Job: %s", name) - err := DeleteJob(conn, name) - if err != nil { - log.Printf("[ERROR] Failed to delete Glue Job %s: %s", name, err) - } + for _, v := range page.Jobs { + r := ResourceJob() + d := r.Data(nil) + d.SetId(aws.StringValue(v.Name)) + + sweepResources = append(sweepResources, sweep.NewSweepResource(r, d, client)) } + return !lastPage }) + + if sweep.SkipSweepError(err) { + log.Printf("[WARN] Skipping Glue Job sweep for %s: %s", region, err) + return nil + } + if err != nil { - if sweep.SkipSweepError(err) { - log.Printf("[WARN] Skipping Glue Job sweep for %s: %s", region, err) - return nil - } - return fmt.Errorf("Error retrieving Glue Jobs: %s", err) + return fmt.Errorf("error listing Glue Jobs (%s): %w", region, err) + } + + err = sweep.SweepOrchestrator(sweepResources) + + if err != nil { + return fmt.Errorf("error sweeping Glue Jobs (%s): %w", region, err) } return nil From 5db15d52fe3a24ec9294924a61eb94f098c6812f Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Wed, 10 Aug 2022 12:31:21 -0400 Subject: [PATCH 13/13] Get 'TestAccGlueJob_executionClass' passing. --- internal/service/glue/job_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/service/glue/job_test.go b/internal/service/glue/job_test.go index c8d5ea5a0f8f..6254365f3173 100644 --- a/internal/service/glue/job_test.go +++ b/internal/service/glue/job_test.go @@ -925,10 +925,12 @@ resource "aws_glue_job" "test" { func testAccJobConfig_executionClass(rName, executionClass string) string { return acctest.ConfigCompose(testAccJobConfig_base(rName), fmt.Sprintf(` resource "aws_glue_job" "test" { - execution_class = %[2]q - max_capacity = 10 - name = %[1]q - role_arn = aws_iam_role.test.arn + execution_class = %[2]q + name = %[1]q + number_of_workers = 2 + role_arn = aws_iam_role.test.arn + worker_type = "G.1X" + glue_version = "3.0" command { script_location = "testscriptlocation"