From a2359072b71dc29c515b04271900d73c97f1bbaa Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Tue, 12 Mar 2019 09:53:08 +0100 Subject: [PATCH 01/12] Add resource aws_glue_dev_endpoint --- aws/provider.go | 1 + aws/resource_aws_glue_dev_endpoint.go | 446 +++++++++++++ aws/resource_aws_glue_dev_endpoint_test.go | 701 +++++++++++++++++++++ website/docs/r/glue_dev_endpoint.markdown | 81 +++ 4 files changed, 1229 insertions(+) create mode 100644 aws/resource_aws_glue_dev_endpoint.go create mode 100644 aws/resource_aws_glue_dev_endpoint_test.go create mode 100644 website/docs/r/glue_dev_endpoint.markdown diff --git a/aws/provider.go b/aws/provider.go index 976a61f49a0..34533b76eaf 100644 --- a/aws/provider.go +++ b/aws/provider.go @@ -656,6 +656,7 @@ func Provider() *schema.Provider { "aws_glue_catalog_table": resourceAwsGlueCatalogTable(), "aws_glue_classifier": resourceAwsGlueClassifier(), "aws_glue_connection": resourceAwsGlueConnection(), + "aws_glue_dev_endpoint": resourceAwsGlueDevEndpoint(), "aws_glue_crawler": resourceAwsGlueCrawler(), "aws_glue_data_catalog_encryption_settings": resourceAwsGlueDataCatalogEncryptionSettings(), "aws_glue_job": resourceAwsGlueJob(), diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go new file mode 100644 index 00000000000..cc8ed89925a --- /dev/null +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -0,0 +1,446 @@ +package aws + +import ( + "fmt" + "log" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/glue" + + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/helper/schema" +) + +func resourceAwsGlueDevEndpoint() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsGlueDevEndpointCreate, + Read: resourceAwsGlueDevEndpointRead, + Update: resourceAwsDevEndpointUpdate, + Delete: resourceAwsDevEndpointDelete, + Importer: &schema.ResourceImporter{ + State: schema.ImportStatePassthrough, + }, + + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ForceNew: true, + }, + + "extra_jars_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + + "extra_python_libs_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + + "number_of_nodes": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Default: 5, + }, + + "public_key": { + Type: schema.TypeString, + Optional: true, + }, + + "public_keys": { + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + MaxItems: 5, + }, + + "role_arn": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + + "security_configuration": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + + "security_group_ids": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + }, + + "subnet_id": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + + "private_address": { + Type: schema.TypeString, + Computed: true, + }, + + "public_address": { + Type: schema.TypeString, + Computed: true, + }, + + "yarn_endpoint_address": { + Type: schema.TypeString, + Computed: true, + }, + + "zeppelin_remote_spark_interpreter_port": { + Type: schema.TypeInt, + Computed: true, + }, + + "availability_zone": { + Type: schema.TypeString, + Computed: true, + }, + + "vpc_id": { + Type: schema.TypeString, + Computed: true, + }, + + "status": { + Type: schema.TypeString, + Computed: true, + }, + + "failure_reason": { + Type: schema.TypeString, + Computed: true, + }, + }, + } +} + +func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + var name string + if v, ok := d.GetOk("name"); ok { + name = v.(string) + } else { + name = resource.UniqueId() + } + + createOpts := &glue.CreateDevEndpointInput{ + EndpointName: aws.String(name), + RoleArn: aws.String(d.Get("role_arn").(string)), + } + + if v, ok := d.GetOk("extra_jars_s3_path"); ok { + createOpts.SetExtraJarsS3Path(v.(string)) + } + + if v, ok := d.GetOk("extra_python_libs_s3_path"); ok { + createOpts.SetExtraPythonLibsS3Path(v.(string)) + } + + if v, ok := d.GetOk("number_of_nodes"); ok { + createOpts.SetNumberOfNodes(int64(v.(int))) + } + + if v, ok := d.GetOk("public_key"); ok { + createOpts.SetPublicKey(v.(string)) + } + + if v, ok := d.GetOk("public_keys"); ok { + publicKeys := expandStringSet(v.(*schema.Set)) + createOpts.SetPublicKeys(publicKeys) + } + + if v, ok := d.GetOk("security_configuration"); ok { + createOpts.SetSecurityConfiguration(v.(string)) + } + + if v, ok := d.GetOk("security_group_ids"); ok { + securityGroupIDs := expandStringSet(v.(*schema.Set)) + createOpts.SetSecurityGroupIds(securityGroupIDs) + } + + if v, ok := d.GetOk("subnet_id"); ok { + createOpts.SetSubnetId(v.(string)) + } + log.Printf("[DEBUG] Glue dev endpoint create config: %#v", *createOpts) + err := resource.Retry(1*time.Minute, func() *resource.RetryError { + _, err := conn.CreateDevEndpoint(createOpts) + if err != nil { + // Retry for IAM eventual consistency + if isAWSErr(err, glue.ErrCodeInvalidInputException, "should be given assume role permissions for Glue Service") { + return resource.RetryableError(err) + } + + return resource.NonRetryableError(err) + } + return nil + }) + + if err != nil { + return fmt.Errorf("error creating Glue dev endpoint: %s", err) + } + + d.SetId(name) + log.Printf("[INFO] Glue dev endpoint ID: %s", d.Id()) + + log.Printf("[DEBUG] Waiting for Glue dev endpoint (%s) to become available", d.Id()) + stateConf := &resource.StateChangeConf{ + Pending: []string{ + "PROVISIONING", + }, + Target: []string{"READY"}, + Refresh: glueDevEndpointStateRefreshFunc(conn, d.Id()), + Timeout: 10 * time.Minute, + } + if _, err := stateConf.WaitForState(); err != nil { + return fmt.Errorf("error while waiting for Glue dev endpoint (%s) to become available: %s", d.Id(), err) + } + + return resourceAwsGlueDevEndpointRead(d, meta) +} + +func resourceAwsGlueDevEndpointRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + request := &glue.GetDevEndpointInput{ + EndpointName: aws.String(d.Id()), + } + + output, err := conn.GetDevEndpoint(request) + if err != nil { + if glueErr, ok := err.(awserr.Error); ok && glueErr.Code() == glue.ErrCodeEntityNotFoundException { + log.Printf("[INFO] unable to find Glue dev endpoint and therfore it is removed from the state: %s", d.Id()) + d.SetId("") + return nil + } + return fmt.Errorf("error finding Glue dev endpoint %s: %s", d.Id(), err) + } + + endpoint := output.DevEndpoint + if endpoint == nil { + return fmt.Errorf("Glue dev endpoint (%s) is nil: %s", d.Id(), err) + } + + if err := d.Set("name", endpoint.EndpointName); err != nil { + return fmt.Errorf("error setting name for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("extra_jars_s3_path", endpoint.ExtraJarsS3Path); err != nil { + return fmt.Errorf("error setting extra_jars_s3_path for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("extra_python_libs_s3_path", endpoint.ExtraPythonLibsS3Path); err != nil { + return fmt.Errorf("error setting extra_python_libs_s3_path for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("number_of_nodes", endpoint.NumberOfNodes); err != nil { + return fmt.Errorf("error setting number_of_nodes for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_key", endpoint.PublicKey); err != nil { + return fmt.Errorf("error setting public_key for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_keys", flattenStringSet(endpoint.PublicKeys)); err != nil { + return fmt.Errorf("error setting public_keys for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("role_arn", endpoint.RoleArn); err != nil { + return fmt.Errorf("error setting role_arn for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("security_configuration", endpoint.SecurityConfiguration); err != nil { + return fmt.Errorf("error setting security_configuration for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("security_group_ids", flattenStringSet(endpoint.SecurityGroupIds)); err != nil { + return fmt.Errorf("error setting security_group_ids for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("subnet_id", endpoint.SubnetId); err != nil { + return fmt.Errorf("error setting subnet_id for Glue dev endpoint (%s): %s", d.Id(), err) + } + + // extra attributes + if err := d.Set("private_address", endpoint.PrivateAddress); err != nil { + return fmt.Errorf("error setting private_address for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_address", endpoint.PublicAddress); err != nil { + return fmt.Errorf("error setting public_address for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("yarn_endpoint_address", endpoint.YarnEndpointAddress); err != nil { + return fmt.Errorf("error setting yarn_endpoint_address for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("zeppelin_remote_spark_interpreter_port", endpoint.ZeppelinRemoteSparkInterpreterPort); err != nil { + return fmt.Errorf("error setting zeppelin_remote_spark_interpreter_port for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("availability_zone", endpoint.AvailabilityZone); err != nil { + return fmt.Errorf("error setting availability_zone for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("vpc_id", endpoint.VpcId); err != nil { + return fmt.Errorf("error setting vpc_id for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("status", endpoint.Status); err != nil { + return fmt.Errorf("error setting status for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("failure_reason", endpoint.FailureReason); err != nil { + return fmt.Errorf("error setting failure_reason for Glue dev endpoint (%s): %s", d.Id(), err) + } + return nil +} + +func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + updateOpts := &glue.UpdateDevEndpointInput{ + EndpointName: aws.String(d.Get("name").(string)), + } + + hasChanged := false + + if d.HasChange("public_keys") { + oldRaw, newRaw := d.GetChange("public_keys") + old := oldRaw.(*schema.Set) + new := newRaw.(*schema.Set) + create, remove := diffPublicKeys(expandStringSet(old), expandStringSet(new)) + updateOpts.SetAddPublicKeys(create) + updateOpts.SetDeletePublicKeys(remove) + + hasChanged = true + } + + if d.HasChange("public_key") { + updateOpts.SetPublicKey(d.Get("public_key").(string)) + hasChanged = true + } + + customLibs := &glue.DevEndpointCustomLibraries{} + + if d.HasChange("extra_jars_s3_path") { + customLibs.SetExtraJarsS3Path(d.Get("extra_jars_s3_path").(string)) + updateOpts.SetCustomLibraries(customLibs) + updateOpts.SetUpdateEtlLibraries(true) + hasChanged = true + } + + if d.HasChange("extra_python_libs_s3_path") { + customLibs.SetExtraPythonLibsS3Path(d.Get("extra_python_libs_s3_path").(string)) + updateOpts.SetCustomLibraries(customLibs) + updateOpts.SetUpdateEtlLibraries(true) + hasChanged = true + } + + if hasChanged { + _, err := conn.UpdateDevEndpoint(updateOpts) + if err != nil { + return fmt.Errorf("error updating Glue dev endpoint: %s", err) + } + } + + return resourceAwsGlueDevEndpointRead(d, meta) +} + +func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + deleteOpts := &glue.DeleteDevEndpointInput{ + EndpointName: aws.String(d.Id()), + } + log.Printf("[INFO] Deleting Glue dev endpoint: %s", d.Id()) + + return resource.Retry(5*time.Minute, func() *resource.RetryError { + _, err := conn.DeleteDevEndpoint(deleteOpts) + if err == nil { + return nil + } + + glueErr, ok := err.(awserr.Error) + if !ok { + return resource.NonRetryableError(err) + } + + if glueErr.Code() == glue.ErrCodeEntityNotFoundException { + return nil + } + + return resource.NonRetryableError(fmt.Errorf("error deleting Glue dev endpoint: %s", err)) + }) +} + +func glueDevEndpointStateRefreshFunc(conn *glue.Glue, name string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + getDevEndpointInput := &glue.GetDevEndpointInput{ + EndpointName: aws.String(name), + } + endpoint, err := conn.GetDevEndpoint(getDevEndpointInput) + if err != nil { + if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + return nil, "", nil + } + + return nil, "", err + } + + if endpoint == nil { + return nil, "", nil + } + + return endpoint, *endpoint.DevEndpoint.Status, nil + } +} + +func diffPublicKeys(oldKeys, newKeys []*string) ([]*string, []*string) { + var create []*string + var remove []*string + + for _, oldKey := range oldKeys { + found := false + for _, newKey := range newKeys { + if oldKey == newKey { + found = true + break + } + } + if !found { + remove = append(remove, oldKey) + } + } + + for _, newKey := range newKeys { + found := false + for _, oldKey := range oldKeys { + if oldKey == newKey { + found = true + break + } + } + if !found { + create = append(create, newKey) + } + } + + return create, remove +} diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go new file mode 100644 index 00000000000..aa5e2cd25f8 --- /dev/null +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -0,0 +1,701 @@ +package aws + +import ( + "fmt" + "log" + "regexp" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +const ( + GlueDevEndpointResourcePrefix = "tf-acc-test" + publicKey1 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD3F6tyPEFEzV0LX3X8BsXdMsQz1x2cEikKDEY0aIj41qgxMCP/iteneqXSIFZBp5vizPvaoIR3Um9xK7PGoW8giupGn+EPuxIA4cDM4vzOqOkiMPhz5XK0whEjkVzTo4+S0puvDZuwIsdiW9mxhJc7tgBNL0cYlWSYVkz4G/fslNfRPW5mYAM49f4fhtxPb5ok4Q2Lg9dPKVHO/Bgeu5woMc7RY0p1ej6D4CKFE6lymSDJpW0YHX/wqE9+cfEauh7xZcG0q9t2ta6F6fmX0agvpFyZo8aFbXeUBr7osSCJNgvavWbM/06niWrOvYX2xwWdhXmXSrbX8ZbabVohBK41 foo1@bar.com" + publicKey2 = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAq6U3HQYC4g8WzU147gZZ7CKQH8TgYn3chZGRPxaGmHW1RUwsyEs0nmombmIhwxudhJ4ehjqXsDLoQpd6+c7BuLgTMvbv8LgE9LX53vnljFe1dsObsr/fYLvpU9LTlo8HgHAqO5ibNdrAUvV31ronzCZhms/Gyfdaue88Fd0/YnsZVGeOZPayRkdOHSpqme2CBrpa8myBeL1CWl0LkDG4+YCURjbaelfyZlIApLYKy3FcCan9XQFKaL32MJZwCgzfOvWIMtYcU8QtXMgnA3/I3gXk8YDUJv5P4lj0s/PJXuTM8DygVAUtebNwPuinS7wwonm5FXcWMuVGsVpG5K7FGQ== foo2@bar.com" + publicKey3 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtCk0lzMj1gPEOjdfQ37AIxCyETqJBubaMWuB4bgvGHp8LvEghr2YDl2bml1JrE1EOcZhPnIwgyucryXKA959sTUlgbvaFN7vmpVze56Q9tVU6BJQxOdaRoy5FcQMET9LB6SdbXk+V4CkDMsQNaFXezpg98HgCj+V7+bBWsfI6U63IESlWKK7kraCom8EWxkQk4mk9fizE2I+KrtiqN4xcah02LFG6IMnS+Xy3CDhcpZeYzWOV6zhcf675UJOdg/pLgQbUhhiwTOJFgRo8IcvE3iBrRMz508ppx6vLLr8J+3B8ujykc+/3ZSGfQfx6rO+OuSskhG5FLI6icbQBtBzf foo3@bar.com" + publicKey4 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD3F6tyPEFEzV0LX3X8BsXdMsQz1x2cEikKDEY0aIj41qgxMCP/iteneqXSIFZBp5vizPvaoIR3Um9xK7PGoW8giupGn+EPuxIA4cDM4vzOqOkiMPhz5XK0whEjkVzTo4+S0puvDZuwIsdiW9mxhJc7tgBNL0cYlWSYVkz4G/fslNfRPW5mYAM49f4fhtxPb5ok4Q2Lg9dPKVHO/Bgeu5woMc7RY0p1ej6D4CKFE6lymSDJpW0YHX/wqE9+cfEauh7xZcG0q9t2ta6F6fmX0agvpFyZo8aFbXeUBr7osSCJNgvavWbM/06niWrOvYX2xwWdhXmXSrbX8ZbabVohBK41 foo4@bar.com" +) + +func init() { + resource.AddTestSweepers("aws_glue_dev_endpoint", &resource.Sweeper{ + Name: "aws_glue_dev_endpoint", + F: testSweepGlueDevEndpoint, + }) +} + +func testSweepGlueDevEndpoint(region string) error { + client, err := sharedClientForRegion(region) + if err != nil { + return fmt.Errorf("error getting client: %s", err) + } + conn := client.(*AWSClient).glueconn + + input := &glue.GetDevEndpointsInput{} + err = conn.GetDevEndpointsPages(input, func(page *glue.GetDevEndpointsOutput, lastPage bool) bool { + if len(page.DevEndpoints) == 0 { + log.Printf("[INFO] No Glue Dev Endpoints to sweep") + return false + } + for _, endpoint := range page.DevEndpoints { + name := aws.StringValue(endpoint.EndpointName) + if !strings.HasPrefix(name, GlueDevEndpointResourcePrefix) { + log.Printf("[INFO] Skipping Glue Dev Endpoint: %s", name) + continue + } + + log.Printf("[INFO] Deleting Glue Dev Endpoint: %s", name) + _, err := conn.DeleteDevEndpoint(&glue.DeleteDevEndpointInput{ + EndpointName: aws.String(name), + }) + if err != nil { + log.Printf("[ERROR] Failed to delete Glue Dev Endpoint %s: %s", name, err) + } + } + return !lastPage + }) + if err != nil { + if testSweepSkipSweepError(err) { + log.Printf("[WARN] Skipping Glue Dev Endpoint sweep for %s: %s", region, err) + return nil + } + return fmt.Errorf("error retrieving Glue Dev Endpoint: %s", err) + } + + return nil +} + +func TestAccGlueDevEndpoint_Basic(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_Basic(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestMatchResourceAttr(resourceName, "role_arn", regexp.MustCompile(`^arn:[^:]+:iam::[^:]+:role/AWSGlueServiceRole-tf-acc-test-[0-9]+$`)), + resource.TestCheckResourceAttr(resourceName, "status", "READY"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraJarsS3Path := "foo" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraPythonLibsS3Path := "foo" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "2"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKeys(rName, publicKey1, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_SecurityConfiguration(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "security_configuration", rName), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Note: Either none or both of subnetId and securityGroupIds must be specified. +func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "vpc_security_group_ids.#", "1"), + resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "data.aws_security_group.default", "id"), + resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "aws_subnet.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "availability_zone", "aws_subnet.test", "availability_zone"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_PublicKey(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + ), + }, + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey2), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_ExtraJarsS3Path(t *testing.T) { + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraJarsS3Path := "foo" + extraJarsS3PathUpdated := "bar" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3PathUpdated), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_ExtraPythonLibsS3Path(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraPythonLibsS3Path := "foo" + extraPythonLibsS3PathUpdated := "bar" + + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3PathUpdated), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_PublicKeys(t *testing.T) { + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKeys(rName, publicKey1, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_Update_PublicKeys(rName, publicKey1, publicKey3, publicKey4), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccCheckAWSGlueDevEndpointExists(resourceName string, endpoint *glue.DevEndpoint) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[resourceName] + if !ok { + return fmt.Errorf("not found: %s", resourceName) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("no ID is set") + } + + conn := testAccProvider.Meta().(*AWSClient).glueconn + output, err := conn.GetDevEndpoint(&glue.GetDevEndpointInput{ + EndpointName: aws.String(rs.Primary.ID), + }) + + if err != nil { + return err + } + + if output == nil { + return fmt.Errorf("no Glue Dev Endpoint") + } + + *endpoint = *output.DevEndpoint + + return nil + } +} + +func testAccCheckAWSGlueDevEndpointDestroy(s *terraform.State) error { + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_glue_dev_endpoint" { + continue + } + + conn := testAccProvider.Meta().(*AWSClient).glueconn + output, err := conn.GetDevEndpoint(&glue.GetDevEndpointInput{ + EndpointName: aws.String(rs.Primary.ID), + }) + + if err != nil { + if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + return nil + } + return err + } + + endpoint := output.DevEndpoint + if endpoint != nil && aws.StringValue(endpoint.EndpointName) == rs.Primary.ID { + return fmt.Errorf("the Glue Dev Endpoint %s still exists", rs.Primary.ID) + } + + return nil + } + + return nil +} + +func testAccGlueDevEndpointConfig_Base(rName string) string { + return fmt.Sprintf(` +resource "aws_iam_role" "test" { + name = "AWSGlueServiceRole-%s" + assume_role_policy = "${data.aws_iam_policy_document.test.json}" +} + +data "aws_iam_policy_document" "test" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy_attachment" "test-AWSGlueServiceRole" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" + role = "${aws_iam_role.test.name}" +} +`, rName) +} + +func testAccGlueDevEndpointConfig_Basic(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" +} +`, rName) +} + +func testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName string, extraJarsS3Path string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + extra_jars_s3_path = %q +} +`, rName, extraJarsS3Path) +} + +func testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName string, extraPythonLibsS3Path string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + extra_python_libs_s3_path = %q +} +`, rName, extraPythonLibsS3Path) +} + +func testAccGlueDevEndpointConfig_NumberOfNodes(rName string, numberOfNodes int) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + number_of_nodes = %d +} +`, rName, numberOfNodes) +} + +func testAccGlueDevEndpointConfig_PublicKey(rName string, publicKey string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + public_key = "%s" +} +`, rName, publicKey) +} + +func testAccGlueDevEndpointConfig_PublicKeys(rName string, publicKey1 string, publicKey2 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + public_keys = ["%s", "%s"] +} +`, rName, publicKey1, publicKey2) +} + +func testAccGlueDevEndpointConfig_Update_PublicKeys(rName string, publicKey1 string, publicKey2 string, publicKey3 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + public_keys = ["%s", "%s", "%s"] +} +`, rName, publicKey1, publicKey2, publicKey3) +} + +func testAccGlueDevEndpointConfig_SecurityConfiguration(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + security_configuration = "${aws_glue_security_configuration.test.name}" +} + +resource "aws_glue_security_configuration" "test" { + name = %q + + encryption_configuration { + cloudwatch_encryption { + cloudwatch_encryption_mode = "DISABLED" + } + + job_bookmarks_encryption { + job_bookmarks_encryption_mode = "DISABLED" + } + + s3_encryption { + s3_encryption_mode = "DISABLED" + } + } +} +`, rName, rName) +} + +func testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + subnet_id = "${aws_subnet.test.id}" + security_group_ids = ["${data.aws_security_group.default.id}"] +} + +resource "aws_vpc" "test" { + cidr_block = "10.0.0.0/16" + tags = { + Name = %q + } +} + +resource "aws_subnet" "test" { + vpc_id = "${aws_vpc.test.id}" + cidr_block = "10.0.1.0/24" + availability_zone = "us-west-2a" + + tags = { + Name = %q + } +} + +data "aws_security_group" "default" { + vpc_id = "${aws_vpc.test.id}" + name = "default" +} + +data "aws_vpc_endpoint_service" "s3" { + service = "s3" +} + +resource "aws_vpc_endpoint" "test" { + service_name = "${data.aws_vpc_endpoint_service.s3.service_name}" + vpc_id = "${aws_vpc.test.id}" +} + +resource "aws_internet_gateway" "test" { + vpc_id = "${aws_vpc.test.id}" +} + +resource "aws_eip" "test" { + vpc = true +} + +resource "aws_nat_gateway" "test" { + allocation_id = "${aws_eip.test.id}" + subnet_id = "${aws_subnet.test.id}" + + tags = { + Name = %q + } + + depends_on = ["aws_internet_gateway.test"] +} +`, rName, rName, rName, rName) +} diff --git a/website/docs/r/glue_dev_endpoint.markdown b/website/docs/r/glue_dev_endpoint.markdown new file mode 100644 index 00000000000..7ef68117bb1 --- /dev/null +++ b/website/docs/r/glue_dev_endpoint.markdown @@ -0,0 +1,81 @@ +--- +layout: "aws" +page_title: "AWS: aws_glue_dev_endpoint" +sidebar_current: "docs-aws-resource-glue-dev-endpoint" +description: |- + Provides a Glue Development Endpoint resource. +--- + +# aws_glue_dev_endpoint + +Provides a Glue Development Endpoint resource. + +## Example Usage + +Basic usage: + +```hcl +resource "aws_glue_dev_endpoint" "de" { + name = "foo" + role_arn = "${aws_iam_role.test.arn}" +} + +resource "aws_iam_role" "de" { + name = "AWSGlueServiceRole-foo" + assume_role_policy = "${data.aws_iam_policy_document.de.json}" +} + +data "aws_iam_policy_document" "de" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy_attachment" "foo-AWSGlueServiceRole" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" + role = "${aws_iam_role.de.name}" +} + +``` + +## Argument Reference + +The following arguments are supported: + +* `name` - (Optional) The name of the Glue Development Endpoint (must be unique). If omitted, Terraform will assign a random, unique name. +* `extra_jars_s3_path` - (Optional) Path to one or more Java Jars in an S3 bucket that should be loaded in your DevEndpoint. +* `extra_python_libs_s3_path` - (Optional) Path(s) to one or more Python libraries in an S3 bucket that should be loaded in your DevEndpoint. Multiple values must be complete paths separated by a comma. +* `number_of_nodes` - (Optional) The number of AWS Glue Data Processing Units (DPUs) to allocate to this DevEndpoint. +* `public_key` - (Optional) The public key to be used by this DevEndpoint for authentication. +* `public_keys` - (Optional) A list of public keys to be used by the DevEndpoints for authentication. +* `role_arn` - (Required) The IAM role for the DevEndpoint. +* `security_configuration` - (Optional) The name of the SecurityConfiguration structure to be used with this DevEndpoint. +* `security_group_ids` - (Optional) Security group IDs for the security groups to be used by the new DevEndpoint. +* `subnet_id` - (Optional) The subnet ID for the new DevEndpoint to use. + +## Attributes Reference + +The following attributes are exported: + +* `name` - The name of the new Glue Development Endpoint. +* `private_address` - A private IP address to access the DevEndpoint within a VPC, if the DevEndpoint is created within one. +* `public_address` - The public IP address used by this DevEndpoint. The PublicAddress field is present only when you create a non-VPC DevEndpoint. +* `yarn_endpoint_address` - The YARN endpoint address used by this DevEndpoint. +* `zeppelin_remote_spark_interpreter_port` - The Apache Zeppelin port for the remote Apache Spark interpreter. +* `availability_zone` - The AWS availability zone where this DevEndpoint is located. +* `vpc_id` - he ID of the VPC used by this DevEndpoint. +* `status` - The current status of this DevEndpoint. +* `failure_reason` - The reason for a current failure in this DevEndpoint. + +## Import + +SageMaker Glue Development Endpoint can be imported using the `name`, e.g. + +``` +$ terraform import aws_glue_dev_endpoint.de foo +``` \ No newline at end of file From 2ef02595136b17ba7b269a15bce2055fb9df6058 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Sat, 21 Dec 2019 21:00:05 +0100 Subject: [PATCH 02/12] Update Glue Dev Endpoint to AWS API changes Add glue_version, worker_type, etc. arguments --- aws/arguments_test.go | 88 ++++ aws/publicKeys_test.go | 67 +++ aws/resource_aws_glue_dev_endpoint.go | 353 +++++++++++---- aws/resource_aws_glue_dev_endpoint_test.go | 478 ++++++++++++++++----- aws/validators.go | 10 + website/docs/r/glue_dev_endpoint.markdown | 42 +- 6 files changed, 810 insertions(+), 228 deletions(-) create mode 100644 aws/arguments_test.go create mode 100644 aws/publicKeys_test.go diff --git a/aws/arguments_test.go b/aws/arguments_test.go new file mode 100644 index 00000000000..88c6af26fe5 --- /dev/null +++ b/aws/arguments_test.go @@ -0,0 +1,88 @@ +package aws + +import ( + "reflect" + "testing" + + "github.com/aws/aws-sdk-go/aws" +) + +func TestDiffArguments(t *testing.T) { + tests := []struct { + name string + oldArgs, newArgs map[string]interface{} + expectedCreate map[string]string + expectedRemove []string + }{ + { + name: "empty old and new arguments", + oldArgs: map[string]interface{}{}, + newArgs: map[string]interface{}{}, + expectedCreate: map[string]string{}, + expectedRemove: []string{}, + }, + { + name: "old and new arguments are the same", + oldArgs: map[string]interface{}{ + "foo": "foo-val", + "bar": "bar-val", + }, + newArgs: map[string]interface{}{ + "foo": "foo-val", + "bar": "bar-val", + }, + expectedCreate: map[string]string{}, + expectedRemove: []string{}, + }, + { + name: "same argument with new value created", + oldArgs: map[string]interface{}{}, + newArgs: map[string]interface{}{ + "foo": "foo-val", + }, + expectedCreate: map[string]string{ + "foo": "foo-val", + }, + expectedRemove: []string{}, + }, + { + name: "old argument deleted", + oldArgs: map[string]interface{}{ + "foo": "foo-val", + }, + newArgs: map[string]interface{}{}, + expectedCreate: map[string]string{}, + expectedRemove: []string{"foo"}, + }, + { + name: "some old and new arguments overlap", + oldArgs: map[string]interface{}{ + "foo": "foo-val", + "bar": "bar-val", + }, + newArgs: map[string]interface{}{ + "foo": "foo-val", + "bar": "baz-val", + }, + expectedCreate: map[string]string{ + "bar": "baz-val", + }, + expectedRemove: []string{"bar"}, + }, + } + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + create, remove := diffArguments(tc.oldArgs, tc.newArgs) + cr := aws.StringValueMap(create) + rm := aws.StringValueSlice(remove) + + if !reflect.DeepEqual(cr, tc.expectedCreate) { + t.Fatalf("%d: bad create: %#v", i, cr) + } + if !reflect.DeepEqual(rm, tc.expectedRemove) { + t.Fatalf("%d: bad remove: %#v", i, rm) + } + + }) + } +} diff --git a/aws/publicKeys_test.go b/aws/publicKeys_test.go new file mode 100644 index 00000000000..81e039ab4f7 --- /dev/null +++ b/aws/publicKeys_test.go @@ -0,0 +1,67 @@ +package aws + +import ( + "reflect" + "testing" + + "github.com/aws/aws-sdk-go/aws" +) + +func TestDiffPublicKeys(t *testing.T) { + tests := []struct { + name string + oldKeys, newKeys []*string + expectedCreate, expectedRemove []string + }{ + { + name: "empty old and new keys", + oldKeys: []*string{}, + newKeys: []*string{}, + expectedCreate: []string{}, + expectedRemove: []string{}, + }, + { + name: "old and new keys are the same", + oldKeys: []*string{aws.String("foo"), aws.String("bar")}, + newKeys: []*string{aws.String("foo"), aws.String("bar")}, + expectedCreate: []string{}, + expectedRemove: []string{}, + }, + { + name: "new key created", + oldKeys: []*string{}, + newKeys: []*string{aws.String("foo")}, + expectedCreate: []string{"foo"}, + expectedRemove: []string{}, + }, + { + name: "old key deleted", + oldKeys: []*string{aws.String("foo")}, + newKeys: []*string{}, + expectedCreate: []string{}, + expectedRemove: []string{"foo"}, + }, + { + name: "some old and new keys overlap", + oldKeys: []*string{aws.String("foo"), aws.String("bar")}, + newKeys: []*string{aws.String("bar"), aws.String("baz")}, + expectedCreate: []string{"baz"}, + expectedRemove: []string{"foo"}, + }, + } + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + create, remove := diffPublicKeys(tc.oldKeys, tc.newKeys) + cr := aws.StringValueSlice(create) + rm := aws.StringValueSlice(remove) + + if !reflect.DeepEqual(cr, tc.expectedCreate) { + t.Fatalf("%d: bad create: %#v", i, cr) + } + if !reflect.DeepEqual(rm, tc.expectedRemove) { + t.Fatalf("%d: bad remove: %#v", i, rm) + } + + }) + } +} diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index cc8ed89925a..0550b1f5c9a 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -6,11 +6,13 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/glue" - - "github.com/hashicorp/terraform/helper/resource" - "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/helper/validation" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags" ) func resourceAwsGlueDevEndpoint() *schema.Resource { @@ -24,11 +26,14 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { }, Schema: map[string]*schema.Schema{ - "name": { - Type: schema.TypeString, + "arguments": { + Type: schema.TypeMap, Optional: true, + }, + + "arn": { + Type: schema.TypeString, Computed: true, - ForceNew: true, }, "extra_jars_s3_path": { @@ -41,24 +46,52 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { Optional: true, }, - "number_of_nodes": { - Type: schema.TypeInt, + "glue_version": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validateGlueVersion, + }, + + "name": { + Type: schema.TypeString, Optional: true, + Computed: true, ForceNew: true, - Default: 5, + }, + + "number_of_nodes": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"number_of_workers", "worker_type"}, + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return new == "0" + }, + ValidateFunc: validation.IntAtLeast(2), + }, + + "number_of_workers": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + ValidateFunc: validation.IntAtLeast(2), + ConflictsWith: []string{"number_of_nodes"}, }, "public_key": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ConflictsWith: []string{"public_keys"}, }, "public_keys": { - Type: schema.TypeSet, - Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, - Set: schema.HashString, - MaxItems: 5, + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + ConflictsWith: []string{"public_key"}, + MaxItems: 5, }, "role_arn": { @@ -87,6 +120,8 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { ForceNew: true, }, + "tags": tagsSchema(), + "private_address": { Type: schema.TypeString, Computed: true, @@ -107,6 +142,18 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { Computed: true, }, + "worker_type": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice([]string{ + glue.WorkerTypeG1x, + glue.WorkerTypeG2x, + glue.WorkerTypeStandard, + }, false), + ConflictsWith: []string{"number_of_nodes"}, + ForceNew: true, + }, + "availability_zone": { Type: schema.TypeString, Computed: true, @@ -140,52 +187,77 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) name = resource.UniqueId() } - createOpts := &glue.CreateDevEndpointInput{ + input := &glue.CreateDevEndpointInput{ EndpointName: aws.String(name), RoleArn: aws.String(d.Get("role_arn").(string)), + Tags: keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().GlueTags(), + } + + if kv, ok := d.GetOk("arguments"); ok { + arguments := make(map[string]string) + for k, v := range kv.(map[string]interface{}) { + arguments[k] = v.(string) + } + input.Arguments = aws.StringMap(arguments) } if v, ok := d.GetOk("extra_jars_s3_path"); ok { - createOpts.SetExtraJarsS3Path(v.(string)) + input.ExtraJarsS3Path = aws.String(v.(string)) } if v, ok := d.GetOk("extra_python_libs_s3_path"); ok { - createOpts.SetExtraPythonLibsS3Path(v.(string)) + input.ExtraPythonLibsS3Path = aws.String(v.(string)) + } + + if v, ok := d.GetOk("glue_version"); ok { + input.GlueVersion = aws.String(v.(string)) } if v, ok := d.GetOk("number_of_nodes"); ok { - createOpts.SetNumberOfNodes(int64(v.(int))) + input.NumberOfNodes = aws.Int64(int64(v.(int))) + } + + if v, ok := d.GetOk("number_of_workers"); ok { + input.NumberOfWorkers = aws.Int64(int64(v.(int))) } if v, ok := d.GetOk("public_key"); ok { - createOpts.SetPublicKey(v.(string)) + input.PublicKey = aws.String(v.(string)) } if v, ok := d.GetOk("public_keys"); ok { publicKeys := expandStringSet(v.(*schema.Set)) - createOpts.SetPublicKeys(publicKeys) + input.PublicKeys = publicKeys } if v, ok := d.GetOk("security_configuration"); ok { - createOpts.SetSecurityConfiguration(v.(string)) + input.SecurityConfiguration = aws.String(v.(string)) } if v, ok := d.GetOk("security_group_ids"); ok { securityGroupIDs := expandStringSet(v.(*schema.Set)) - createOpts.SetSecurityGroupIds(securityGroupIDs) + input.SecurityGroupIds = securityGroupIDs } if v, ok := d.GetOk("subnet_id"); ok { - createOpts.SetSubnetId(v.(string)) + input.SubnetId = aws.String(v.(string)) + } + + if v, ok := d.GetOk("worker_type"); ok { + input.WorkerType = aws.String(v.(string)) } - log.Printf("[DEBUG] Glue dev endpoint create config: %#v", *createOpts) + + log.Printf("[DEBUG] Creating Glue Dev Endpoint: %#v", *input) err := resource.Retry(1*time.Minute, func() *resource.RetryError { - _, err := conn.CreateDevEndpoint(createOpts) + _, err := conn.CreateDevEndpoint(input) if err != nil { // Retry for IAM eventual consistency if isAWSErr(err, glue.ErrCodeInvalidInputException, "should be given assume role permissions for Glue Service") { return resource.RetryableError(err) } + if isAWSErr(err, glue.ErrCodeInvalidInputException, "S3 endpoint and NAT validation has failed for subnetId") { + return resource.RetryableError(err) + } return resource.NonRetryableError(err) } @@ -193,23 +265,22 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) }) if err != nil { - return fmt.Errorf("error creating Glue dev endpoint: %s", err) + return fmt.Errorf("error creating Glue Dev Endpoint: %s", err) } d.SetId(name) - log.Printf("[INFO] Glue dev endpoint ID: %s", d.Id()) - log.Printf("[DEBUG] Waiting for Glue dev endpoint (%s) to become available", d.Id()) + log.Printf("[DEBUG] Waiting for Glue Dev Endpoint (%s) to become available", d.Id()) stateConf := &resource.StateChangeConf{ Pending: []string{ "PROVISIONING", }, Target: []string{"READY"}, Refresh: glueDevEndpointStateRefreshFunc(conn, d.Id()), - Timeout: 10 * time.Minute, + Timeout: 15 * time.Minute, } if _, err := stateConf.WaitForState(); err != nil { - return fmt.Errorf("error while waiting for Glue dev endpoint (%s) to become available: %s", d.Id(), err) + return fmt.Errorf("error while waiting for Glue Dev Endpoint (%s) to become available: %s", d.Id(), err) } return resourceAwsGlueDevEndpointRead(d, meta) @@ -224,139 +295,204 @@ func resourceAwsGlueDevEndpointRead(d *schema.ResourceData, meta interface{}) er output, err := conn.GetDevEndpoint(request) if err != nil { - if glueErr, ok := err.(awserr.Error); ok && glueErr.Code() == glue.ErrCodeEntityNotFoundException { - log.Printf("[INFO] unable to find Glue dev endpoint and therfore it is removed from the state: %s", d.Id()) + if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + log.Printf("[WARN] Glue Dev Endpoint (%s) not found, removing from state", d.Id()) d.SetId("") return nil } - return fmt.Errorf("error finding Glue dev endpoint %s: %s", d.Id(), err) + return fmt.Errorf("error reading Glue Dev Endpoint (%s): %s", d.Id(), err) } endpoint := output.DevEndpoint if endpoint == nil { - return fmt.Errorf("Glue dev endpoint (%s) is nil: %s", d.Id(), err) + log.Printf("[WARN] Glue Dev Endpoint (%s) not found, removing from state", d.Id()) + d.SetId("") + return nil } - if err := d.Set("name", endpoint.EndpointName); err != nil { - return fmt.Errorf("error setting name for Glue dev endpoint (%s): %s", d.Id(), err) + endpointARN := arn.ARN{ + Partition: meta.(*AWSClient).partition, + Service: "glue", + Region: meta.(*AWSClient).region, + AccountID: meta.(*AWSClient).accountid, + Resource: fmt.Sprintf("devEndpoint/%s", d.Id()), + }.String() + + if err := d.Set("arn", endpointARN); err != nil { + return fmt.Errorf("error setting arn for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("arguments", aws.StringValueMap(endpoint.Arguments)); err != nil { + return fmt.Errorf("error setting arguments for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("availability_zone", endpoint.AvailabilityZone); err != nil { + return fmt.Errorf("error setting availability_zone for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("extra_jars_s3_path", endpoint.ExtraJarsS3Path); err != nil { - return fmt.Errorf("error setting extra_jars_s3_path for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting extra_jars_s3_path for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("extra_python_libs_s3_path", endpoint.ExtraPythonLibsS3Path); err != nil { - return fmt.Errorf("error setting extra_python_libs_s3_path for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting extra_python_libs_s3_path for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("failure_reason", endpoint.FailureReason); err != nil { + return fmt.Errorf("error setting failure_reason for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("glue_version", endpoint.GlueVersion); err != nil { + return fmt.Errorf("error setting glue_version for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("name", endpoint.EndpointName); err != nil { + return fmt.Errorf("error setting name for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("number_of_nodes", endpoint.NumberOfNodes); err != nil { - return fmt.Errorf("error setting number_of_nodes for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting number_of_nodes for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("number_of_workers", endpoint.NumberOfWorkers); err != nil { + return fmt.Errorf("error setting number_of_workers for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("private_address", endpoint.PrivateAddress); err != nil { + return fmt.Errorf("error setting private_address for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_address", endpoint.PublicAddress); err != nil { + return fmt.Errorf("error setting public_address for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("public_key", endpoint.PublicKey); err != nil { - return fmt.Errorf("error setting public_key for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting public_key for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("public_keys", flattenStringSet(endpoint.PublicKeys)); err != nil { - return fmt.Errorf("error setting public_keys for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting public_keys for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("role_arn", endpoint.RoleArn); err != nil { - return fmt.Errorf("error setting role_arn for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting role_arn for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("security_configuration", endpoint.SecurityConfiguration); err != nil { - return fmt.Errorf("error setting security_configuration for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting security_configuration for Glue Dev Endpoint (%s): %s", d.Id(), err) } if err := d.Set("security_group_ids", flattenStringSet(endpoint.SecurityGroupIds)); err != nil { - return fmt.Errorf("error setting security_group_ids for Glue dev endpoint (%s): %s", d.Id(), err) + return fmt.Errorf("error setting security_group_ids for Glue Dev Endpoint (%s): %s", d.Id(), err) } - if err := d.Set("subnet_id", endpoint.SubnetId); err != nil { - return fmt.Errorf("error setting subnet_id for Glue dev endpoint (%s): %s", d.Id(), err) + if err := d.Set("status", endpoint.Status); err != nil { + return fmt.Errorf("error setting status for Glue Dev Endpoint (%s): %s", d.Id(), err) } - // extra attributes - if err := d.Set("private_address", endpoint.PrivateAddress); err != nil { - return fmt.Errorf("error setting private_address for Glue dev endpoint (%s): %s", d.Id(), err) + if err := d.Set("subnet_id", endpoint.SubnetId); err != nil { + return fmt.Errorf("error setting subnet_id for Glue Dev Endpoint (%s): %s", d.Id(), err) } - if err := d.Set("public_address", endpoint.PublicAddress); err != nil { - return fmt.Errorf("error setting public_address for Glue dev endpoint (%s): %s", d.Id(), err) + if err := d.Set("vpc_id", endpoint.VpcId); err != nil { + return fmt.Errorf("error setting vpc_id for Glue Dev Endpoint (%s): %s", d.Id(), err) } - if err := d.Set("yarn_endpoint_address", endpoint.YarnEndpointAddress); err != nil { - return fmt.Errorf("error setting yarn_endpoint_address for Glue dev endpoint (%s): %s", d.Id(), err) + if err := d.Set("worker_type", endpoint.WorkerType); err != nil { + return fmt.Errorf("error setting worker_type for Glue Dev Endpoint (%s): %s", d.Id(), err) } - if err := d.Set("zeppelin_remote_spark_interpreter_port", endpoint.ZeppelinRemoteSparkInterpreterPort); err != nil { - return fmt.Errorf("error setting zeppelin_remote_spark_interpreter_port for Glue dev endpoint (%s): %s", d.Id(), err) - } + tags, err := keyvaluetags.GlueListTags(conn, endpointARN) - if err := d.Set("availability_zone", endpoint.AvailabilityZone); err != nil { - return fmt.Errorf("error setting availability_zone for Glue dev endpoint (%s): %s", d.Id(), err) + if err != nil { + return fmt.Errorf("error listing tags for Glue Dev Endpoint (%s): %s", endpointARN, err) } - if err := d.Set("vpc_id", endpoint.VpcId); err != nil { - return fmt.Errorf("error setting vpc_id for Glue dev endpoint (%s): %s", d.Id(), err) + if err := d.Set("tags", tags.IgnoreAws().Map()); err != nil { + return fmt.Errorf("error setting tags: %s", err) } - if err := d.Set("status", endpoint.Status); err != nil { - return fmt.Errorf("error setting status for Glue dev endpoint (%s): %s", d.Id(), err) + if err := d.Set("yarn_endpoint_address", endpoint.YarnEndpointAddress); err != nil { + return fmt.Errorf("error setting yarn_endpoint_address for Glue Dev Endpoint (%s): %s", d.Id(), err) } - if err := d.Set("failure_reason", endpoint.FailureReason); err != nil { - return fmt.Errorf("error setting failure_reason for Glue dev endpoint (%s): %s", d.Id(), err) + if err := d.Set("zeppelin_remote_spark_interpreter_port", endpoint.ZeppelinRemoteSparkInterpreterPort); err != nil { + return fmt.Errorf("error setting zeppelin_remote_spark_interpreter_port for Glue Dev Endpoint (%s): %s", d.Id(), err) } + return nil } func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).glueconn - updateOpts := &glue.UpdateDevEndpointInput{ + input := &glue.UpdateDevEndpointInput{ EndpointName: aws.String(d.Get("name").(string)), } hasChanged := false - if d.HasChange("public_keys") { - oldRaw, newRaw := d.GetChange("public_keys") - old := oldRaw.(*schema.Set) - new := newRaw.(*schema.Set) - create, remove := diffPublicKeys(expandStringSet(old), expandStringSet(new)) - updateOpts.SetAddPublicKeys(create) - updateOpts.SetDeletePublicKeys(remove) + customLibs := &glue.DevEndpointCustomLibraries{} + + if d.HasChange("arguments") { + oldRaw, newRaw := d.GetChange("arguments") + old := oldRaw.(map[string]interface{}) + new := newRaw.(map[string]interface{}) + create, remove := diffArguments(old, new) + input.AddArguments = create + input.DeleteArguments = remove hasChanged = true } - if d.HasChange("public_key") { - updateOpts.SetPublicKey(d.Get("public_key").(string)) + if d.HasChange("extra_jars_s3_path") { + customLibs.ExtraJarsS3Path = aws.String(d.Get("extra_jars_s3_path").(string)) + input.CustomLibraries = customLibs + input.UpdateEtlLibraries = aws.Bool(true) + hasChanged = true } - customLibs := &glue.DevEndpointCustomLibraries{} + if d.HasChange("extra_python_libs_s3_path") { + customLibs.ExtraPythonLibsS3Path = aws.String(d.Get("extra_python_libs_s3_path").(string)) + input.CustomLibraries = customLibs + input.UpdateEtlLibraries = aws.Bool(true) - if d.HasChange("extra_jars_s3_path") { - customLibs.SetExtraJarsS3Path(d.Get("extra_jars_s3_path").(string)) - updateOpts.SetCustomLibraries(customLibs) - updateOpts.SetUpdateEtlLibraries(true) hasChanged = true } - if d.HasChange("extra_python_libs_s3_path") { - customLibs.SetExtraPythonLibsS3Path(d.Get("extra_python_libs_s3_path").(string)) - updateOpts.SetCustomLibraries(customLibs) - updateOpts.SetUpdateEtlLibraries(true) + if d.HasChange("public_key") { + input.PublicKey = aws.String(d.Get("public_key").(string)) + + hasChanged = true + } + + if d.HasChange("public_keys") { + oldRaw, newRaw := d.GetChange("public_keys") + old := oldRaw.(*schema.Set) + new := newRaw.(*schema.Set) + create, remove := diffPublicKeys(expandStringSet(old), expandStringSet(new)) + input.AddPublicKeys = create + log.Printf("[DEBUG] expectedCreate public keys: %v", create) + input.DeletePublicKeys = remove + log.Printf("[DEBUG] remove public keys: %v", remove) + hasChanged = true } if hasChanged { - _, err := conn.UpdateDevEndpoint(updateOpts) + log.Printf("[DEBUG] Updating Glue Dev Endpoint: %s", input) + + _, err := conn.UpdateDevEndpoint(input) if err != nil { - return fmt.Errorf("error updating Glue dev endpoint: %s", err) + return fmt.Errorf("error updating Glue Dev Endpoint: %s", err) + } + } + + if d.HasChange("tags") { + o, n := d.GetChange("tags") + if err := keyvaluetags.GlueUpdateTags(conn, d.Get("arn").(string), o, n); err != nil { + return fmt.Errorf("error updating tags: %s", err) } } @@ -369,7 +505,7 @@ func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) erro deleteOpts := &glue.DeleteDevEndpointInput{ EndpointName: aws.String(d.Id()), } - log.Printf("[INFO] Deleting Glue dev endpoint: %s", d.Id()) + log.Printf("[INFO] Deleting Glue Dev Endpoint: %s", d.Id()) return resource.Retry(5*time.Minute, func() *resource.RetryError { _, err := conn.DeleteDevEndpoint(deleteOpts) @@ -386,7 +522,7 @@ func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) erro return nil } - return resource.NonRetryableError(fmt.Errorf("error deleting Glue dev endpoint: %s", err)) + return resource.NonRetryableError(fmt.Errorf("error deleting Glue Dev Endpoint: %s", err)) }) } @@ -419,7 +555,7 @@ func diffPublicKeys(oldKeys, newKeys []*string) ([]*string, []*string) { for _, oldKey := range oldKeys { found := false for _, newKey := range newKeys { - if oldKey == newKey { + if *oldKey == *newKey { found = true break } @@ -432,7 +568,7 @@ func diffPublicKeys(oldKeys, newKeys []*string) ([]*string, []*string) { for _, newKey := range newKeys { found := false for _, oldKey := range oldKeys { - if oldKey == newKey { + if *oldKey == *newKey { found = true break } @@ -444,3 +580,38 @@ func diffPublicKeys(oldKeys, newKeys []*string) ([]*string, []*string) { return create, remove } + +func diffArguments(oldArgs, newArgs map[string]interface{}) (map[string]*string, []*string) { + var create = make(map[string]*string) + var remove []*string + + for oldArgKey, oldArgVal := range oldArgs { + found := false + for newArgKey, newArgVal := range newArgs { + if oldArgKey == newArgKey && + oldArgVal.(string) == newArgVal.(string) { + found = true + break + } + } + if !found { + remove = append(remove, &oldArgKey) + } + } + + for newArgKey, newArgVal := range newArgs { + found := false + for oldArgKey, oldArgVal := range oldArgs { + if oldArgKey == newArgKey && + oldArgVal.(string) == newArgVal.(string) { + found = true + break + } + } + if !found { + create[newArgKey] = aws.String(newArgVal.(string)) + } + } + + return create, remove +} diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index aa5e2cd25f8..b0e59a440bd 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -9,9 +9,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" - "github.com/hashicorp/terraform/helper/acctest" - "github.com/hashicorp/terraform/helper/resource" - "github.com/hashicorp/terraform/terraform" + "github.com/hashicorp/terraform-plugin-sdk/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/terraform" ) const ( @@ -71,9 +71,8 @@ func testSweepGlueDevEndpoint(region string) error { } func TestAccGlueDevEndpoint_Basic(t *testing.T) { - t.Skip() - var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) resourceName := "aws_glue_dev_endpoint.test" @@ -87,9 +86,13 @@ func TestAccGlueDevEndpoint_Basic(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("devEndpoint/%s", rName)), resource.TestCheckResourceAttr(resourceName, "name", rName), resource.TestMatchResourceAttr(resourceName, "role_arn", regexp.MustCompile(`^arn:[^:]+:iam::[^:]+:role/AWSGlueServiceRole-tf-acc-test-[0-9]+$`)), resource.TestCheckResourceAttr(resourceName, "status", "READY"), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "0"), + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), + resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), ), }, { @@ -101,12 +104,10 @@ func TestAccGlueDevEndpoint_Basic(t *testing.T) { }) } -func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { - t.Skip() - +func TestAccGlueDevEndpoint_Arguments(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) - extraJarsS3Path := "foo" resourceName := "aws_glue_dev_endpoint.test" resource.ParallelTest(t, resource.TestCase{ @@ -115,11 +116,22 @@ func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), + Config: testAccGlueDevEndpointConfig_Arguments2(rName, "bar", "python"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "2"), + resource.TestCheckResourceAttr(resourceName, "arguments.--foo", "bar"), + resource.TestCheckResourceAttr(resourceName, "arguments.--job-language", "python"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_Arguments(rName, "baz"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "arguments.%", "1"), + resource.TestCheckResourceAttr(resourceName, "arguments.--foo", "baz"), ), }, { @@ -131,12 +143,12 @@ func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { }) } -func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { - t.Skip() - +func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) - extraPythonLibsS3Path := "foo" + extraJarsS3Path := "foo" + extraJarsS3PathUpdated := "bar" resourceName := "aws_glue_dev_endpoint.test" resource.ParallelTest(t, resource.TestCase{ @@ -145,11 +157,19 @@ func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3PathUpdated), ), }, { @@ -161,11 +181,13 @@ func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { }) } -func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { - t.Skip() - +func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraPythonLibsS3Path := "foo" + extraPythonLibsS3PathUpdated := "bar" + resourceName := "aws_glue_dev_endpoint.test" resource.ParallelTest(t, resource.TestCase{ @@ -174,11 +196,19 @@ func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 2), + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "2"), + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3PathUpdated), ), }, { @@ -190,10 +220,9 @@ func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { }) } -func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { - t.Skip() - +func TestAccGlueDevEndpoint_GlueVersion(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) resourceName := "aws_glue_dev_endpoint.test" @@ -203,11 +232,23 @@ func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), + Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "1"), + ExpectError: regexp.MustCompile(`attribute glue_version must match version pattern X.X`), + }, + { + Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "1.0"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + resource.TestCheckResourceAttr(resourceName, "glue_version", "1.0"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "0.9"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "glue_version", "0.9"), ), }, { @@ -219,10 +260,9 @@ func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { }) } -func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { - t.Skip() - +func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) resourceName := "aws_glue_dev_endpoint.test" @@ -232,11 +272,23 @@ func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_PublicKeys(rName, publicKey1, publicKey2), + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 1), + ExpectError: regexp.MustCompile(`expected number_of_nodes to be at least`), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 2), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 5), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), ), }, { @@ -248,10 +300,9 @@ func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { }) } -func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { - t.Skip() - +func TestAccGlueDevEndpoint_NumberOfWorkers(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) resourceName := "aws_glue_dev_endpoint.test" @@ -261,11 +312,23 @@ func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_SecurityConfiguration(rName), + Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 1), + ExpectError: regexp.MustCompile(`expected number_of_workers to be at least`), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 2), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "security_configuration", rName), + resource.TestCheckResourceAttr(resourceName, "number_of_workers", "2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 5), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "number_of_workers", "5"), ), }, { @@ -277,9 +340,9 @@ func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { }) } -// Note: Either none or both of subnetId and securityGroupIds must be specified. -func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { +func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) resourceName := "aws_glue_dev_endpoint.test" @@ -289,15 +352,19 @@ func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName), + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "vpc_security_group_ids.#", "1"), - resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "data.aws_security_group.default", "id"), - resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "aws_subnet.test", "id"), - resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"), - resource.TestCheckResourceAttrPair(resourceName, "availability_zone", "aws_subnet.test", "availability_zone"), + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + ), + }, + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey2), ), }, { @@ -309,10 +376,9 @@ func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { }) } -func TestAccGlueDevEndpoint_Update_PublicKey(t *testing.T) { - t.Skip() - +func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) resourceName := "aws_glue_dev_endpoint.test" @@ -322,19 +388,27 @@ func TestAccGlueDevEndpoint_Update_PublicKey(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), + Config: testAccGlueDevEndpointConfig_PublicKeys2(rName, publicKey1, publicKey2), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), ), }, { - Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey2), + Config: testAccGlueDevEndpointConfig_PublicKeys3(rName, publicKey1, publicKey3, publicKey4), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_key", publicKey2), + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_PublicKeys4(rName, publicKey1, publicKey1, publicKey3, publicKey4), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), ), }, { @@ -346,11 +420,10 @@ func TestAccGlueDevEndpoint_Update_PublicKey(t *testing.T) { }) } -func TestAccGlueDevEndpoint_Update_ExtraJarsS3Path(t *testing.T) { +func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) - extraJarsS3Path := "foo" - extraJarsS3PathUpdated := "bar" resourceName := "aws_glue_dev_endpoint.test" resource.ParallelTest(t, resource.TestCase{ @@ -359,19 +432,43 @@ func TestAccGlueDevEndpoint_Update_ExtraJarsS3Path(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), + Config: testAccGlueDevEndpointConfig_SecurityConfiguration(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + resource.TestCheckResourceAttr(resourceName, "security_configuration", rName), ), }, { - Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3PathUpdated), + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Note: Either none or both of subnetId and securityGroupIds must be specified. +func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3PathUpdated), + resource.TestCheckResourceAttr(resourceName, "security_group_ids.#", "1"), + resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "aws_subnet.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "availability_zone", "aws_subnet.test", "availability_zone"), ), }, { @@ -383,14 +480,10 @@ func TestAccGlueDevEndpoint_Update_ExtraJarsS3Path(t *testing.T) { }) } -func TestAccGlueDevEndpoint_Update_ExtraPythonLibsS3Path(t *testing.T) { - t.Skip() +func TestAccGlueDevEndpoint_Tags(t *testing.T) { + var endpoint1, endpoint2, endpoint3 glue.DevEndpoint - var endpoint glue.DevEndpoint rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) - extraPythonLibsS3Path := "foo" - extraPythonLibsS3PathUpdated := "bar" - resourceName := "aws_glue_dev_endpoint.test" resource.ParallelTest(t, resource.TestCase{ @@ -399,32 +492,42 @@ func TestAccGlueDevEndpoint_Update_ExtraPythonLibsS3Path(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), + Config: testAccAWSGlueDevEndpointConfig_Tags1(rName, "key1", "value1"), Check: resource.ComposeTestCheckFunc( - testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - - resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint1), + resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), + resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"), ), }, { - Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3PathUpdated), + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccAWSGlueDevEndpointConfig_Tags2(rName, "key1", "value1updated", "key2", "value2"), Check: resource.ComposeTestCheckFunc( - testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - - resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3PathUpdated), + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint2), + resource.TestCheckResourceAttr(resourceName, "tags.%", "2"), + resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"), + resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, + Config: testAccAWSGlueDevEndpointConfig_Tags1(rName, "key2", "value2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint3), + resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), + resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), + ), }, }, }) } -func TestAccGlueDevEndpoint_Update_PublicKeys(t *testing.T) { +func TestAccGlueDevEndpoint_WorkerType(t *testing.T) { var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) resourceName := "aws_glue_dev_endpoint.test" @@ -434,19 +537,27 @@ func TestAccGlueDevEndpoint_Update_PublicKeys(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_PublicKeys(rName, publicKey1, publicKey2), + Config: testAccGlueDevEndpointConfig_WorkerType_Standard(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), + resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeStandard), ), }, { - Config: testAccGlueDevEndpointConfig_Update_PublicKeys(rName, publicKey1, publicKey3, publicKey4), + Config: testAccGlueDevEndpointConfig_WorkerType(rName, glue.WorkerTypeG1x), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), + resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeG1x), + ), + }, + { + Config: testAccGlueDevEndpointConfig_WorkerType(rName, glue.WorkerTypeG2x), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeG2x), ), }, { @@ -520,11 +631,11 @@ func testAccCheckAWSGlueDevEndpointDestroy(s *terraform.State) error { func testAccGlueDevEndpointConfig_Base(rName string) string { return fmt.Sprintf(` resource "aws_iam_role" "test" { - name = "AWSGlueServiceRole-%s" - assume_role_policy = "${data.aws_iam_policy_document.test.json}" + name = "AWSGlueServiceRole-%s" + assume_role_policy = "${data.aws_iam_policy_document.service.json}" } -data "aws_iam_policy_document" "test" { +data "aws_iam_policy_document" "service" { statement { actions = ["sts:AssumeRole"] @@ -535,11 +646,28 @@ data "aws_iam_policy_document" "test" { } } -resource "aws_iam_role_policy_attachment" "test-AWSGlueServiceRole" { +resource "aws_iam_policy" "test" { + name = %q + policy = "${data.aws_iam_policy_document.test.json}" +} + +data "aws_iam_policy_document" "test" { + statement { + actions = ["ec2:DescribeSecurityGroups", "ec2:DescribeSubnets"] + resources = ["*"] + } +} + +resource "aws_iam_role_policy_attachment" "test" { + role = "${aws_iam_role.test.name}" + policy_arn = "${aws_iam_policy.test.arn}" +} + +resource "aws_iam_role_policy_attachment" "glue_service_role" { policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" role = "${aws_iam_role.test.name}" } -`, rName) +`, rName, rName) } func testAccGlueDevEndpointConfig_Basic(rName string) string { @@ -551,6 +679,31 @@ resource "aws_glue_dev_endpoint" "test" { `, rName) } +func testAccGlueDevEndpointConfig_Arguments(rName, arguments string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + arguments = { + "--foo" = "%s" + } +} +`, rName, arguments) +} + +func testAccGlueDevEndpointConfig_Arguments2(rName, arguments1, arguments2 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + arguments = { + "--foo" = "%s" + "--job-language" = "%s" + } +} +`, rName, arguments1, arguments2) +} + func testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName string, extraJarsS3Path string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { @@ -571,6 +724,16 @@ resource "aws_glue_dev_endpoint" "test" { `, rName, extraPythonLibsS3Path) } +func testAccGlueDevEndpointConfig_GlueVersion(rName string, glueVersion string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = "${aws_iam_role.test.arn}" + glue_version = %[2]q +} +`, rName, glueVersion) +} + func testAccGlueDevEndpointConfig_NumberOfNodes(rName string, numberOfNodes int) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { @@ -581,6 +744,17 @@ resource "aws_glue_dev_endpoint" "test" { `, rName, numberOfNodes) } +func testAccGlueDevEndpointConfig_NumberOfWorkers(rName string, numberOfWorkers int) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + worker_type = "G.1X" + number_of_workers = %d +} +`, rName, numberOfWorkers) +} + func testAccGlueDevEndpointConfig_PublicKey(rName string, publicKey string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { @@ -591,26 +765,36 @@ resource "aws_glue_dev_endpoint" "test" { `, rName, publicKey) } -func testAccGlueDevEndpointConfig_PublicKeys(rName string, publicKey1 string, publicKey2 string) string { +func testAccGlueDevEndpointConfig_PublicKeys2(rName string, publicKey1 string, publicKey2 string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = "${aws_iam_role.test.arn}" - public_keys = ["%s", "%s"] + public_keys = [%[2]q, %[3]q] } `, rName, publicKey1, publicKey2) } -func testAccGlueDevEndpointConfig_Update_PublicKeys(rName string, publicKey1 string, publicKey2 string, publicKey3 string) string { +func testAccGlueDevEndpointConfig_PublicKeys3(rName string, publicKey1 string, publicKey2 string, publicKey3 string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = "${aws_iam_role.test.arn}" - public_keys = ["%s", "%s", "%s"] + public_keys = [%[2]q, %[3]q, %[4]q] } `, rName, publicKey1, publicKey2, publicKey3) } +func testAccGlueDevEndpointConfig_PublicKeys4(rName string, publicKey1 string, publicKey2 string, publicKey3 string, publicKey4 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = "${aws_iam_role.test.arn}" + public_keys = [%[2]q, %[3]q, %[4]q, %[5]q] +} +`, rName, publicKey1, publicKey2, publicKey3, publicKey4) +} + func testAccGlueDevEndpointConfig_SecurityConfiguration(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { @@ -645,57 +829,117 @@ resource "aws_glue_dev_endpoint" "test" { name = %q role_arn = "${aws_iam_role.test.arn}" subnet_id = "${aws_subnet.test.id}" - security_group_ids = ["${data.aws_security_group.default.id}"] + security_group_ids = ["${aws_security_group.test.id}"] +} + +resource "aws_vpc_endpoint" "s3" { + vpc_id = "${aws_vpc.test.id}" + service_name = "${data.aws_vpc_endpoint_service.s3.service_name}" +} + +data "aws_vpc_endpoint_service" "s3" { + service = "s3" +} + +resource "aws_vpc_endpoint_route_table_association" "test" { + vpc_endpoint_id = "${aws_vpc_endpoint.s3.id}" + route_table_id = "${aws_vpc.test.main_route_table_id}" } resource "aws_vpc" "test" { - cidr_block = "10.0.0.0/16" + cidr_block = "10.0.0.0/16" + tags = { Name = %q } } resource "aws_subnet" "test" { - vpc_id = "${aws_vpc.test.id}" - cidr_block = "10.0.1.0/24" + vpc_id = "${aws_vpc.test.id}" + cidr_block = "10.0.1.0/24" availability_zone = "us-west-2a" tags = { Name = %q } -} -data "aws_security_group" "default" { - vpc_id = "${aws_vpc.test.id}" - name = "default" + timeouts { + delete = "40m" + } + depends_on = ["aws_iam_role_policy_attachment.glue_service_role"] } -data "aws_vpc_endpoint_service" "s3" { - service = "s3" -} +resource "aws_security_group" "test" { + name = %q + vpc_id = "${aws_vpc.test.id}" -resource "aws_vpc_endpoint" "test" { - service_name = "${data.aws_vpc_endpoint_service.s3.service_name}" - vpc_id = "${aws_vpc.test.id}" -} + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } -resource "aws_internet_gateway" "test" { - vpc_id = "${aws_vpc.test.id}" + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + timeouts { + delete = "40m" + } + depends_on = ["aws_iam_role_policy_attachment.glue_service_role"] +} +`, rName, rName, rName, rName) } -resource "aws_eip" "test" { - vpc = true +func testAccAWSGlueDevEndpointConfig_Tags1(rName, tagKey1, tagValue1 string) string { + return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = "${aws_iam_role.test.arn}" + + tags = { + %[2]q = %[3]q + } +} +`, rName, tagKey1, tagValue1) } -resource "aws_nat_gateway" "test" { - allocation_id = "${aws_eip.test.id}" - subnet_id = "${aws_subnet.test.id}" +func testAccAWSGlueDevEndpointConfig_Tags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string { + return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = "${aws_iam_role.test.arn}" tags = { - Name = %q + %[2]q = %[3]q + %[4]q = %[5]q } +} +`, rName, tagKey1, tagValue1, tagKey2, tagValue2) +} + +func testAccGlueDevEndpointConfig_WorkerType(rName, workerType string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = "${aws_iam_role.test.arn}" + worker_type = %[2]q + number_of_workers = 2 +} +`, rName, workerType) +} - depends_on = ["aws_internet_gateway.test"] +func testAccGlueDevEndpointConfig_WorkerType_Standard(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = "${aws_iam_role.test.arn}" + worker_type = %[2]q + number_of_workers = 2 } -`, rName, rName, rName, rName) +`, rName, glue.WorkerTypeStandard) } diff --git a/aws/validators.go b/aws/validators.go index 730a08addfc..aefa8fcda72 100644 --- a/aws/validators.go +++ b/aws/validators.go @@ -434,6 +434,16 @@ func validateElbNamePrefix(v interface{}, k string) (ws []string, errors []error return } +func validateGlueVersion(v interface{}, k string) (ws []string, errors []error) { + value := v.(string) + if !regexp.MustCompile(`^\w+\.\w+$`).MatchString(value) { + errors = append(errors, fmt.Errorf( + "attribute %s must match version pattern X.X: %s", + k, value)) + } + return +} + func validateSagemakerName(v interface{}, k string) (ws []string, errors []error) { value := v.(string) if !regexp.MustCompile(`^[0-9A-Za-z-]+$`).MatchString(value) { diff --git a/website/docs/r/glue_dev_endpoint.markdown b/website/docs/r/glue_dev_endpoint.markdown index 7ef68117bb1..0b109867c67 100644 --- a/website/docs/r/glue_dev_endpoint.markdown +++ b/website/docs/r/glue_dev_endpoint.markdown @@ -1,7 +1,7 @@ --- +subcategory: "Glue" layout: "aws" page_title: "AWS: aws_glue_dev_endpoint" -sidebar_current: "docs-aws-resource-glue-dev-endpoint" description: |- Provides a Glue Development Endpoint resource. --- @@ -40,41 +40,43 @@ resource "aws_iam_role_policy_attachment" "foo-AWSGlueServiceRole" { policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" role = "${aws_iam_role.de.name}" } - ``` ## Argument Reference The following arguments are supported: -* `name` - (Optional) The name of the Glue Development Endpoint (must be unique). If omitted, Terraform will assign a random, unique name. -* `extra_jars_s3_path` - (Optional) Path to one or more Java Jars in an S3 bucket that should be loaded in your DevEndpoint. -* `extra_python_libs_s3_path` - (Optional) Path(s) to one or more Python libraries in an S3 bucket that should be loaded in your DevEndpoint. Multiple values must be complete paths separated by a comma. -* `number_of_nodes` - (Optional) The number of AWS Glue Data Processing Units (DPUs) to allocate to this DevEndpoint. -* `public_key` - (Optional) The public key to be used by this DevEndpoint for authentication. -* `public_keys` - (Optional) A list of public keys to be used by the DevEndpoints for authentication. -* `role_arn` - (Required) The IAM role for the DevEndpoint. -* `security_configuration` - (Optional) The name of the SecurityConfiguration structure to be used with this DevEndpoint. -* `security_group_ids` - (Optional) Security group IDs for the security groups to be used by the new DevEndpoint. -* `subnet_id` - (Optional) The subnet ID for the new DevEndpoint to use. +* `name` - (Optional) The name of this endpoint (must be unique). If omitted, Terraform will assign a random, unique name. +* `extra_jars_s3_path` - (Optional) Path to one or more Java Jars in an S3 bucket that should be loaded in this endpoint. +* `extra_python_libs_s3_path` - (Optional) Path(s) to one or more Python libraries in an S3 bucket that should be loaded in this endpoint. Multiple values must be complete paths separated by a comma. +* `glue_version` - (Optional) - Specifies the versions of Python and Apache Spark to use. Defaults to AWS Glue version 0.9. +* `number_of_nodes` - (Optional) The number of AWS Glue Data Processing Units (DPUs) to allocate to this endpoint. Conflicts with `worker_type`. +* `number_of_workers` - (Optional) The number of workers of a defined worker type that are allocated to this endpoint. This field is available only when you choose worker type G.1X or G.2X. +* `public_key` - (Optional) The public key to be used by this endpoint for authentication. +* `public_keys` - (Optional) A list of public keys to be used by this endpoint for authentication. +* `role_arn` - (Required) The IAM role for this endpoint. +* `security_configuration` - (Optional) The name of the Security Configuration structure to be used with this endpoint. +* `security_group_ids` - (Optional) Security group IDs for the security groups to be used by this endpoint. +* `subnet_id` - (Optional) The subnet ID for the new endpoint to use. +* `worker_type` - (Optional) The type of predefined worker that is allocated to this endpoint. Accepts a value of Standard, G.1X, or G.2X. ## Attributes Reference The following attributes are exported: * `name` - The name of the new Glue Development Endpoint. -* `private_address` - A private IP address to access the DevEndpoint within a VPC, if the DevEndpoint is created within one. -* `public_address` - The public IP address used by this DevEndpoint. The PublicAddress field is present only when you create a non-VPC DevEndpoint. -* `yarn_endpoint_address` - The YARN endpoint address used by this DevEndpoint. +* `private_address` - A private IP address to access the Glue Development Endpoint within a VPC, if this endpoint is created within one. +* `public_address` - The public IP address used by this endpoint. The PublicAddress field is present only when you create a non-VPC endpoint. +* `yarn_endpoint_address` - The YARN endpoint address used by this endpoint. * `zeppelin_remote_spark_interpreter_port` - The Apache Zeppelin port for the remote Apache Spark interpreter. -* `availability_zone` - The AWS availability zone where this DevEndpoint is located. -* `vpc_id` - he ID of the VPC used by this DevEndpoint. -* `status` - The current status of this DevEndpoint. -* `failure_reason` - The reason for a current failure in this DevEndpoint. +* `availability_zone` - The AWS availability zone where this endpoint is located. +* `vpc_id` - he ID of the VPC used by this endpoint. +* `status` - The current status of this endpoint. +* `failure_reason` - The reason for a current failure in this endpoint. ## Import -SageMaker Glue Development Endpoint can be imported using the `name`, e.g. +Glue Development Endpoint can be imported using the `name`, e.g. ``` $ terraform import aws_glue_dev_endpoint.de foo From 7b034488c518a7341d7a28adaae757e317b0f115 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Fri, 18 Sep 2020 22:16:31 +0200 Subject: [PATCH 03/12] Switch to Terraform Plugin SDK Version 2 --- aws/resource_aws_glue_dev_endpoint.go | 6 +++--- aws/resource_aws_glue_dev_endpoint_test.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index 0550b1f5c9a..7ce4c88613d 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -9,9 +9,9 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/glue" - "github.com/hashicorp/terraform-plugin-sdk/helper/resource" - "github.com/hashicorp/terraform-plugin-sdk/helper/schema" - "github.com/hashicorp/terraform-plugin-sdk/helper/validation" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" "github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags" ) diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index b0e59a440bd..d7703554c2e 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -9,9 +9,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" - "github.com/hashicorp/terraform-plugin-sdk/helper/acctest" - "github.com/hashicorp/terraform-plugin-sdk/helper/resource" - "github.com/hashicorp/terraform-plugin-sdk/terraform" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" ) const ( From b3371a288242ea58e93cd329f6aee6ddf79ffc85 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Fri, 18 Sep 2020 22:24:30 +0200 Subject: [PATCH 04/12] Switch to Terraform 0.12 Syntax --- aws/resource_aws_glue_dev_endpoint_test.go | 68 +++++++++++----------- website/docs/r/glue_dev_endpoint.markdown | 10 ++-- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index d7703554c2e..348a0315c52 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -632,7 +632,7 @@ func testAccGlueDevEndpointConfig_Base(rName string) string { return fmt.Sprintf(` resource "aws_iam_role" "test" { name = "AWSGlueServiceRole-%s" - assume_role_policy = "${data.aws_iam_policy_document.service.json}" + assume_role_policy = data.aws_iam_policy_document.service.json } data "aws_iam_policy_document" "service" { @@ -648,7 +648,7 @@ data "aws_iam_policy_document" "service" { resource "aws_iam_policy" "test" { name = %q - policy = "${data.aws_iam_policy_document.test.json}" + policy = data.aws_iam_policy_document.test.json } data "aws_iam_policy_document" "test" { @@ -659,13 +659,13 @@ data "aws_iam_policy_document" "test" { } resource "aws_iam_role_policy_attachment" "test" { - role = "${aws_iam_role.test.name}" - policy_arn = "${aws_iam_policy.test.arn}" + role = aws_iam_role.test.name + policy_arn = aws_iam_policy.test.arn } resource "aws_iam_role_policy_attachment" "glue_service_role" { policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" - role = "${aws_iam_role.test.name}" + role = aws_iam_role.test.name } `, rName, rName) } @@ -674,7 +674,7 @@ func testAccGlueDevEndpointConfig_Basic(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn } `, rName) } @@ -683,7 +683,7 @@ func testAccGlueDevEndpointConfig_Arguments(rName, arguments string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn arguments = { "--foo" = "%s" } @@ -695,7 +695,7 @@ func testAccGlueDevEndpointConfig_Arguments2(rName, arguments1, arguments2 strin return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn arguments = { "--foo" = "%s" "--job-language" = "%s" @@ -708,7 +708,7 @@ func testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName string, extraJarsS3Path return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn extra_jars_s3_path = %q } `, rName, extraJarsS3Path) @@ -718,7 +718,7 @@ func testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName string, extraPytho return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn extra_python_libs_s3_path = %q } `, rName, extraPythonLibsS3Path) @@ -728,7 +728,7 @@ func testAccGlueDevEndpointConfig_GlueVersion(rName string, glueVersion string) return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn glue_version = %[2]q } `, rName, glueVersion) @@ -738,7 +738,7 @@ func testAccGlueDevEndpointConfig_NumberOfNodes(rName string, numberOfNodes int) return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn number_of_nodes = %d } `, rName, numberOfNodes) @@ -748,7 +748,7 @@ func testAccGlueDevEndpointConfig_NumberOfWorkers(rName string, numberOfWorkers return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn worker_type = "G.1X" number_of_workers = %d } @@ -759,7 +759,7 @@ func testAccGlueDevEndpointConfig_PublicKey(rName string, publicKey string) stri return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn public_key = "%s" } `, rName, publicKey) @@ -769,7 +769,7 @@ func testAccGlueDevEndpointConfig_PublicKeys2(rName string, publicKey1 string, p return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q] } `, rName, publicKey1, publicKey2) @@ -779,7 +779,7 @@ func testAccGlueDevEndpointConfig_PublicKeys3(rName string, publicKey1 string, p return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q, %[4]q] } `, rName, publicKey1, publicKey2, publicKey3) @@ -789,7 +789,7 @@ func testAccGlueDevEndpointConfig_PublicKeys4(rName string, publicKey1 string, p return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q, %[4]q, %[5]q] } `, rName, publicKey1, publicKey2, publicKey3, publicKey4) @@ -799,8 +799,8 @@ func testAccGlueDevEndpointConfig_SecurityConfiguration(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" - security_configuration = "${aws_glue_security_configuration.test.name}" + role_arn = aws_iam_role.test.arn + security_configuration = aws_glue_security_configuration.test.name } resource "aws_glue_security_configuration" "test" { @@ -827,14 +827,14 @@ func testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName string) string return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %q - role_arn = "${aws_iam_role.test.arn}" - subnet_id = "${aws_subnet.test.id}" - security_group_ids = ["${aws_security_group.test.id}"] + role_arn = aws_iam_role.test.arn + subnet_id = aws_subnet.test.id + security_group_ids = [aws_security_group.test.id] } resource "aws_vpc_endpoint" "s3" { - vpc_id = "${aws_vpc.test.id}" - service_name = "${data.aws_vpc_endpoint_service.s3.service_name}" + vpc_id = aws_vpc.test.id + service_name = data.aws_vpc_endpoint_service.s3.service_name } data "aws_vpc_endpoint_service" "s3" { @@ -842,8 +842,8 @@ data "aws_vpc_endpoint_service" "s3" { } resource "aws_vpc_endpoint_route_table_association" "test" { - vpc_endpoint_id = "${aws_vpc_endpoint.s3.id}" - route_table_id = "${aws_vpc.test.main_route_table_id}" + vpc_endpoint_id = aws_vpc_endpoint.s3.id + route_table_id = aws_vpc.test.main_route_table_id } resource "aws_vpc" "test" { @@ -855,7 +855,7 @@ resource "aws_vpc" "test" { } resource "aws_subnet" "test" { - vpc_id = "${aws_vpc.test.id}" + vpc_id = aws_vpc.test.id cidr_block = "10.0.1.0/24" availability_zone = "us-west-2a" @@ -866,12 +866,12 @@ resource "aws_subnet" "test" { timeouts { delete = "40m" } - depends_on = ["aws_iam_role_policy_attachment.glue_service_role"] + depends_on = [aws_iam_role_policy_attachment.glue_service_role] } resource "aws_security_group" "test" { name = %q - vpc_id = "${aws_vpc.test.id}" + vpc_id = aws_vpc.test.id ingress { from_port = 0 @@ -890,7 +890,7 @@ resource "aws_security_group" "test" { timeouts { delete = "40m" } - depends_on = ["aws_iam_role_policy_attachment.glue_service_role"] + depends_on = [aws_iam_role_policy_attachment.glue_service_role] } `, rName, rName, rName, rName) } @@ -899,7 +899,7 @@ func testAccAWSGlueDevEndpointConfig_Tags1(rName, tagKey1, tagValue1 string) str return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn tags = { %[2]q = %[3]q @@ -912,7 +912,7 @@ func testAccAWSGlueDevEndpointConfig_Tags2(rName, tagKey1, tagValue1, tagKey2, t return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn tags = { %[2]q = %[3]q @@ -926,7 +926,7 @@ func testAccGlueDevEndpointConfig_WorkerType(rName, workerType string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn worker_type = %[2]q number_of_workers = 2 } @@ -937,7 +937,7 @@ func testAccGlueDevEndpointConfig_WorkerType_Standard(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q - role_arn = "${aws_iam_role.test.arn}" + role_arn = aws_iam_role.test.arn worker_type = %[2]q number_of_workers = 2 } diff --git a/website/docs/r/glue_dev_endpoint.markdown b/website/docs/r/glue_dev_endpoint.markdown index 0b109867c67..3e867a8098e 100644 --- a/website/docs/r/glue_dev_endpoint.markdown +++ b/website/docs/r/glue_dev_endpoint.markdown @@ -16,13 +16,13 @@ Basic usage: ```hcl resource "aws_glue_dev_endpoint" "de" { - name = "foo" - role_arn = "${aws_iam_role.test.arn}" + name = "foo" + role_arn = aws_iam_role.de.arn } resource "aws_iam_role" "de" { - name = "AWSGlueServiceRole-foo" - assume_role_policy = "${data.aws_iam_policy_document.de.json}" + name = "AWSGlueServiceRole-foo" + assume_role_policy = data.aws_iam_policy_document.de.json } data "aws_iam_policy_document" "de" { @@ -38,7 +38,7 @@ data "aws_iam_policy_document" "de" { resource "aws_iam_role_policy_attachment" "foo-AWSGlueServiceRole" { policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" - role = "${aws_iam_role.de.name}" + role = aws_iam_role.de.name } ``` From 0ee148f6e392704b7ca419365949e84ea91432b5 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Wed, 14 Oct 2020 16:59:18 +0200 Subject: [PATCH 05/12] Please the linters --- aws/resource_aws_glue_dev_endpoint.go | 29 ++++---- aws/resource_aws_glue_dev_endpoint_test.go | 86 +++++++++++----------- website/docs/r/glue_dev_endpoint.markdown | 11 ++- 3 files changed, 64 insertions(+), 62 deletions(-) diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index 7ce4c88613d..6cab3f8d1fa 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -7,7 +7,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/arn" - "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/glue" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -29,6 +28,7 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { "arguments": { Type: schema.TypeMap, Optional: true, + Elem: schema.TypeString, }, "arn": { @@ -264,6 +264,10 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) return nil }) + if isResourceTimeoutError(err) { + _, err = conn.CreateDevEndpoint(input) + } + if err != nil { return fmt.Errorf("error creating Glue Dev Endpoint: %s", err) } @@ -288,6 +292,7 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) func resourceAwsGlueDevEndpointRead(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).glueconn + ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig request := &glue.GetDevEndpointInput{ EndpointName: aws.String(d.Id()), @@ -408,7 +413,7 @@ func resourceAwsGlueDevEndpointRead(d *schema.ResourceData, meta interface{}) er return fmt.Errorf("error listing tags for Glue Dev Endpoint (%s): %s", endpointARN, err) } - if err := d.Set("tags", tags.IgnoreAws().Map()); err != nil { + if err := d.Set("tags", tags.IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil { return fmt.Errorf("error setting tags: %s", err) } @@ -505,25 +510,19 @@ func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) erro deleteOpts := &glue.DeleteDevEndpointInput{ EndpointName: aws.String(d.Id()), } + log.Printf("[INFO] Deleting Glue Dev Endpoint: %s", d.Id()) - return resource.Retry(5*time.Minute, func() *resource.RetryError { - _, err := conn.DeleteDevEndpoint(deleteOpts) - if err == nil { + _, err := conn.DeleteDevEndpoint(deleteOpts) + if err != nil { + if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { return nil } - glueErr, ok := err.(awserr.Error) - if !ok { - return resource.NonRetryableError(err) - } - - if glueErr.Code() == glue.ErrCodeEntityNotFoundException { - return nil - } + return err + } - return resource.NonRetryableError(fmt.Errorf("error deleting Glue Dev Endpoint: %s", err)) - }) + return nil } func glueDevEndpointStateRefreshFunc(conn *glue.Glue, name string) resource.StateRefreshFunc { diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index 348a0315c52..831dd2071f8 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -88,7 +88,7 @@ func TestAccGlueDevEndpoint_Basic(t *testing.T) { testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("devEndpoint/%s", rName)), resource.TestCheckResourceAttr(resourceName, "name", rName), - resource.TestMatchResourceAttr(resourceName, "role_arn", regexp.MustCompile(`^arn:[^:]+:iam::[^:]+:role/AWSGlueServiceRole-tf-acc-test-[0-9]+$`)), + testAccMatchResourceAttrGlobalARN(resourceName, "role_arn", "iam", regexp.MustCompile(`role/AWSGlueServiceRole-tf-acc-test-[0-9]+$`)), resource.TestCheckResourceAttr(resourceName, "status", "READY"), resource.TestCheckResourceAttr(resourceName, "arguments.%", "0"), resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), @@ -673,7 +673,7 @@ resource "aws_iam_role_policy_attachment" "glue_service_role" { func testAccGlueDevEndpointConfig_Basic(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %q role_arn = aws_iam_role.test.arn } `, rName) @@ -682,7 +682,7 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_Arguments(rName, arguments string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %q role_arn = aws_iam_role.test.arn arguments = { "--foo" = "%s" @@ -694,10 +694,10 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_Arguments2(rName, arguments1, arguments2 string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %q role_arn = aws_iam_role.test.arn arguments = { - "--foo" = "%s" + "--foo" = "%s" "--job-language" = "%s" } } @@ -707,8 +707,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName string, extraJarsS3Path string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q - role_arn = aws_iam_role.test.arn + name = %q + role_arn = aws_iam_role.test.arn extra_jars_s3_path = %q } `, rName, extraJarsS3Path) @@ -717,8 +717,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName string, extraPythonLibsS3Path string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q - role_arn = aws_iam_role.test.arn + name = %q + role_arn = aws_iam_role.test.arn extra_python_libs_s3_path = %q } `, rName, extraPythonLibsS3Path) @@ -727,8 +727,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_GlueVersion(rName string, glueVersion string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q - role_arn = aws_iam_role.test.arn + name = %[1]q + role_arn = aws_iam_role.test.arn glue_version = %[2]q } `, rName, glueVersion) @@ -737,8 +737,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_NumberOfNodes(rName string, numberOfNodes int) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q - role_arn = aws_iam_role.test.arn + name = %q + role_arn = aws_iam_role.test.arn number_of_nodes = %d } `, rName, numberOfNodes) @@ -747,9 +747,9 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_NumberOfWorkers(rName string, numberOfWorkers int) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q - role_arn = aws_iam_role.test.arn - worker_type = "G.1X" + name = %q + role_arn = aws_iam_role.test.arn + worker_type = "G.1X" number_of_workers = %d } `, rName, numberOfWorkers) @@ -758,8 +758,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_PublicKey(rName string, publicKey string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q - role_arn = aws_iam_role.test.arn + name = %q + role_arn = aws_iam_role.test.arn public_key = "%s" } `, rName, publicKey) @@ -768,8 +768,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_PublicKeys2(rName string, publicKey1 string, publicKey2 string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q - role_arn = aws_iam_role.test.arn + name = %[1]q + role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q] } `, rName, publicKey1, publicKey2) @@ -778,8 +778,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_PublicKeys3(rName string, publicKey1 string, publicKey2 string, publicKey3 string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q - role_arn = aws_iam_role.test.arn + name = %[1]q + role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q, %[4]q] } `, rName, publicKey1, publicKey2, publicKey3) @@ -788,8 +788,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_PublicKeys4(rName string, publicKey1 string, publicKey2 string, publicKey3 string, publicKey4 string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q - role_arn = aws_iam_role.test.arn + name = %[1]q + role_arn = aws_iam_role.test.arn public_keys = [%[2]q, %[3]q, %[4]q, %[5]q] } `, rName, publicKey1, publicKey2, publicKey3, publicKey4) @@ -798,8 +798,8 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_SecurityConfiguration(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q - role_arn = aws_iam_role.test.arn + name = %q + role_arn = aws_iam_role.test.arn security_configuration = aws_glue_security_configuration.test.name } @@ -826,9 +826,9 @@ resource "aws_glue_security_configuration" "test" { func testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q - role_arn = aws_iam_role.test.arn - subnet_id = aws_subnet.test.id + name = %q + role_arn = aws_iam_role.test.arn + subnet_id = aws_subnet.test.id security_group_ids = [aws_security_group.test.id] } @@ -847,7 +847,7 @@ resource "aws_vpc_endpoint_route_table_association" "test" { } resource "aws_vpc" "test" { - cidr_block = "10.0.0.0/16" + cidr_block = "10.0.0.0/16" tags = { Name = %q @@ -855,8 +855,8 @@ resource "aws_vpc" "test" { } resource "aws_subnet" "test" { - vpc_id = aws_vpc.test.id - cidr_block = "10.0.1.0/24" + vpc_id = aws_vpc.test.id + cidr_block = "10.0.1.0/24" availability_zone = "us-west-2a" tags = { @@ -870,8 +870,8 @@ resource "aws_subnet" "test" { } resource "aws_security_group" "test" { - name = %q - vpc_id = aws_vpc.test.id + name = %q + vpc_id = aws_vpc.test.id ingress { from_port = 0 @@ -882,7 +882,7 @@ resource "aws_security_group" "test" { egress { from_port = 0 - to_port = 0 + to_port = 0 protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } @@ -898,7 +898,7 @@ resource "aws_security_group" "test" { func testAccAWSGlueDevEndpointConfig_Tags1(rName, tagKey1, tagValue1 string) string { return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q + name = %[1]q role_arn = aws_iam_role.test.arn tags = { @@ -911,7 +911,7 @@ resource "aws_glue_dev_endpoint" "test" { func testAccAWSGlueDevEndpointConfig_Tags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string { return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q + name = %[1]q role_arn = aws_iam_role.test.arn tags = { @@ -925,9 +925,9 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_WorkerType(rName, workerType string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q - role_arn = aws_iam_role.test.arn - worker_type = %[2]q + name = %[1]q + role_arn = aws_iam_role.test.arn + worker_type = %[2]q number_of_workers = 2 } `, rName, workerType) @@ -936,9 +936,9 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_WorkerType_Standard(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %[1]q - role_arn = aws_iam_role.test.arn - worker_type = %[2]q + name = %[1]q + role_arn = aws_iam_role.test.arn + worker_type = %[2]q number_of_workers = 2 } `, rName, glue.WorkerTypeStandard) diff --git a/website/docs/r/glue_dev_endpoint.markdown b/website/docs/r/glue_dev_endpoint.markdown index 3e867a8098e..77142dde6f3 100644 --- a/website/docs/r/glue_dev_endpoint.markdown +++ b/website/docs/r/glue_dev_endpoint.markdown @@ -46,10 +46,11 @@ resource "aws_iam_role_policy_attachment" "foo-AWSGlueServiceRole" { The following arguments are supported: -* `name` - (Optional) The name of this endpoint (must be unique). If omitted, Terraform will assign a random, unique name. +* `arguments` - (Optional) A map of arguments used to configure the endpoint. * `extra_jars_s3_path` - (Optional) Path to one or more Java Jars in an S3 bucket that should be loaded in this endpoint. * `extra_python_libs_s3_path` - (Optional) Path(s) to one or more Python libraries in an S3 bucket that should be loaded in this endpoint. Multiple values must be complete paths separated by a comma. * `glue_version` - (Optional) - Specifies the versions of Python and Apache Spark to use. Defaults to AWS Glue version 0.9. +* `name` - (Optional) The name of this endpoint (must be unique). If omitted, Terraform will assign a random, unique name. * `number_of_nodes` - (Optional) The number of AWS Glue Data Processing Units (DPUs) to allocate to this endpoint. Conflicts with `worker_type`. * `number_of_workers` - (Optional) The number of workers of a defined worker type that are allocated to this endpoint. This field is available only when you choose worker type G.1X or G.2X. * `public_key` - (Optional) The public key to be used by this endpoint for authentication. @@ -58,14 +59,16 @@ The following arguments are supported: * `security_configuration` - (Optional) The name of the Security Configuration structure to be used with this endpoint. * `security_group_ids` - (Optional) Security group IDs for the security groups to be used by this endpoint. * `subnet_id` - (Optional) The subnet ID for the new endpoint to use. +* `tags` - (Optional) Key-value map of resource tags. * `worker_type` - (Optional) The type of predefined worker that is allocated to this endpoint. Accepts a value of Standard, G.1X, or G.2X. ## Attributes Reference The following attributes are exported: -* `name` - The name of the new Glue Development Endpoint. -* `private_address` - A private IP address to access the Glue Development Endpoint within a VPC, if this endpoint is created within one. +* `arn` - The ARN of the endpoint. +* `name` - The name of the new endpoint. +* `private_address` - A private IP address to access the endpoint within a VPC, if this endpoint is created within one. * `public_address` - The public IP address used by this endpoint. The PublicAddress field is present only when you create a non-VPC endpoint. * `yarn_endpoint_address` - The YARN endpoint address used by this endpoint. * `zeppelin_remote_spark_interpreter_port` - The Apache Zeppelin port for the remote Apache Spark interpreter. @@ -76,7 +79,7 @@ The following attributes are exported: ## Import -Glue Development Endpoint can be imported using the `name`, e.g. +A Glue Development Endpoint can be imported using the `name`, e.g. ``` $ terraform import aws_glue_dev_endpoint.de foo From adce4a12901970abb71c063fc5e8e8fad62c8859 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Sat, 17 Oct 2020 13:41:12 +0200 Subject: [PATCH 06/12] Address comments --- aws/internal/service/glue/waiter/status.go | 23 ++++ aws/resource_aws_glue_dev_endpoint.go | 132 ++++++++------------- aws/resource_aws_glue_dev_endpoint_test.go | 10 +- aws/validators.go | 10 -- 4 files changed, 72 insertions(+), 103 deletions(-) diff --git a/aws/internal/service/glue/waiter/status.go b/aws/internal/service/glue/waiter/status.go index e7093e36246..c800a4695fc 100644 --- a/aws/internal/service/glue/waiter/status.go +++ b/aws/internal/service/glue/waiter/status.go @@ -3,6 +3,7 @@ package waiter import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/aws-sdk-go-base/tfawserr" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" ) @@ -52,3 +53,25 @@ func TriggerStatus(conn *glue.Glue, triggerName string) resource.StateRefreshFun return output, aws.StringValue(output.Trigger.State), nil } } + +func GlueDevEndpointStatus(conn *glue.Glue, name string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + getDevEndpointInput := &glue.GetDevEndpointInput{ + EndpointName: aws.String(name), + } + endpoint, err := conn.GetDevEndpoint(getDevEndpointInput) + if err != nil { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + return nil, "", nil + } + + return nil, "", err + } + + if endpoint == nil { + return nil, "", nil + } + + return endpoint, aws.StringValue(endpoint.DevEndpoint.Status), nil + } +} diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index 6cab3f8d1fa..19502c3d43c 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -3,8 +3,11 @@ package aws import ( "fmt" "log" + "regexp" "time" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/glue/waiter" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/service/glue" @@ -47,10 +50,18 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { }, "glue_version": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - ValidateFunc: validateGlueVersion, + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) { + value := v.(string) + if !regexp.MustCompile(`^\w+\.\w+$`).MatchString(value) { + errors = append(errors, fmt.Errorf( + "attribute %s must match version pattern X.X: %s", + k, value)) + } + return + }, }, "name": { @@ -95,9 +106,10 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { }, "role_arn": { - Type: schema.TypeString, - Required: true, - ForceNew: true, + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validateArn, }, "security_configuration": { @@ -107,17 +119,19 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { }, "security_group_ids": { - Type: schema.TypeSet, - Optional: true, - ForceNew: true, - Elem: &schema.Schema{Type: schema.TypeString}, - Set: schema.HashString, + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + RequiredWith: []string{"subnet_id"}, }, "subnet_id": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, + Type: schema.TypeString, + Optional: true, + ForceNew: true, + RequiredWith: []string{"security_group_ids"}, }, "tags": tagsSchema(), @@ -143,13 +157,9 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { }, "worker_type": { - Type: schema.TypeString, - Optional: true, - ValidateFunc: validation.StringInSlice([]string{ - glue.WorkerTypeG1x, - glue.WorkerTypeG2x, - glue.WorkerTypeStandard, - }, false), + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice(glue.WorkerType_Values(), false), ConflictsWith: []string{"number_of_nodes"}, ForceNew: true, }, @@ -280,7 +290,7 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) "PROVISIONING", }, Target: []string{"READY"}, - Refresh: glueDevEndpointStateRefreshFunc(conn, d.Id()), + Refresh: waiter.GlueDevEndpointStatus(conn, d.Id()), Timeout: 15 * time.Minute, } if _, err := stateConf.WaitForState(); err != nil { @@ -473,13 +483,22 @@ func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) erro } if d.HasChange("public_keys") { - oldRaw, newRaw := d.GetChange("public_keys") - old := oldRaw.(*schema.Set) - new := newRaw.(*schema.Set) - create, remove := diffPublicKeys(expandStringSet(old), expandStringSet(new)) - input.AddPublicKeys = create + o, n := d.GetChange("public_keys") + if o == nil { + o = new(schema.Set) + } + if n == nil { + n = new(schema.Set) + } + os := o.(*schema.Set) + ns := n.(*schema.Set) + remove := os.Difference(ns).List() + create := ns.Difference(os).List() + + input.AddPublicKeys = expandStringList(create) log.Printf("[DEBUG] expectedCreate public keys: %v", create) - input.DeletePublicKeys = remove + + input.DeletePublicKeys = expandStringList(remove) log.Printf("[DEBUG] remove public keys: %v", remove) hasChanged = true @@ -525,61 +544,6 @@ func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) erro return nil } -func glueDevEndpointStateRefreshFunc(conn *glue.Glue, name string) resource.StateRefreshFunc { - return func() (interface{}, string, error) { - getDevEndpointInput := &glue.GetDevEndpointInput{ - EndpointName: aws.String(name), - } - endpoint, err := conn.GetDevEndpoint(getDevEndpointInput) - if err != nil { - if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { - return nil, "", nil - } - - return nil, "", err - } - - if endpoint == nil { - return nil, "", nil - } - - return endpoint, *endpoint.DevEndpoint.Status, nil - } -} - -func diffPublicKeys(oldKeys, newKeys []*string) ([]*string, []*string) { - var create []*string - var remove []*string - - for _, oldKey := range oldKeys { - found := false - for _, newKey := range newKeys { - if *oldKey == *newKey { - found = true - break - } - } - if !found { - remove = append(remove, oldKey) - } - } - - for _, newKey := range newKeys { - found := false - for _, oldKey := range oldKeys { - if *oldKey == *newKey { - found = true - break - } - } - if !found { - create = append(create, newKey) - } - } - - return create, remove -} - func diffArguments(oldArgs, newArgs map[string]interface{}) (map[string]*string, []*string) { var create = make(map[string]*string) var remove []*string diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index 831dd2071f8..cf6af9afd2c 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -85,10 +85,9 @@ func TestAccGlueDevEndpoint_Basic(t *testing.T) { Config: testAccGlueDevEndpointConfig_Basic(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("devEndpoint/%s", rName)), resource.TestCheckResourceAttr(resourceName, "name", rName), - testAccMatchResourceAttrGlobalARN(resourceName, "role_arn", "iam", regexp.MustCompile(`role/AWSGlueServiceRole-tf-acc-test-[0-9]+$`)), + resource.TestCheckResourceAttrPair(resourceName, "role_arn", "aws_iam_role.test", "role_arn"), resource.TestCheckResourceAttr(resourceName, "status", "READY"), resource.TestCheckResourceAttr(resourceName, "arguments.%", "0"), resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), @@ -119,7 +118,6 @@ func TestAccGlueDevEndpoint_Arguments(t *testing.T) { Config: testAccGlueDevEndpointConfig_Arguments2(rName, "bar", "python"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "arguments.%", "2"), resource.TestCheckResourceAttr(resourceName, "arguments.--foo", "bar"), resource.TestCheckResourceAttr(resourceName, "arguments.--job-language", "python"), @@ -129,7 +127,6 @@ func TestAccGlueDevEndpoint_Arguments(t *testing.T) { Config: testAccGlueDevEndpointConfig_Arguments(rName, "baz"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "arguments.%", "1"), resource.TestCheckResourceAttr(resourceName, "arguments.--foo", "baz"), ), @@ -160,7 +157,6 @@ func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), ), }, @@ -168,7 +164,6 @@ func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3PathUpdated), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3PathUpdated), ), }, @@ -187,7 +182,6 @@ func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) extraPythonLibsS3Path := "foo" extraPythonLibsS3PathUpdated := "bar" - resourceName := "aws_glue_dev_endpoint.test" resource.ParallelTest(t, resource.TestCase{ @@ -199,7 +193,6 @@ func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), ), }, @@ -207,7 +200,6 @@ func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3PathUpdated), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3PathUpdated), ), }, diff --git a/aws/validators.go b/aws/validators.go index aefa8fcda72..730a08addfc 100644 --- a/aws/validators.go +++ b/aws/validators.go @@ -434,16 +434,6 @@ func validateElbNamePrefix(v interface{}, k string) (ws []string, errors []error return } -func validateGlueVersion(v interface{}, k string) (ws []string, errors []error) { - value := v.(string) - if !regexp.MustCompile(`^\w+\.\w+$`).MatchString(value) { - errors = append(errors, fmt.Errorf( - "attribute %s must match version pattern X.X: %s", - k, value)) - } - return -} - func validateSagemakerName(v interface{}, k string) (ws []string, errors []error) { value := v.(string) if !regexp.MustCompile(`^[0-9A-Za-z-]+$`).MatchString(value) { From 88d3b45b6be9170f6dede3bf64b0f9da5209e8c2 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Sat, 24 Oct 2020 13:37:44 +0200 Subject: [PATCH 07/12] Clean up TestAccGlueDevEndpoint_Arguments --- aws/publicKeys_test.go | 67 ---------------------- aws/resource_aws_glue_dev_endpoint_test.go | 46 +++++++++------ 2 files changed, 27 insertions(+), 86 deletions(-) delete mode 100644 aws/publicKeys_test.go diff --git a/aws/publicKeys_test.go b/aws/publicKeys_test.go deleted file mode 100644 index 81e039ab4f7..00000000000 --- a/aws/publicKeys_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package aws - -import ( - "reflect" - "testing" - - "github.com/aws/aws-sdk-go/aws" -) - -func TestDiffPublicKeys(t *testing.T) { - tests := []struct { - name string - oldKeys, newKeys []*string - expectedCreate, expectedRemove []string - }{ - { - name: "empty old and new keys", - oldKeys: []*string{}, - newKeys: []*string{}, - expectedCreate: []string{}, - expectedRemove: []string{}, - }, - { - name: "old and new keys are the same", - oldKeys: []*string{aws.String("foo"), aws.String("bar")}, - newKeys: []*string{aws.String("foo"), aws.String("bar")}, - expectedCreate: []string{}, - expectedRemove: []string{}, - }, - { - name: "new key created", - oldKeys: []*string{}, - newKeys: []*string{aws.String("foo")}, - expectedCreate: []string{"foo"}, - expectedRemove: []string{}, - }, - { - name: "old key deleted", - oldKeys: []*string{aws.String("foo")}, - newKeys: []*string{}, - expectedCreate: []string{}, - expectedRemove: []string{"foo"}, - }, - { - name: "some old and new keys overlap", - oldKeys: []*string{aws.String("foo"), aws.String("bar")}, - newKeys: []*string{aws.String("bar"), aws.String("baz")}, - expectedCreate: []string{"baz"}, - expectedRemove: []string{"foo"}, - }, - } - for i, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - create, remove := diffPublicKeys(tc.oldKeys, tc.newKeys) - cr := aws.StringValueSlice(create) - rm := aws.StringValueSlice(remove) - - if !reflect.DeepEqual(cr, tc.expectedCreate) { - t.Fatalf("%d: bad create: %#v", i, cr) - } - if !reflect.DeepEqual(rm, tc.expectedRemove) { - t.Fatalf("%d: bad remove: %#v", i, rm) - } - - }) - } -} diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index cf6af9afd2c..466153d5884 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -115,26 +115,34 @@ func TestAccGlueDevEndpoint_Arguments(t *testing.T) { CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, Steps: []resource.TestStep{ { - Config: testAccGlueDevEndpointConfig_Arguments2(rName, "bar", "python"), + Config: testAccGlueDevEndpointConfig_Arguments(rName, "--arg1", "value1"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "arguments.%", "2"), - resource.TestCheckResourceAttr(resourceName, "arguments.--foo", "bar"), - resource.TestCheckResourceAttr(resourceName, "arguments.--job-language", "python"), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "1"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg1", "value1"), ), }, { - Config: testAccGlueDevEndpointConfig_Arguments(rName, "baz"), + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccGlueDevEndpointConfig_Arguments2(rName, "--arg1", "value1updated", "--arg2", "value2"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "arguments.%", "1"), - resource.TestCheckResourceAttr(resourceName, "arguments.--foo", "baz"), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "2"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg1", "value1updated"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg2", "value2"), ), }, { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, + Config: testAccGlueDevEndpointConfig_Arguments(rName, "--arg2", "value2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "1"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg2", "value2"), + ), }, }, }) @@ -671,29 +679,29 @@ resource "aws_glue_dev_endpoint" "test" { `, rName) } -func testAccGlueDevEndpointConfig_Arguments(rName, arguments string) string { +func testAccGlueDevEndpointConfig_Arguments(rName, argKey, argValue string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn arguments = { - "--foo" = "%s" + %[2]q = %[3]q } } -`, rName, arguments) +`, rName, argKey, argValue) } -func testAccGlueDevEndpointConfig_Arguments2(rName, arguments1, arguments2 string) string { +func testAccGlueDevEndpointConfig_Arguments2(rName, argKey1, argValue1, argKey2, argValue2 string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn arguments = { - "--foo" = "%s" - "--job-language" = "%s" + %[2]q = %[3]q + %[4]q = %[5]q } } -`, rName, arguments1, arguments2) +`, rName, argKey1, argValue1, argKey2, argValue2) } func testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName string, extraJarsS3Path string) string { From 86bc3cc80ab1ac248a0fe5861d37dbb6b1ed375a Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Sat, 24 Oct 2020 13:57:54 +0200 Subject: [PATCH 08/12] Fix and simplify diffArguments func Remove for in for loop --- aws/arguments_test.go | 53 +++++++++++++++------------ aws/resource_aws_glue_dev_endpoint.go | 44 +++++++++++++--------- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/aws/arguments_test.go b/aws/arguments_test.go index 88c6af26fe5..9caab7d9c23 100644 --- a/aws/arguments_test.go +++ b/aws/arguments_test.go @@ -24,50 +24,56 @@ func TestDiffArguments(t *testing.T) { { name: "old and new arguments are the same", oldArgs: map[string]interface{}{ - "foo": "foo-val", - "bar": "bar-val", + "arg1": "value1", + "arg2": "value2", }, newArgs: map[string]interface{}{ - "foo": "foo-val", - "bar": "bar-val", + "arg1": "value1", + "arg2": "value2", }, expectedCreate: map[string]string{}, expectedRemove: []string{}, }, { - name: "same argument with new value created", - oldArgs: map[string]interface{}{}, + name: "argument updated", + oldArgs: map[string]interface{}{ + "arg1": "value1", + "arg2": "value2", + }, newArgs: map[string]interface{}{ - "foo": "foo-val", + "arg1": "value1updated", + "arg2": "value2", }, expectedCreate: map[string]string{ - "foo": "foo-val", + "arg1": "value1updated", }, - expectedRemove: []string{}, + expectedRemove: []string{"arg1"}, }, { - name: "old argument deleted", + name: "argument added", oldArgs: map[string]interface{}{ - "foo": "foo-val", + "arg1": "value1", }, - newArgs: map[string]interface{}{}, - expectedCreate: map[string]string{}, - expectedRemove: []string{"foo"}, + newArgs: map[string]interface{}{ + "arg1": "value1", + "arg2": "value2", + }, + expectedCreate: map[string]string{ + "arg2": "value2", + }, + expectedRemove: []string{}, }, { - name: "some old and new arguments overlap", + name: "argument deleted", oldArgs: map[string]interface{}{ - "foo": "foo-val", - "bar": "bar-val", + "arg1": "value1", + "arg2": "value2", }, newArgs: map[string]interface{}{ - "foo": "foo-val", - "bar": "baz-val", + "arg2": "value2", }, - expectedCreate: map[string]string{ - "bar": "baz-val", - }, - expectedRemove: []string{"bar"}, + expectedCreate: map[string]string{}, + expectedRemove: []string{"arg1"}, }, } for i, tc := range tests { @@ -82,7 +88,6 @@ func TestDiffArguments(t *testing.T) { if !reflect.DeepEqual(rm, tc.expectedRemove) { t.Fatalf("%d: bad remove: %#v", i, rm) } - }) } } diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index 19502c3d43c..1b1379d1cd6 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -506,8 +506,22 @@ func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) erro if hasChanged { log.Printf("[DEBUG] Updating Glue Dev Endpoint: %s", input) + err := resource.Retry(5*time.Minute, func() *resource.RetryError { + _, err := conn.UpdateDevEndpoint(input) + if err != nil { + if isAWSErr(err, glue.ErrCodeInvalidInputException, "another concurrent update operation") { + return resource.RetryableError(err) + } + + return resource.NonRetryableError(err) + } + return nil + }) + + if isResourceTimeoutError(err) { + _, err = conn.UpdateDevEndpoint(input) + } - _, err := conn.UpdateDevEndpoint(input) if err != nil { return fmt.Errorf("error updating Glue Dev Endpoint: %s", err) } @@ -549,29 +563,23 @@ func diffArguments(oldArgs, newArgs map[string]interface{}) (map[string]*string, var remove []*string for oldArgKey, oldArgVal := range oldArgs { - found := false - for newArgKey, newArgVal := range newArgs { - if oldArgKey == newArgKey && - oldArgVal.(string) == newArgVal.(string) { - found = true - break + tmpKey := oldArgKey + + if val, ok := newArgs[oldArgKey]; ok { + if val.(string) != oldArgVal.(string) { + remove = append(remove, &tmpKey) } - } - if !found { - remove = append(remove, &oldArgKey) + } else { + remove = append(remove, &tmpKey) } } for newArgKey, newArgVal := range newArgs { - found := false - for oldArgKey, oldArgVal := range oldArgs { - if oldArgKey == newArgKey && - oldArgVal.(string) == newArgVal.(string) { - found = true - break + if val, ok := oldArgs[newArgKey]; ok { + if val.(string) != newArgVal.(string) { + create[newArgKey] = aws.String(newArgVal.(string)) } - } - if !found { + } else { create[newArgKey] = aws.String(newArgVal.(string)) } } From 3bde6004a63c036b6cff16fb7528443d8c290db6 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Sat, 24 Oct 2020 14:03:07 +0200 Subject: [PATCH 09/12] Add TestAccGlueDevEndpoint_disappears --- aws/resource_aws_glue_dev_endpoint_test.go | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index 466153d5884..0df7abf4d2b 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -569,6 +569,29 @@ func TestAccGlueDevEndpoint_WorkerType(t *testing.T) { }) } +func TestAccGlueDevEndpoint_disappears(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_Basic(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + testAccCheckResourceDisappears(testAccProvider, resourceAwsGlueDevEndpoint(), resourceName), + ), + ExpectNonEmptyPlan: true, + }, + }, + }) +} + func testAccCheckAWSGlueDevEndpointExists(resourceName string, endpoint *glue.DevEndpoint) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[resourceName] From 0eee437df8913bb84616d0a328d5e01664e98006 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Sun, 1 Nov 2020 21:18:06 +0100 Subject: [PATCH 10/12] Address comments of 2nd feedback round --- aws/arguments_test.go | 93 ------------------ aws/internal/service/glue/waiter/waiter.go | 40 ++++++++ aws/resource_aws_glue_dev_endpoint.go | 109 +++++---------------- aws/resource_aws_glue_dev_endpoint_test.go | 46 +++------ website/docs/r/glue_dev_endpoint.markdown | 18 ++-- 5 files changed, 90 insertions(+), 216 deletions(-) delete mode 100644 aws/arguments_test.go diff --git a/aws/arguments_test.go b/aws/arguments_test.go deleted file mode 100644 index 9caab7d9c23..00000000000 --- a/aws/arguments_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package aws - -import ( - "reflect" - "testing" - - "github.com/aws/aws-sdk-go/aws" -) - -func TestDiffArguments(t *testing.T) { - tests := []struct { - name string - oldArgs, newArgs map[string]interface{} - expectedCreate map[string]string - expectedRemove []string - }{ - { - name: "empty old and new arguments", - oldArgs: map[string]interface{}{}, - newArgs: map[string]interface{}{}, - expectedCreate: map[string]string{}, - expectedRemove: []string{}, - }, - { - name: "old and new arguments are the same", - oldArgs: map[string]interface{}{ - "arg1": "value1", - "arg2": "value2", - }, - newArgs: map[string]interface{}{ - "arg1": "value1", - "arg2": "value2", - }, - expectedCreate: map[string]string{}, - expectedRemove: []string{}, - }, - { - name: "argument updated", - oldArgs: map[string]interface{}{ - "arg1": "value1", - "arg2": "value2", - }, - newArgs: map[string]interface{}{ - "arg1": "value1updated", - "arg2": "value2", - }, - expectedCreate: map[string]string{ - "arg1": "value1updated", - }, - expectedRemove: []string{"arg1"}, - }, - { - name: "argument added", - oldArgs: map[string]interface{}{ - "arg1": "value1", - }, - newArgs: map[string]interface{}{ - "arg1": "value1", - "arg2": "value2", - }, - expectedCreate: map[string]string{ - "arg2": "value2", - }, - expectedRemove: []string{}, - }, - { - name: "argument deleted", - oldArgs: map[string]interface{}{ - "arg1": "value1", - "arg2": "value2", - }, - newArgs: map[string]interface{}{ - "arg2": "value2", - }, - expectedCreate: map[string]string{}, - expectedRemove: []string{"arg1"}, - }, - } - for i, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - create, remove := diffArguments(tc.oldArgs, tc.newArgs) - cr := aws.StringValueMap(create) - rm := aws.StringValueSlice(remove) - - if !reflect.DeepEqual(cr, tc.expectedCreate) { - t.Fatalf("%d: bad create: %#v", i, cr) - } - if !reflect.DeepEqual(rm, tc.expectedRemove) { - t.Fatalf("%d: bad remove: %#v", i, rm) - } - }) - } -} diff --git a/aws/internal/service/glue/waiter/waiter.go b/aws/internal/service/glue/waiter/waiter.go index bc0f6f4ab86..379ec3776a1 100644 --- a/aws/internal/service/glue/waiter/waiter.go +++ b/aws/internal/service/glue/waiter/waiter.go @@ -73,3 +73,43 @@ func TriggerDeleted(conn *glue.Glue, triggerName string) (*glue.GetTriggerOutput return nil, err } + +// GlueDevEndpointCreated waits for a Glue Dev Endpoint to become available. +func GlueDevEndpointCreated(conn *glue.Glue, devEndpointId string) (*glue.GetDevEndpointOutput, error) { + stateConf := &resource.StateChangeConf{ + Pending: []string{ + "PROVISIONING", + }, + Target: []string{"READY"}, + Refresh: GlueDevEndpointStatus(conn, devEndpointId), + Timeout: 15 * time.Minute, + } + + outputRaw, err := stateConf.WaitForState() + + if output, ok := outputRaw.(*glue.GetDevEndpointOutput); ok { + return output, err + } + + return nil, err +} + +// GlueDevEndpointDeleted waits for a Glue Dev Endpoint to become terminated. +func GlueDevEndpointDeleted(conn *glue.Glue, devEndpointId string) (*glue.GetDevEndpointOutput, error) { + stateConf := &resource.StateChangeConf{ + Pending: []string{ + "TERMINATING", + }, + Target: []string{"TERMINATED"}, + Refresh: GlueDevEndpointStatus(conn, devEndpointId), + Timeout: 15 * time.Minute, + } + + outputRaw, err := stateConf.WaitForState() + + if output, ok := outputRaw.(*glue.GetDevEndpointOutput); ok { + return output, err + } + + return nil, err +} diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index 1b1379d1cd6..7893b28504c 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -33,44 +33,30 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { Optional: true, Elem: schema.TypeString, }, - "arn": { Type: schema.TypeString, Computed: true, }, - "extra_jars_s3_path": { Type: schema.TypeString, Optional: true, }, - "extra_python_libs_s3_path": { Type: schema.TypeString, Optional: true, }, - "glue_version": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) { - value := v.(string) - if !regexp.MustCompile(`^\w+\.\w+$`).MatchString(value) { - errors = append(errors, fmt.Errorf( - "attribute %s must match version pattern X.X: %s", - k, value)) - } - return - }, + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validation.StringMatch(regexp.MustCompile(`^\w+\.\w+$`), "must match version pattern X.X"), }, - "name": { - Type: schema.TypeString, - Optional: true, - Computed: true, - ForceNew: true, + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validation.NoZeroValues, }, - "number_of_nodes": { Type: schema.TypeInt, Optional: true, @@ -81,7 +67,6 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { }, ValidateFunc: validation.IntAtLeast(2), }, - "number_of_workers": { Type: schema.TypeInt, Optional: true, @@ -89,13 +74,11 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { ValidateFunc: validation.IntAtLeast(2), ConflictsWith: []string{"number_of_nodes"}, }, - "public_key": { Type: schema.TypeString, Optional: true, ConflictsWith: []string{"public_keys"}, }, - "public_keys": { Type: schema.TypeSet, Optional: true, @@ -104,20 +87,17 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { ConflictsWith: []string{"public_key"}, MaxItems: 5, }, - "role_arn": { Type: schema.TypeString, Required: true, ForceNew: true, ValidateFunc: validateArn, }, - "security_configuration": { Type: schema.TypeString, Optional: true, ForceNew: true, }, - "security_group_ids": { Type: schema.TypeSet, Optional: true, @@ -126,36 +106,29 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { Set: schema.HashString, RequiredWith: []string{"subnet_id"}, }, - "subnet_id": { Type: schema.TypeString, Optional: true, ForceNew: true, RequiredWith: []string{"security_group_ids"}, }, - "tags": tagsSchema(), - "private_address": { Type: schema.TypeString, Computed: true, }, - "public_address": { Type: schema.TypeString, Computed: true, }, - "yarn_endpoint_address": { Type: schema.TypeString, Computed: true, }, - "zeppelin_remote_spark_interpreter_port": { Type: schema.TypeInt, Computed: true, }, - "worker_type": { Type: schema.TypeString, Optional: true, @@ -163,22 +136,18 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { ConflictsWith: []string{"number_of_nodes"}, ForceNew: true, }, - "availability_zone": { Type: schema.TypeString, Computed: true, }, - "vpc_id": { Type: schema.TypeString, Computed: true, }, - "status": { Type: schema.TypeString, Computed: true, }, - "failure_reason": { Type: schema.TypeString, Computed: true, @@ -189,13 +158,7 @@ func resourceAwsGlueDevEndpoint() *schema.Resource { func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).glueconn - - var name string - if v, ok := d.GetOk("name"); ok { - name = v.(string) - } else { - name = resource.UniqueId() - } + name := d.Get("name").(string) input := &glue.CreateDevEndpointInput{ EndpointName: aws.String(name), @@ -285,15 +248,7 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) d.SetId(name) log.Printf("[DEBUG] Waiting for Glue Dev Endpoint (%s) to become available", d.Id()) - stateConf := &resource.StateChangeConf{ - Pending: []string{ - "PROVISIONING", - }, - Target: []string{"READY"}, - Refresh: waiter.GlueDevEndpointStatus(conn, d.Id()), - Timeout: 15 * time.Minute, - } - if _, err := stateConf.WaitForState(); err != nil { + if _, err := waiter.GlueDevEndpointCreated(conn, d.Id()); err != nil { return fmt.Errorf("error while waiting for Glue Dev Endpoint (%s) to become available: %s", d.Id(), err) } @@ -453,9 +408,15 @@ func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) erro oldRaw, newRaw := d.GetChange("arguments") old := oldRaw.(map[string]interface{}) new := newRaw.(map[string]interface{}) - create, remove := diffArguments(old, new) + create, remove := diffStringMaps(old, new) + + removeKeys := make([]*string, 0) + for k := range remove { + removeKeys = append(removeKeys, &k) + } + input.AddArguments = create - input.DeleteArguments = remove + input.DeleteArguments = removeKeys hasChanged = true } @@ -552,37 +513,17 @@ func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) erro return nil } - return err + return fmt.Errorf("error deleting Glue Dev Endpoint (%s): %s", d.Id(), err) } - return nil -} - -func diffArguments(oldArgs, newArgs map[string]interface{}) (map[string]*string, []*string) { - var create = make(map[string]*string) - var remove []*string - - for oldArgKey, oldArgVal := range oldArgs { - tmpKey := oldArgKey - - if val, ok := newArgs[oldArgKey]; ok { - if val.(string) != oldArgVal.(string) { - remove = append(remove, &tmpKey) - } - } else { - remove = append(remove, &tmpKey) + log.Printf("[DEBUG] Waiting for Glue Dev Endpoint (%s) to become terminated", d.Id()) + if _, err := waiter.GlueDevEndpointDeleted(conn, d.Id()); err != nil { + if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + return nil } - } - for newArgKey, newArgVal := range newArgs { - if val, ok := oldArgs[newArgKey]; ok { - if val.(string) != newArgVal.(string) { - create[newArgKey] = aws.String(newArgVal.(string)) - } - } else { - create[newArgKey] = aws.String(newArgVal.(string)) - } + return fmt.Errorf("error while waiting for Glue Dev Endpoint (%s) to become terminated: %s", d.Id(), err) } - return create, remove + return nil } diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index 0df7abf4d2b..143f27f7fcd 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -239,7 +239,6 @@ func TestAccGlueDevEndpoint_GlueVersion(t *testing.T) { Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "1.0"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "glue_version", "1.0"), ), }, @@ -247,7 +246,6 @@ func TestAccGlueDevEndpoint_GlueVersion(t *testing.T) { Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "0.9"), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "glue_version", "0.9"), ), }, @@ -279,7 +277,6 @@ func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 2), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "2"), ), }, @@ -287,7 +284,6 @@ func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 5), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), ), }, @@ -319,7 +315,6 @@ func TestAccGlueDevEndpoint_NumberOfWorkers(t *testing.T) { Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 2), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "number_of_workers", "2"), ), }, @@ -327,7 +322,6 @@ func TestAccGlueDevEndpoint_NumberOfWorkers(t *testing.T) { Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 5), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "number_of_workers", "5"), ), }, @@ -355,7 +349,6 @@ func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), ), }, @@ -363,7 +356,6 @@ func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey2), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_key", publicKey2), ), }, @@ -391,7 +383,6 @@ func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { Config: testAccGlueDevEndpointConfig_PublicKeys2(rName, publicKey1, publicKey2), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), ), }, @@ -399,7 +390,6 @@ func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { Config: testAccGlueDevEndpointConfig_PublicKeys3(rName, publicKey1, publicKey3, publicKey4), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), ), }, @@ -407,7 +397,6 @@ func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { Config: testAccGlueDevEndpointConfig_PublicKeys4(rName, publicKey1, publicKey1, publicKey3, publicKey4), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), ), }, @@ -435,7 +424,6 @@ func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { Config: testAccGlueDevEndpointConfig_SecurityConfiguration(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "security_configuration", rName), ), }, @@ -464,7 +452,6 @@ func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { Config: testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "security_group_ids.#", "1"), resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "aws_subnet.test", "id"), resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"), @@ -540,7 +527,6 @@ func TestAccGlueDevEndpoint_WorkerType(t *testing.T) { Config: testAccGlueDevEndpointConfig_WorkerType_Standard(rName), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeStandard), ), }, @@ -548,7 +534,6 @@ func TestAccGlueDevEndpoint_WorkerType(t *testing.T) { Config: testAccGlueDevEndpointConfig_WorkerType(rName, glue.WorkerTypeG1x), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeG1x), ), }, @@ -556,7 +541,6 @@ func TestAccGlueDevEndpoint_WorkerType(t *testing.T) { Config: testAccGlueDevEndpointConfig_WorkerType(rName, glue.WorkerTypeG2x), Check: resource.ComposeTestCheckFunc( testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), - resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeG2x), ), }, @@ -654,7 +638,7 @@ func testAccCheckAWSGlueDevEndpointDestroy(s *terraform.State) error { func testAccGlueDevEndpointConfig_Base(rName string) string { return fmt.Sprintf(` resource "aws_iam_role" "test" { - name = "AWSGlueServiceRole-%s" + name = "AWSGlueServiceRole-%[1]s" assume_role_policy = data.aws_iam_policy_document.service.json } @@ -670,7 +654,7 @@ data "aws_iam_policy_document" "service" { } resource "aws_iam_policy" "test" { - name = %q + name = %[1]q policy = data.aws_iam_policy_document.test.json } @@ -687,10 +671,12 @@ resource "aws_iam_role_policy_attachment" "test" { } resource "aws_iam_role_policy_attachment" "glue_service_role" { - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AWSGlueServiceRole" role = aws_iam_role.test.name } -`, rName, rName) + +data "aws_partition" "current" {} +`, rName) } func testAccGlueDevEndpointConfig_Basic(rName string) string { @@ -821,13 +807,13 @@ resource "aws_glue_dev_endpoint" "test" { func testAccGlueDevEndpointConfig_SecurityConfiguration(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn security_configuration = aws_glue_security_configuration.test.name } resource "aws_glue_security_configuration" "test" { - name = %q + name = %[1]q encryption_configuration { cloudwatch_encryption { @@ -843,13 +829,13 @@ resource "aws_glue_security_configuration" "test" { } } } -`, rName, rName) +`, rName) } func testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName string) string { return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { - name = %q + name = %[1]q role_arn = aws_iam_role.test.arn subnet_id = aws_subnet.test.id security_group_ids = [aws_security_group.test.id] @@ -873,7 +859,7 @@ resource "aws_vpc" "test" { cidr_block = "10.0.0.0/16" tags = { - Name = %q + Name = %[1]q } } @@ -883,7 +869,7 @@ resource "aws_subnet" "test" { availability_zone = "us-west-2a" tags = { - Name = %q + Name = %[1]q } timeouts { @@ -893,7 +879,7 @@ resource "aws_subnet" "test" { } resource "aws_security_group" "test" { - name = %q + name = %[1]q vpc_id = aws_vpc.test.id ingress { @@ -915,7 +901,7 @@ resource "aws_security_group" "test" { } depends_on = [aws_iam_role_policy_attachment.glue_service_role] } -`, rName, rName, rName, rName) +`, rName) } func testAccAWSGlueDevEndpointConfig_Tags1(rName, tagKey1, tagValue1 string) string { @@ -961,8 +947,8 @@ func testAccGlueDevEndpointConfig_WorkerType_Standard(rName string) string { resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn - worker_type = %[2]q + worker_type = "Standard" number_of_workers = 2 } -`, rName, glue.WorkerTypeStandard) +`, rName) } diff --git a/website/docs/r/glue_dev_endpoint.markdown b/website/docs/r/glue_dev_endpoint.markdown index 77142dde6f3..437fc3290de 100644 --- a/website/docs/r/glue_dev_endpoint.markdown +++ b/website/docs/r/glue_dev_endpoint.markdown @@ -15,17 +15,17 @@ Provides a Glue Development Endpoint resource. Basic usage: ```hcl -resource "aws_glue_dev_endpoint" "de" { +resource "aws_glue_dev_endpoint" "example" { name = "foo" - role_arn = aws_iam_role.de.arn + role_arn = aws_iam_role.example.arn } -resource "aws_iam_role" "de" { +resource "aws_iam_role" "example" { name = "AWSGlueServiceRole-foo" - assume_role_policy = data.aws_iam_policy_document.de.json + assume_role_policy = data.aws_iam_policy_document.example.json } -data "aws_iam_policy_document" "de" { +data "aws_iam_policy_document" "example" { statement { actions = ["sts:AssumeRole"] @@ -36,9 +36,9 @@ data "aws_iam_policy_document" "de" { } } -resource "aws_iam_role_policy_attachment" "foo-AWSGlueServiceRole" { +resource "aws_iam_role_policy_attachment" "example-AWSGlueServiceRole" { policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" - role = aws_iam_role.de.name + role = aws_iam_role.example.name } ``` @@ -50,7 +50,7 @@ The following arguments are supported: * `extra_jars_s3_path` - (Optional) Path to one or more Java Jars in an S3 bucket that should be loaded in this endpoint. * `extra_python_libs_s3_path` - (Optional) Path(s) to one or more Python libraries in an S3 bucket that should be loaded in this endpoint. Multiple values must be complete paths separated by a comma. * `glue_version` - (Optional) - Specifies the versions of Python and Apache Spark to use. Defaults to AWS Glue version 0.9. -* `name` - (Optional) The name of this endpoint (must be unique). If omitted, Terraform will assign a random, unique name. +* `name` - (Required) The name of this endpoint. It must be unique in your account. * `number_of_nodes` - (Optional) The number of AWS Glue Data Processing Units (DPUs) to allocate to this endpoint. Conflicts with `worker_type`. * `number_of_workers` - (Optional) The number of workers of a defined worker type that are allocated to this endpoint. This field is available only when you choose worker type G.1X or G.2X. * `public_key` - (Optional) The public key to be used by this endpoint for authentication. @@ -82,5 +82,5 @@ The following attributes are exported: A Glue Development Endpoint can be imported using the `name`, e.g. ``` -$ terraform import aws_glue_dev_endpoint.de foo +$ terraform import aws_glue_dev_endpoint.example foo ``` \ No newline at end of file From 404f176fe9003d5c10423c86f85f6dc5da7d2a43 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Tue, 3 Nov 2020 14:56:03 +0100 Subject: [PATCH 11/12] Fix more comments --- aws/resource_aws_glue_dev_endpoint.go | 24 +++++++++------------- aws/resource_aws_glue_dev_endpoint_test.go | 6 +++--- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index 7893b28504c..d1dc82e8117 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -6,8 +6,6 @@ import ( "regexp" "time" - "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/glue/waiter" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/service/glue" @@ -15,6 +13,8 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" "github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/glue/waiter" + iamwaiter "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/iam/waiter" ) func resourceAwsGlueDevEndpoint() *schema.Resource { @@ -166,12 +166,8 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) Tags: keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().GlueTags(), } - if kv, ok := d.GetOk("arguments"); ok { - arguments := make(map[string]string) - for k, v := range kv.(map[string]interface{}) { - arguments[k] = v.(string) - } - input.Arguments = aws.StringMap(arguments) + if v, ok := d.GetOk("arguments"); ok { + input.Arguments = stringMapToPointers(v.(map[string]interface{})) } if v, ok := d.GetOk("extra_jars_s3_path"); ok { @@ -221,7 +217,7 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) } log.Printf("[DEBUG] Creating Glue Dev Endpoint: %#v", *input) - err := resource.Retry(1*time.Minute, func() *resource.RetryError { + err := resource.Retry(iamwaiter.PropagationTimeout, func() *resource.RetryError { _, err := conn.CreateDevEndpoint(input) if err != nil { // Retry for IAM eventual consistency @@ -408,7 +404,7 @@ func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) erro oldRaw, newRaw := d.GetChange("arguments") old := oldRaw.(map[string]interface{}) new := newRaw.(map[string]interface{}) - create, remove := diffStringMaps(old, new) + create, remove, _ := diffStringMaps(old, new) removeKeys := make([]*string, 0) for k := range remove { @@ -453,13 +449,13 @@ func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) erro } os := o.(*schema.Set) ns := n.(*schema.Set) - remove := os.Difference(ns).List() - create := ns.Difference(os).List() + remove := os.Difference(ns) + create := ns.Difference(os) - input.AddPublicKeys = expandStringList(create) + input.AddPublicKeys = expandStringSet(create) log.Printf("[DEBUG] expectedCreate public keys: %v", create) - input.DeletePublicKeys = expandStringList(remove) + input.DeletePublicKeys = expandStringSet(remove) log.Printf("[DEBUG] remove public keys: %v", remove) hasChanged = true diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index 143f27f7fcd..d1a044c6236 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -833,7 +833,7 @@ resource "aws_glue_security_configuration" "test" { } func testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName string) string { - return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` + return composeConfig(testAccAvailableAZsNoOptInConfig(), testAccGlueDevEndpointConfig_Base(rName), fmt.Sprintf(` resource "aws_glue_dev_endpoint" "test" { name = %[1]q role_arn = aws_iam_role.test.arn @@ -866,7 +866,7 @@ resource "aws_vpc" "test" { resource "aws_subnet" "test" { vpc_id = aws_vpc.test.id cidr_block = "10.0.1.0/24" - availability_zone = "us-west-2a" + availability_zone = data.aws_availability_zones.available.names[0] tags = { Name = %[1]q @@ -901,7 +901,7 @@ resource "aws_security_group" "test" { } depends_on = [aws_iam_role_policy_attachment.glue_service_role] } -`, rName) +`, rName)) } func testAccAWSGlueDevEndpointConfig_Tags1(rName, tagKey1, tagValue1 string) string { From 153d7872d363a11f3cf7c065b33e3664b7572e68 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Sun, 8 Nov 2020 14:58:29 +0100 Subject: [PATCH 12/12] Fix tests after feedback --- aws/internal/service/glue/waiter/waiter.go | 6 ++---- aws/resource_aws_glue_dev_endpoint.go | 7 ++++--- aws/resource_aws_glue_dev_endpoint_test.go | 7 ++++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/aws/internal/service/glue/waiter/waiter.go b/aws/internal/service/glue/waiter/waiter.go index 379ec3776a1..3ca45d967cb 100644 --- a/aws/internal/service/glue/waiter/waiter.go +++ b/aws/internal/service/glue/waiter/waiter.go @@ -97,10 +97,8 @@ func GlueDevEndpointCreated(conn *glue.Glue, devEndpointId string) (*glue.GetDev // GlueDevEndpointDeleted waits for a Glue Dev Endpoint to become terminated. func GlueDevEndpointDeleted(conn *glue.Glue, devEndpointId string) (*glue.GetDevEndpointOutput, error) { stateConf := &resource.StateChangeConf{ - Pending: []string{ - "TERMINATING", - }, - Target: []string{"TERMINATED"}, + Pending: []string{"TERMINATING"}, + Target: []string{}, Refresh: GlueDevEndpointStatus(conn, devEndpointId), Timeout: 15 * time.Minute, } diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go index d1dc82e8117..35384afeee8 100644 --- a/aws/resource_aws_glue_dev_endpoint.go +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/aws-sdk-go-base/tfawserr" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" @@ -261,7 +262,7 @@ func resourceAwsGlueDevEndpointRead(d *schema.ResourceData, meta interface{}) er output, err := conn.GetDevEndpoint(request) if err != nil { - if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { log.Printf("[WARN] Glue Dev Endpoint (%s) not found, removing from state", d.Id()) d.SetId("") return nil @@ -505,7 +506,7 @@ func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) erro _, err := conn.DeleteDevEndpoint(deleteOpts) if err != nil { - if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { return nil } @@ -514,7 +515,7 @@ func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) erro log.Printf("[DEBUG] Waiting for Glue Dev Endpoint (%s) to become terminated", d.Id()) if _, err := waiter.GlueDevEndpointDeleted(conn, d.Id()); err != nil { - if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { return nil } diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go index d1a044c6236..3b6dee5a44a 100644 --- a/aws/resource_aws_glue_dev_endpoint_test.go +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/aws-sdk-go-base/tfawserr" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" @@ -87,7 +88,7 @@ func TestAccGlueDevEndpoint_Basic(t *testing.T) { testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("devEndpoint/%s", rName)), resource.TestCheckResourceAttr(resourceName, "name", rName), - resource.TestCheckResourceAttrPair(resourceName, "role_arn", "aws_iam_role.test", "role_arn"), + resource.TestCheckResourceAttrPair(resourceName, "role_arn", "aws_iam_role.test", "arn"), resource.TestCheckResourceAttr(resourceName, "status", "READY"), resource.TestCheckResourceAttr(resourceName, "arguments.%", "0"), resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), @@ -233,7 +234,7 @@ func TestAccGlueDevEndpoint_GlueVersion(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "1"), - ExpectError: regexp.MustCompile(`attribute glue_version must match version pattern X.X`), + ExpectError: regexp.MustCompile(`must match version pattern X.X`), }, { Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "1.0"), @@ -618,7 +619,7 @@ func testAccCheckAWSGlueDevEndpointDestroy(s *terraform.State) error { }) if err != nil { - if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { return nil } return err