diff --git a/aws/internal/service/glue/waiter/status.go b/aws/internal/service/glue/waiter/status.go index e7093e36246..c800a4695fc 100644 --- a/aws/internal/service/glue/waiter/status.go +++ b/aws/internal/service/glue/waiter/status.go @@ -3,6 +3,7 @@ package waiter import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/aws-sdk-go-base/tfawserr" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" ) @@ -52,3 +53,25 @@ func TriggerStatus(conn *glue.Glue, triggerName string) resource.StateRefreshFun return output, aws.StringValue(output.Trigger.State), nil } } + +func GlueDevEndpointStatus(conn *glue.Glue, name string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + getDevEndpointInput := &glue.GetDevEndpointInput{ + EndpointName: aws.String(name), + } + endpoint, err := conn.GetDevEndpoint(getDevEndpointInput) + if err != nil { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + return nil, "", nil + } + + return nil, "", err + } + + if endpoint == nil { + return nil, "", nil + } + + return endpoint, aws.StringValue(endpoint.DevEndpoint.Status), nil + } +} diff --git a/aws/internal/service/glue/waiter/waiter.go b/aws/internal/service/glue/waiter/waiter.go index bc0f6f4ab86..3ca45d967cb 100644 --- a/aws/internal/service/glue/waiter/waiter.go +++ b/aws/internal/service/glue/waiter/waiter.go @@ -73,3 +73,41 @@ func TriggerDeleted(conn *glue.Glue, triggerName string) (*glue.GetTriggerOutput return nil, err } + +// GlueDevEndpointCreated waits for a Glue Dev Endpoint to become available. +func GlueDevEndpointCreated(conn *glue.Glue, devEndpointId string) (*glue.GetDevEndpointOutput, error) { + stateConf := &resource.StateChangeConf{ + Pending: []string{ + "PROVISIONING", + }, + Target: []string{"READY"}, + Refresh: GlueDevEndpointStatus(conn, devEndpointId), + Timeout: 15 * time.Minute, + } + + outputRaw, err := stateConf.WaitForState() + + if output, ok := outputRaw.(*glue.GetDevEndpointOutput); ok { + return output, err + } + + return nil, err +} + +// GlueDevEndpointDeleted waits for a Glue Dev Endpoint to become terminated. +func GlueDevEndpointDeleted(conn *glue.Glue, devEndpointId string) (*glue.GetDevEndpointOutput, error) { + stateConf := &resource.StateChangeConf{ + Pending: []string{"TERMINATING"}, + Target: []string{}, + Refresh: GlueDevEndpointStatus(conn, devEndpointId), + Timeout: 15 * time.Minute, + } + + outputRaw, err := stateConf.WaitForState() + + if output, ok := outputRaw.(*glue.GetDevEndpointOutput); ok { + return output, err + } + + return nil, err +} diff --git a/aws/provider.go b/aws/provider.go index 976a61f49a0..34533b76eaf 100644 --- a/aws/provider.go +++ b/aws/provider.go @@ -656,6 +656,7 @@ func Provider() *schema.Provider { "aws_glue_catalog_table": resourceAwsGlueCatalogTable(), "aws_glue_classifier": resourceAwsGlueClassifier(), "aws_glue_connection": resourceAwsGlueConnection(), + "aws_glue_dev_endpoint": resourceAwsGlueDevEndpoint(), "aws_glue_crawler": resourceAwsGlueCrawler(), "aws_glue_data_catalog_encryption_settings": resourceAwsGlueDataCatalogEncryptionSettings(), "aws_glue_job": resourceAwsGlueJob(), diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go new file mode 100644 index 00000000000..35384afeee8 --- /dev/null +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -0,0 +1,526 @@ +package aws + +import ( + "fmt" + "log" + "regexp" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" + "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/aws-sdk-go-base/tfawserr" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags" + "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/glue/waiter" + iamwaiter "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/iam/waiter" +) + +func resourceAwsGlueDevEndpoint() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsGlueDevEndpointCreate, + Read: resourceAwsGlueDevEndpointRead, + Update: resourceAwsDevEndpointUpdate, + Delete: resourceAwsDevEndpointDelete, + Importer: &schema.ResourceImporter{ + State: schema.ImportStatePassthrough, + }, + + Schema: map[string]*schema.Schema{ + "arguments": { + Type: schema.TypeMap, + Optional: true, + Elem: schema.TypeString, + }, + "arn": { + Type: schema.TypeString, + Computed: true, + }, + "extra_jars_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + "extra_python_libs_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + "glue_version": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validation.StringMatch(regexp.MustCompile(`^\w+\.\w+$`), "must match version pattern X.X"), + }, + "name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validation.NoZeroValues, + }, + "number_of_nodes": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"number_of_workers", "worker_type"}, + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return new == "0" + }, + ValidateFunc: validation.IntAtLeast(2), + }, + "number_of_workers": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + ValidateFunc: validation.IntAtLeast(2), + ConflictsWith: []string{"number_of_nodes"}, + }, + "public_key": { + Type: schema.TypeString, + Optional: true, + ConflictsWith: []string{"public_keys"}, + }, + "public_keys": { + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + ConflictsWith: []string{"public_key"}, + MaxItems: 5, + }, + "role_arn": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validateArn, + }, + "security_configuration": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "security_group_ids": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + RequiredWith: []string{"subnet_id"}, + }, + "subnet_id": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + RequiredWith: []string{"security_group_ids"}, + }, + "tags": tagsSchema(), + "private_address": { + Type: schema.TypeString, + Computed: true, + }, + "public_address": { + Type: schema.TypeString, + Computed: true, + }, + "yarn_endpoint_address": { + Type: schema.TypeString, + Computed: true, + }, + "zeppelin_remote_spark_interpreter_port": { + Type: schema.TypeInt, + Computed: true, + }, + "worker_type": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice(glue.WorkerType_Values(), false), + ConflictsWith: []string{"number_of_nodes"}, + ForceNew: true, + }, + "availability_zone": { + Type: schema.TypeString, + Computed: true, + }, + "vpc_id": { + Type: schema.TypeString, + Computed: true, + }, + "status": { + Type: schema.TypeString, + Computed: true, + }, + "failure_reason": { + Type: schema.TypeString, + Computed: true, + }, + }, + } +} + +func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + name := d.Get("name").(string) + + input := &glue.CreateDevEndpointInput{ + EndpointName: aws.String(name), + RoleArn: aws.String(d.Get("role_arn").(string)), + Tags: keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().GlueTags(), + } + + if v, ok := d.GetOk("arguments"); ok { + input.Arguments = stringMapToPointers(v.(map[string]interface{})) + } + + if v, ok := d.GetOk("extra_jars_s3_path"); ok { + input.ExtraJarsS3Path = aws.String(v.(string)) + } + + if v, ok := d.GetOk("extra_python_libs_s3_path"); ok { + input.ExtraPythonLibsS3Path = aws.String(v.(string)) + } + + if v, ok := d.GetOk("glue_version"); ok { + input.GlueVersion = aws.String(v.(string)) + } + + if v, ok := d.GetOk("number_of_nodes"); ok { + input.NumberOfNodes = aws.Int64(int64(v.(int))) + } + + if v, ok := d.GetOk("number_of_workers"); ok { + input.NumberOfWorkers = aws.Int64(int64(v.(int))) + } + + if v, ok := d.GetOk("public_key"); ok { + input.PublicKey = aws.String(v.(string)) + } + + if v, ok := d.GetOk("public_keys"); ok { + publicKeys := expandStringSet(v.(*schema.Set)) + input.PublicKeys = publicKeys + } + + if v, ok := d.GetOk("security_configuration"); ok { + input.SecurityConfiguration = aws.String(v.(string)) + } + + if v, ok := d.GetOk("security_group_ids"); ok { + securityGroupIDs := expandStringSet(v.(*schema.Set)) + input.SecurityGroupIds = securityGroupIDs + } + + if v, ok := d.GetOk("subnet_id"); ok { + input.SubnetId = aws.String(v.(string)) + } + + if v, ok := d.GetOk("worker_type"); ok { + input.WorkerType = aws.String(v.(string)) + } + + log.Printf("[DEBUG] Creating Glue Dev Endpoint: %#v", *input) + err := resource.Retry(iamwaiter.PropagationTimeout, func() *resource.RetryError { + _, err := conn.CreateDevEndpoint(input) + if err != nil { + // Retry for IAM eventual consistency + if isAWSErr(err, glue.ErrCodeInvalidInputException, "should be given assume role permissions for Glue Service") { + return resource.RetryableError(err) + } + if isAWSErr(err, glue.ErrCodeInvalidInputException, "S3 endpoint and NAT validation has failed for subnetId") { + return resource.RetryableError(err) + } + + return resource.NonRetryableError(err) + } + return nil + }) + + if isResourceTimeoutError(err) { + _, err = conn.CreateDevEndpoint(input) + } + + if err != nil { + return fmt.Errorf("error creating Glue Dev Endpoint: %s", err) + } + + d.SetId(name) + + log.Printf("[DEBUG] Waiting for Glue Dev Endpoint (%s) to become available", d.Id()) + if _, err := waiter.GlueDevEndpointCreated(conn, d.Id()); err != nil { + return fmt.Errorf("error while waiting for Glue Dev Endpoint (%s) to become available: %s", d.Id(), err) + } + + return resourceAwsGlueDevEndpointRead(d, meta) +} + +func resourceAwsGlueDevEndpointRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig + + request := &glue.GetDevEndpointInput{ + EndpointName: aws.String(d.Id()), + } + + output, err := conn.GetDevEndpoint(request) + if err != nil { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + log.Printf("[WARN] Glue Dev Endpoint (%s) not found, removing from state", d.Id()) + d.SetId("") + return nil + } + return fmt.Errorf("error reading Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + endpoint := output.DevEndpoint + if endpoint == nil { + log.Printf("[WARN] Glue Dev Endpoint (%s) not found, removing from state", d.Id()) + d.SetId("") + return nil + } + + endpointARN := arn.ARN{ + Partition: meta.(*AWSClient).partition, + Service: "glue", + Region: meta.(*AWSClient).region, + AccountID: meta.(*AWSClient).accountid, + Resource: fmt.Sprintf("devEndpoint/%s", d.Id()), + }.String() + + if err := d.Set("arn", endpointARN); err != nil { + return fmt.Errorf("error setting arn for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("arguments", aws.StringValueMap(endpoint.Arguments)); err != nil { + return fmt.Errorf("error setting arguments for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("availability_zone", endpoint.AvailabilityZone); err != nil { + return fmt.Errorf("error setting availability_zone for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("extra_jars_s3_path", endpoint.ExtraJarsS3Path); err != nil { + return fmt.Errorf("error setting extra_jars_s3_path for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("extra_python_libs_s3_path", endpoint.ExtraPythonLibsS3Path); err != nil { + return fmt.Errorf("error setting extra_python_libs_s3_path for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("failure_reason", endpoint.FailureReason); err != nil { + return fmt.Errorf("error setting failure_reason for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("glue_version", endpoint.GlueVersion); err != nil { + return fmt.Errorf("error setting glue_version for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("name", endpoint.EndpointName); err != nil { + return fmt.Errorf("error setting name for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("number_of_nodes", endpoint.NumberOfNodes); err != nil { + return fmt.Errorf("error setting number_of_nodes for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("number_of_workers", endpoint.NumberOfWorkers); err != nil { + return fmt.Errorf("error setting number_of_workers for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("private_address", endpoint.PrivateAddress); err != nil { + return fmt.Errorf("error setting private_address for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_address", endpoint.PublicAddress); err != nil { + return fmt.Errorf("error setting public_address for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_key", endpoint.PublicKey); err != nil { + return fmt.Errorf("error setting public_key for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_keys", flattenStringSet(endpoint.PublicKeys)); err != nil { + return fmt.Errorf("error setting public_keys for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("role_arn", endpoint.RoleArn); err != nil { + return fmt.Errorf("error setting role_arn for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("security_configuration", endpoint.SecurityConfiguration); err != nil { + return fmt.Errorf("error setting security_configuration for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("security_group_ids", flattenStringSet(endpoint.SecurityGroupIds)); err != nil { + return fmt.Errorf("error setting security_group_ids for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("status", endpoint.Status); err != nil { + return fmt.Errorf("error setting status for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("subnet_id", endpoint.SubnetId); err != nil { + return fmt.Errorf("error setting subnet_id for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("vpc_id", endpoint.VpcId); err != nil { + return fmt.Errorf("error setting vpc_id for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("worker_type", endpoint.WorkerType); err != nil { + return fmt.Errorf("error setting worker_type for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + tags, err := keyvaluetags.GlueListTags(conn, endpointARN) + + if err != nil { + return fmt.Errorf("error listing tags for Glue Dev Endpoint (%s): %s", endpointARN, err) + } + + if err := d.Set("tags", tags.IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil { + return fmt.Errorf("error setting tags: %s", err) + } + + if err := d.Set("yarn_endpoint_address", endpoint.YarnEndpointAddress); err != nil { + return fmt.Errorf("error setting yarn_endpoint_address for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("zeppelin_remote_spark_interpreter_port", endpoint.ZeppelinRemoteSparkInterpreterPort); err != nil { + return fmt.Errorf("error setting zeppelin_remote_spark_interpreter_port for Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + return nil +} + +func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + input := &glue.UpdateDevEndpointInput{ + EndpointName: aws.String(d.Get("name").(string)), + } + + hasChanged := false + + customLibs := &glue.DevEndpointCustomLibraries{} + + if d.HasChange("arguments") { + oldRaw, newRaw := d.GetChange("arguments") + old := oldRaw.(map[string]interface{}) + new := newRaw.(map[string]interface{}) + create, remove, _ := diffStringMaps(old, new) + + removeKeys := make([]*string, 0) + for k := range remove { + removeKeys = append(removeKeys, &k) + } + + input.AddArguments = create + input.DeleteArguments = removeKeys + + hasChanged = true + } + + if d.HasChange("extra_jars_s3_path") { + customLibs.ExtraJarsS3Path = aws.String(d.Get("extra_jars_s3_path").(string)) + input.CustomLibraries = customLibs + input.UpdateEtlLibraries = aws.Bool(true) + + hasChanged = true + } + + if d.HasChange("extra_python_libs_s3_path") { + customLibs.ExtraPythonLibsS3Path = aws.String(d.Get("extra_python_libs_s3_path").(string)) + input.CustomLibraries = customLibs + input.UpdateEtlLibraries = aws.Bool(true) + + hasChanged = true + } + + if d.HasChange("public_key") { + input.PublicKey = aws.String(d.Get("public_key").(string)) + + hasChanged = true + } + + if d.HasChange("public_keys") { + o, n := d.GetChange("public_keys") + if o == nil { + o = new(schema.Set) + } + if n == nil { + n = new(schema.Set) + } + os := o.(*schema.Set) + ns := n.(*schema.Set) + remove := os.Difference(ns) + create := ns.Difference(os) + + input.AddPublicKeys = expandStringSet(create) + log.Printf("[DEBUG] expectedCreate public keys: %v", create) + + input.DeletePublicKeys = expandStringSet(remove) + log.Printf("[DEBUG] remove public keys: %v", remove) + + hasChanged = true + } + + if hasChanged { + log.Printf("[DEBUG] Updating Glue Dev Endpoint: %s", input) + err := resource.Retry(5*time.Minute, func() *resource.RetryError { + _, err := conn.UpdateDevEndpoint(input) + if err != nil { + if isAWSErr(err, glue.ErrCodeInvalidInputException, "another concurrent update operation") { + return resource.RetryableError(err) + } + + return resource.NonRetryableError(err) + } + return nil + }) + + if isResourceTimeoutError(err) { + _, err = conn.UpdateDevEndpoint(input) + } + + if err != nil { + return fmt.Errorf("error updating Glue Dev Endpoint: %s", err) + } + } + + if d.HasChange("tags") { + o, n := d.GetChange("tags") + if err := keyvaluetags.GlueUpdateTags(conn, d.Get("arn").(string), o, n); err != nil { + return fmt.Errorf("error updating tags: %s", err) + } + } + + return resourceAwsGlueDevEndpointRead(d, meta) +} + +func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + deleteOpts := &glue.DeleteDevEndpointInput{ + EndpointName: aws.String(d.Id()), + } + + log.Printf("[INFO] Deleting Glue Dev Endpoint: %s", d.Id()) + + _, err := conn.DeleteDevEndpoint(deleteOpts) + if err != nil { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + return nil + } + + return fmt.Errorf("error deleting Glue Dev Endpoint (%s): %s", d.Id(), err) + } + + log.Printf("[DEBUG] Waiting for Glue Dev Endpoint (%s) to become terminated", d.Id()) + if _, err := waiter.GlueDevEndpointDeleted(conn, d.Id()); err != nil { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + return nil + } + + return fmt.Errorf("error while waiting for Glue Dev Endpoint (%s) to become terminated: %s", d.Id(), err) + } + + return nil +} diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go new file mode 100644 index 00000000000..3b6dee5a44a --- /dev/null +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -0,0 +1,955 @@ +package aws + +import ( + "fmt" + "log" + "regexp" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/aws-sdk-go-base/tfawserr" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +const ( + GlueDevEndpointResourcePrefix = "tf-acc-test" + publicKey1 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD3F6tyPEFEzV0LX3X8BsXdMsQz1x2cEikKDEY0aIj41qgxMCP/iteneqXSIFZBp5vizPvaoIR3Um9xK7PGoW8giupGn+EPuxIA4cDM4vzOqOkiMPhz5XK0whEjkVzTo4+S0puvDZuwIsdiW9mxhJc7tgBNL0cYlWSYVkz4G/fslNfRPW5mYAM49f4fhtxPb5ok4Q2Lg9dPKVHO/Bgeu5woMc7RY0p1ej6D4CKFE6lymSDJpW0YHX/wqE9+cfEauh7xZcG0q9t2ta6F6fmX0agvpFyZo8aFbXeUBr7osSCJNgvavWbM/06niWrOvYX2xwWdhXmXSrbX8ZbabVohBK41 foo1@bar.com" + publicKey2 = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAq6U3HQYC4g8WzU147gZZ7CKQH8TgYn3chZGRPxaGmHW1RUwsyEs0nmombmIhwxudhJ4ehjqXsDLoQpd6+c7BuLgTMvbv8LgE9LX53vnljFe1dsObsr/fYLvpU9LTlo8HgHAqO5ibNdrAUvV31ronzCZhms/Gyfdaue88Fd0/YnsZVGeOZPayRkdOHSpqme2CBrpa8myBeL1CWl0LkDG4+YCURjbaelfyZlIApLYKy3FcCan9XQFKaL32MJZwCgzfOvWIMtYcU8QtXMgnA3/I3gXk8YDUJv5P4lj0s/PJXuTM8DygVAUtebNwPuinS7wwonm5FXcWMuVGsVpG5K7FGQ== foo2@bar.com" + publicKey3 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtCk0lzMj1gPEOjdfQ37AIxCyETqJBubaMWuB4bgvGHp8LvEghr2YDl2bml1JrE1EOcZhPnIwgyucryXKA959sTUlgbvaFN7vmpVze56Q9tVU6BJQxOdaRoy5FcQMET9LB6SdbXk+V4CkDMsQNaFXezpg98HgCj+V7+bBWsfI6U63IESlWKK7kraCom8EWxkQk4mk9fizE2I+KrtiqN4xcah02LFG6IMnS+Xy3CDhcpZeYzWOV6zhcf675UJOdg/pLgQbUhhiwTOJFgRo8IcvE3iBrRMz508ppx6vLLr8J+3B8ujykc+/3ZSGfQfx6rO+OuSskhG5FLI6icbQBtBzf foo3@bar.com" + publicKey4 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD3F6tyPEFEzV0LX3X8BsXdMsQz1x2cEikKDEY0aIj41qgxMCP/iteneqXSIFZBp5vizPvaoIR3Um9xK7PGoW8giupGn+EPuxIA4cDM4vzOqOkiMPhz5XK0whEjkVzTo4+S0puvDZuwIsdiW9mxhJc7tgBNL0cYlWSYVkz4G/fslNfRPW5mYAM49f4fhtxPb5ok4Q2Lg9dPKVHO/Bgeu5woMc7RY0p1ej6D4CKFE6lymSDJpW0YHX/wqE9+cfEauh7xZcG0q9t2ta6F6fmX0agvpFyZo8aFbXeUBr7osSCJNgvavWbM/06niWrOvYX2xwWdhXmXSrbX8ZbabVohBK41 foo4@bar.com" +) + +func init() { + resource.AddTestSweepers("aws_glue_dev_endpoint", &resource.Sweeper{ + Name: "aws_glue_dev_endpoint", + F: testSweepGlueDevEndpoint, + }) +} + +func testSweepGlueDevEndpoint(region string) error { + client, err := sharedClientForRegion(region) + if err != nil { + return fmt.Errorf("error getting client: %s", err) + } + conn := client.(*AWSClient).glueconn + + input := &glue.GetDevEndpointsInput{} + err = conn.GetDevEndpointsPages(input, func(page *glue.GetDevEndpointsOutput, lastPage bool) bool { + if len(page.DevEndpoints) == 0 { + log.Printf("[INFO] No Glue Dev Endpoints to sweep") + return false + } + for _, endpoint := range page.DevEndpoints { + name := aws.StringValue(endpoint.EndpointName) + if !strings.HasPrefix(name, GlueDevEndpointResourcePrefix) { + log.Printf("[INFO] Skipping Glue Dev Endpoint: %s", name) + continue + } + + log.Printf("[INFO] Deleting Glue Dev Endpoint: %s", name) + _, err := conn.DeleteDevEndpoint(&glue.DeleteDevEndpointInput{ + EndpointName: aws.String(name), + }) + if err != nil { + log.Printf("[ERROR] Failed to delete Glue Dev Endpoint %s: %s", name, err) + } + } + return !lastPage + }) + if err != nil { + if testSweepSkipSweepError(err) { + log.Printf("[WARN] Skipping Glue Dev Endpoint sweep for %s: %s", region, err) + return nil + } + return fmt.Errorf("error retrieving Glue Dev Endpoint: %s", err) + } + + return nil +} + +func TestAccGlueDevEndpoint_Basic(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_Basic(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("devEndpoint/%s", rName)), + resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestCheckResourceAttrPair(resourceName, "role_arn", "aws_iam_role.test", "arn"), + resource.TestCheckResourceAttr(resourceName, "status", "READY"), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "0"), + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), + resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Arguments(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_Arguments(rName, "--arg1", "value1"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "1"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg1", "value1"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccGlueDevEndpointConfig_Arguments2(rName, "--arg1", "value1updated", "--arg2", "value2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "2"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg1", "value1updated"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg2", "value2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_Arguments(rName, "--arg2", "value2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "arguments.%", "1"), + resource.TestCheckResourceAttr(resourceName, "arguments.--arg2", "value2"), + ), + }, + }, + }) +} + +func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraJarsS3Path := "foo" + extraJarsS3PathUpdated := "bar" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3PathUpdated), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraPythonLibsS3Path := "foo" + extraPythonLibsS3PathUpdated := "bar" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3PathUpdated), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_GlueVersion(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "1"), + ExpectError: regexp.MustCompile(`must match version pattern X.X`), + }, + { + Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "1.0"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "glue_version", "1.0"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_GlueVersion(rName, "0.9"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "glue_version", "0.9"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 1), + ExpectError: regexp.MustCompile(`expected number_of_nodes to be at least`), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 5), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "5"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_NumberOfWorkers(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 1), + ExpectError: regexp.MustCompile(`expected number_of_workers to be at least`), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "number_of_workers", "2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_NumberOfWorkers(rName, 5), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "number_of_workers", "5"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + ), + }, + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey2), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKeys2(rName, publicKey1, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_PublicKeys3(rName, publicKey1, publicKey3, publicKey4), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_PublicKeys4(rName, publicKey1, publicKey1, publicKey3, publicKey4), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_SecurityConfiguration(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "security_configuration", rName), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Note: Either none or both of subnetId and securityGroupIds must be specified. +func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "security_group_ids.#", "1"), + resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "aws_subnet.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "availability_zone", "aws_subnet.test", "availability_zone"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Tags(t *testing.T) { + var endpoint1, endpoint2, endpoint3 glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSGlueDevEndpointConfig_Tags1(rName, "key1", "value1"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint1), + resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), + resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccAWSGlueDevEndpointConfig_Tags2(rName, "key1", "value1updated", "key2", "value2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint2), + resource.TestCheckResourceAttr(resourceName, "tags.%", "2"), + resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"), + resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), + ), + }, + { + Config: testAccAWSGlueDevEndpointConfig_Tags1(rName, "key2", "value2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint3), + resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), + resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), + ), + }, + }, + }) +} + +func TestAccGlueDevEndpoint_WorkerType(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_WorkerType_Standard(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeStandard), + ), + }, + { + Config: testAccGlueDevEndpointConfig_WorkerType(rName, glue.WorkerTypeG1x), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeG1x), + ), + }, + { + Config: testAccGlueDevEndpointConfig_WorkerType(rName, glue.WorkerTypeG2x), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + resource.TestCheckResourceAttr(resourceName, "worker_type", glue.WorkerTypeG2x), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_disappears(t *testing.T) { + var endpoint glue.DevEndpoint + + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_Basic(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + testAccCheckResourceDisappears(testAccProvider, resourceAwsGlueDevEndpoint(), resourceName), + ), + ExpectNonEmptyPlan: true, + }, + }, + }) +} + +func testAccCheckAWSGlueDevEndpointExists(resourceName string, endpoint *glue.DevEndpoint) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[resourceName] + if !ok { + return fmt.Errorf("not found: %s", resourceName) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("no ID is set") + } + + conn := testAccProvider.Meta().(*AWSClient).glueconn + output, err := conn.GetDevEndpoint(&glue.GetDevEndpointInput{ + EndpointName: aws.String(rs.Primary.ID), + }) + + if err != nil { + return err + } + + if output == nil { + return fmt.Errorf("no Glue Dev Endpoint") + } + + *endpoint = *output.DevEndpoint + + return nil + } +} + +func testAccCheckAWSGlueDevEndpointDestroy(s *terraform.State) error { + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_glue_dev_endpoint" { + continue + } + + conn := testAccProvider.Meta().(*AWSClient).glueconn + output, err := conn.GetDevEndpoint(&glue.GetDevEndpointInput{ + EndpointName: aws.String(rs.Primary.ID), + }) + + if err != nil { + if tfawserr.ErrCodeEquals(err, glue.ErrCodeEntityNotFoundException) { + return nil + } + return err + } + + endpoint := output.DevEndpoint + if endpoint != nil && aws.StringValue(endpoint.EndpointName) == rs.Primary.ID { + return fmt.Errorf("the Glue Dev Endpoint %s still exists", rs.Primary.ID) + } + + return nil + } + + return nil +} + +func testAccGlueDevEndpointConfig_Base(rName string) string { + return fmt.Sprintf(` +resource "aws_iam_role" "test" { + name = "AWSGlueServiceRole-%[1]s" + assume_role_policy = data.aws_iam_policy_document.service.json +} + +data "aws_iam_policy_document" "service" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + } +} + +resource "aws_iam_policy" "test" { + name = %[1]q + policy = data.aws_iam_policy_document.test.json +} + +data "aws_iam_policy_document" "test" { + statement { + actions = ["ec2:DescribeSecurityGroups", "ec2:DescribeSubnets"] + resources = ["*"] + } +} + +resource "aws_iam_role_policy_attachment" "test" { + role = aws_iam_role.test.name + policy_arn = aws_iam_policy.test.arn +} + +resource "aws_iam_role_policy_attachment" "glue_service_role" { + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AWSGlueServiceRole" + role = aws_iam_role.test.name +} + +data "aws_partition" "current" {} +`, rName) +} + +func testAccGlueDevEndpointConfig_Basic(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = aws_iam_role.test.arn +} +`, rName) +} + +func testAccGlueDevEndpointConfig_Arguments(rName, argKey, argValue string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + arguments = { + %[2]q = %[3]q + } +} +`, rName, argKey, argValue) +} + +func testAccGlueDevEndpointConfig_Arguments2(rName, argKey1, argValue1, argKey2, argValue2 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + arguments = { + %[2]q = %[3]q + %[4]q = %[5]q + } +} +`, rName, argKey1, argValue1, argKey2, argValue2) +} + +func testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName string, extraJarsS3Path string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = aws_iam_role.test.arn + extra_jars_s3_path = %q +} +`, rName, extraJarsS3Path) +} + +func testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName string, extraPythonLibsS3Path string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = aws_iam_role.test.arn + extra_python_libs_s3_path = %q +} +`, rName, extraPythonLibsS3Path) +} + +func testAccGlueDevEndpointConfig_GlueVersion(rName string, glueVersion string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + glue_version = %[2]q +} +`, rName, glueVersion) +} + +func testAccGlueDevEndpointConfig_NumberOfNodes(rName string, numberOfNodes int) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = aws_iam_role.test.arn + number_of_nodes = %d +} +`, rName, numberOfNodes) +} + +func testAccGlueDevEndpointConfig_NumberOfWorkers(rName string, numberOfWorkers int) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = aws_iam_role.test.arn + worker_type = "G.1X" + number_of_workers = %d +} +`, rName, numberOfWorkers) +} + +func testAccGlueDevEndpointConfig_PublicKey(rName string, publicKey string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = aws_iam_role.test.arn + public_key = "%s" +} +`, rName, publicKey) +} + +func testAccGlueDevEndpointConfig_PublicKeys2(rName string, publicKey1 string, publicKey2 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + public_keys = [%[2]q, %[3]q] +} +`, rName, publicKey1, publicKey2) +} + +func testAccGlueDevEndpointConfig_PublicKeys3(rName string, publicKey1 string, publicKey2 string, publicKey3 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + public_keys = [%[2]q, %[3]q, %[4]q] +} +`, rName, publicKey1, publicKey2, publicKey3) +} + +func testAccGlueDevEndpointConfig_PublicKeys4(rName string, publicKey1 string, publicKey2 string, publicKey3 string, publicKey4 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + public_keys = [%[2]q, %[3]q, %[4]q, %[5]q] +} +`, rName, publicKey1, publicKey2, publicKey3, publicKey4) +} + +func testAccGlueDevEndpointConfig_SecurityConfiguration(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + security_configuration = aws_glue_security_configuration.test.name +} + +resource "aws_glue_security_configuration" "test" { + name = %[1]q + + encryption_configuration { + cloudwatch_encryption { + cloudwatch_encryption_mode = "DISABLED" + } + + job_bookmarks_encryption { + job_bookmarks_encryption_mode = "DISABLED" + } + + s3_encryption { + s3_encryption_mode = "DISABLED" + } + } +} +`, rName) +} + +func testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName string) string { + return composeConfig(testAccAvailableAZsNoOptInConfig(), testAccGlueDevEndpointConfig_Base(rName), fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + subnet_id = aws_subnet.test.id + security_group_ids = [aws_security_group.test.id] +} + +resource "aws_vpc_endpoint" "s3" { + vpc_id = aws_vpc.test.id + service_name = data.aws_vpc_endpoint_service.s3.service_name +} + +data "aws_vpc_endpoint_service" "s3" { + service = "s3" +} + +resource "aws_vpc_endpoint_route_table_association" "test" { + vpc_endpoint_id = aws_vpc_endpoint.s3.id + route_table_id = aws_vpc.test.main_route_table_id +} + +resource "aws_vpc" "test" { + cidr_block = "10.0.0.0/16" + + tags = { + Name = %[1]q + } +} + +resource "aws_subnet" "test" { + vpc_id = aws_vpc.test.id + cidr_block = "10.0.1.0/24" + availability_zone = data.aws_availability_zones.available.names[0] + + tags = { + Name = %[1]q + } + + timeouts { + delete = "40m" + } + depends_on = [aws_iam_role_policy_attachment.glue_service_role] +} + +resource "aws_security_group" "test" { + name = %[1]q + vpc_id = aws_vpc.test.id + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + timeouts { + delete = "40m" + } + depends_on = [aws_iam_role_policy_attachment.glue_service_role] +} +`, rName)) +} + +func testAccAWSGlueDevEndpointConfig_Tags1(rName, tagKey1, tagValue1 string) string { + return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + + tags = { + %[2]q = %[3]q + } +} +`, rName, tagKey1, tagValue1) +} + +func testAccAWSGlueDevEndpointConfig_Tags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string { + return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + + tags = { + %[2]q = %[3]q + %[4]q = %[5]q + } +} +`, rName, tagKey1, tagValue1, tagKey2, tagValue2) +} + +func testAccGlueDevEndpointConfig_WorkerType(rName, workerType string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + worker_type = %[2]q + number_of_workers = 2 +} +`, rName, workerType) +} + +func testAccGlueDevEndpointConfig_WorkerType_Standard(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %[1]q + role_arn = aws_iam_role.test.arn + worker_type = "Standard" + number_of_workers = 2 +} +`, rName) +} diff --git a/website/docs/r/glue_dev_endpoint.markdown b/website/docs/r/glue_dev_endpoint.markdown new file mode 100644 index 00000000000..437fc3290de --- /dev/null +++ b/website/docs/r/glue_dev_endpoint.markdown @@ -0,0 +1,86 @@ +--- +subcategory: "Glue" +layout: "aws" +page_title: "AWS: aws_glue_dev_endpoint" +description: |- + Provides a Glue Development Endpoint resource. +--- + +# aws_glue_dev_endpoint + +Provides a Glue Development Endpoint resource. + +## Example Usage + +Basic usage: + +```hcl +resource "aws_glue_dev_endpoint" "example" { + name = "foo" + role_arn = aws_iam_role.example.arn +} + +resource "aws_iam_role" "example" { + name = "AWSGlueServiceRole-foo" + assume_role_policy = data.aws_iam_policy_document.example.json +} + +data "aws_iam_policy_document" "example" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy_attachment" "example-AWSGlueServiceRole" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" + role = aws_iam_role.example.name +} +``` + +## Argument Reference + +The following arguments are supported: + +* `arguments` - (Optional) A map of arguments used to configure the endpoint. +* `extra_jars_s3_path` - (Optional) Path to one or more Java Jars in an S3 bucket that should be loaded in this endpoint. +* `extra_python_libs_s3_path` - (Optional) Path(s) to one or more Python libraries in an S3 bucket that should be loaded in this endpoint. Multiple values must be complete paths separated by a comma. +* `glue_version` - (Optional) - Specifies the versions of Python and Apache Spark to use. Defaults to AWS Glue version 0.9. +* `name` - (Required) The name of this endpoint. It must be unique in your account. +* `number_of_nodes` - (Optional) The number of AWS Glue Data Processing Units (DPUs) to allocate to this endpoint. Conflicts with `worker_type`. +* `number_of_workers` - (Optional) The number of workers of a defined worker type that are allocated to this endpoint. This field is available only when you choose worker type G.1X or G.2X. +* `public_key` - (Optional) The public key to be used by this endpoint for authentication. +* `public_keys` - (Optional) A list of public keys to be used by this endpoint for authentication. +* `role_arn` - (Required) The IAM role for this endpoint. +* `security_configuration` - (Optional) The name of the Security Configuration structure to be used with this endpoint. +* `security_group_ids` - (Optional) Security group IDs for the security groups to be used by this endpoint. +* `subnet_id` - (Optional) The subnet ID for the new endpoint to use. +* `tags` - (Optional) Key-value map of resource tags. +* `worker_type` - (Optional) The type of predefined worker that is allocated to this endpoint. Accepts a value of Standard, G.1X, or G.2X. + +## Attributes Reference + +The following attributes are exported: + +* `arn` - The ARN of the endpoint. +* `name` - The name of the new endpoint. +* `private_address` - A private IP address to access the endpoint within a VPC, if this endpoint is created within one. +* `public_address` - The public IP address used by this endpoint. The PublicAddress field is present only when you create a non-VPC endpoint. +* `yarn_endpoint_address` - The YARN endpoint address used by this endpoint. +* `zeppelin_remote_spark_interpreter_port` - The Apache Zeppelin port for the remote Apache Spark interpreter. +* `availability_zone` - The AWS availability zone where this endpoint is located. +* `vpc_id` - he ID of the VPC used by this endpoint. +* `status` - The current status of this endpoint. +* `failure_reason` - The reason for a current failure in this endpoint. + +## Import + +A Glue Development Endpoint can be imported using the `name`, e.g. + +``` +$ terraform import aws_glue_dev_endpoint.example foo +``` \ No newline at end of file