From ca0158c10c4ab68f2fe1d603dc63081bdd8e3f11 Mon Sep 17 00:00:00 2001 From: Jan-Christoph Kuester Date: Tue, 12 Mar 2019 09:53:08 +0100 Subject: [PATCH] Add resource aws_glue_dev_endpoint --- aws/provider.go | 1 + aws/resource_aws_glue_dev_endpoint.go | 446 +++++++++++++ aws/resource_aws_glue_dev_endpoint_test.go | 701 +++++++++++++++++++++ website/aws.erb | 3 + website/docs/r/glue_dev_endpoint.markdown | 81 +++ 5 files changed, 1232 insertions(+) create mode 100644 aws/resource_aws_glue_dev_endpoint.go create mode 100644 aws/resource_aws_glue_dev_endpoint_test.go create mode 100644 website/docs/r/glue_dev_endpoint.markdown diff --git a/aws/provider.go b/aws/provider.go index 68a03f329df5..fff951f110b9 100644 --- a/aws/provider.go +++ b/aws/provider.go @@ -476,6 +476,7 @@ func Provider() terraform.ResourceProvider { "aws_glue_catalog_table": resourceAwsGlueCatalogTable(), "aws_glue_classifier": resourceAwsGlueClassifier(), "aws_glue_connection": resourceAwsGlueConnection(), + "aws_glue_dev_endpoint": resourceAwsGlueDevEndpoint(), "aws_glue_crawler": resourceAwsGlueCrawler(), "aws_glue_job": resourceAwsGlueJob(), "aws_glue_security_configuration": resourceAwsGlueSecurityConfiguration(), diff --git a/aws/resource_aws_glue_dev_endpoint.go b/aws/resource_aws_glue_dev_endpoint.go new file mode 100644 index 000000000000..cc8ed89925a5 --- /dev/null +++ b/aws/resource_aws_glue_dev_endpoint.go @@ -0,0 +1,446 @@ +package aws + +import ( + "fmt" + "log" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/glue" + + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/helper/schema" +) + +func resourceAwsGlueDevEndpoint() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsGlueDevEndpointCreate, + Read: resourceAwsGlueDevEndpointRead, + Update: resourceAwsDevEndpointUpdate, + Delete: resourceAwsDevEndpointDelete, + Importer: &schema.ResourceImporter{ + State: schema.ImportStatePassthrough, + }, + + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ForceNew: true, + }, + + "extra_jars_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + + "extra_python_libs_s3_path": { + Type: schema.TypeString, + Optional: true, + }, + + "number_of_nodes": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Default: 5, + }, + + "public_key": { + Type: schema.TypeString, + Optional: true, + }, + + "public_keys": { + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + MaxItems: 5, + }, + + "role_arn": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + + "security_configuration": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + + "security_group_ids": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + }, + + "subnet_id": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + + "private_address": { + Type: schema.TypeString, + Computed: true, + }, + + "public_address": { + Type: schema.TypeString, + Computed: true, + }, + + "yarn_endpoint_address": { + Type: schema.TypeString, + Computed: true, + }, + + "zeppelin_remote_spark_interpreter_port": { + Type: schema.TypeInt, + Computed: true, + }, + + "availability_zone": { + Type: schema.TypeString, + Computed: true, + }, + + "vpc_id": { + Type: schema.TypeString, + Computed: true, + }, + + "status": { + Type: schema.TypeString, + Computed: true, + }, + + "failure_reason": { + Type: schema.TypeString, + Computed: true, + }, + }, + } +} + +func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + var name string + if v, ok := d.GetOk("name"); ok { + name = v.(string) + } else { + name = resource.UniqueId() + } + + createOpts := &glue.CreateDevEndpointInput{ + EndpointName: aws.String(name), + RoleArn: aws.String(d.Get("role_arn").(string)), + } + + if v, ok := d.GetOk("extra_jars_s3_path"); ok { + createOpts.SetExtraJarsS3Path(v.(string)) + } + + if v, ok := d.GetOk("extra_python_libs_s3_path"); ok { + createOpts.SetExtraPythonLibsS3Path(v.(string)) + } + + if v, ok := d.GetOk("number_of_nodes"); ok { + createOpts.SetNumberOfNodes(int64(v.(int))) + } + + if v, ok := d.GetOk("public_key"); ok { + createOpts.SetPublicKey(v.(string)) + } + + if v, ok := d.GetOk("public_keys"); ok { + publicKeys := expandStringSet(v.(*schema.Set)) + createOpts.SetPublicKeys(publicKeys) + } + + if v, ok := d.GetOk("security_configuration"); ok { + createOpts.SetSecurityConfiguration(v.(string)) + } + + if v, ok := d.GetOk("security_group_ids"); ok { + securityGroupIDs := expandStringSet(v.(*schema.Set)) + createOpts.SetSecurityGroupIds(securityGroupIDs) + } + + if v, ok := d.GetOk("subnet_id"); ok { + createOpts.SetSubnetId(v.(string)) + } + log.Printf("[DEBUG] Glue dev endpoint create config: %#v", *createOpts) + err := resource.Retry(1*time.Minute, func() *resource.RetryError { + _, err := conn.CreateDevEndpoint(createOpts) + if err != nil { + // Retry for IAM eventual consistency + if isAWSErr(err, glue.ErrCodeInvalidInputException, "should be given assume role permissions for Glue Service") { + return resource.RetryableError(err) + } + + return resource.NonRetryableError(err) + } + return nil + }) + + if err != nil { + return fmt.Errorf("error creating Glue dev endpoint: %s", err) + } + + d.SetId(name) + log.Printf("[INFO] Glue dev endpoint ID: %s", d.Id()) + + log.Printf("[DEBUG] Waiting for Glue dev endpoint (%s) to become available", d.Id()) + stateConf := &resource.StateChangeConf{ + Pending: []string{ + "PROVISIONING", + }, + Target: []string{"READY"}, + Refresh: glueDevEndpointStateRefreshFunc(conn, d.Id()), + Timeout: 10 * time.Minute, + } + if _, err := stateConf.WaitForState(); err != nil { + return fmt.Errorf("error while waiting for Glue dev endpoint (%s) to become available: %s", d.Id(), err) + } + + return resourceAwsGlueDevEndpointRead(d, meta) +} + +func resourceAwsGlueDevEndpointRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + request := &glue.GetDevEndpointInput{ + EndpointName: aws.String(d.Id()), + } + + output, err := conn.GetDevEndpoint(request) + if err != nil { + if glueErr, ok := err.(awserr.Error); ok && glueErr.Code() == glue.ErrCodeEntityNotFoundException { + log.Printf("[INFO] unable to find Glue dev endpoint and therfore it is removed from the state: %s", d.Id()) + d.SetId("") + return nil + } + return fmt.Errorf("error finding Glue dev endpoint %s: %s", d.Id(), err) + } + + endpoint := output.DevEndpoint + if endpoint == nil { + return fmt.Errorf("Glue dev endpoint (%s) is nil: %s", d.Id(), err) + } + + if err := d.Set("name", endpoint.EndpointName); err != nil { + return fmt.Errorf("error setting name for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("extra_jars_s3_path", endpoint.ExtraJarsS3Path); err != nil { + return fmt.Errorf("error setting extra_jars_s3_path for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("extra_python_libs_s3_path", endpoint.ExtraPythonLibsS3Path); err != nil { + return fmt.Errorf("error setting extra_python_libs_s3_path for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("number_of_nodes", endpoint.NumberOfNodes); err != nil { + return fmt.Errorf("error setting number_of_nodes for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_key", endpoint.PublicKey); err != nil { + return fmt.Errorf("error setting public_key for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_keys", flattenStringSet(endpoint.PublicKeys)); err != nil { + return fmt.Errorf("error setting public_keys for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("role_arn", endpoint.RoleArn); err != nil { + return fmt.Errorf("error setting role_arn for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("security_configuration", endpoint.SecurityConfiguration); err != nil { + return fmt.Errorf("error setting security_configuration for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("security_group_ids", flattenStringSet(endpoint.SecurityGroupIds)); err != nil { + return fmt.Errorf("error setting security_group_ids for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("subnet_id", endpoint.SubnetId); err != nil { + return fmt.Errorf("error setting subnet_id for Glue dev endpoint (%s): %s", d.Id(), err) + } + + // extra attributes + if err := d.Set("private_address", endpoint.PrivateAddress); err != nil { + return fmt.Errorf("error setting private_address for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("public_address", endpoint.PublicAddress); err != nil { + return fmt.Errorf("error setting public_address for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("yarn_endpoint_address", endpoint.YarnEndpointAddress); err != nil { + return fmt.Errorf("error setting yarn_endpoint_address for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("zeppelin_remote_spark_interpreter_port", endpoint.ZeppelinRemoteSparkInterpreterPort); err != nil { + return fmt.Errorf("error setting zeppelin_remote_spark_interpreter_port for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("availability_zone", endpoint.AvailabilityZone); err != nil { + return fmt.Errorf("error setting availability_zone for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("vpc_id", endpoint.VpcId); err != nil { + return fmt.Errorf("error setting vpc_id for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("status", endpoint.Status); err != nil { + return fmt.Errorf("error setting status for Glue dev endpoint (%s): %s", d.Id(), err) + } + + if err := d.Set("failure_reason", endpoint.FailureReason); err != nil { + return fmt.Errorf("error setting failure_reason for Glue dev endpoint (%s): %s", d.Id(), err) + } + return nil +} + +func resourceAwsDevEndpointUpdate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + updateOpts := &glue.UpdateDevEndpointInput{ + EndpointName: aws.String(d.Get("name").(string)), + } + + hasChanged := false + + if d.HasChange("public_keys") { + oldRaw, newRaw := d.GetChange("public_keys") + old := oldRaw.(*schema.Set) + new := newRaw.(*schema.Set) + create, remove := diffPublicKeys(expandStringSet(old), expandStringSet(new)) + updateOpts.SetAddPublicKeys(create) + updateOpts.SetDeletePublicKeys(remove) + + hasChanged = true + } + + if d.HasChange("public_key") { + updateOpts.SetPublicKey(d.Get("public_key").(string)) + hasChanged = true + } + + customLibs := &glue.DevEndpointCustomLibraries{} + + if d.HasChange("extra_jars_s3_path") { + customLibs.SetExtraJarsS3Path(d.Get("extra_jars_s3_path").(string)) + updateOpts.SetCustomLibraries(customLibs) + updateOpts.SetUpdateEtlLibraries(true) + hasChanged = true + } + + if d.HasChange("extra_python_libs_s3_path") { + customLibs.SetExtraPythonLibsS3Path(d.Get("extra_python_libs_s3_path").(string)) + updateOpts.SetCustomLibraries(customLibs) + updateOpts.SetUpdateEtlLibraries(true) + hasChanged = true + } + + if hasChanged { + _, err := conn.UpdateDevEndpoint(updateOpts) + if err != nil { + return fmt.Errorf("error updating Glue dev endpoint: %s", err) + } + } + + return resourceAwsGlueDevEndpointRead(d, meta) +} + +func resourceAwsDevEndpointDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).glueconn + + deleteOpts := &glue.DeleteDevEndpointInput{ + EndpointName: aws.String(d.Id()), + } + log.Printf("[INFO] Deleting Glue dev endpoint: %s", d.Id()) + + return resource.Retry(5*time.Minute, func() *resource.RetryError { + _, err := conn.DeleteDevEndpoint(deleteOpts) + if err == nil { + return nil + } + + glueErr, ok := err.(awserr.Error) + if !ok { + return resource.NonRetryableError(err) + } + + if glueErr.Code() == glue.ErrCodeEntityNotFoundException { + return nil + } + + return resource.NonRetryableError(fmt.Errorf("error deleting Glue dev endpoint: %s", err)) + }) +} + +func glueDevEndpointStateRefreshFunc(conn *glue.Glue, name string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + getDevEndpointInput := &glue.GetDevEndpointInput{ + EndpointName: aws.String(name), + } + endpoint, err := conn.GetDevEndpoint(getDevEndpointInput) + if err != nil { + if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + return nil, "", nil + } + + return nil, "", err + } + + if endpoint == nil { + return nil, "", nil + } + + return endpoint, *endpoint.DevEndpoint.Status, nil + } +} + +func diffPublicKeys(oldKeys, newKeys []*string) ([]*string, []*string) { + var create []*string + var remove []*string + + for _, oldKey := range oldKeys { + found := false + for _, newKey := range newKeys { + if oldKey == newKey { + found = true + break + } + } + if !found { + remove = append(remove, oldKey) + } + } + + for _, newKey := range newKeys { + found := false + for _, oldKey := range oldKeys { + if oldKey == newKey { + found = true + break + } + } + if !found { + create = append(create, newKey) + } + } + + return create, remove +} diff --git a/aws/resource_aws_glue_dev_endpoint_test.go b/aws/resource_aws_glue_dev_endpoint_test.go new file mode 100644 index 000000000000..aa5e2cd25f8f --- /dev/null +++ b/aws/resource_aws_glue_dev_endpoint_test.go @@ -0,0 +1,701 @@ +package aws + +import ( + "fmt" + "log" + "regexp" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/glue" + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +const ( + GlueDevEndpointResourcePrefix = "tf-acc-test" + publicKey1 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD3F6tyPEFEzV0LX3X8BsXdMsQz1x2cEikKDEY0aIj41qgxMCP/iteneqXSIFZBp5vizPvaoIR3Um9xK7PGoW8giupGn+EPuxIA4cDM4vzOqOkiMPhz5XK0whEjkVzTo4+S0puvDZuwIsdiW9mxhJc7tgBNL0cYlWSYVkz4G/fslNfRPW5mYAM49f4fhtxPb5ok4Q2Lg9dPKVHO/Bgeu5woMc7RY0p1ej6D4CKFE6lymSDJpW0YHX/wqE9+cfEauh7xZcG0q9t2ta6F6fmX0agvpFyZo8aFbXeUBr7osSCJNgvavWbM/06niWrOvYX2xwWdhXmXSrbX8ZbabVohBK41 foo1@bar.com" + publicKey2 = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAq6U3HQYC4g8WzU147gZZ7CKQH8TgYn3chZGRPxaGmHW1RUwsyEs0nmombmIhwxudhJ4ehjqXsDLoQpd6+c7BuLgTMvbv8LgE9LX53vnljFe1dsObsr/fYLvpU9LTlo8HgHAqO5ibNdrAUvV31ronzCZhms/Gyfdaue88Fd0/YnsZVGeOZPayRkdOHSpqme2CBrpa8myBeL1CWl0LkDG4+YCURjbaelfyZlIApLYKy3FcCan9XQFKaL32MJZwCgzfOvWIMtYcU8QtXMgnA3/I3gXk8YDUJv5P4lj0s/PJXuTM8DygVAUtebNwPuinS7wwonm5FXcWMuVGsVpG5K7FGQ== foo2@bar.com" + publicKey3 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtCk0lzMj1gPEOjdfQ37AIxCyETqJBubaMWuB4bgvGHp8LvEghr2YDl2bml1JrE1EOcZhPnIwgyucryXKA959sTUlgbvaFN7vmpVze56Q9tVU6BJQxOdaRoy5FcQMET9LB6SdbXk+V4CkDMsQNaFXezpg98HgCj+V7+bBWsfI6U63IESlWKK7kraCom8EWxkQk4mk9fizE2I+KrtiqN4xcah02LFG6IMnS+Xy3CDhcpZeYzWOV6zhcf675UJOdg/pLgQbUhhiwTOJFgRo8IcvE3iBrRMz508ppx6vLLr8J+3B8ujykc+/3ZSGfQfx6rO+OuSskhG5FLI6icbQBtBzf foo3@bar.com" + publicKey4 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD3F6tyPEFEzV0LX3X8BsXdMsQz1x2cEikKDEY0aIj41qgxMCP/iteneqXSIFZBp5vizPvaoIR3Um9xK7PGoW8giupGn+EPuxIA4cDM4vzOqOkiMPhz5XK0whEjkVzTo4+S0puvDZuwIsdiW9mxhJc7tgBNL0cYlWSYVkz4G/fslNfRPW5mYAM49f4fhtxPb5ok4Q2Lg9dPKVHO/Bgeu5woMc7RY0p1ej6D4CKFE6lymSDJpW0YHX/wqE9+cfEauh7xZcG0q9t2ta6F6fmX0agvpFyZo8aFbXeUBr7osSCJNgvavWbM/06niWrOvYX2xwWdhXmXSrbX8ZbabVohBK41 foo4@bar.com" +) + +func init() { + resource.AddTestSweepers("aws_glue_dev_endpoint", &resource.Sweeper{ + Name: "aws_glue_dev_endpoint", + F: testSweepGlueDevEndpoint, + }) +} + +func testSweepGlueDevEndpoint(region string) error { + client, err := sharedClientForRegion(region) + if err != nil { + return fmt.Errorf("error getting client: %s", err) + } + conn := client.(*AWSClient).glueconn + + input := &glue.GetDevEndpointsInput{} + err = conn.GetDevEndpointsPages(input, func(page *glue.GetDevEndpointsOutput, lastPage bool) bool { + if len(page.DevEndpoints) == 0 { + log.Printf("[INFO] No Glue Dev Endpoints to sweep") + return false + } + for _, endpoint := range page.DevEndpoints { + name := aws.StringValue(endpoint.EndpointName) + if !strings.HasPrefix(name, GlueDevEndpointResourcePrefix) { + log.Printf("[INFO] Skipping Glue Dev Endpoint: %s", name) + continue + } + + log.Printf("[INFO] Deleting Glue Dev Endpoint: %s", name) + _, err := conn.DeleteDevEndpoint(&glue.DeleteDevEndpointInput{ + EndpointName: aws.String(name), + }) + if err != nil { + log.Printf("[ERROR] Failed to delete Glue Dev Endpoint %s: %s", name, err) + } + } + return !lastPage + }) + if err != nil { + if testSweepSkipSweepError(err) { + log.Printf("[WARN] Skipping Glue Dev Endpoint sweep for %s: %s", region, err) + return nil + } + return fmt.Errorf("error retrieving Glue Dev Endpoint: %s", err) + } + + return nil +} + +func TestAccGlueDevEndpoint_Basic(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_Basic(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestMatchResourceAttr(resourceName, "role_arn", regexp.MustCompile(`^arn:[^:]+:iam::[^:]+:role/AWSGlueServiceRole-tf-acc-test-[0-9]+$`)), + resource.TestCheckResourceAttr(resourceName, "status", "READY"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_ExtraJarsS3Path(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraJarsS3Path := "foo" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_ExtraPythonLibsS3Path(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraPythonLibsS3Path := "foo" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_NumberOfNodes(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_NumberOfNodes(rName, 2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "number_of_nodes", "2"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_PublicKey(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_PublicKeys(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKeys(rName, publicKey1, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_SecurityConfiguration(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_SecurityConfiguration(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "security_configuration", rName), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +// Note: Either none or both of subnetId and securityGroupIds must be specified. +func TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs(t *testing.T) { + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "vpc_security_group_ids.#", "1"), + resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "data.aws_security_group.default", "id"), + resource.TestCheckResourceAttrPair(resourceName, "subnet_id", "aws_subnet.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "vpc_id", "aws_vpc.test", "id"), + resource.TestCheckResourceAttrPair(resourceName, "availability_zone", "aws_subnet.test", "availability_zone"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_PublicKey(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey1), + ), + }, + { + Config: testAccGlueDevEndpointConfig_PublicKey(rName, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_key", publicKey2), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_ExtraJarsS3Path(t *testing.T) { + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraJarsS3Path := "foo" + extraJarsS3PathUpdated := "bar" + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName, extraJarsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_jars_s3_path", extraJarsS3PathUpdated), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_ExtraPythonLibsS3Path(t *testing.T) { + t.Skip() + + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + extraPythonLibsS3Path := "foo" + extraPythonLibsS3PathUpdated := "bar" + + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3Path), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3Path), + ), + }, + { + Config: testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName, extraPythonLibsS3PathUpdated), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "extra_python_libs_s3_path", extraPythonLibsS3PathUpdated), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccGlueDevEndpoint_Update_PublicKeys(t *testing.T) { + var endpoint glue.DevEndpoint + rName := acctest.RandomWithPrefix(GlueDevEndpointResourcePrefix) + resourceName := "aws_glue_dev_endpoint.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueDevEndpointDestroy, + Steps: []resource.TestStep{ + { + Config: testAccGlueDevEndpointConfig_PublicKeys(rName, publicKey1, publicKey2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "2"), + ), + }, + { + Config: testAccGlueDevEndpointConfig_Update_PublicKeys(rName, publicKey1, publicKey3, publicKey4), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueDevEndpointExists(resourceName, &endpoint), + + resource.TestCheckResourceAttr(resourceName, "public_keys.#", "3"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccCheckAWSGlueDevEndpointExists(resourceName string, endpoint *glue.DevEndpoint) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[resourceName] + if !ok { + return fmt.Errorf("not found: %s", resourceName) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("no ID is set") + } + + conn := testAccProvider.Meta().(*AWSClient).glueconn + output, err := conn.GetDevEndpoint(&glue.GetDevEndpointInput{ + EndpointName: aws.String(rs.Primary.ID), + }) + + if err != nil { + return err + } + + if output == nil { + return fmt.Errorf("no Glue Dev Endpoint") + } + + *endpoint = *output.DevEndpoint + + return nil + } +} + +func testAccCheckAWSGlueDevEndpointDestroy(s *terraform.State) error { + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_glue_dev_endpoint" { + continue + } + + conn := testAccProvider.Meta().(*AWSClient).glueconn + output, err := conn.GetDevEndpoint(&glue.GetDevEndpointInput{ + EndpointName: aws.String(rs.Primary.ID), + }) + + if err != nil { + if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") { + return nil + } + return err + } + + endpoint := output.DevEndpoint + if endpoint != nil && aws.StringValue(endpoint.EndpointName) == rs.Primary.ID { + return fmt.Errorf("the Glue Dev Endpoint %s still exists", rs.Primary.ID) + } + + return nil + } + + return nil +} + +func testAccGlueDevEndpointConfig_Base(rName string) string { + return fmt.Sprintf(` +resource "aws_iam_role" "test" { + name = "AWSGlueServiceRole-%s" + assume_role_policy = "${data.aws_iam_policy_document.test.json}" +} + +data "aws_iam_policy_document" "test" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy_attachment" "test-AWSGlueServiceRole" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" + role = "${aws_iam_role.test.name}" +} +`, rName) +} + +func testAccGlueDevEndpointConfig_Basic(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" +} +`, rName) +} + +func testAccGlueDevEndpointConfig_ExtraJarsS3Path(rName string, extraJarsS3Path string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + extra_jars_s3_path = %q +} +`, rName, extraJarsS3Path) +} + +func testAccGlueDevEndpointConfig_ExtraPythonLibsS3Path(rName string, extraPythonLibsS3Path string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + extra_python_libs_s3_path = %q +} +`, rName, extraPythonLibsS3Path) +} + +func testAccGlueDevEndpointConfig_NumberOfNodes(rName string, numberOfNodes int) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + number_of_nodes = %d +} +`, rName, numberOfNodes) +} + +func testAccGlueDevEndpointConfig_PublicKey(rName string, publicKey string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + public_key = "%s" +} +`, rName, publicKey) +} + +func testAccGlueDevEndpointConfig_PublicKeys(rName string, publicKey1 string, publicKey2 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + public_keys = ["%s", "%s"] +} +`, rName, publicKey1, publicKey2) +} + +func testAccGlueDevEndpointConfig_Update_PublicKeys(rName string, publicKey1 string, publicKey2 string, publicKey3 string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + public_keys = ["%s", "%s", "%s"] +} +`, rName, publicKey1, publicKey2, publicKey3) +} + +func testAccGlueDevEndpointConfig_SecurityConfiguration(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + security_configuration = "${aws_glue_security_configuration.test.name}" +} + +resource "aws_glue_security_configuration" "test" { + name = %q + + encryption_configuration { + cloudwatch_encryption { + cloudwatch_encryption_mode = "DISABLED" + } + + job_bookmarks_encryption { + job_bookmarks_encryption_mode = "DISABLED" + } + + s3_encryption { + s3_encryption_mode = "DISABLED" + } + } +} +`, rName, rName) +} + +func testAccGlueDevEndpointConfig_SubnetID_SecurityGroupIDs(rName string) string { + return testAccGlueDevEndpointConfig_Base(rName) + fmt.Sprintf(` +resource "aws_glue_dev_endpoint" "test" { + name = %q + role_arn = "${aws_iam_role.test.arn}" + subnet_id = "${aws_subnet.test.id}" + security_group_ids = ["${data.aws_security_group.default.id}"] +} + +resource "aws_vpc" "test" { + cidr_block = "10.0.0.0/16" + tags = { + Name = %q + } +} + +resource "aws_subnet" "test" { + vpc_id = "${aws_vpc.test.id}" + cidr_block = "10.0.1.0/24" + availability_zone = "us-west-2a" + + tags = { + Name = %q + } +} + +data "aws_security_group" "default" { + vpc_id = "${aws_vpc.test.id}" + name = "default" +} + +data "aws_vpc_endpoint_service" "s3" { + service = "s3" +} + +resource "aws_vpc_endpoint" "test" { + service_name = "${data.aws_vpc_endpoint_service.s3.service_name}" + vpc_id = "${aws_vpc.test.id}" +} + +resource "aws_internet_gateway" "test" { + vpc_id = "${aws_vpc.test.id}" +} + +resource "aws_eip" "test" { + vpc = true +} + +resource "aws_nat_gateway" "test" { + allocation_id = "${aws_eip.test.id}" + subnet_id = "${aws_subnet.test.id}" + + tags = { + Name = %q + } + + depends_on = ["aws_internet_gateway.test"] +} +`, rName, rName, rName, rName) +} diff --git a/website/aws.erb b/website/aws.erb index 108184c6ed89..65ce6302e150 100644 --- a/website/aws.erb +++ b/website/aws.erb @@ -1511,6 +1511,9 @@
  • aws_glue_crawler
    • +
  • + aws_glue_dev_endpoint +
  • aws_glue_job
    • diff --git a/website/docs/r/glue_dev_endpoint.markdown b/website/docs/r/glue_dev_endpoint.markdown new file mode 100644 index 000000000000..7ef68117bb17 --- /dev/null +++ b/website/docs/r/glue_dev_endpoint.markdown @@ -0,0 +1,81 @@ +--- +layout: "aws" +page_title: "AWS: aws_glue_dev_endpoint" +sidebar_current: "docs-aws-resource-glue-dev-endpoint" +description: |- + Provides a Glue Development Endpoint resource. +--- + +# aws_glue_dev_endpoint + +Provides a Glue Development Endpoint resource. + +## Example Usage + +Basic usage: + +```hcl +resource "aws_glue_dev_endpoint" "de" { + name = "foo" + role_arn = "${aws_iam_role.test.arn}" +} + +resource "aws_iam_role" "de" { + name = "AWSGlueServiceRole-foo" + assume_role_policy = "${data.aws_iam_policy_document.de.json}" +} + +data "aws_iam_policy_document" "de" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy_attachment" "foo-AWSGlueServiceRole" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" + role = "${aws_iam_role.de.name}" +} + +``` + +## Argument Reference + +The following arguments are supported: + +* `name` - (Optional) The name of the Glue Development Endpoint (must be unique). If omitted, Terraform will assign a random, unique name. +* `extra_jars_s3_path` - (Optional) Path to one or more Java Jars in an S3 bucket that should be loaded in your DevEndpoint. +* `extra_python_libs_s3_path` - (Optional) Path(s) to one or more Python libraries in an S3 bucket that should be loaded in your DevEndpoint. Multiple values must be complete paths separated by a comma. +* `number_of_nodes` - (Optional) The number of AWS Glue Data Processing Units (DPUs) to allocate to this DevEndpoint. +* `public_key` - (Optional) The public key to be used by this DevEndpoint for authentication. +* `public_keys` - (Optional) A list of public keys to be used by the DevEndpoints for authentication. +* `role_arn` - (Required) The IAM role for the DevEndpoint. +* `security_configuration` - (Optional) The name of the SecurityConfiguration structure to be used with this DevEndpoint. +* `security_group_ids` - (Optional) Security group IDs for the security groups to be used by the new DevEndpoint. +* `subnet_id` - (Optional) The subnet ID for the new DevEndpoint to use. + +## Attributes Reference + +The following attributes are exported: + +* `name` - The name of the new Glue Development Endpoint. +* `private_address` - A private IP address to access the DevEndpoint within a VPC, if the DevEndpoint is created within one. +* `public_address` - The public IP address used by this DevEndpoint. The PublicAddress field is present only when you create a non-VPC DevEndpoint. +* `yarn_endpoint_address` - The YARN endpoint address used by this DevEndpoint. +* `zeppelin_remote_spark_interpreter_port` - The Apache Zeppelin port for the remote Apache Spark interpreter. +* `availability_zone` - The AWS availability zone where this DevEndpoint is located. +* `vpc_id` - he ID of the VPC used by this DevEndpoint. +* `status` - The current status of this DevEndpoint. +* `failure_reason` - The reason for a current failure in this DevEndpoint. + +## Import + +SageMaker Glue Development Endpoint can be imported using the `name`, e.g. + +``` +$ terraform import aws_glue_dev_endpoint.de foo +``` \ No newline at end of file