resource/aws_glue_dev_endpoint: Catch additional IAM eventual consist…

…ency error and fix VPC testing (#16232)

Reference: #16170
Reference: https://docs.aws.amazon.com/glue/latest/dg/set-up-vpc-dns.html

The `FAILED` status in GovCloud was from this reason: `"FailureReason":"Reverse dns resolution of ip 10.0.1.153 failed"`, because Glue requires VPCs to have DNS hostnames support enabled. Added the API failure reason to the waiter error messaging when available. Also fixed up some of the testing configuration to remove extraneous IAM handling.

Previously:

```
=== CONT  TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs
    resource_aws_glue_dev_endpoint_test.go:447: Step 1/2 error: Error running apply: 2020/11/16 22:57:14 [DEBUG] Using modified User-Agent: Terraform/0.12.29 HashiCorp-terraform-exec/0.10.0

        Error: error creating Glue Dev Endpoint: InvalidInputException: Failed to call ec2:DescribeSubnets: User: arn:aws:sts::056518428541:assumed-role/GlueJobExecutorCoralService-adminEC2Role-1URQ1VJPS7RAI/i-0b29c5b9f851d3f90 is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::187416307283:role/AWSGlueServiceRole-tf-acc-test-1350868708791253218

--- FAIL: TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs (25.32s)

=== CONT  TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs
    resource_aws_glue_dev_endpoint_test.go:447: Step 1/2 error: Error running apply: 2020/11/12 13:49:01 [DEBUG] Using modified User-Agent: Terraform/0.12.29 HashiCorp-terraform-exec/0.10.0

        Error: error while waiting for Glue Dev Endpoint (tf-acc-test-831422846705845092) to become available: unexpected state 'FAILED', wanted target 'READY'. last error: %!s(<nil>)

--- FAIL: TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs (690.11s)

=== CONT  TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs
TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs: resource_aws_glue_dev_endpoint_test.go:447: Step 1/2 error: Error running pre-apply refresh: 2020/11/16 13:33:06 [DEBUG] Using modified User-Agent: Terraform/0.12.29 HashiCorp-terraform-exec/0.10.0
Error: multiple VPC Endpoint Services matched; use additional constraints to reduce matches to a single VPC Endpoint Service
--- FAIL: TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs (2.32s)
```

Output from acceptance testing in AWS Commercial:

```
--- PASS: TestAccGlueDevEndpoint_Arguments (631.30s)
--- PASS: TestAccGlueDevEndpoint_Basic (540.64s)
--- PASS: TestAccGlueDevEndpoint_disappears (529.64s)
--- PASS: TestAccGlueDevEndpoint_ExtraJarsS3Path (524.98s)
--- PASS: TestAccGlueDevEndpoint_ExtraPythonLibsS3Path (504.74s)
--- PASS: TestAccGlueDevEndpoint_GlueVersion (975.71s)
--- PASS: TestAccGlueDevEndpoint_NumberOfNodes (575.51s)
--- PASS: TestAccGlueDevEndpoint_NumberOfWorkers (1123.20s)
--- PASS: TestAccGlueDevEndpoint_PublicKey (684.84s)
--- PASS: TestAccGlueDevEndpoint_PublicKeys (716.43s)
--- PASS: TestAccGlueDevEndpoint_SecurityConfiguration (540.04s)
--- PASS: TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs (1055.05s)
--- PASS: TestAccGlueDevEndpoint_Tags (628.04s)
--- PASS: TestAccGlueDevEndpoint_WorkerType (1040.59s)
```

Output from acceptance testing in AWS GovCloud (US):

```
--- PASS: TestAccGlueDevEndpoint_Arguments (594.05s)
--- PASS: TestAccGlueDevEndpoint_Basic (580.48s)
--- PASS: TestAccGlueDevEndpoint_disappears (483.68s)
--- PASS: TestAccGlueDevEndpoint_ExtraJarsS3Path (528.97s)
--- PASS: TestAccGlueDevEndpoint_ExtraPythonLibsS3Path (668.35s)
--- PASS: TestAccGlueDevEndpoint_GlueVersion (800.76s)
--- PASS: TestAccGlueDevEndpoint_NumberOfNodes (894.37s)
--- PASS: TestAccGlueDevEndpoint_NumberOfWorkers (828.21s)
--- PASS: TestAccGlueDevEndpoint_PublicKey (490.94s)
--- PASS: TestAccGlueDevEndpoint_PublicKeys (492.38s)
--- PASS: TestAccGlueDevEndpoint_SubnetID_SecurityGroupIDs (998.87s)
--- PASS: TestAccGlueDevEndpoint_Tags (483.63s)
--- PASS: TestAccGlueDevEndpoint_WorkerType (799.62s)
```

aws/internal/service/glue/waiter/status.go

@@ -1,6 +1,8 @@
 package waiter
 
 import (
+	"fmt"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/glue"
 	"github.com/hashicorp/aws-sdk-go-base/tfawserr"
@@ -68,10 +70,14 @@ func GlueDevEndpointStatus(conn *glue.Glue, name string) resource.StateRefreshFu
 			return nil, "", err
 		}
 
-		if endpoint == nil {
+		if endpoint == nil || endpoint.DevEndpoint == nil {
 			return nil, "", nil
 		}
 
+		if aws.StringValue(endpoint.DevEndpoint.Status) == "FAILED" && endpoint.DevEndpoint.FailureReason != nil {
+			return endpoint, aws.StringValue(endpoint.DevEndpoint.Status), fmt.Errorf("%s", aws.StringValue(endpoint.DevEndpoint.FailureReason))
+		}
+
 		return endpoint, aws.StringValue(endpoint.DevEndpoint.Status), nil
 	}
 }

aws/resource_aws_glue_dev_endpoint.go

@@ -222,10 +222,13 @@ func resourceAwsGlueDevEndpointCreate(d *schema.ResourceData, meta interface{})
 		_, err := conn.CreateDevEndpoint(input)
 		if err != nil {
 			// Retry for IAM eventual consistency
-			if isAWSErr(err, glue.ErrCodeInvalidInputException, "should be given assume role permissions for Glue Service") {
+			if tfawserr.ErrMessageContains(err, glue.ErrCodeInvalidInputException, "should be given assume role permissions for Glue Service") {
 				return resource.RetryableError(err)
 			}
-			if isAWSErr(err, glue.ErrCodeInvalidInputException, "S3 endpoint and NAT validation has failed for subnetId") {
+			if tfawserr.ErrMessageContains(err, glue.ErrCodeInvalidInputException, "is not authorized to perform") {
+				return resource.RetryableError(err)
+			}
+			if tfawserr.ErrMessageContains(err, glue.ErrCodeInvalidInputException, "S3 endpoint and NAT validation has failed for subnetId") {
 				return resource.RetryableError(err)
 			}
 

aws/resource_aws_glue_dev_endpoint_test.go

@@ -649,28 +649,11 @@ data "aws_iam_policy_document" "service" {
 
     principals {
       type        = "Service"
-      identifiers = ["glue.amazonaws.com"]
+      identifiers = ["glue.${data.aws_partition.current.dns_suffix}"]
     }
   }
 }
 
-resource "aws_iam_policy" "test" {
-  name   = %[1]q
-  policy = data.aws_iam_policy_document.test.json
-}
-
-data "aws_iam_policy_document" "test" {
-  statement {
-    actions   = ["ec2:DescribeSecurityGroups", "ec2:DescribeSubnets"]
-    resources = ["*"]
-  }
-}
-
-resource "aws_iam_role_policy_attachment" "test" {
-  role       = aws_iam_role.test.name
-  policy_arn = aws_iam_policy.test.arn
-}
-
 resource "aws_iam_role_policy_attachment" "glue_service_role" {
   policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AWSGlueServiceRole"
   role       = aws_iam_role.test.name
@@ -848,7 +831,8 @@ resource "aws_vpc_endpoint" "s3" {
 }
 
 data "aws_vpc_endpoint_service" "s3" {
-  service = "s3"
+  service      = "s3"
+  service_type = "Gateway"
 }
 
 resource "aws_vpc_endpoint_route_table_association" "test" {
@@ -857,7 +841,9 @@ resource "aws_vpc_endpoint_route_table_association" "test" {
 }
 
 resource "aws_vpc" "test" {
-  cidr_block = "10.0.0.0/16"
+  cidr_block           = "10.0.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
 
   tags = {
     Name = %[1]q