Skip to content

Commit

Permalink
resource/aws_datasync_task: Allow UNAVAILABLE as pending status durin…
Browse files Browse the repository at this point in the history 
…g creation (#15949)

Reference: #15947

Especially for new EFS Mount Targets, the task can temporarily transition from CREATING to UNAVAILABLE (mount failures) then eventually it can transition to AVAILABLE. This also simplifies and updates the base testing configuration based on the current DataSync user guide.

Previously:

```
=== CONT  TestAccAWSDataSyncTask_basic
TestAccAWSDataSyncTask_basic: resource_aws_datasync_task_test.go:85: Step 1/2 error: Error running apply:
Error: error waiting for DataSync Task (arn:aws:datasync:us-west-2:*******:task/task-0e819038fac1ebf6e) creation: waiting for DataSync Task (arn:aws:datasync:us-west-2:*******:task/task-0e819038fac1ebf6e) creation: last status (UNAVAILABLE), error code (MountFailure), error detail: Failed to connect to an NFS server on host fs-7c85ce79.efs.us-west-2.amazonaws.com. Ensure that your DataSync agent has a route to your NFS server and can make a TCP connection to port 2049. Please also ensure that the NFS service is running and listening on port 2049. After resolving any networking issues, retry your task.
--- FAIL: TestAccAWSDataSyncTask_basic (252.60s)
```

Output from acceptance testing in AWS Commercial:

```
--- PASS: TestAccAWSDataSyncTask_basic (229.43s)
--- PASS: TestAccAWSDataSyncTask_CloudWatchLogGroupARN (227.57s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_AtimeMtime (348.45s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_BytesPerSecond (266.03s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_Gid (240.71s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_PosixPermissions (287.10s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_PreserveDeletedFiles (240.21s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_PreserveDevices (349.08s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_Uid (365.25s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_VerifyMode (356.74s)
--- PASS: TestAccAWSDataSyncTask_disappears (227.63s)
```

Output from acceptance testing in AWS GovCloud (US):

```
--- PASS: TestAccAWSDataSyncTask_basic (240.55s)
--- PASS: TestAccAWSDataSyncTask_CloudWatchLogGroupARN (272.05s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_AtimeMtime (276.25s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_BytesPerSecond (289.87s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_Gid (301.43s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_PosixPermissions (286.56s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_PreserveDeletedFiles (299.53s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_PreserveDevices (300.01s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_Uid (312.04s)
--- PASS: TestAccAWSDataSyncTask_DefaultSyncOptions_VerifyMode (344.80s)
--- PASS: TestAccAWSDataSyncTask_disappears (246.44s)
```
  • Loading branch information
@bflad
bflad authored Nov 5, 2020
1 parent 59e60af commit 6be55d7
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 83 deletions.
37 changes: 37 additions & 0 deletions aws/internal/service/datasync/waiter/status.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package waiter

import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/datasync"
"github.com/hashicorp/aws-sdk-go-base/tfawserr"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
)

const (
TaskStatusUnknown = "Unknown"
)

// TaskStatus fetches the Operation and its Status
func TaskStatus(conn *datasync.DataSync, arn string) resource.StateRefreshFunc {
return func() (interface{}, string, error) {
input := &datasync.DescribeTaskInput{
TaskArn: aws.String(arn),
}

output, err := conn.DescribeTask(input)

if tfawserr.ErrMessageContains(err, datasync.ErrCodeInvalidRequestException, "not found") {
return nil, "", nil
}

if err != nil {
return output, TaskStatusUnknown, err
}

if output == nil {
return output, TaskStatusUnknown, nil
}

return output, aws.StringValue(output.Status), nil
}
}
49 changes: 49 additions & 0 deletions aws/internal/service/datasync/waiter/waiter.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package waiter

import (
"fmt"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/datasync"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
)

// TaskStatusAvailable waits for a Task to return Available
func TaskStatusAvailable(conn *datasync.DataSync, arn string, timeout time.Duration) (*datasync.DescribeTaskOutput, error) {
stateConf := &resource.StateChangeConf{
Pending: []string{
datasync.TaskStatusCreating,
datasync.TaskStatusUnavailable,
},
Target: []string{
datasync.TaskStatusAvailable,
datasync.TaskStatusRunning,
},
Refresh: TaskStatus(conn, arn),
Timeout: timeout,
}

outputRaw, err := stateConf.WaitForState()

if output, ok := outputRaw.(*datasync.DescribeTaskOutput); ok {
if err != nil && output != nil && output.ErrorCode != nil && output.ErrorDetail != nil {
newErr := fmt.Errorf("%s: %s", aws.StringValue(output.ErrorCode), aws.StringValue(output.ErrorDetail))

switch e := err.(type) {
case *resource.TimeoutError:
if e.LastError == nil {
e.LastError = newErr
}
case *resource.UnexpectedStateError:
if e.LastError == nil {
e.LastError = newErr
}
}
}

return output, err
}

return nil, err
}
50 changes: 5 additions & 45 deletions aws/resource_aws_datasync_task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/datasync"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/datasync/waiter"
)

func resourceAwsDataSyncTask() *schema.Resource {
Expand Down Expand Up @@ -176,55 +176,15 @@ func resourceAwsDataSyncTaskCreate(d *schema.ResourceData, meta interface{}) err

log.Printf("[DEBUG] Creating DataSync Task: %s", input)
output, err := conn.CreateTask(input)

if err != nil {
return fmt.Errorf("error creating DataSync Task: %s", err)
return fmt.Errorf("error creating DataSync Task: %w", err)
}

d.SetId(aws.StringValue(output.TaskArn))

// Task creation can take a few minutes\
taskInput := &datasync.DescribeTaskInput{
TaskArn: aws.String(d.Id()),
}
var taskOutput *datasync.DescribeTaskOutput
err = resource.Retry(d.Timeout(schema.TimeoutCreate), func() *resource.RetryError {
taskOutput, err := conn.DescribeTask(taskInput)

if isAWSErr(err, "InvalidRequestException", "not found") {
return resource.RetryableError(err)
}

if err != nil {
return resource.NonRetryableError(err)
}

if aws.StringValue(taskOutput.Status) == datasync.TaskStatusAvailable || aws.StringValue(taskOutput.Status) == datasync.TaskStatusRunning {
return nil
}

err = fmt.Errorf("waiting for DataSync Task (%s) creation: last status (%s), error code (%s), error detail: %s",
d.Id(), aws.StringValue(taskOutput.Status), aws.StringValue(taskOutput.ErrorCode), aws.StringValue(taskOutput.ErrorDetail))

if aws.StringValue(taskOutput.Status) == datasync.TaskStatusCreating {
return resource.RetryableError(err)
}

return resource.NonRetryableError(err) // should only happen if err != nil
})
if isResourceTimeoutError(err) {
taskOutput, err = conn.DescribeTask(taskInput)
if isAWSErr(err, "InvalidRequestException", "not found") {
return fmt.Errorf("Task not found after creation: %s", err)
}
if err != nil {
return fmt.Errorf("Error describing task after creation: %s", err)
}
if aws.StringValue(taskOutput.Status) == datasync.TaskStatusCreating {
return fmt.Errorf("Data sync task status has not finished creating")
}
}
if err != nil {
return fmt.Errorf("error waiting for DataSync Task (%s) creation: %s", d.Id(), err)
if _, err := waiter.TaskStatusAvailable(conn, d.Id(), d.Timeout(schema.TimeoutCreate)); err != nil {
return fmt.Errorf("error waiting for DataSync Task (%s) creation: %w", d.Id(), err)
}

return resourceAwsDataSyncTaskRead(d, meta)
Expand Down
54 changes: 16 additions & 38 deletions aws/resource_aws_datasync_task_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -672,15 +672,13 @@ resource "aws_datasync_location_s3" "destination" {
}

func testAccAWSDataSyncTaskConfigSourceLocationNfsBase(rName string) string {
return fmt.Sprintf(`
data "aws_ami" "source-aws-thinstaller" {
most_recent = true
owners = ["amazon"]
filter {
name = "name"
values = ["aws-thinstaller-*"]
}
return composeConfig(
// Reference: https://docs.aws.amazon.com/datasync/latest/userguide/agent-requirements.html
testAccAvailableEc2InstanceTypeForAvailabilityZone("aws_subnet.source.availability_zone", "m5.2xlarge", "m5.4xlarge"),
fmt.Sprintf(`
# Reference: https://docs.aws.amazon.com/datasync/latest/userguide/deploy-agents.html
data "aws_ssm_parameter" "aws_service_datasync_ami" {
name = "/aws/service/datasync/ami"
}
resource "aws_vpc" "source" {
Expand All @@ -693,15 +691,6 @@ resource "aws_vpc" "source" {
}
}
resource "aws_vpc_dhcp_options" "source" {
domain_name_servers = ["AmazonProvidedDNS"]
}
resource "aws_vpc_dhcp_options_association" "source" {
dhcp_options_id = aws_vpc_dhcp_options.source.id
vpc_id = aws_vpc.source.id
}
resource "aws_subnet" "source" {
cidr_block = "10.0.0.0/24"
vpc_id = aws_vpc.source.id
Expand All @@ -719,8 +708,8 @@ resource "aws_internet_gateway" "source" {
}
}
resource "aws_route_table" "source" {
vpc_id = aws_vpc.source.id
resource "aws_default_route_table" "source" {
default_route_table_id = aws_vpc.source.default_route_table_id
route {
cidr_block = "0.0.0.0/0"
Expand All @@ -732,11 +721,6 @@ resource "aws_route_table" "source" {
}
}
resource "aws_route_table_association" "source" {
subnet_id = aws_subnet.source.id
route_table_id = aws_route_table.source.id
}
resource "aws_security_group" "source" {
vpc_id = aws_vpc.source.id
Expand Down Expand Up @@ -771,17 +755,14 @@ resource "aws_efs_mount_target" "source" {
resource "aws_instance" "source" {
depends_on = [
aws_internet_gateway.source,
aws_vpc_dhcp_options_association.source,
aws_default_route_table.source,
]
ami = data.aws_ami.source-aws-thinstaller.id
ami = data.aws_ssm_parameter.aws_service_datasync_ami.value
associate_public_ip_address = true
# Default instance type from sync.sh
instance_type = "c5.2xlarge"
vpc_security_group_ids = [aws_security_group.source.id]
subnet_id = aws_subnet.source.id
instance_type = data.aws_ec2_instance_type_offering.available.instance_type
vpc_security_group_ids = [aws_security_group.source.id]
subnet_id = aws_subnet.source.id
tags = {
Name = "tf-acc-test-datasync-task"
Expand All @@ -793,18 +774,15 @@ resource "aws_datasync_agent" "source" {
name = %q
}
# Using EFS File System DNS name due to DNS propagation delays
resource "aws_datasync_location_nfs" "source" {
depends_on = [aws_efs_mount_target.source]
server_hostname = aws_efs_file_system.source.dns_name
server_hostname = aws_efs_mount_target.source.dns_name
subdirectory = "/"
on_prem_config {
agent_arns = [aws_datasync_agent.source.arn]
}
}
`, rName)
`, rName))
}

func testAccAWSDataSyncTaskConfig(rName string) string {
Expand Down

0 comments on commit 6be55d7

Please sign in to comment.