Add configurable timeout for VM PowerOn operations

vsphere/internal/helper/viapi/vim_helper.go

@@ -60,6 +60,28 @@ func IsManagedObjectNotFoundError(err error) bool {
 	return false
 }
 
+// IsInvalidStateError checks an error to see if it's of the
+// InvalidState type.
+func IsInvalidStateError(err error) bool {
+	if f, ok := vimSoapFault(err); ok {
+		if _, ok := f.(types.InvalidState); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// IsInvalidPowerStateError checks an error to see if it's of the
+// InvalidState type.
+func IsInvalidPowerStateError(err error) bool {
+	if f, ok := vimSoapFault(err); ok {
+		if _, ok := f.(types.InvalidPowerState); ok {
+			return true
+		}
+	}
+	return false
+}
+
 // isNotFoundError checks an error to see if it's of the NotFoundError type.
 //
 // Note this is different from the other "not found" faults and is an error

vsphere/internal/helper/virtualmachine/virtual_machine_helper.go

@@ -376,10 +376,8 @@ func skipIPAddrForWaiter(ip net.IP, ignoredGuestIPs []interface{}) bool {
 	return false
 }
 
-func blockUntilReadyForMethod(method string, vm *object.VirtualMachine, waitFor time.Duration) error {
+func blockUntilReadyForMethod(method string, vm *object.VirtualMachine, ctx context.Context) error {
 	log.Printf("[DEBUG] blockUntilReadyForMethod: Going to block until %q is no longer in the Disabled Methods list for vm %s", method, vm.Reference().Value)
-	ctx, cancel := context.WithTimeout(context.TODO(), waitFor)
-	defer cancel()
 
 	for {
 		vprops, err := Properties(vm)
@@ -480,12 +478,20 @@ func Customize(vm *object.VirtualMachine, spec types.CustomizationSpec) error {
 }
 
 // PowerOn wraps powering on a VM and the waiting for the subsequent task.
-func PowerOn(vm *object.VirtualMachine) error {
-	log.Printf("[DEBUG] Powering on virtual machine %q", vm.InventoryPath)
-	ctx, cancel := context.WithTimeout(context.Background(), provider.DefaultAPITimeout)
+func PowerOn(vm *object.VirtualMachine, pTimeout time.Duration) error {
+	vmPath := vm.InventoryPath
+	log.Printf("[DEBUG] Powering on virtual machine %q", vmPath)
+	var ctxTimeout time.Duration
+	if pTimeout > provider.DefaultAPITimeout {
+		ctxTimeout = pTimeout
+	} else {
+		ctxTimeout = provider.DefaultAPITimeout
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout)
 	defer cancel()
 
-	err := blockUntilReadyForMethod("PowerOnVM_Task", vm, provider.DefaultAPITimeout)
+	err := blockUntilReadyForMethod("PowerOnVM_Task", vm, ctx)
 	if err != nil {
 		return err
 	}
@@ -494,16 +500,42 @@ func PowerOn(vm *object.VirtualMachine) error {
 	// is in a state that can be started we have noticed that vsphere will randomly fail to
 	// power on the vm with "InvalidState" errors.
 	//
-	// We're adding a small delay here to avoid this issue.
-	time.Sleep(powerOnWaitMilli * time.Millisecond)
+	// We're adding a small loop that will try to power on the VM until we hit a timeout
+	// or manage to call PowerOnVM_Task successfully.
 
-	task, err := vm.PowerOn(ctx)
-	if err != nil {
-		return err
+powerLoop:
+	for {
+		select {
+		case <-time.After(500 * time.Millisecond):
+			vprops, err := Properties(vm)
+			if err != nil {
+				return fmt.Errorf("cannot fetch properties of created virtual machine: %s", err)
+			}
+			if vprops.Runtime.PowerState == types.VirtualMachinePowerStatePoweredOff {
+				log.Printf("[DEBUG] VM %q is powered off, attempting to power on.", vmPath)
+				task, err := vm.PowerOn(ctx)
+				if err != nil {
+					log.Printf("[DEBUG] Failed to submit PowerOn task for vm %q. Error: %s", vmPath, err)
+					return fmt.Errorf("failed to submit poweron task for vm %q: %s", vmPath, err)
+				}
+				err = task.Wait(ctx)
+				if err != nil {
+					if err.Error() == "The operation is not allowed in the current state." {
+						log.Printf("[DEBUG] vm %q cannot be powered on in the current state", vmPath)
+						continue powerLoop
+					} else {
+						log.Printf("[DEBUG] PowerOn task for vm %q failed. Error: %s", vmPath, err)
+						return fmt.Errorf("powerOn task for vm %q failed: %s", vmPath, err)
+					}
+				}
+				log.Printf("[DEBUG] PowerOn task for VM %q was successful.", vmPath)
+				break powerLoop
+			}
+		case <-ctx.Done():
+			return fmt.Errorf("timed out while trying to power on vm %q", vmPath)
+		}
 	}
-	tctx, tcancel := context.WithTimeout(context.Background(), provider.DefaultAPITimeout)
-	defer tcancel()
-	return task.Wait(tctx)
+	return nil
 }
 
 // PowerOff wraps powering off a VM and the waiting for the subsequent task.

vsphere/resource_vsphere_virtual_machine.go

@@ -149,6 +149,13 @@ func resourceVSphereVirtualMachine() *schema.Resource {
 			Description:  "The amount of time, in minutes, to wait for a vMotion operation to complete before failing.",
 			ValidateFunc: validation.IntAtLeast(10),
 		},
+		"poweron_timeout": {
+			Type:         schema.TypeInt,
+			Description:  "The amount of time, in seconds, that we will be trying to power on a VM",
+			Default:      300,
+			ValidateFunc: validation.IntAtLeast(300),
+			Optional:     true,
+		},
 		"force_power_off": {
 			Type:        schema.TypeBool,
 			Optional:    true,
@@ -673,7 +680,13 @@ func resourceVSphereVirtualMachineUpdate(d *schema.ResourceData, meta interface{
 		}
 		// Power back on the VM, and wait for network if necessary.
 		if vprops.Runtime.PowerState != types.VirtualMachinePowerStatePoweredOn {
-			if err := virtualmachine.PowerOn(vm); err != nil {
+			pTimeoutStr := fmt.Sprintf("%ds", d.Get("poweron_timeout").(int))
+			pTimeout, err := time.ParseDuration(pTimeoutStr)
+			if err != nil {
+				return fmt.Errorf("failed to parse poweron_timeout as a valid duration: %s", err)
+			}
+			// Start the virtual machine
+			if err := virtualmachine.PowerOn(vm, pTimeout); err != nil {
 				return fmt.Errorf("error powering on virtual machine: %s", err)
 			}
 			err = virtualmachine.WaitForGuestIP(
@@ -1104,8 +1117,13 @@ func resourceVSphereVirtualMachineCreateBare(d *schema.ResourceData, meta interf
 	log.Printf("[DEBUG] VM %q - UUID is %q", vm.InventoryPath, vprops.Config.Uuid)
 	d.SetId(vprops.Config.Uuid)
 
+	pTimeoutStr := fmt.Sprintf("%ds", d.Get("poweron_timeout").(int))
+	pTimeout, err := time.ParseDuration(pTimeoutStr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse poweron_timeout as a valid duration: %s", err)
+	}
 	// Start the virtual machine
-	if err := virtualmachine.PowerOn(vm); err != nil {
+	if err := virtualmachine.PowerOn(vm, pTimeout); err != nil {
 		return nil, fmt.Errorf("error powering on virtual machine: %s", err)
 	}
 	return vm, nil
@@ -1329,7 +1347,8 @@ func resourceVSphereVirtualMachineCreateClone(d *schema.ResourceData, meta inter
 		}
 	}
 	// Finally time to power on the virtual machine!
-	if err := virtualmachine.PowerOn(vm); err != nil {
+	pTimeout := time.Duration(d.Get("poweron_timeout").(int)) * time.Second
+	if err := virtualmachine.PowerOn(vm, pTimeout); err != nil {
 		return nil, fmt.Errorf("error powering on virtual machine: %s", err)
 	}
 	// If we customized, wait on customization.