diff --git a/managed/build.sbt b/managed/build.sbt index 9ea561e5635e..bf3442393ec9 100644 --- a/managed/build.sbt +++ b/managed/build.sbt @@ -228,6 +228,7 @@ libraryDependencies ++= Seq( "com.squareup.okhttp3" % "mockwebserver" % "4.9.2" % Test, "io.grpc" % "grpc-testing" % "1.48.0" % Test, "io.zonky.test" % "embedded-postgres" % "2.0.1" % Test, + "org.springframework" % "spring-test" % "5.3.9" % Test, ) // Clear default resolvers. diff --git a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java index bf378225c733..9c54e245560d 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java +++ b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java @@ -3,12 +3,15 @@ import static com.yugabyte.yw.common.Util.getUUIDRepresentation; import static java.lang.Math.max; import static play.mvc.Http.Status.BAD_REQUEST; +import static play.mvc.Http.Status.CONFLICT; import static play.mvc.Http.Status.INTERNAL_SERVER_ERROR; import static play.mvc.Http.Status.PRECONDITION_FAILED; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.yugabyte.yw.commissioner.Commissioner; +import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteBackupYb; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.StorageUtil; import com.yugabyte.yw.common.StorageUtilFactory; @@ -21,17 +24,28 @@ import com.yugabyte.yw.common.config.UniverseConfKeys; import com.yugabyte.yw.common.customer.config.CustomerConfigService; import com.yugabyte.yw.common.services.YBClientService; +import com.yugabyte.yw.forms.BackupRequestParams; import com.yugabyte.yw.forms.BackupRequestParams.KeyspaceTable; import com.yugabyte.yw.forms.BackupTableParams; +import com.yugabyte.yw.forms.DeleteBackupParams; +import com.yugabyte.yw.forms.DeleteBackupParams.DeleteBackupInfo; +import com.yugabyte.yw.forms.PlatformResults.YBPTask; +import com.yugabyte.yw.forms.RestoreBackupParams; import com.yugabyte.yw.forms.RestoreBackupParams.BackupStorageInfo; import com.yugabyte.yw.forms.RestorePreflightParams; import com.yugabyte.yw.forms.RestorePreflightResponse; import com.yugabyte.yw.models.Backup; import com.yugabyte.yw.models.Backup.BackupCategory; +import com.yugabyte.yw.models.Backup.BackupState; +import com.yugabyte.yw.models.Customer; +import com.yugabyte.yw.models.CustomerTask; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.configs.CustomerConfig; +import com.yugabyte.yw.models.configs.CustomerConfig.ConfigState; import com.yugabyte.yw.models.configs.data.CustomerConfigData; import com.yugabyte.yw.models.configs.data.CustomerConfigStorageData; +import com.yugabyte.yw.models.helpers.TaskType; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -42,6 +56,7 @@ import java.util.Set; import java.util.UUID; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -58,12 +73,14 @@ @Slf4j @Singleton public class BackupHelper { + private static final String VALID_OWNER_REGEX = "^[\\pL_][\\pL\\pM_0-9]*$"; private YbcManager ybcManager; private YBClientService ybClientService; private CustomerConfigService customerConfigService; private RuntimeConfGetter confGetter; private StorageUtilFactory storageUtilFactory; + @Inject Commissioner commissioner; @Inject public BackupHelper( @@ -71,12 +88,18 @@ public BackupHelper( YBClientService ybClientService, CustomerConfigService customerConfigService, RuntimeConfGetter confGetter, - StorageUtilFactory storageUtilFactory) { + StorageUtilFactory storageUtilFactory, + Commissioner commisssioner) { this.ybcManager = ybcManager; this.ybClientService = ybClientService; this.customerConfigService = customerConfigService; this.confGetter = confGetter; this.storageUtilFactory = storageUtilFactory; + // this.commissioner = commissioner; + } + + public String getValidOwnerRegex() { + return VALID_OWNER_REGEX; } public void validateIncrementalScheduleFrequency( @@ -100,6 +123,193 @@ public void validateIncrementalScheduleFrequency( } } + public UUID createBackupTask(UUID customerUUID, BackupRequestParams taskParams) { + Customer customer = Customer.getOrBadRequest(customerUUID); + // Validate universe UUID + Universe universe = Universe.getOrBadRequest(taskParams.getUniverseUUID(), customer); + taskParams.customerUUID = customerUUID; + + if (universe + .getConfig() + .getOrDefault(Universe.TAKE_BACKUPS, "true") + .equalsIgnoreCase("false")) { + throw new PlatformServiceException(BAD_REQUEST, "Taking backups on the universe is disabled"); + } + + if (universe.getUniverseDetails().updateInProgress) { + throw new PlatformServiceException( + CONFLICT, + String.format( + "Cannot run Backup task since the universe %s is currently in a locked state.", + taskParams.getUniverseUUID().toString())); + } + if ((universe.getLiveTServersInPrimaryCluster().size() < taskParams.parallelDBBackups) + || taskParams.parallelDBBackups <= 0) { + throw new PlatformServiceException( + BAD_REQUEST, + String.format( + "invalid parallel backups value provided for universe %s", + universe.getUniverseUUID())); + } + validateBackupRequest(taskParams.keyspaceTableList, universe, taskParams.backupType); + + if (taskParams.timeBeforeDelete != 0L && taskParams.expiryTimeUnit == null) { + throw new PlatformServiceException(BAD_REQUEST, "Please provide time unit for backup expiry"); + } + + if (taskParams.storageConfigUUID == null) { + throw new PlatformServiceException( + BAD_REQUEST, "Missing StorageConfig UUID: " + taskParams.storageConfigUUID); + } + CustomerConfig customerConfig = + customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); + + if (!customerConfig.getState().equals(ConfigState.Active)) { + throw new PlatformServiceException( + BAD_REQUEST, "Cannot create backup as config is queued for deletion."); + } + if (taskParams.baseBackupUUID != null) { + Backup previousBackup = + Backup.getLastSuccessfulBackupInChain(customerUUID, taskParams.baseBackupUUID); + if (!universe.isYbcEnabled()) { + throw new PlatformServiceException( + BAD_REQUEST, "Incremental backup not allowed for non-YBC universes"); + } else if (previousBackup == null) { + throw new PlatformServiceException( + BAD_REQUEST, "No previous successful backup found, please trigger a new base backup."); + } + validateStorageConfigOnBackup(customerConfig, previousBackup); + } else { + validateStorageConfig(customerConfig); + } + UUID taskUUID = commissioner.submit(TaskType.CreateBackup, taskParams); + log.info("Submitted task to universe {}, task uuid = {}.", universe.getName(), taskUUID); + CustomerTask.create( + customer, + taskParams.getUniverseUUID(), + taskUUID, + CustomerTask.TargetType.Backup, + CustomerTask.TaskType.Create, + universe.getName()); + log.info("Saved task uuid {} in customer tasks for universe {}", taskUUID, universe.getName()); + return taskUUID; + } + + public UUID createRestoreTask(UUID customerUUID, RestoreBackupParams taskParams) { + Customer customer = Customer.getOrBadRequest(customerUUID); + taskParams.backupStorageInfoList.forEach( + bSI -> { + if (StringUtils.isNotBlank(bSI.newOwner) + && !Pattern.matches(VALID_OWNER_REGEX, bSI.newOwner)) { + throw new PlatformServiceException( + BAD_REQUEST, "Invalid owner rename during restore operation"); + } + }); + + taskParams.customerUUID = customerUUID; + taskParams.prefixUUID = UUID.randomUUID(); + UUID universeUUID = taskParams.getUniverseUUID(); + Universe universe = Universe.getOrBadRequest(universeUUID, customer); + if (CollectionUtils.isEmpty(taskParams.backupStorageInfoList)) { + throw new PlatformServiceException(BAD_REQUEST, "Backup information not provided"); + } + validateRestoreOverwrites(taskParams.backupStorageInfoList, universe, taskParams.category); + CustomerConfig customerConfig = + customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); + if (!customerConfig.getState().equals(ConfigState.Active)) { + throw new PlatformServiceException( + BAD_REQUEST, "Cannot restore backup as config is queued for deletion."); + } + // Even though we check with default location below, this is needed to validate + // regional locations, because their validity is not known to us when we send restore + // request with a config. + validateStorageConfig(customerConfig); + CustomerConfigStorageData configData = + (CustomerConfigStorageData) customerConfig.getDataObject(); + + storageUtilFactory + .getStorageUtil(customerConfig.getName()) + .validateStorageConfigOnLocationsList( + configData, + taskParams + .backupStorageInfoList + .parallelStream() + .map(bSI -> bSI.storageLocation) + .collect(Collectors.toSet())); + + if (taskParams.category.equals(BackupCategory.YB_CONTROLLER) && !universe.isYbcEnabled()) { + throw new PlatformServiceException( + BAD_REQUEST, "Cannot restore the ybc backup as ybc is not installed on the universe"); + } + UUID taskUUID = commissioner.submit(TaskType.RestoreBackup, taskParams); + CustomerTask.create( + customer, + universeUUID, + taskUUID, + CustomerTask.TargetType.Universe, + CustomerTask.TaskType.Restore, + universe.getName()); + return taskUUID; + } + + public List createDeleteBackupTasks( + UUID customerUUID, DeleteBackupParams deleteBackupParams) { + Customer customer = Customer.getOrBadRequest(customerUUID); + List taskList = new ArrayList<>(); + for (DeleteBackupInfo deleteBackupInfo : deleteBackupParams.deleteBackupInfos) { + UUID backupUUID = deleteBackupInfo.backupUUID; + Backup backup = Backup.maybeGet(customerUUID, backupUUID).orElse(null); + if (backup == null) { + log.error("Can not delete {} backup as it is not present in the database.", backupUUID); + } else { + if (Backup.IN_PROGRESS_STATES.contains(backup.getState())) { + log.error( + "Backup {} is in the state {}. Deletion is not allowed", + backupUUID, + backup.getState()); + } else { + UUID storageConfigUUID = deleteBackupInfo.storageConfigUUID; + if (storageConfigUUID == null) { + // Pick default backup storage config to delete the backup if not provided. + storageConfigUUID = backup.getBackupInfo().storageConfigUUID; + } + if (backup.isIncrementalBackup() && backup.getState().equals(BackupState.Completed)) { + // Currently, we don't allow users to delete successful standalone incremental backups. + // They can only delete the full backup, along which all the incremental backups + // will also be deleted. + log.error( + "Cannot delete backup {} as it in {} state", + backup.getBackupUUID(), + backup.getState()); + continue; + } + BackupTableParams params = backup.getBackupInfo(); + params.storageConfigUUID = storageConfigUUID; + backup.updateBackupInfo(params); + DeleteBackupYb.Params taskParams = new DeleteBackupYb.Params(); + taskParams.customerUUID = customerUUID; + taskParams.backupUUID = backupUUID; + taskParams.deleteForcefully = deleteBackupParams.deleteForcefully; + UUID taskUUID = commissioner.submit(TaskType.DeleteBackupYb, taskParams); + log.info("Saved task uuid {} in customer tasks for backup {}.", taskUUID, backupUUID); + String target = + !StringUtils.isEmpty(backup.getUniverseName()) + ? backup.getUniverseName() + : String.format("univ-%s", backup.getUniverseUUID().toString()); + CustomerTask.create( + customer, + backup.getUniverseUUID(), + taskUUID, + CustomerTask.TargetType.Backup, + CustomerTask.TaskType.Delete, + target); + taskList.add(new YBPTask(taskUUID, taskParams.backupUUID)); + } + } + } + return taskList; + } + public void validateStorageConfigOnBackup(Backup backup) { CustomerConfig config = customerConfigService.getOrBadRequest( diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java b/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java index 9f8a613db664..d63bff08f52a 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.yugabyte.yw.commissioner.Commissioner; import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteBackup; -import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteBackupYb; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.StorageUtilFactory; import com.yugabyte.yw.common.TaskInfoManager; @@ -18,7 +17,6 @@ import com.yugabyte.yw.forms.BackupRequestParams; import com.yugabyte.yw.forms.BackupTableParams; import com.yugabyte.yw.forms.DeleteBackupParams; -import com.yugabyte.yw.forms.DeleteBackupParams.DeleteBackupInfo; import com.yugabyte.yw.forms.EditBackupParams; import com.yugabyte.yw.forms.PlatformResults; import com.yugabyte.yw.forms.PlatformResults.YBPError; @@ -36,7 +34,6 @@ import com.yugabyte.yw.forms.paging.RestorePagedApiQuery; import com.yugabyte.yw.models.Audit; import com.yugabyte.yw.models.Backup; -import com.yugabyte.yw.models.Backup.BackupCategory; import com.yugabyte.yw.models.Backup.BackupState; import com.yugabyte.yw.models.Backup.StorageConfigType; import com.yugabyte.yw.models.CommonBackupInfo; @@ -49,7 +46,6 @@ import com.yugabyte.yw.models.configs.CustomerConfig; import com.yugabyte.yw.models.configs.CustomerConfig.ConfigState; import com.yugabyte.yw.models.configs.CustomerConfig.ConfigType; -import com.yugabyte.yw.models.configs.data.CustomerConfigStorageData; import com.yugabyte.yw.models.extended.UserWithFeatures; import com.yugabyte.yw.models.filters.BackupFilter; import com.yugabyte.yw.models.filters.RestoreFilter; @@ -70,9 +66,7 @@ import java.util.Objects; import java.util.UUID; import java.util.regex.Pattern; -import java.util.stream.Collectors; import javax.inject.Inject; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,7 +79,6 @@ public class BackupsController extends AuthenticatedController { public static final Logger LOG = LoggerFactory.getLogger(BackupsController.class); private static final int maxRetryCount = 5; - private static final String VALID_OWNER_REGEX = "^[\\pL_][\\pL\\pM_0-9]*$"; private final Commissioner commissioner; private final CustomerConfigService customerConfigService; @@ -239,86 +232,11 @@ public Result fetchBackupsByTaskUUID(UUID customerUUID, UUID universeUUID, UUID dataType = "com.yugabyte.yw.forms.BackupRequestParams", paramType = "body") }) - // Rename this to createBackup on completion public Result createBackupYb(UUID customerUUID, Http.Request request) { - // Validate customer UUID - Customer customer = Customer.getOrBadRequest(customerUUID); - BackupRequestParams taskParams = parseJsonAndValidate(request, BackupRequestParams.class); - - // Validate universe UUID - Universe universe = Universe.getOrBadRequest(taskParams.getUniverseUUID(), customer); - taskParams.customerUUID = customerUUID; - - if (universe - .getConfig() - .getOrDefault(Universe.TAKE_BACKUPS, "true") - .equalsIgnoreCase("false")) { - throw new PlatformServiceException(BAD_REQUEST, "Taking backups on the universe is disabled"); - } - - if (universe.getUniverseDetails().updateInProgress) { - throw new PlatformServiceException( - CONFLICT, - String.format( - "Cannot run Backup task since the universe %s is currently in a locked state.", - taskParams.getUniverseUUID().toString())); - } - - if ((universe.getLiveTServersInPrimaryCluster().size() < taskParams.parallelDBBackups) - || taskParams.parallelDBBackups <= 0) { - throw new PlatformServiceException( - BAD_REQUEST, - String.format( - "invalid parallel backups value provided for universe %s", - universe.getUniverseUUID())); - } - - backupHelper.validateBackupRequest( - taskParams.keyspaceTableList, universe, taskParams.backupType); - - if (taskParams.timeBeforeDelete != 0L && taskParams.expiryTimeUnit == null) { - throw new PlatformServiceException(BAD_REQUEST, "Please provide time unit for backup expiry"); - } - - if (taskParams.storageConfigUUID == null) { - throw new PlatformServiceException( - BAD_REQUEST, "Missing StorageConfig UUID: " + taskParams.storageConfigUUID); - } - CustomerConfig customerConfig = - customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); - if (!customerConfig.getState().equals(ConfigState.Active)) { - throw new PlatformServiceException( - BAD_REQUEST, "Cannot create backup as config is queued for deletion."); - } - - if (taskParams.baseBackupUUID != null) { - Backup previousBackup = - Backup.getLastSuccessfulBackupInChain(customerUUID, taskParams.baseBackupUUID); - if (!universe.isYbcEnabled()) { - throw new PlatformServiceException( - BAD_REQUEST, "Incremental backup not allowed for non-YBC universes"); - } else if (previousBackup == null) { - throw new PlatformServiceException( - BAD_REQUEST, "No previous successful backup found, please trigger a new base backup."); - } - backupHelper.validateStorageConfigOnBackup(customerConfig, previousBackup); - } else { - backupHelper.validateStorageConfig(customerConfig); - } - - UUID taskUUID = commissioner.submit(TaskType.CreateBackup, taskParams); - LOG.info("Submitted task to universe {}, task uuid = {}.", universe.getName(), taskUUID); - CustomerTask.create( - customer, - taskParams.getUniverseUUID(), - taskUUID, - CustomerTask.TargetType.Backup, - CustomerTask.TaskType.Create, - universe.getName()); - LOG.info("Saved task uuid {} in customer tasks for universe {}", taskUUID, universe.getName()); - auditService().createAuditEntry(request, Json.toJson(taskParams), taskUUID); - return new YBPTask(taskUUID).asResult(); + UUID taskUuid = backupHelper.createBackupTask(customerUUID, taskParams); + auditService().createAuditEntry(request, Json.toJson(taskParams), taskUuid); + return new YBPTask(taskUuid).asResult(); } @ApiOperation( @@ -479,70 +397,18 @@ private void validateScheduleTaskParams(BackupRequestParams taskParams, UUID cus dataType = "com.yugabyte.yw.forms.RestoreBackupParams", required = true)) public Result restoreBackup(UUID customerUUID, Http.Request request) { - Customer customer = Customer.getOrBadRequest(customerUUID); RestoreBackupParams taskParams = parseJsonAndValidate(request, RestoreBackupParams.class); - taskParams.backupStorageInfoList.forEach( - bSI -> { - if (StringUtils.isNotBlank(bSI.newOwner) - && !Pattern.matches(VALID_OWNER_REGEX, bSI.newOwner)) { - throw new PlatformServiceException( - BAD_REQUEST, "Invalid owner rename during restore operation"); - } - }); - - taskParams.customerUUID = customerUUID; - taskParams.prefixUUID = UUID.randomUUID(); + UUID taskUuid = backupHelper.createRestoreTask(customerUUID, taskParams); UUID universeUUID = taskParams.getUniverseUUID(); - Universe universe = Universe.getOrBadRequest(universeUUID, customer); - if (CollectionUtils.isEmpty(taskParams.backupStorageInfoList)) { - throw new PlatformServiceException(BAD_REQUEST, "Backup information not provided"); - } - backupHelper.validateRestoreOverwrites( - taskParams.backupStorageInfoList, universe, taskParams.category); - CustomerConfig customerConfig = - customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); - if (!customerConfig.getState().equals(ConfigState.Active)) { - throw new PlatformServiceException( - BAD_REQUEST, "Cannot restore backup as config is queued for deletion."); - } - // Even though we check with default location below, this is needed to validate - // regional locations, because their validity is not known to us when we send restore - // request with a config. - backupHelper.validateStorageConfig(customerConfig); - CustomerConfigStorageData configData = - (CustomerConfigStorageData) customerConfig.getDataObject(); - - storageUtilFactory - .getStorageUtil(customerConfig.getName()) - .validateStorageConfigOnLocationsList( - configData, - taskParams - .backupStorageInfoList - .parallelStream() - .map(bSI -> bSI.storageLocation) - .collect(Collectors.toSet())); - - if (taskParams.category.equals(BackupCategory.YB_CONTROLLER) && !universe.isYbcEnabled()) { - throw new PlatformServiceException( - BAD_REQUEST, "Cannot restore the ybc backup as ybc is not installed on the universe"); - } - UUID taskUUID = commissioner.submit(TaskType.RestoreBackup, taskParams); - CustomerTask.create( - customer, - universeUUID, - taskUUID, - CustomerTask.TargetType.Universe, - CustomerTask.TaskType.Restore, - universe.getName()); auditService() .createAuditEntryWithReqBody( request, Audit.TargetType.Universe, universeUUID.toString(), Audit.ActionType.RestoreBackup, - taskUUID); - return new YBPTask(taskUUID).asResult(); + taskUuid); + return new YBPTask(taskUuid).asResult(); } @ApiOperation( @@ -574,7 +440,8 @@ public Result restore(UUID customerUUID, UUID universeUUID, Http.Request request } if (taskParams.newOwner != null) { - if (!Pattern.matches(VALID_OWNER_REGEX, taskParams.newOwner)) { + String validOwnerRegex = backupHelper.getValidOwnerRegex(); + if (!Pattern.matches(validOwnerRegex, taskParams.newOwner)) { throw new PlatformServiceException( BAD_REQUEST, "Invalid owner rename during restore operation"); } @@ -723,64 +590,15 @@ public Result delete(UUID customerUUID, Http.Request request) { public Result deleteYb(UUID customerUUID, Http.Request request) { Customer customer = Customer.getOrBadRequest(customerUUID); DeleteBackupParams deleteBackupParams = parseJsonAndValidate(request, DeleteBackupParams.class); - List taskList = new ArrayList<>(); - for (DeleteBackupInfo deleteBackupInfo : deleteBackupParams.deleteBackupInfos) { - UUID backupUUID = deleteBackupInfo.backupUUID; - Backup backup = Backup.maybeGet(customerUUID, backupUUID).orElse(null); - if (backup == null) { - LOG.error("Can not delete {} backup as it is not present in the database.", backupUUID); - } else { - if (Backup.IN_PROGRESS_STATES.contains(backup.getState())) { - LOG.error( - "Backup {} is in the state {}. Deletion is not allowed", - backupUUID, - backup.getState()); - } else { - UUID storageConfigUUID = deleteBackupInfo.storageConfigUUID; - if (storageConfigUUID == null) { - // Pick default backup storage config to delete the backup if not provided. - storageConfigUUID = backup.getBackupInfo().storageConfigUUID; - } - if (backup.isIncrementalBackup() && backup.getState().equals(BackupState.Completed)) { - // Currently, we don't allow users to delete successful standalone incremental backups. - // They can only delete the full backup, along which all the incremental backups - // will also be deleted. - LOG.error( - "Cannot delete backup {} as it in {} state", - backup.getBackupUUID(), - backup.getState()); - continue; - } - BackupTableParams params = backup.getBackupInfo(); - params.storageConfigUUID = storageConfigUUID; - backup.updateBackupInfo(params); - DeleteBackupYb.Params taskParams = new DeleteBackupYb.Params(); - taskParams.customerUUID = customerUUID; - taskParams.backupUUID = backupUUID; - taskParams.deleteForcefully = deleteBackupParams.deleteForcefully; - UUID taskUUID = commissioner.submit(TaskType.DeleteBackupYb, taskParams); - LOG.info("Saved task uuid {} in customer tasks for backup {}.", taskUUID, backupUUID); - String target = - !StringUtils.isEmpty(backup.getUniverseName()) - ? backup.getUniverseName() - : String.format("univ-%s", backup.getUniverseUUID().toString()); - CustomerTask.create( - customer, - backup.getUniverseUUID(), - taskUUID, - CustomerTask.TargetType.Backup, - CustomerTask.TaskType.Delete, - target); - taskList.add(new YBPTask(taskUUID, taskParams.backupUUID)); - auditService() - .createAuditEntryWithReqBody( - request, - Audit.TargetType.Backup, - Objects.toString(backup.getBackupUUID(), null), - Audit.ActionType.Delete, - taskUUID); - } - } + List taskList = backupHelper.createDeleteBackupTasks(customerUUID, deleteBackupParams); + for (YBPTask task : taskList) { + auditService() + .createAuditEntryWithReqBody( + request, + Audit.TargetType.Backup, + Objects.toString(task.resourceUUID, null), + Audit.ActionType.Delete, + task.taskUUID); } if (taskList.size() == 0) { auditService() diff --git a/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java b/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java index 229b8669b38a..bb5c7971868d 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.when; import com.google.protobuf.ByteString; +import com.yugabyte.yw.commissioner.Commissioner; import com.yugabyte.yw.commissioner.Common.CloudType; import com.yugabyte.yw.common.FakeDBApplication; import com.yugabyte.yw.common.ModelFactory; @@ -76,6 +77,7 @@ public void setup() { testUniverse = ModelFactory.createUniverse(testCustomer.getId()); mockConfigService = mock(CustomerConfigService.class); mockRuntimeConfGetter = mock(RuntimeConfGetter.class); + mockCommissioner = mock(Commissioner.class); spyBackupHelper = Mockito.spy( new BackupHelper( @@ -83,7 +85,8 @@ public void setup() { mockService, mockConfigService, mockRuntimeConfGetter, - mockStorageUtilFactory)); + mockStorageUtilFactory, + mockCommissioner)); when(mockStorageUtilFactory.getStorageUtil(eq("S3"))).thenReturn(mockAWSUtil); when(mockStorageUtilFactory.getStorageUtil(eq("NFS"))).thenReturn(mockNfsUtil); } diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java index ac56aff4146c..23ed2d9eef1e 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java @@ -18,6 +18,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -37,6 +38,7 @@ import com.yugabyte.yw.common.ModelFactory; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.Util; +import com.yugabyte.yw.common.customer.config.CustomerConfigService; import com.yugabyte.yw.forms.BackupRequestParams; import com.yugabyte.yw.forms.BackupTableParams; import com.yugabyte.yw.forms.RestoreBackupParams; @@ -74,6 +76,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; import play.libs.Json; import play.mvc.Result; @@ -224,7 +227,12 @@ public void testFetchBackupsByTaskUUIDWithSingleEntry() { public void testCreateBackup() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST21"); UUID fakeTaskUUID = UUID.randomUUID(); + CustomerConfigService mockCCS = mock(CustomerConfigService.class); + when(mockCCS.getOrBadRequest(any(), any())).thenReturn(customerConfig); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); + ReflectionTestUtils.setField(mockBackupHelper, "customerConfigService", mockCCS); + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); ObjectNode bodyJson = Json.newObject(); bodyJson.put("universeUUID", defaultUniverse.getUniverseUUID().toString()); bodyJson.put("backupType", "PGSQL_TABLE_TYPE"); @@ -724,6 +732,7 @@ public void testCreateIncrementalScheduleBackupAsyncWithBaseBackup() { public void testCreateBackupValidationFailed() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST25"); UUID fakeTaskUUID = UUID.randomUUID(); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); doThrow(new PlatformServiceException(BAD_REQUEST, "error")) .when(mockBackupHelper) @@ -744,6 +753,8 @@ public void testCreateBackupWithUniverseDisabled() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST25"); ObjectNode bodyJson = Json.newObject(); Map config = defaultUniverse.getConfig(); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); + config.put(Universe.TAKE_BACKUPS, "false"); defaultUniverse.updateConfig(config); defaultUniverse.update(); @@ -761,6 +772,7 @@ public void testCreateBackupWithUniverseDisabled() { public void testCreateBackupWithoutTimeUnit() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST26"); UUID fakeTaskUUID = UUID.randomUUID(); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); ObjectNode bodyJson = Json.newObject(); bodyJson.put("universeUUID", defaultUniverse.getUniverseUUID().toString()); @@ -885,6 +897,7 @@ private Result editBackup(Users user, ObjectNode bodyJson, UUID backupUUID) { public void testRestoreBackupWithInvalidUniverseUUID() { UUID universeUUID = UUID.randomUUID(); JsonNode bodyJson = Json.newObject(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); Result result = assertPlatformException(() -> restoreBackup(universeUUID, bodyJson, null)); assertEquals(BAD_REQUEST, result.status()); @@ -898,6 +911,8 @@ public void testRestoreBackupWithInvalidParams() { BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = UUID.randomUUID(); bp.setUniverseUUID(UUID.randomUUID()); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); bodyJson.put("actionType", "RESTORE"); @@ -915,6 +930,8 @@ public void testRestoreBackupWithoutStorageLocation() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST2"); BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = customerConfig.getConfigUUID(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + bp.setUniverseUUID(UUID.randomUUID()); Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); @@ -936,6 +953,7 @@ public void testRestoreBackupWithInvalidStorageUUID() { BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = UUID.randomUUID(); bp.setUniverseUUID(UUID.randomUUID()); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); Backup b = Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); bodyJson.put("keyspace", "mock_ks"); @@ -956,6 +974,7 @@ public void testRestoreBackupWithInvalidStorageUUID() { public void testRestoreBackupWithReadOnlyUser() { Users user = ModelFactory.testUser(defaultCustomer, "tc@test.com", Users.Role.ReadOnly); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); bp.storageConfigUUID = UUID.randomUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup b = Backup.create(defaultCustomer.getUuid(), bp); @@ -979,6 +998,7 @@ public void testRestoreBackupWithValidParams() { bp.setUniverseUUID(defaultUniverse.getUniverseUUID()); Backup b = Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); long maxReqSizeInBytes = app.config().getMemorySize("play.http.parser.maxMemoryBuffer").toBytes(); @@ -1021,6 +1041,8 @@ public void testRestoreBackupWithValidParams() { public void testRestoreBackupRequestTooLarge() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST5"); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup.create(defaultCustomer.getUuid(), bp); @@ -1049,6 +1071,9 @@ public void testRestoreBackupRequestTooLarge() { public void testRestoreBackupWithInvalidOwner() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST5"); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + when(mockBackupHelper.getValidOwnerRegex()).thenCallRealMethod(); + bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(defaultUniverse.getUniverseUUID()); Backup.create(defaultCustomer.getUuid(), bp); @@ -1211,6 +1236,8 @@ public void testDeleteFailedBackup() { public void testDeleteBackupYbWithValidState(BackupState state) { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST6"); BackupTableParams bp = new BackupTableParams(); + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); @@ -1241,6 +1268,7 @@ public void testDeleteBackupYbWithValidState(BackupState state) { public void testDeleteBackupYbWithInvalidState(BackupState state) { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST6"); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); @@ -1265,6 +1293,8 @@ public void testDeleteBackupYbWithCustomCustomerStorageConfig() { UUID invalidStorageConfigUUID = UUID.randomUUID(); BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = invalidStorageConfigUUID; + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); bp.setUniverseUUID(UUID.randomUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); backup.transitionState(BackupState.Completed); @@ -1300,6 +1330,8 @@ public void testDeleteBackupDuplicateTask() { bp.setUniverseUUID(defaultUniverse.getUniverseUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); backup.transitionState(BackupState.Completed); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); + List backupUUIDList = new ArrayList<>(); backupUUIDList.add(backup.getBackupUUID().toString()); UUID fakeTaskUUID = UUID.randomUUID(); @@ -1814,6 +1846,8 @@ public void testDeleteFailedIncrementalBackup() { List backupUUIDList = new ArrayList<>(); backupUUIDList.add(backup.getBackupUUID().toString()); UUID fakeTaskUUID = UUID.randomUUID(); + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); ObjectNode resultNode = Json.newObject(); ArrayNode arrayNode = resultNode.putArray("backups");