From a68e98c8ee7179ea0271c991e95006f9cf757974 Mon Sep 17 00:00:00 2001 From: Aman Nijhawan Date: Wed, 26 Jul 2023 06:35:59 +0000 Subject: [PATCH] [PLAT-8859] Refactoring backup controller for create and restore to add helper methods that allow calling from non API context. Summary: Refactoring backup controller for create and restore to add helper methods that allow calling from non API ocntext. This will make calling these functions from the backup reconciler much easier and allow us to not remimplement this code. This will also keep backup/restore functionality consistent between operator and the API clients. Test Plan: itest build Fixed Failing UT ``` OpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader classes because bootstrap classpath has been appended [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectNFSNonYBC(false, true) [0] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectNFSNonYBC(true, false) [1] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectNFSNonYBC(true, true) [2] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectNFSNonYBC(false, false) [3] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testValidateBackupRequestYCQLSameKeyspaceInvalid started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testValidateBackupRequestYCQLSameKeyspaceValid started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testValidateBackupRequestYCQLDifferentKeyspaceValid started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYSQLBackupObjectExists(true, YB_CONTROLLER) [0] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYSQLBackupObjectExists(false, YB_CONTROLLER) [1] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYSQLBackupObjectExists(true, YB_BACKUP_SCRIPT) [2] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYSQLBackupObjectExists(false, YB_BACKUP_SCRIPT) [3] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectYBCWithYCQLIndexTables(backup/ybc_success_file_with_index_tables.json, true, false) [0] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectYBCWithYCQLIndexTables(backup/ybc_success_file_with_index_tables.json, true, true) [1] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectYBCWithYCQLIndexTables(backup/ybc_success_file_with_index_tables.json, false, true) [2] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectYBCWithYCQLIndexTables(backup/ybc_success_file_with_index_tables.json, false, false) [3] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectS3NonYBC(false, true) [0] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectS3NonYBC(true, false) [1] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectS3NonYBC(true, true) [2] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightWithoutBackupObjectS3NonYBC(false, false) [3] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testValidateMapToRestoreWithUniverseNonRedisYBC_OverwriteYCQL started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testValidateMapToRestoreWithUniverseNonRedisYBC_OverwriteYSQL started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYCQLBackupObjectExists(true, YB_CONTROLLER) [0] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYCQLBackupObjectExists(false, YB_CONTROLLER) [1] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYCQLBackupObjectExists(true, YB_BACKUP_SCRIPT) [2] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testRestorePreflightResponseWithYCQLBackupObjectExists(false, YB_BACKUP_SCRIPT) [3] started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testValidateMapToRestoreWithUniverseNonRedisYBC_NoOverwriteYCQL started [info] Test com.yugabyte.yw.common.backuprestore.BackupHelperTest.testValidateMapToRestoreWithUniverseNonRedisYBC_NoOverwriteYSQL started [info] Test run finished: 0 failed, 0 ignored, 27 total, 86.92s [info] Passed: Total 27, Failed 0, Errors 0, Passed 27 ``` Reviewers: #yba-api-review, hzare Reviewed By: #yba-api-review, hzare Subscribers: sneelakantan, hzare, yugaware Differential Revision: https://phorge.dev.yugabyte.com/D27207 --- managed/build.sbt | 1 + .../yw/common/backuprestore/BackupHelper.java | 212 ++++++++++++++++- .../yw/controllers/BackupsController.java | 216 ++---------------- .../backuprestore/BackupHelperTest.java | 5 +- .../yw/controllers/BackupsControllerTest.java | 34 +++ 5 files changed, 267 insertions(+), 201 deletions(-) diff --git a/managed/build.sbt b/managed/build.sbt index 9ea561e5635e..bf3442393ec9 100644 --- a/managed/build.sbt +++ b/managed/build.sbt @@ -228,6 +228,7 @@ libraryDependencies ++= Seq( "com.squareup.okhttp3" % "mockwebserver" % "4.9.2" % Test, "io.grpc" % "grpc-testing" % "1.48.0" % Test, "io.zonky.test" % "embedded-postgres" % "2.0.1" % Test, + "org.springframework" % "spring-test" % "5.3.9" % Test, ) // Clear default resolvers. diff --git a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java index bf378225c733..9c54e245560d 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java +++ b/managed/src/main/java/com/yugabyte/yw/common/backuprestore/BackupHelper.java @@ -3,12 +3,15 @@ import static com.yugabyte.yw.common.Util.getUUIDRepresentation; import static java.lang.Math.max; import static play.mvc.Http.Status.BAD_REQUEST; +import static play.mvc.Http.Status.CONFLICT; import static play.mvc.Http.Status.INTERNAL_SERVER_ERROR; import static play.mvc.Http.Status.PRECONDITION_FAILED; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.yugabyte.yw.commissioner.Commissioner; +import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteBackupYb; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.StorageUtil; import com.yugabyte.yw.common.StorageUtilFactory; @@ -21,17 +24,28 @@ import com.yugabyte.yw.common.config.UniverseConfKeys; import com.yugabyte.yw.common.customer.config.CustomerConfigService; import com.yugabyte.yw.common.services.YBClientService; +import com.yugabyte.yw.forms.BackupRequestParams; import com.yugabyte.yw.forms.BackupRequestParams.KeyspaceTable; import com.yugabyte.yw.forms.BackupTableParams; +import com.yugabyte.yw.forms.DeleteBackupParams; +import com.yugabyte.yw.forms.DeleteBackupParams.DeleteBackupInfo; +import com.yugabyte.yw.forms.PlatformResults.YBPTask; +import com.yugabyte.yw.forms.RestoreBackupParams; import com.yugabyte.yw.forms.RestoreBackupParams.BackupStorageInfo; import com.yugabyte.yw.forms.RestorePreflightParams; import com.yugabyte.yw.forms.RestorePreflightResponse; import com.yugabyte.yw.models.Backup; import com.yugabyte.yw.models.Backup.BackupCategory; +import com.yugabyte.yw.models.Backup.BackupState; +import com.yugabyte.yw.models.Customer; +import com.yugabyte.yw.models.CustomerTask; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.configs.CustomerConfig; +import com.yugabyte.yw.models.configs.CustomerConfig.ConfigState; import com.yugabyte.yw.models.configs.data.CustomerConfigData; import com.yugabyte.yw.models.configs.data.CustomerConfigStorageData; +import com.yugabyte.yw.models.helpers.TaskType; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -42,6 +56,7 @@ import java.util.Set; import java.util.UUID; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -58,12 +73,14 @@ @Slf4j @Singleton public class BackupHelper { + private static final String VALID_OWNER_REGEX = "^[\\pL_][\\pL\\pM_0-9]*$"; private YbcManager ybcManager; private YBClientService ybClientService; private CustomerConfigService customerConfigService; private RuntimeConfGetter confGetter; private StorageUtilFactory storageUtilFactory; + @Inject Commissioner commissioner; @Inject public BackupHelper( @@ -71,12 +88,18 @@ public BackupHelper( YBClientService ybClientService, CustomerConfigService customerConfigService, RuntimeConfGetter confGetter, - StorageUtilFactory storageUtilFactory) { + StorageUtilFactory storageUtilFactory, + Commissioner commisssioner) { this.ybcManager = ybcManager; this.ybClientService = ybClientService; this.customerConfigService = customerConfigService; this.confGetter = confGetter; this.storageUtilFactory = storageUtilFactory; + // this.commissioner = commissioner; + } + + public String getValidOwnerRegex() { + return VALID_OWNER_REGEX; } public void validateIncrementalScheduleFrequency( @@ -100,6 +123,193 @@ public void validateIncrementalScheduleFrequency( } } + public UUID createBackupTask(UUID customerUUID, BackupRequestParams taskParams) { + Customer customer = Customer.getOrBadRequest(customerUUID); + // Validate universe UUID + Universe universe = Universe.getOrBadRequest(taskParams.getUniverseUUID(), customer); + taskParams.customerUUID = customerUUID; + + if (universe + .getConfig() + .getOrDefault(Universe.TAKE_BACKUPS, "true") + .equalsIgnoreCase("false")) { + throw new PlatformServiceException(BAD_REQUEST, "Taking backups on the universe is disabled"); + } + + if (universe.getUniverseDetails().updateInProgress) { + throw new PlatformServiceException( + CONFLICT, + String.format( + "Cannot run Backup task since the universe %s is currently in a locked state.", + taskParams.getUniverseUUID().toString())); + } + if ((universe.getLiveTServersInPrimaryCluster().size() < taskParams.parallelDBBackups) + || taskParams.parallelDBBackups <= 0) { + throw new PlatformServiceException( + BAD_REQUEST, + String.format( + "invalid parallel backups value provided for universe %s", + universe.getUniverseUUID())); + } + validateBackupRequest(taskParams.keyspaceTableList, universe, taskParams.backupType); + + if (taskParams.timeBeforeDelete != 0L && taskParams.expiryTimeUnit == null) { + throw new PlatformServiceException(BAD_REQUEST, "Please provide time unit for backup expiry"); + } + + if (taskParams.storageConfigUUID == null) { + throw new PlatformServiceException( + BAD_REQUEST, "Missing StorageConfig UUID: " + taskParams.storageConfigUUID); + } + CustomerConfig customerConfig = + customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); + + if (!customerConfig.getState().equals(ConfigState.Active)) { + throw new PlatformServiceException( + BAD_REQUEST, "Cannot create backup as config is queued for deletion."); + } + if (taskParams.baseBackupUUID != null) { + Backup previousBackup = + Backup.getLastSuccessfulBackupInChain(customerUUID, taskParams.baseBackupUUID); + if (!universe.isYbcEnabled()) { + throw new PlatformServiceException( + BAD_REQUEST, "Incremental backup not allowed for non-YBC universes"); + } else if (previousBackup == null) { + throw new PlatformServiceException( + BAD_REQUEST, "No previous successful backup found, please trigger a new base backup."); + } + validateStorageConfigOnBackup(customerConfig, previousBackup); + } else { + validateStorageConfig(customerConfig); + } + UUID taskUUID = commissioner.submit(TaskType.CreateBackup, taskParams); + log.info("Submitted task to universe {}, task uuid = {}.", universe.getName(), taskUUID); + CustomerTask.create( + customer, + taskParams.getUniverseUUID(), + taskUUID, + CustomerTask.TargetType.Backup, + CustomerTask.TaskType.Create, + universe.getName()); + log.info("Saved task uuid {} in customer tasks for universe {}", taskUUID, universe.getName()); + return taskUUID; + } + + public UUID createRestoreTask(UUID customerUUID, RestoreBackupParams taskParams) { + Customer customer = Customer.getOrBadRequest(customerUUID); + taskParams.backupStorageInfoList.forEach( + bSI -> { + if (StringUtils.isNotBlank(bSI.newOwner) + && !Pattern.matches(VALID_OWNER_REGEX, bSI.newOwner)) { + throw new PlatformServiceException( + BAD_REQUEST, "Invalid owner rename during restore operation"); + } + }); + + taskParams.customerUUID = customerUUID; + taskParams.prefixUUID = UUID.randomUUID(); + UUID universeUUID = taskParams.getUniverseUUID(); + Universe universe = Universe.getOrBadRequest(universeUUID, customer); + if (CollectionUtils.isEmpty(taskParams.backupStorageInfoList)) { + throw new PlatformServiceException(BAD_REQUEST, "Backup information not provided"); + } + validateRestoreOverwrites(taskParams.backupStorageInfoList, universe, taskParams.category); + CustomerConfig customerConfig = + customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); + if (!customerConfig.getState().equals(ConfigState.Active)) { + throw new PlatformServiceException( + BAD_REQUEST, "Cannot restore backup as config is queued for deletion."); + } + // Even though we check with default location below, this is needed to validate + // regional locations, because their validity is not known to us when we send restore + // request with a config. + validateStorageConfig(customerConfig); + CustomerConfigStorageData configData = + (CustomerConfigStorageData) customerConfig.getDataObject(); + + storageUtilFactory + .getStorageUtil(customerConfig.getName()) + .validateStorageConfigOnLocationsList( + configData, + taskParams + .backupStorageInfoList + .parallelStream() + .map(bSI -> bSI.storageLocation) + .collect(Collectors.toSet())); + + if (taskParams.category.equals(BackupCategory.YB_CONTROLLER) && !universe.isYbcEnabled()) { + throw new PlatformServiceException( + BAD_REQUEST, "Cannot restore the ybc backup as ybc is not installed on the universe"); + } + UUID taskUUID = commissioner.submit(TaskType.RestoreBackup, taskParams); + CustomerTask.create( + customer, + universeUUID, + taskUUID, + CustomerTask.TargetType.Universe, + CustomerTask.TaskType.Restore, + universe.getName()); + return taskUUID; + } + + public List createDeleteBackupTasks( + UUID customerUUID, DeleteBackupParams deleteBackupParams) { + Customer customer = Customer.getOrBadRequest(customerUUID); + List taskList = new ArrayList<>(); + for (DeleteBackupInfo deleteBackupInfo : deleteBackupParams.deleteBackupInfos) { + UUID backupUUID = deleteBackupInfo.backupUUID; + Backup backup = Backup.maybeGet(customerUUID, backupUUID).orElse(null); + if (backup == null) { + log.error("Can not delete {} backup as it is not present in the database.", backupUUID); + } else { + if (Backup.IN_PROGRESS_STATES.contains(backup.getState())) { + log.error( + "Backup {} is in the state {}. Deletion is not allowed", + backupUUID, + backup.getState()); + } else { + UUID storageConfigUUID = deleteBackupInfo.storageConfigUUID; + if (storageConfigUUID == null) { + // Pick default backup storage config to delete the backup if not provided. + storageConfigUUID = backup.getBackupInfo().storageConfigUUID; + } + if (backup.isIncrementalBackup() && backup.getState().equals(BackupState.Completed)) { + // Currently, we don't allow users to delete successful standalone incremental backups. + // They can only delete the full backup, along which all the incremental backups + // will also be deleted. + log.error( + "Cannot delete backup {} as it in {} state", + backup.getBackupUUID(), + backup.getState()); + continue; + } + BackupTableParams params = backup.getBackupInfo(); + params.storageConfigUUID = storageConfigUUID; + backup.updateBackupInfo(params); + DeleteBackupYb.Params taskParams = new DeleteBackupYb.Params(); + taskParams.customerUUID = customerUUID; + taskParams.backupUUID = backupUUID; + taskParams.deleteForcefully = deleteBackupParams.deleteForcefully; + UUID taskUUID = commissioner.submit(TaskType.DeleteBackupYb, taskParams); + log.info("Saved task uuid {} in customer tasks for backup {}.", taskUUID, backupUUID); + String target = + !StringUtils.isEmpty(backup.getUniverseName()) + ? backup.getUniverseName() + : String.format("univ-%s", backup.getUniverseUUID().toString()); + CustomerTask.create( + customer, + backup.getUniverseUUID(), + taskUUID, + CustomerTask.TargetType.Backup, + CustomerTask.TaskType.Delete, + target); + taskList.add(new YBPTask(taskUUID, taskParams.backupUUID)); + } + } + } + return taskList; + } + public void validateStorageConfigOnBackup(Backup backup) { CustomerConfig config = customerConfigService.getOrBadRequest( diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java b/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java index 9f8a613db664..d63bff08f52a 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/BackupsController.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.yugabyte.yw.commissioner.Commissioner; import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteBackup; -import com.yugabyte.yw.commissioner.tasks.subtasks.DeleteBackupYb; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.StorageUtilFactory; import com.yugabyte.yw.common.TaskInfoManager; @@ -18,7 +17,6 @@ import com.yugabyte.yw.forms.BackupRequestParams; import com.yugabyte.yw.forms.BackupTableParams; import com.yugabyte.yw.forms.DeleteBackupParams; -import com.yugabyte.yw.forms.DeleteBackupParams.DeleteBackupInfo; import com.yugabyte.yw.forms.EditBackupParams; import com.yugabyte.yw.forms.PlatformResults; import com.yugabyte.yw.forms.PlatformResults.YBPError; @@ -36,7 +34,6 @@ import com.yugabyte.yw.forms.paging.RestorePagedApiQuery; import com.yugabyte.yw.models.Audit; import com.yugabyte.yw.models.Backup; -import com.yugabyte.yw.models.Backup.BackupCategory; import com.yugabyte.yw.models.Backup.BackupState; import com.yugabyte.yw.models.Backup.StorageConfigType; import com.yugabyte.yw.models.CommonBackupInfo; @@ -49,7 +46,6 @@ import com.yugabyte.yw.models.configs.CustomerConfig; import com.yugabyte.yw.models.configs.CustomerConfig.ConfigState; import com.yugabyte.yw.models.configs.CustomerConfig.ConfigType; -import com.yugabyte.yw.models.configs.data.CustomerConfigStorageData; import com.yugabyte.yw.models.extended.UserWithFeatures; import com.yugabyte.yw.models.filters.BackupFilter; import com.yugabyte.yw.models.filters.RestoreFilter; @@ -70,9 +66,7 @@ import java.util.Objects; import java.util.UUID; import java.util.regex.Pattern; -import java.util.stream.Collectors; import javax.inject.Inject; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,7 +79,6 @@ public class BackupsController extends AuthenticatedController { public static final Logger LOG = LoggerFactory.getLogger(BackupsController.class); private static final int maxRetryCount = 5; - private static final String VALID_OWNER_REGEX = "^[\\pL_][\\pL\\pM_0-9]*$"; private final Commissioner commissioner; private final CustomerConfigService customerConfigService; @@ -239,86 +232,11 @@ public Result fetchBackupsByTaskUUID(UUID customerUUID, UUID universeUUID, UUID dataType = "com.yugabyte.yw.forms.BackupRequestParams", paramType = "body") }) - // Rename this to createBackup on completion public Result createBackupYb(UUID customerUUID, Http.Request request) { - // Validate customer UUID - Customer customer = Customer.getOrBadRequest(customerUUID); - BackupRequestParams taskParams = parseJsonAndValidate(request, BackupRequestParams.class); - - // Validate universe UUID - Universe universe = Universe.getOrBadRequest(taskParams.getUniverseUUID(), customer); - taskParams.customerUUID = customerUUID; - - if (universe - .getConfig() - .getOrDefault(Universe.TAKE_BACKUPS, "true") - .equalsIgnoreCase("false")) { - throw new PlatformServiceException(BAD_REQUEST, "Taking backups on the universe is disabled"); - } - - if (universe.getUniverseDetails().updateInProgress) { - throw new PlatformServiceException( - CONFLICT, - String.format( - "Cannot run Backup task since the universe %s is currently in a locked state.", - taskParams.getUniverseUUID().toString())); - } - - if ((universe.getLiveTServersInPrimaryCluster().size() < taskParams.parallelDBBackups) - || taskParams.parallelDBBackups <= 0) { - throw new PlatformServiceException( - BAD_REQUEST, - String.format( - "invalid parallel backups value provided for universe %s", - universe.getUniverseUUID())); - } - - backupHelper.validateBackupRequest( - taskParams.keyspaceTableList, universe, taskParams.backupType); - - if (taskParams.timeBeforeDelete != 0L && taskParams.expiryTimeUnit == null) { - throw new PlatformServiceException(BAD_REQUEST, "Please provide time unit for backup expiry"); - } - - if (taskParams.storageConfigUUID == null) { - throw new PlatformServiceException( - BAD_REQUEST, "Missing StorageConfig UUID: " + taskParams.storageConfigUUID); - } - CustomerConfig customerConfig = - customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); - if (!customerConfig.getState().equals(ConfigState.Active)) { - throw new PlatformServiceException( - BAD_REQUEST, "Cannot create backup as config is queued for deletion."); - } - - if (taskParams.baseBackupUUID != null) { - Backup previousBackup = - Backup.getLastSuccessfulBackupInChain(customerUUID, taskParams.baseBackupUUID); - if (!universe.isYbcEnabled()) { - throw new PlatformServiceException( - BAD_REQUEST, "Incremental backup not allowed for non-YBC universes"); - } else if (previousBackup == null) { - throw new PlatformServiceException( - BAD_REQUEST, "No previous successful backup found, please trigger a new base backup."); - } - backupHelper.validateStorageConfigOnBackup(customerConfig, previousBackup); - } else { - backupHelper.validateStorageConfig(customerConfig); - } - - UUID taskUUID = commissioner.submit(TaskType.CreateBackup, taskParams); - LOG.info("Submitted task to universe {}, task uuid = {}.", universe.getName(), taskUUID); - CustomerTask.create( - customer, - taskParams.getUniverseUUID(), - taskUUID, - CustomerTask.TargetType.Backup, - CustomerTask.TaskType.Create, - universe.getName()); - LOG.info("Saved task uuid {} in customer tasks for universe {}", taskUUID, universe.getName()); - auditService().createAuditEntry(request, Json.toJson(taskParams), taskUUID); - return new YBPTask(taskUUID).asResult(); + UUID taskUuid = backupHelper.createBackupTask(customerUUID, taskParams); + auditService().createAuditEntry(request, Json.toJson(taskParams), taskUuid); + return new YBPTask(taskUuid).asResult(); } @ApiOperation( @@ -479,70 +397,18 @@ private void validateScheduleTaskParams(BackupRequestParams taskParams, UUID cus dataType = "com.yugabyte.yw.forms.RestoreBackupParams", required = true)) public Result restoreBackup(UUID customerUUID, Http.Request request) { - Customer customer = Customer.getOrBadRequest(customerUUID); RestoreBackupParams taskParams = parseJsonAndValidate(request, RestoreBackupParams.class); - taskParams.backupStorageInfoList.forEach( - bSI -> { - if (StringUtils.isNotBlank(bSI.newOwner) - && !Pattern.matches(VALID_OWNER_REGEX, bSI.newOwner)) { - throw new PlatformServiceException( - BAD_REQUEST, "Invalid owner rename during restore operation"); - } - }); - - taskParams.customerUUID = customerUUID; - taskParams.prefixUUID = UUID.randomUUID(); + UUID taskUuid = backupHelper.createRestoreTask(customerUUID, taskParams); UUID universeUUID = taskParams.getUniverseUUID(); - Universe universe = Universe.getOrBadRequest(universeUUID, customer); - if (CollectionUtils.isEmpty(taskParams.backupStorageInfoList)) { - throw new PlatformServiceException(BAD_REQUEST, "Backup information not provided"); - } - backupHelper.validateRestoreOverwrites( - taskParams.backupStorageInfoList, universe, taskParams.category); - CustomerConfig customerConfig = - customerConfigService.getOrBadRequest(customerUUID, taskParams.storageConfigUUID); - if (!customerConfig.getState().equals(ConfigState.Active)) { - throw new PlatformServiceException( - BAD_REQUEST, "Cannot restore backup as config is queued for deletion."); - } - // Even though we check with default location below, this is needed to validate - // regional locations, because their validity is not known to us when we send restore - // request with a config. - backupHelper.validateStorageConfig(customerConfig); - CustomerConfigStorageData configData = - (CustomerConfigStorageData) customerConfig.getDataObject(); - - storageUtilFactory - .getStorageUtil(customerConfig.getName()) - .validateStorageConfigOnLocationsList( - configData, - taskParams - .backupStorageInfoList - .parallelStream() - .map(bSI -> bSI.storageLocation) - .collect(Collectors.toSet())); - - if (taskParams.category.equals(BackupCategory.YB_CONTROLLER) && !universe.isYbcEnabled()) { - throw new PlatformServiceException( - BAD_REQUEST, "Cannot restore the ybc backup as ybc is not installed on the universe"); - } - UUID taskUUID = commissioner.submit(TaskType.RestoreBackup, taskParams); - CustomerTask.create( - customer, - universeUUID, - taskUUID, - CustomerTask.TargetType.Universe, - CustomerTask.TaskType.Restore, - universe.getName()); auditService() .createAuditEntryWithReqBody( request, Audit.TargetType.Universe, universeUUID.toString(), Audit.ActionType.RestoreBackup, - taskUUID); - return new YBPTask(taskUUID).asResult(); + taskUuid); + return new YBPTask(taskUuid).asResult(); } @ApiOperation( @@ -574,7 +440,8 @@ public Result restore(UUID customerUUID, UUID universeUUID, Http.Request request } if (taskParams.newOwner != null) { - if (!Pattern.matches(VALID_OWNER_REGEX, taskParams.newOwner)) { + String validOwnerRegex = backupHelper.getValidOwnerRegex(); + if (!Pattern.matches(validOwnerRegex, taskParams.newOwner)) { throw new PlatformServiceException( BAD_REQUEST, "Invalid owner rename during restore operation"); } @@ -723,64 +590,15 @@ public Result delete(UUID customerUUID, Http.Request request) { public Result deleteYb(UUID customerUUID, Http.Request request) { Customer customer = Customer.getOrBadRequest(customerUUID); DeleteBackupParams deleteBackupParams = parseJsonAndValidate(request, DeleteBackupParams.class); - List taskList = new ArrayList<>(); - for (DeleteBackupInfo deleteBackupInfo : deleteBackupParams.deleteBackupInfos) { - UUID backupUUID = deleteBackupInfo.backupUUID; - Backup backup = Backup.maybeGet(customerUUID, backupUUID).orElse(null); - if (backup == null) { - LOG.error("Can not delete {} backup as it is not present in the database.", backupUUID); - } else { - if (Backup.IN_PROGRESS_STATES.contains(backup.getState())) { - LOG.error( - "Backup {} is in the state {}. Deletion is not allowed", - backupUUID, - backup.getState()); - } else { - UUID storageConfigUUID = deleteBackupInfo.storageConfigUUID; - if (storageConfigUUID == null) { - // Pick default backup storage config to delete the backup if not provided. - storageConfigUUID = backup.getBackupInfo().storageConfigUUID; - } - if (backup.isIncrementalBackup() && backup.getState().equals(BackupState.Completed)) { - // Currently, we don't allow users to delete successful standalone incremental backups. - // They can only delete the full backup, along which all the incremental backups - // will also be deleted. - LOG.error( - "Cannot delete backup {} as it in {} state", - backup.getBackupUUID(), - backup.getState()); - continue; - } - BackupTableParams params = backup.getBackupInfo(); - params.storageConfigUUID = storageConfigUUID; - backup.updateBackupInfo(params); - DeleteBackupYb.Params taskParams = new DeleteBackupYb.Params(); - taskParams.customerUUID = customerUUID; - taskParams.backupUUID = backupUUID; - taskParams.deleteForcefully = deleteBackupParams.deleteForcefully; - UUID taskUUID = commissioner.submit(TaskType.DeleteBackupYb, taskParams); - LOG.info("Saved task uuid {} in customer tasks for backup {}.", taskUUID, backupUUID); - String target = - !StringUtils.isEmpty(backup.getUniverseName()) - ? backup.getUniverseName() - : String.format("univ-%s", backup.getUniverseUUID().toString()); - CustomerTask.create( - customer, - backup.getUniverseUUID(), - taskUUID, - CustomerTask.TargetType.Backup, - CustomerTask.TaskType.Delete, - target); - taskList.add(new YBPTask(taskUUID, taskParams.backupUUID)); - auditService() - .createAuditEntryWithReqBody( - request, - Audit.TargetType.Backup, - Objects.toString(backup.getBackupUUID(), null), - Audit.ActionType.Delete, - taskUUID); - } - } + List taskList = backupHelper.createDeleteBackupTasks(customerUUID, deleteBackupParams); + for (YBPTask task : taskList) { + auditService() + .createAuditEntryWithReqBody( + request, + Audit.TargetType.Backup, + Objects.toString(task.resourceUUID, null), + Audit.ActionType.Delete, + task.taskUUID); } if (taskList.size() == 0) { auditService() diff --git a/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java b/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java index 229b8669b38a..bb5c7971868d 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/backuprestore/BackupHelperTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.when; import com.google.protobuf.ByteString; +import com.yugabyte.yw.commissioner.Commissioner; import com.yugabyte.yw.commissioner.Common.CloudType; import com.yugabyte.yw.common.FakeDBApplication; import com.yugabyte.yw.common.ModelFactory; @@ -76,6 +77,7 @@ public void setup() { testUniverse = ModelFactory.createUniverse(testCustomer.getId()); mockConfigService = mock(CustomerConfigService.class); mockRuntimeConfGetter = mock(RuntimeConfGetter.class); + mockCommissioner = mock(Commissioner.class); spyBackupHelper = Mockito.spy( new BackupHelper( @@ -83,7 +85,8 @@ public void setup() { mockService, mockConfigService, mockRuntimeConfGetter, - mockStorageUtilFactory)); + mockStorageUtilFactory, + mockCommissioner)); when(mockStorageUtilFactory.getStorageUtil(eq("S3"))).thenReturn(mockAWSUtil); when(mockStorageUtilFactory.getStorageUtil(eq("NFS"))).thenReturn(mockNfsUtil); } diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java index ac56aff4146c..23ed2d9eef1e 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/BackupsControllerTest.java @@ -18,6 +18,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -37,6 +38,7 @@ import com.yugabyte.yw.common.ModelFactory; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.Util; +import com.yugabyte.yw.common.customer.config.CustomerConfigService; import com.yugabyte.yw.forms.BackupRequestParams; import com.yugabyte.yw.forms.BackupTableParams; import com.yugabyte.yw.forms.RestoreBackupParams; @@ -74,6 +76,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; import play.libs.Json; import play.mvc.Result; @@ -224,7 +227,12 @@ public void testFetchBackupsByTaskUUIDWithSingleEntry() { public void testCreateBackup() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST21"); UUID fakeTaskUUID = UUID.randomUUID(); + CustomerConfigService mockCCS = mock(CustomerConfigService.class); + when(mockCCS.getOrBadRequest(any(), any())).thenReturn(customerConfig); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); + ReflectionTestUtils.setField(mockBackupHelper, "customerConfigService", mockCCS); + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); ObjectNode bodyJson = Json.newObject(); bodyJson.put("universeUUID", defaultUniverse.getUniverseUUID().toString()); bodyJson.put("backupType", "PGSQL_TABLE_TYPE"); @@ -724,6 +732,7 @@ public void testCreateIncrementalScheduleBackupAsyncWithBaseBackup() { public void testCreateBackupValidationFailed() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST25"); UUID fakeTaskUUID = UUID.randomUUID(); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); doThrow(new PlatformServiceException(BAD_REQUEST, "error")) .when(mockBackupHelper) @@ -744,6 +753,8 @@ public void testCreateBackupWithUniverseDisabled() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST25"); ObjectNode bodyJson = Json.newObject(); Map config = defaultUniverse.getConfig(); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); + config.put(Universe.TAKE_BACKUPS, "false"); defaultUniverse.updateConfig(config); defaultUniverse.update(); @@ -761,6 +772,7 @@ public void testCreateBackupWithUniverseDisabled() { public void testCreateBackupWithoutTimeUnit() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST26"); UUID fakeTaskUUID = UUID.randomUUID(); + when(mockBackupHelper.createBackupTask(any(), any())).thenCallRealMethod(); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); ObjectNode bodyJson = Json.newObject(); bodyJson.put("universeUUID", defaultUniverse.getUniverseUUID().toString()); @@ -885,6 +897,7 @@ private Result editBackup(Users user, ObjectNode bodyJson, UUID backupUUID) { public void testRestoreBackupWithInvalidUniverseUUID() { UUID universeUUID = UUID.randomUUID(); JsonNode bodyJson = Json.newObject(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); Result result = assertPlatformException(() -> restoreBackup(universeUUID, bodyJson, null)); assertEquals(BAD_REQUEST, result.status()); @@ -898,6 +911,8 @@ public void testRestoreBackupWithInvalidParams() { BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = UUID.randomUUID(); bp.setUniverseUUID(UUID.randomUUID()); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); bodyJson.put("actionType", "RESTORE"); @@ -915,6 +930,8 @@ public void testRestoreBackupWithoutStorageLocation() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST2"); BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = customerConfig.getConfigUUID(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + bp.setUniverseUUID(UUID.randomUUID()); Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); @@ -936,6 +953,7 @@ public void testRestoreBackupWithInvalidStorageUUID() { BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = UUID.randomUUID(); bp.setUniverseUUID(UUID.randomUUID()); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); Backup b = Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); bodyJson.put("keyspace", "mock_ks"); @@ -956,6 +974,7 @@ public void testRestoreBackupWithInvalidStorageUUID() { public void testRestoreBackupWithReadOnlyUser() { Users user = ModelFactory.testUser(defaultCustomer, "tc@test.com", Users.Role.ReadOnly); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); bp.storageConfigUUID = UUID.randomUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup b = Backup.create(defaultCustomer.getUuid(), bp); @@ -979,6 +998,7 @@ public void testRestoreBackupWithValidParams() { bp.setUniverseUUID(defaultUniverse.getUniverseUUID()); Backup b = Backup.create(defaultCustomer.getUuid(), bp); ObjectNode bodyJson = Json.newObject(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); long maxReqSizeInBytes = app.config().getMemorySize("play.http.parser.maxMemoryBuffer").toBytes(); @@ -1021,6 +1041,8 @@ public void testRestoreBackupWithValidParams() { public void testRestoreBackupRequestTooLarge() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST5"); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup.create(defaultCustomer.getUuid(), bp); @@ -1049,6 +1071,9 @@ public void testRestoreBackupRequestTooLarge() { public void testRestoreBackupWithInvalidOwner() { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST5"); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createRestoreTask(any(), any())).thenCallRealMethod(); + when(mockBackupHelper.getValidOwnerRegex()).thenCallRealMethod(); + bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(defaultUniverse.getUniverseUUID()); Backup.create(defaultCustomer.getUuid(), bp); @@ -1211,6 +1236,8 @@ public void testDeleteFailedBackup() { public void testDeleteBackupYbWithValidState(BackupState state) { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST6"); BackupTableParams bp = new BackupTableParams(); + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); @@ -1241,6 +1268,7 @@ public void testDeleteBackupYbWithValidState(BackupState state) { public void testDeleteBackupYbWithInvalidState(BackupState state) { CustomerConfig customerConfig = ModelFactory.createS3StorageConfig(defaultCustomer, "TEST6"); BackupTableParams bp = new BackupTableParams(); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); bp.storageConfigUUID = customerConfig.getConfigUUID(); bp.setUniverseUUID(UUID.randomUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); @@ -1265,6 +1293,8 @@ public void testDeleteBackupYbWithCustomCustomerStorageConfig() { UUID invalidStorageConfigUUID = UUID.randomUUID(); BackupTableParams bp = new BackupTableParams(); bp.storageConfigUUID = invalidStorageConfigUUID; + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); bp.setUniverseUUID(UUID.randomUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); backup.transitionState(BackupState.Completed); @@ -1300,6 +1330,8 @@ public void testDeleteBackupDuplicateTask() { bp.setUniverseUUID(defaultUniverse.getUniverseUUID()); Backup backup = Backup.create(defaultCustomer.getUuid(), bp); backup.transitionState(BackupState.Completed); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); + List backupUUIDList = new ArrayList<>(); backupUUIDList.add(backup.getBackupUUID().toString()); UUID fakeTaskUUID = UUID.randomUUID(); @@ -1814,6 +1846,8 @@ public void testDeleteFailedIncrementalBackup() { List backupUUIDList = new ArrayList<>(); backupUUIDList.add(backup.getBackupUUID().toString()); UUID fakeTaskUUID = UUID.randomUUID(); + ReflectionTestUtils.setField(mockBackupHelper, "commissioner", mockCommissioner); + when(mockBackupHelper.createDeleteBackupTasks(any(), any())).thenCallRealMethod(); when(mockCommissioner.submit(any(), any())).thenReturn(fakeTaskUUID); ObjectNode resultNode = Json.newObject(); ArrayNode arrayNode = resultNode.putArray("backups");