diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index f3c6c1ea92..32c20b6eb5 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -723,13 +723,13 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B s.arkClient.ArkV1(), s.arkClient.ArkV1(), restorer, - config.BackupStorageProvider.CloudProviderConfig, - config.BackupStorageProvider.Bucket, s.sharedInformerFactory.Ark().V1().Backups(), + s.sharedInformerFactory.Ark().V1().BackupStorageLocations(), s.blockStore != nil, s.logger, s.logLevel, s.pluginRegistry, + s.defaultBackupLocation, s.metrics, ) diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 7746b378ef..4fe2842b01 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -71,23 +71,24 @@ var nonRestorableResources = []string{ } type restoreController struct { - namespace string - restoreClient arkv1client.RestoresGetter - backupClient arkv1client.BackupsGetter - restorer restore.Restorer - objectStoreConfig api.CloudProviderConfig - bucket string - pvProviderExists bool - backupLister listers.BackupLister - backupListerSynced cache.InformerSynced - restoreLister listers.RestoreLister - restoreListerSynced cache.InformerSynced - syncHandler func(restoreName string) error - queue workqueue.RateLimitingInterface - logger logrus.FieldLogger - logLevel logrus.Level - pluginRegistry plugin.Registry - metrics *metrics.ServerMetrics + namespace string + restoreClient arkv1client.RestoresGetter + backupClient arkv1client.BackupsGetter + restorer restore.Restorer + pvProviderExists bool + backupLister listers.BackupLister + backupListerSynced cache.InformerSynced + restoreLister listers.RestoreLister + restoreListerSynced cache.InformerSynced + backupLocationLister listers.BackupStorageLocationLister + backupLocationListerSynced cache.InformerSynced + syncHandler func(restoreName string) error + queue workqueue.RateLimitingInterface + logger logrus.FieldLogger + logLevel logrus.Level + pluginRegistry plugin.Registry + defaultBackupLocation string + metrics *metrics.ServerMetrics getBackup cloudprovider.GetBackupFunc downloadBackup cloudprovider.DownloadBackupFunc @@ -102,33 +103,34 @@ func NewRestoreController( restoreClient arkv1client.RestoresGetter, backupClient arkv1client.BackupsGetter, restorer restore.Restorer, - objectStoreConfig api.CloudProviderConfig, - bucket string, backupInformer informers.BackupInformer, + backupLocationInformer informers.BackupStorageLocationInformer, pvProviderExists bool, logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry, + defaultBackupLocation string, metrics *metrics.ServerMetrics, ) Interface { c := &restoreController{ - namespace: namespace, - restoreClient: restoreClient, - backupClient: backupClient, - restorer: restorer, - objectStoreConfig: objectStoreConfig, - bucket: bucket, - pvProviderExists: pvProviderExists, - backupLister: backupInformer.Lister(), - backupListerSynced: backupInformer.Informer().HasSynced, - restoreLister: restoreInformer.Lister(), - restoreListerSynced: restoreInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"), - logger: logger, - logLevel: logLevel, - pluginRegistry: pluginRegistry, - metrics: metrics, + namespace: namespace, + restoreClient: restoreClient, + backupClient: backupClient, + restorer: restorer, + pvProviderExists: pvProviderExists, + backupLister: backupInformer.Lister(), + backupListerSynced: backupInformer.Informer().HasSynced, + restoreLister: restoreInformer.Lister(), + restoreListerSynced: restoreInformer.Informer().HasSynced, + backupLocationLister: backupLocationInformer.Lister(), + backupLocationListerSynced: backupLocationInformer.Informer().HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "restore"), + logger: logger, + logLevel: logLevel, + pluginRegistry: pluginRegistry, + defaultBackupLocation: defaultBackupLocation, + metrics: metrics, getBackup: cloudprovider.GetBackup, downloadBackup: cloudprovider.DownloadBackup, @@ -193,7 +195,7 @@ func (c *restoreController) Run(ctx context.Context, numWorkers int) error { defer c.logger.Info("Shutting down RestoreController") c.logger.Info("Waiting for caches to sync") - if !cache.WaitForCacheSync(ctx.Done(), c.backupListerSynced, c.restoreListerSynced) { + if !cache.WaitForCacheSync(ctx.Done(), c.backupListerSynced, c.restoreListerSynced, c.backupLocationListerSynced) { return errors.New("timed out waiting for caches to sync") } c.logger.Info("Caches are synced") @@ -283,28 +285,25 @@ func (c *restoreController) processRestore(key string) error { pluginManager := c.newPluginManager(logContext, logContext.Level, c.pluginRegistry) defer pluginManager.CleanupClients() - objectStore, err := getObjectStore(c.objectStoreConfig, pluginManager) - if err != nil { - return errors.Wrap(err, "error initializing object store") - } - actions, err := pluginManager.GetRestoreItemActions() if err != nil { return errors.Wrap(err, "error initializing restore item actions") } - // complete & validate restore - if restore.Status.ValidationErrors = c.completeAndValidate(objectStore, restore); len(restore.Status.ValidationErrors) > 0 { + // validate the restore and fetch the backup + info := c.validateAndComplete(restore, pluginManager) + backupScheduleName := restore.Spec.ScheduleName + // Register attempts after validation so we don't have to fetch the backup multiple times + c.metrics.RegisterRestoreAttempt(backupScheduleName) + + if len(restore.Status.ValidationErrors) > 0 { restore.Status.Phase = api.RestorePhaseFailedValidation + c.metrics.RegisterRestoreValidationFailed(backupScheduleName) } else { restore.Status.Phase = api.RestorePhaseInProgress } - backupScheduleName := restore.Spec.ScheduleName - // Register attempts after validation so we don't have to fetch the backup multiple times - c.metrics.RegisterRestoreAttempt(backupScheduleName) - - // update status + // patch to update status and persist to API updatedRestore, err := patchRestore(original, restore, c.restoreClient) if err != nil { return errors.Wrapf(err, "error updating Restore phase to %s", restore.Status.Phase) @@ -314,15 +313,16 @@ func (c *restoreController) processRestore(key string) error { restore = updatedRestore.DeepCopy() if restore.Status.Phase == api.RestorePhaseFailedValidation { - c.metrics.RegisterRestoreValidationFailed(backupScheduleName) return nil } + logContext.Debug("Running restore") + // execution & upload of restore restoreWarnings, restoreErrors, restoreFailure := c.runRestore( restore, actions, - objectStore, + info, ) restore.Status.Warnings = len(restoreWarnings.Ark) + len(restoreWarnings.Cluster) @@ -355,7 +355,13 @@ func (c *restoreController) processRestore(key string) error { return nil } -func (c *restoreController) completeAndValidate(objectStore cloudprovider.ObjectStore, restore *api.Restore) []string { +type backupInfo struct { + bucketName string + backup *api.Backup + objectStore cloudprovider.ObjectStore +} + +func (c *restoreController) validateAndComplete(restore *api.Restore, pluginManager plugin.Manager) backupInfo { // add non-restorable resources to restore's excluded resources excludedResources := sets.NewString(restore.Spec.ExcludedResources...) for _, nonrestorable := range nonRestorableResources { @@ -363,34 +369,34 @@ func (c *restoreController) completeAndValidate(objectStore cloudprovider.Object restore.Spec.ExcludedResources = append(restore.Spec.ExcludedResources, nonrestorable) } } - var validationErrors []string // validate that included resources don't contain any non-restorable resources includedResources := sets.NewString(restore.Spec.IncludedResources...) for _, nonRestorableResource := range nonRestorableResources { if includedResources.Has(nonRestorableResource) { - validationErrors = append(validationErrors, fmt.Sprintf("%v are non-restorable resources", nonRestorableResource)) + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("%v are non-restorable resources", nonRestorableResource)) } } // validate included/excluded resources for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedResources, restore.Spec.ExcludedResources) { - validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err)) + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded resource lists: %v", err)) } // validate included/excluded namespaces for _, err := range collections.ValidateIncludesExcludes(restore.Spec.IncludedNamespaces, restore.Spec.ExcludedNamespaces) { - validationErrors = append(validationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) } // validate that PV provider exists if we're restoring PVs if boolptr.IsSetToTrue(restore.Spec.RestorePVs) && !c.pvProviderExists { - validationErrors = append(validationErrors, "Server is not configured for PV snapshot restores") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Server is not configured for PV snapshot restores") } // validate that exactly one of BackupName and ScheduleName have been specified if !backupXorScheduleProvided(restore) { - return append(validationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") + return backupInfo{} } // if ScheduleName is specified, fill in BackupName with the most recent successful backup from @@ -402,33 +408,33 @@ func (c *restoreController) completeAndValidate(objectStore cloudprovider.Object backups, err := c.backupLister.Backups(c.namespace).List(selector) if err != nil { - return append(validationErrors, "Unable to list backups for schedule") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule") + return backupInfo{} } if len(backups) == 0 { - return append(validationErrors, "No backups found for schedule") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule") } if backup := mostRecentCompletedBackup(backups); backup != nil { restore.Spec.BackupName = backup.Name } else { - return append(validationErrors, "No completed backups found for schedule") + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule") + return backupInfo{} } } - var ( - backup *api.Backup - err error - ) - if backup, err = c.fetchBackup(objectStore, restore.Spec.BackupName); err != nil { - return append(validationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) + info, err := c.fetchBackupInfo(restore.Spec.BackupName, pluginManager) + if err != nil { + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) + return backupInfo{} } // Fill in the ScheduleName so it's easier to consume for metrics. if restore.Spec.ScheduleName == "" { - restore.Spec.ScheduleName = backup.GetLabels()["ark-schedule"] + restore.Spec.ScheduleName = info.backup.GetLabels()["ark-schedule"] } - return validationErrors + return info } // backupXorScheduleProvided returns true if exactly one of BackupName and @@ -462,43 +468,110 @@ func mostRecentCompletedBackup(backups []*api.Backup) *api.Backup { return nil } -func (c *restoreController) fetchBackup(objectStore cloudprovider.ObjectStore, name string) (*api.Backup, error) { - backup, err := c.backupLister.Backups(c.namespace).Get(name) - if err == nil { - return backup, nil +// fetchBackupInfo checks the backup lister for a backup that matches the given name. If it doesn't +// find it, it tries to retrieve it from one of the backup storage locations. +func (c *restoreController) fetchBackupInfo(backupName string, pluginManager plugin.Manager) (backupInfo, error) { + var info backupInfo + var err error + info.backup, err = c.backupLister.Backups(c.namespace).Get(backupName) + if err != nil { + if !apierrors.IsNotFound(err) { + return backupInfo{}, errors.WithStack(err) + } + + logContext := c.logger.WithField("backupName", backupName) + logContext.Debug("Backup not found in backupLister, checking each backup location directly, starting with default...") + return c.fetchFromBackupStorage(backupName, pluginManager) } - if !apierrors.IsNotFound(err) { - return nil, errors.WithStack(err) + location, err := c.backupLocationLister.BackupStorageLocations(c.namespace).Get(info.backup.Spec.StorageLocation) + if err != nil { + return backupInfo{}, errors.WithStack(err) } - logContext := c.logger.WithField("backupName", name) + info.objectStore, err = getObjectStoreForLocation(location, pluginManager) + if err != nil { + return backupInfo{}, errors.Wrap(err, "error initializing object store") + } + info.bucketName = location.Spec.ObjectStorage.Bucket - logContext.Debug("Backup not found in backupLister, checking object storage directly") - backup, err = c.getBackup(objectStore, c.bucket, name) + return info, nil +} + +// fetchFromBackupStorage checks each backup storage location, starting with the default, +// looking for a backup that matches the given backup name. +func (c *restoreController) fetchFromBackupStorage(backupName string, pluginManager plugin.Manager) (backupInfo, error) { + locations, err := c.backupLocationLister.BackupStorageLocations(c.namespace).List(labels.Everything()) if err != nil { - return nil, err + return backupInfo{}, errors.WithStack(err) + } + + orderedLocations := orderedBackupLocations(locations, c.defaultBackupLocation) + + logContext := c.logger.WithField("backupName", backupName) + for _, location := range orderedLocations { + info, err := c.backupInfoForLocation(location, backupName, pluginManager) + if err != nil { + logContext.WithField("locationName", location.Name).WithError(err).Error("Unable to fetch backup from object storage location") + continue + } + return info, nil + } + + return backupInfo{}, errors.New("not able to fetch from backup storage") +} + +func orderedBackupLocations(locations []*api.BackupStorageLocation, defaultLocationName string) []*api.BackupStorageLocation { + var result []*api.BackupStorageLocation + + for i := range locations { + if locations[i].Name == defaultLocationName { + // put the default location first + result = append(result, locations[i]) + // append everything before the default + result = append(result, locations[:i]...) + // append everything after the default + result = append(result, locations[i+1:]...) + + return result + } + } + + return locations +} + +func (c *restoreController) backupInfoForLocation(location *api.BackupStorageLocation, backupName string, pluginManager plugin.Manager) (backupInfo, error) { + objectStore, err := getObjectStoreForLocation(location, pluginManager) + if err != nil { + return backupInfo{}, err + } + + backup, err := c.getBackup(objectStore, location.Spec.ObjectStorage.Bucket, backupName) + if err != nil { + return backupInfo{}, err } // ResourceVersion needs to be cleared in order to create the object in the API backup.ResourceVersion = "" - // Clear out the namespace too, just in case + // Clear out the namespace, in case the backup was made in a different cluster, with a different namespace backup.Namespace = "" - created, createErr := c.backupClient.Backups(c.namespace).Create(backup) - if createErr != nil { - logContext.WithError(errors.WithStack(createErr)).Error("Unable to create API object for Backup") - } else { - backup = created + backupCreated, err := c.backupClient.Backups(c.namespace).Create(backup) + if err != nil { + return backupInfo{}, errors.WithStack(err) } - return backup, nil + return backupInfo{ + bucketName: location.Spec.ObjectStorage.Bucket, + backup: backupCreated, + objectStore: objectStore, + }, nil } func (c *restoreController) runRestore( restore *api.Restore, actions []restore.ItemAction, - objectStore cloudprovider.ObjectStore, + info backupInfo, ) (restoreWarnings, restoreErrors api.RestoreResult, restoreFailure error) { logFile, err := ioutil.TempFile("", "") if err != nil { @@ -530,14 +603,7 @@ func (c *restoreController) runRestore( "backup": restore.Spec.BackupName, }) - backup, err := c.fetchBackup(objectStore, restore.Spec.BackupName) - if err != nil { - logContext.WithError(err).Error("Error getting backup") - restoreErrors.Ark = append(restoreErrors.Ark, err.Error()) - return - } - - backupFile, err := downloadToTempFile(objectStore, c.bucket, restore.Spec.BackupName, c.downloadBackup, c.logger) + backupFile, err := downloadToTempFile(info.objectStore, info.bucketName, restore.Spec.BackupName, c.downloadBackup, c.logger) if err != nil { logContext.WithError(err).Error("Error downloading backup") restoreErrors.Ark = append(restoreErrors.Ark, err.Error()) @@ -558,7 +624,7 @@ func (c *restoreController) runRestore( // Any return statement above this line means a total restore failure // Some failures after this line *may* be a total restore failure logContext.Info("starting restore") - restoreWarnings, restoreErrors = c.restorer.Restore(logContext, restore, backup, backupFile, actions) + restoreWarnings, restoreErrors = c.restorer.Restore(logContext, restore, info.backup, backupFile, actions) logContext.Info("restore completed") // Try to upload the log file. This is best-effort. If we fail, we'll add to the ark errors. @@ -571,7 +637,7 @@ func (c *restoreController) runRestore( return } - if err := c.uploadRestoreLog(objectStore, c.bucket, restore.Spec.BackupName, restore.Name, logFile); err != nil { + if err := c.uploadRestoreLog(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, logFile); err != nil { restoreErrors.Ark = append(restoreErrors.Ark, fmt.Sprintf("error uploading log file to object storage: %v", err)) } @@ -592,7 +658,7 @@ func (c *restoreController) runRestore( logContext.WithError(errors.WithStack(err)).Error("Error resetting results file offset to 0") return } - if err := c.uploadRestoreResults(objectStore, c.bucket, restore.Spec.BackupName, restore.Name, resultsFile); err != nil { + if err := c.uploadRestoreResults(info.objectStore, info.bucketName, restore.Spec.BackupName, restore.Name, resultsFile); err != nil { logContext.WithError(errors.WithStack(err)).Error("Error uploading results files to object storage") } diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index a618e2ee8c..b67d423e21 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -19,17 +19,16 @@ package controller import ( "bytes" "encoding/json" - "errors" "io" "io/ioutil" "testing" "time" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" core "k8s.io/client-go/testing" @@ -47,10 +46,11 @@ import ( arktest "github.com/heptio/ark/pkg/util/test" ) -func TestFetchBackup(t *testing.T) { +func TestFetchBackupInfo(t *testing.T) { tests := []struct { name string backupName string + informerLocations []*api.BackupStorageLocation informerBackups []*api.Backup backupServiceBackup *api.Backup backupServiceError error @@ -58,16 +58,19 @@ func TestFetchBackup(t *testing.T) { expectedErr bool }{ { - name: "lister has backup", - backupName: "backup-1", - informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").Backup}, - expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup, + name: "lister has backup", + backupName: "backup-1", + informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation}, + informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup}, + expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, }, { - name: "backupSvc has backup", + name: "lister does not have a backup, but backupSvc does", backupName: "backup-1", - backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").Backup, - expectedRes: arktest.NewTestBackup().WithName("backup-1").Backup, + backupServiceBackup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, + informerLocations: []*api.BackupStorageLocation{arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation}, + informerBackups: []*api.Backup{arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup}, + expectedRes: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, }, { name: "no backup", @@ -84,26 +87,43 @@ func TestFetchBackup(t *testing.T) { restorer = &fakeRestorer{} sharedInformers = informers.NewSharedInformerFactory(client, 0) logger = arktest.NewLogger() + pluginManager = &pluginmocks.Manager{} + objectStore = &arktest.ObjectStore{} ) + defer restorer.AssertExpectations(t) + defer objectStore.AssertExpectations(t) + c := NewRestoreController( api.DefaultNamespace, sharedInformers.Ark().V1().Restores(), client.ArkV1(), client.ArkV1(), restorer, - api.CloudProviderConfig{}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), false, logger, logrus.InfoLevel, nil, //pluginRegistry + "default", metrics.NewServerMetrics(), ).(*restoreController) + c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { + return pluginManager + } + + if test.backupServiceError == nil { + pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil) + objectStore.On("Init", mock.Anything).Return(nil) - for _, itm := range test.informerBackups { - sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm) + for _, itm := range test.informerLocations { + sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(itm) + } + + for _, itm := range test.informerBackups { + sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(itm) + } } if test.backupServiceBackup != nil || test.backupServiceError != nil { @@ -114,10 +134,10 @@ func TestFetchBackup(t *testing.T) { } } - backup, err := c.fetchBackup(nil, test.backupName) + info, err := c.fetchBackupInfo(test.backupName, pluginManager) if assert.Equal(t, test.expectedErr, err != nil) { - assert.Equal(t, test.expectedRes, backup) + assert.Equal(t, test.expectedRes, info.backup) } }) } @@ -175,13 +195,13 @@ func TestProcessRestoreSkips(t *testing.T) { client.ArkV1(), client.ArkV1(), restorer, - api.CloudProviderConfig{Name: "myCloud"}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), false, // pvProviderExists logger, logrus.InfoLevel, nil, // pluginRegistry + "default", metrics.NewServerMetrics(), ).(*restoreController) c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { @@ -197,10 +217,12 @@ func TestProcessRestoreSkips(t *testing.T) { }) } } + func TestProcessRestore(t *testing.T) { tests := []struct { name string restoreKey string + location *api.BackupStorageLocation restore *api.Restore backup *api.Backup restorerError error @@ -217,16 +239,18 @@ func TestProcessRestore(t *testing.T) { }{ { name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseNew).WithExcludedNamespace("another-1").Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1"}, }, { name: "restore with resource in both includedResources and excludedResources fails validation", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseNew).WithExcludedResource("a-resource").Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: a-resource"}, @@ -246,11 +270,13 @@ func TestProcessRestore(t *testing.T) { expectedValidationErrors: []string{"Either a backup or schedule must be specified as a source for the restore, but not both"}, }, { - name: "valid restore with schedule name gets executed", - restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).WithSchedule("sched-1").Restore, + name: "valid restore with schedule name gets executed", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, + restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).WithSchedule("sched-1").Restore, backup: arktest. NewTestBackup(). WithName("backup-1"). + WithStorageLocation("default"). WithLabel("ark-schedule", "sched-1"). WithPhase(api.BackupPhaseCompleted). Backup, @@ -263,13 +289,14 @@ func TestProcessRestore(t *testing.T) { restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseNew).Restore, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), - expectedValidationErrors: []string{"Error retrieving backup: no backup here"}, + expectedValidationErrors: []string{"Error retrieving backup: not able to fetch from backup storage"}, backupServiceGetBackupError: errors.New("no backup here"), }, { name: "restorer throwing an error causes the restore to fail", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, restorerError: errors.New("blarg"), expectedErr: false, expectedPhase: string(api.RestorePhaseInProgress), @@ -278,16 +305,18 @@ func TestProcessRestore(t *testing.T) { }, { name: "valid restore gets executed", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseInProgress), expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Restore, }, { name: "valid restore with RestorePVs=true gets executed when allowRestoreSnapshots=true", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, allowRestoreSnapshots: true, expectedErr: false, expectedPhase: string(api.RestorePhaseInProgress), @@ -295,16 +324,18 @@ func TestProcessRestore(t *testing.T) { }, { name: "restore with RestorePVs=true fails validation when allowRestoreSnapshots=false", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).WithRestorePVs(true).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Server is not configured for PV snapshot restores"}, }, { name: "restoration of nodes is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -314,8 +345,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of events is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -325,8 +357,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of events.events.k8s.io is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events.events.k8s.io", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -336,8 +369,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of backups.ark.heptio.com is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "backups.ark.heptio.com", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -347,8 +381,9 @@ func TestProcessRestore(t *testing.T) { }, { name: "restoration of restores.ark.heptio.com is not supported", + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "restores.ark.heptio.com", api.RestorePhaseNew).Restore, - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, expectedErr: false, expectedPhase: string(api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ @@ -358,11 +393,12 @@ func TestProcessRestore(t *testing.T) { }, { name: "backup download error results in failed restore", - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, + location: arktest.NewTestBackupStorageLocation().WithName("default").WithProvider("myCloud").WithObjectStorage("bucket").BackupStorageLocation, + restore: NewRestore(api.DefaultNamespace, "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Restore, expectedPhase: string(api.RestorePhaseInProgress), expectedFinalPhase: string(api.RestorePhaseFailed), backupServiceDownloadBackupError: errors.New("Couldn't download backup"), - backup: arktest.NewTestBackup().WithName("backup-1").Backup, + backup: arktest.NewTestBackup().WithName("backup-1").WithStorageLocation("default").Backup, }, } @@ -377,6 +413,7 @@ func TestProcessRestore(t *testing.T) { objectStore = &arktest.ObjectStore{} ) defer restorer.AssertExpectations(t) + defer objectStore.AssertExpectations(t) c := NewRestoreController( @@ -385,23 +422,29 @@ func TestProcessRestore(t *testing.T) { client.ArkV1(), client.ArkV1(), restorer, - api.CloudProviderConfig{Name: "myCloud"}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), test.allowRestoreSnapshots, logger, logrus.InfoLevel, nil, // pluginRegistry + "default", metrics.NewServerMetrics(), ).(*restoreController) c.newPluginManager = func(logger logrus.FieldLogger, logLevel logrus.Level, pluginRegistry plugin.Registry) plugin.Manager { return pluginManager } - if test.restore != nil { + if test.location != nil { + sharedInformers.Ark().V1().BackupStorageLocations().Informer().GetStore().Add(test.location) + } + if test.backup != nil { + sharedInformers.Ark().V1().Backups().Informer().GetStore().Add(test.backup) pluginManager.On("GetObjectStore", "myCloud").Return(objectStore, nil) objectStore.On("Init", mock.Anything).Return(nil) + } + if test.restore != nil { sharedInformers.Ark().V1().Restores().Informer().GetStore().Add(test.restore) // this is necessary so the Patch() call returns the appropriate object @@ -590,11 +633,12 @@ func TestProcessRestore(t *testing.T) { } } -func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { +func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { var ( client = fake.NewSimpleClientset() sharedInformers = informers.NewSharedInformerFactory(client, 0) logger = arktest.NewLogger() + pluginManager = &pluginmocks.Manager{} ) c := NewRestoreController( @@ -603,13 +647,13 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { client.ArkV1(), client.ArkV1(), nil, - api.CloudProviderConfig{Name: "myCloud"}, - "bucket", sharedInformers.Ark().V1().Backups(), + sharedInformers.Ark().V1().BackupStorageLocations(), false, logger, logrus.DebugLevel, nil, + "default", nil, ).(*restoreController) @@ -632,7 +676,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { Backup, )) - errs := c.completeAndValidate(nil, restore) + errs := c.validateAndComplete(restore, pluginManager) assert.Equal(t, []string{"No backups found for schedule"}, errs) assert.Empty(t, restore.Spec.BackupName) @@ -645,7 +689,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { Backup, )) - errs = c.completeAndValidate(nil, restore) + errs = c.validateAndComplete(restore, pluginManager) assert.Equal(t, []string{"No completed backups found for schedule"}, errs) assert.Empty(t, restore.Spec.BackupName) @@ -669,7 +713,7 @@ func TestCompleteAndValidateWhenScheduleNameSpecified(t *testing.T) { Backup, )) - errs = c.completeAndValidate(nil, restore) + errs = c.validateAndComplete(restore, pluginManager) assert.Nil(t, errs) assert.Equal(t, "bar", restore.Spec.BackupName) } diff --git a/pkg/util/test/test_backup_storage_location.go b/pkg/util/test/test_backup_storage_location.go new file mode 100644 index 0000000000..d18d7c4286 --- /dev/null +++ b/pkg/util/test/test_backup_storage_location.go @@ -0,0 +1,68 @@ +/* +Copyright 2017 the Heptio Ark contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/heptio/ark/pkg/apis/ark/v1" +) + +type TestBackupStorageLocation struct { + *v1.BackupStorageLocation +} + +func NewTestBackupStorageLocation() *TestBackupStorageLocation { + return &TestBackupStorageLocation{ + BackupStorageLocation: &v1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.DefaultNamespace, + }, + }, + } +} + +func (b *TestBackupStorageLocation) WithNamespace(namespace string) *TestBackupStorageLocation { + b.Namespace = namespace + return b +} + +func (b *TestBackupStorageLocation) WithName(name string) *TestBackupStorageLocation { + b.Name = name + return b +} + +func (b *TestBackupStorageLocation) WithLabel(key, value string) *TestBackupStorageLocation { + if b.Labels == nil { + b.Labels = make(map[string]string) + } + b.Labels[key] = value + return b +} + +func (b *TestBackupStorageLocation) WithProvider(name string) *TestBackupStorageLocation { + b.Spec.Provider = name + return b +} + +func (b *TestBackupStorageLocation) WithObjectStorage(bucketName string) *TestBackupStorageLocation { + if b.Spec.StorageType.ObjectStorage == nil { + b.Spec.StorageType.ObjectStorage = &v1.ObjectStorageLocation{} + } + b.Spec.ObjectStorage.Bucket = bucketName + return b +}