create pkg/persistence and move relevant code from pkg/cloudprovider …

…into it

Signed-off-by: Steve Kriss <steve@heptio.com>

pkg/controller/backup_controller.go

@@ -47,6 +47,7 @@ import (
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
 	"github.com/heptio/ark/pkg/metrics"
+	"github.com/heptio/ark/pkg/persistence"
 	"github.com/heptio/ark/pkg/plugin"
 	"github.com/heptio/ark/pkg/util/collections"
 	"github.com/heptio/ark/pkg/util/encode"
@@ -437,7 +438,7 @@ func (controller *backupController) runBackup(backup *api.Backup, backupLocation
 		controller.logger.WithError(err).Error("error closing gzippedLogFile")
 	}
 
-	if err := cloudprovider.UploadBackup(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil {
+	if err := persistence.UploadBackup(log, objectStore, backupLocation.Spec.ObjectStorage.Bucket, backup.Name, backupJSONToUpload, backupFileToUpload, logFile); err != nil {
 		errs = append(errs, err)
 	}
 

pkg/controller/backup_deletion_controller.go

@@ -28,6 +28,7 @@ import (
 	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
+	"github.com/heptio/ark/pkg/persistence"
 	"github.com/heptio/ark/pkg/plugin"
 	"github.com/heptio/ark/pkg/restic"
 	"github.com/heptio/ark/pkg/util/kube"
@@ -57,7 +58,7 @@ type backupDeletionController struct {
 	resticMgr                 restic.RepositoryManager
 	podvolumeBackupLister     listers.PodVolumeBackupLister
 	backupLocationLister      listers.BackupStorageLocationLister
-	deleteBackupDir           cloudprovider.DeleteBackupDirFunc
+	deleteBackupDir           persistence.DeleteBackupDirFunc
 	processRequestFunc        func(*v1.DeleteBackupRequest) error
 	clock                     clock.Clock
 	newPluginManager          func(logrus.FieldLogger) plugin.Manager
@@ -94,7 +95,7 @@ func NewBackupDeletionController(
 		// use variables to refer to these functions so they can be
 		// replaced with fakes for testing.
 		newPluginManager: newPluginManager,
-		deleteBackupDir:  cloudprovider.DeleteBackupDir,
+		deleteBackupDir:  persistence.DeleteBackupDir,
 
 		clock: &clock.RealClock{},
 	}

pkg/controller/backup_sync_controller.go

@@ -33,6 +33,7 @@ import (
 	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
+	"github.com/heptio/ark/pkg/persistence"
 	"github.com/heptio/ark/pkg/plugin"
 	"github.com/heptio/ark/pkg/util/kube"
 	"github.com/heptio/ark/pkg/util/stringslice"
@@ -76,7 +77,7 @@ func NewBackupSyncController(
 		// use variables to refer to these functions so they can be
 		// replaced with fakes for testing.
 		newPluginManager: newPluginManager,
-		listCloudBackups: cloudprovider.ListBackups,
+		listCloudBackups: persistence.ListBackups,
 	}
 
 	c.resyncFunc = c.run

pkg/controller/download_request_controller.go

@@ -32,10 +32,10 @@ import (
 	"k8s.io/client-go/tools/cache"
 
 	"github.com/heptio/ark/pkg/apis/ark/v1"
-	"github.com/heptio/ark/pkg/cloudprovider"
 	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
+	"github.com/heptio/ark/pkg/persistence"
 	"github.com/heptio/ark/pkg/plugin"
 	"github.com/heptio/ark/pkg/util/kube"
 )
@@ -47,7 +47,7 @@ type downloadRequestController struct {
 	downloadRequestLister listers.DownloadRequestLister
 	restoreLister         listers.RestoreLister
 	clock                 clock.Clock
-	createSignedURL       cloudprovider.CreateSignedURLFunc
+	createSignedURL       persistence.CreateSignedURLFunc
 	backupLocationLister  listers.BackupStorageLocationLister
 	backupLister          listers.BackupLister
 	newPluginManager      func(logrus.FieldLogger) plugin.Manager
@@ -73,7 +73,7 @@ func NewDownloadRequestController(
 
 		// use variables to refer to these functions so they can be
 		// replaced with fakes for testing.
-		createSignedURL:  cloudprovider.CreateSignedURL,
+		createSignedURL:  persistence.CreateSignedURL,
 		newPluginManager: newPluginManager,
 
 		clock: &clock.RealClock{},

pkg/controller/restore_controller.go

@@ -46,6 +46,7 @@ import (
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
 	"github.com/heptio/ark/pkg/metrics"
+	"github.com/heptio/ark/pkg/persistence"
 	"github.com/heptio/ark/pkg/plugin"
 	"github.com/heptio/ark/pkg/restore"
 	"github.com/heptio/ark/pkg/util/boolptr"
@@ -89,10 +90,10 @@ type restoreController struct {
 	defaultBackupLocation      string
 	metrics                    *metrics.ServerMetrics
 
-	getBackup            cloudprovider.GetBackupFunc
-	downloadBackup       cloudprovider.DownloadBackupFunc
-	uploadRestoreLog     cloudprovider.UploadRestoreLogFunc
-	uploadRestoreResults cloudprovider.UploadRestoreResultsFunc
+	getBackup            persistence.GetBackupFunc
+	downloadBackup       persistence.DownloadBackupFunc
+	uploadRestoreLog     persistence.UploadRestoreLogFunc
+	uploadRestoreResults persistence.UploadRestoreResultsFunc
 	newPluginManager     func(logger logrus.FieldLogger) plugin.Manager
 }
 
@@ -132,10 +133,10 @@ func NewRestoreController(
 		// use variables to refer to these functions so they can be
 		// replaced with fakes for testing.
 		newPluginManager:     newPluginManager,
-		getBackup:            cloudprovider.GetBackup,
-		downloadBackup:       cloudprovider.DownloadBackup,
-		uploadRestoreLog:     cloudprovider.UploadRestoreLog,
-		uploadRestoreResults: cloudprovider.UploadRestoreResults,
+		getBackup:            persistence.GetBackup,
+		downloadBackup:       persistence.DownloadBackup,
+		uploadRestoreLog:     persistence.UploadRestoreLog,
+		uploadRestoreResults: persistence.UploadRestoreResults,
 	}
 
 	c.syncHandler = c.processRestore
@@ -667,7 +668,7 @@ func (c *restoreController) runRestore(
 func downloadToTempFile(
 	objectStore cloudprovider.ObjectStore,
 	bucket, backupName string,
-	downloadBackup cloudprovider.DownloadBackupFunc,
+	downloadBackup persistence.DownloadBackupFunc,
 	logger logrus.FieldLogger,
 ) (*os.File, error) {
 	readCloser, err := downloadBackup(objectStore, bucket, backupName)

pkg/cloudprovider/backup_service.go → pkg/persistence/object_storage_persistence.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cloudprovider
+package persistence
 
 import (
 	"fmt"
@@ -28,6 +28,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	api "github.com/heptio/ark/pkg/apis/ark/v1"
+	"github.com/heptio/ark/pkg/cloudprovider"
 	"github.com/heptio/ark/pkg/generated/clientset/versioned/scheme"
 )
 
@@ -75,7 +76,7 @@ func seekToBeginning(r io.Reader) error {
 	return err
 }
 
-func seekAndPutObject(objectStore ObjectStore, bucket, key string, file io.Reader) error {
+func seekAndPutObject(objectStore cloudprovider.ObjectStore, bucket, key string, file io.Reader) error {
 	if file == nil {
 		return nil
 	}
@@ -87,27 +88,27 @@ func seekAndPutObject(objectStore ObjectStore, bucket, key string, file io.Reade
 	return objectStore.PutObject(bucket, key, file)
 }
 
-func UploadBackupLog(objectStore ObjectStore, bucket, backupName string, log io.Reader) error {
+func UploadBackupLog(objectStore cloudprovider.ObjectStore, bucket, backupName string, log io.Reader) error {
 	logKey := getBackupLogKey(backupName, backupName)
 	return seekAndPutObject(objectStore, bucket, logKey, log)
 }
 
-func UploadBackupMetadata(objectStore ObjectStore, bucket, backupName string, metadata io.Reader) error {
+func UploadBackupMetadata(objectStore cloudprovider.ObjectStore, bucket, backupName string, metadata io.Reader) error {
 	metadataKey := getMetadataKey(backupName)
 	return seekAndPutObject(objectStore, bucket, metadataKey, metadata)
 }
 
-func DeleteBackupMetadata(objectStore ObjectStore, bucket, backupName string) error {
+func DeleteBackupMetadata(objectStore cloudprovider.ObjectStore, bucket, backupName string) error {
 	metadataKey := getMetadataKey(backupName)
 	return objectStore.DeleteObject(bucket, metadataKey)
 }
 
-func UploadBackupData(objectStore ObjectStore, bucket, backupName string, backup io.Reader) error {
+func UploadBackupData(objectStore cloudprovider.ObjectStore, bucket, backupName string, backup io.Reader) error {
 	backupKey := getBackupContentsKey(backupName, backupName)
 	return seekAndPutObject(objectStore, bucket, backupKey, backup)
 }
 
-func UploadBackup(logger logrus.FieldLogger, objectStore ObjectStore, bucket, backupName string, metadata, backup, log io.Reader) error {
+func UploadBackup(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string, metadata, backup, log io.Reader) error {
 	if err := UploadBackupLog(objectStore, bucket, backupName, log); err != nil {
 		// Uploading the log file is best-effort; if it fails, we log the error but it doesn't impact the
 		// backup's status.
@@ -138,16 +139,16 @@ func UploadBackup(logger logrus.FieldLogger, objectStore ObjectStore, bucket, ba
 }
 
 // DownloadBackupFunc is a function that can download backup metadata from a bucket in object storage.
-type DownloadBackupFunc func(objectStore ObjectStore, bucket, backupName string) (io.ReadCloser, error)
+type DownloadBackupFunc func(objectStore cloudprovider.ObjectStore, bucket, backupName string) (io.ReadCloser, error)
 
 // DownloadBackup downloads an Ark backup with the specified object key from object storage via the cloud API.
 // It returns the snapshot metadata and data (separately), or an error if a problem is encountered
 // downloading or reading the file from the cloud API.
-func DownloadBackup(objectStore ObjectStore, bucket, backupName string) (io.ReadCloser, error) {
+func DownloadBackup(objectStore cloudprovider.ObjectStore, bucket, backupName string) (io.ReadCloser, error) {
 	return objectStore.GetObject(bucket, getBackupContentsKey(backupName, backupName))
 }
 
-func ListBackups(logger logrus.FieldLogger, objectStore ObjectStore, bucket string) ([]*api.Backup, error) {
+func ListBackups(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket string) ([]*api.Backup, error) {
 	prefixes, err := objectStore.ListCommonPrefixes(bucket, "/")
 	if err != nil {
 		return nil, err
@@ -172,10 +173,10 @@ func ListBackups(logger logrus.FieldLogger, objectStore ObjectStore, bucket stri
 }
 
 //GetBackupFunc is a function that can retrieve backup metadata from an object store
-type GetBackupFunc func(objectStore ObjectStore, bucket, backupName string) (*api.Backup, error)
+type GetBackupFunc func(objectStore cloudprovider.ObjectStore, bucket, backupName string) (*api.Backup, error)
 
 // GetBackup gets the specified api.Backup from the given bucket in object storage.
-func GetBackup(objectStore ObjectStore, bucket, backupName string) (*api.Backup, error) {
+func GetBackup(objectStore cloudprovider.ObjectStore, bucket, backupName string) (*api.Backup, error) {
 	key := getMetadataKey(backupName)
 
 	res, err := objectStore.GetObject(bucket, key)
@@ -204,10 +205,10 @@ func GetBackup(objectStore ObjectStore, bucket, backupName string) (*api.Backup,
 }
 
 // DeleteBackupDirFunc is a function that can delete a backup directory from a bucket in object storage.
-type DeleteBackupDirFunc func(logger logrus.FieldLogger, objectStore ObjectStore, bucket, backupName string) error
+type DeleteBackupDirFunc func(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error
 
 // DeleteBackupDir deletes all files in object storage for the given backup.
-func DeleteBackupDir(logger logrus.FieldLogger, objectStore ObjectStore, bucket, backupName string) error {
+func DeleteBackupDir(logger logrus.FieldLogger, objectStore cloudprovider.ObjectStore, bucket, backupName string) error {
 	objects, err := objectStore.ListObjects(bucket, backupName+"/")
 	if err != nil {
 		return err
@@ -228,11 +229,11 @@ func DeleteBackupDir(logger logrus.FieldLogger, objectStore ObjectStore, bucket,
 }
 
 // CreateSignedURLFunc is a function that can create a signed URL for an object in object storage.
-type CreateSignedURLFunc func(objectStore ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error)
+type CreateSignedURLFunc func(objectStore cloudprovider.ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error)
 
 // CreateSignedURL creates a pre-signed URL that can be used to download a file from object
 // storage. The URL expires after ttl.
-func CreateSignedURL(objectStore ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) {
+func CreateSignedURL(objectStore cloudprovider.ObjectStore, target api.DownloadTarget, bucket, directory string, ttl time.Duration) (string, error) {
 	switch target.Kind {
 	case api.DownloadTargetKindBackupContents:
 		return objectStore.CreateSignedURL(bucket, getBackupContentsKey(directory, target.Name), ttl)
@@ -248,19 +249,19 @@ func CreateSignedURL(objectStore ObjectStore, target api.DownloadTarget, bucket,
 }
 
 // UploadRestoreLogFunc is a function that can upload a restore log to a bucket in object storage.
-type UploadRestoreLogFunc func(objectStore ObjectStore, bucket, backup, restore string, log io.Reader) error
+type UploadRestoreLogFunc func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, log io.Reader) error
 
 // UploadRestoreLog uploads the restore's log file to object storage.
-func UploadRestoreLog(objectStore ObjectStore, bucket, backup, restore string, log io.Reader) error {
+func UploadRestoreLog(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, log io.Reader) error {
 	key := getRestoreLogKey(backup, restore)
 	return objectStore.PutObject(bucket, key, log)
 }
 
 // UploadRestoreResultsFunc is a function that can upload restore results to a bucket in object storage.
-type UploadRestoreResultsFunc func(objectStore ObjectStore, bucket, backup, restore string, results io.Reader) error
+type UploadRestoreResultsFunc func(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, results io.Reader) error
 
 // UploadRestoreResults uploads the restore's results file to object storage.
-func UploadRestoreResults(objectStore ObjectStore, bucket, backup, restore string, results io.Reader) error {
+func UploadRestoreResults(objectStore cloudprovider.ObjectStore, bucket, backup, restore string, results io.Reader) error {
 	key := getRestoreResultsKey(backup, restore)
 	return objectStore.PutObject(bucket, key, results)
 }

pkg/cloudprovider/backup_service_test.go → pkg/persistence/object_storage_persistence_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cloudprovider
+package persistence
 
 import (
 	"bytes"