diff --git a/drivers/volume/kdmp/kdmp.go b/drivers/volume/kdmp/kdmp.go index 4737c73239..14647a2ef7 100644 --- a/drivers/volume/kdmp/kdmp.go +++ b/drivers/volume/kdmp/kdmp.go @@ -69,8 +69,9 @@ const ( restoreObjectNameKey = utils.KdmpAnnotationPrefix + "restoreobject-name" restoreObjectUIDKey = utils.KdmpAnnotationPrefix + "restoreobject-uid" - pvcNameKey = utils.KdmpAnnotationPrefix + "pvc-name" - pvcUIDKey = utils.KdmpAnnotationPrefix + "pvc-uid" + pvcNameKey = utils.KdmpAnnotationPrefix + "pvc-name" + pvcUIDKey = utils.KdmpAnnotationPrefix + "pvc-uid" + kdmpStorageClassKey = utils.KdmpAnnotationPrefix + "storage-class" // pvcProvisionerAnnotation is the annotation on PVC which has the // provisioner name pvcProvisionerAnnotation = "volume.beta.kubernetes.io/storage-provisioner" @@ -266,6 +267,7 @@ func (k *kdmp) StartBackup(backup *storkapi.ApplicationBackup, labels[utils.ApplicationBackupCRUIDKey] = utils.GetValidLabel(utils.GetShortUID(string(backup.UID))) labels[pvcNameKey] = utils.GetValidLabel(pvc.Name) labels[pvcUIDKey] = utils.GetValidLabel(utils.GetShortUID(string(pvc.UID))) + labels[kdmpStorageClassKey] = volumeInfo.StorageClass // If backup from px-backup, update the backup object details in the label if val, ok := backup.Annotations[utils.PxbackupAnnotationCreateByKey]; ok { if val == utils.PxbackupAnnotationCreateByValue { diff --git a/go.mod b/go.mod index 3e6b418ffa..131967e79c 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,8 @@ require ( ) require ( - github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 + github.com/evanphx/json-patch v5.6.0+incompatible + github.com/portworx/kdmp v0.4.1-0.20231123040148-68149a57dc24 k8s.io/utils v0.0.0-20230505201702-9f6742963106 kubevirt.io/api v1.0.0 ) @@ -121,7 +122,6 @@ require ( github.com/donovanhide/eventsource v0.0.0-20171031113327-3ed64d21fb0b // indirect github.com/educlos/testrail v0.0.0-20210915115134-adb5e6f62a6d // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/color v1.13.0 // indirect @@ -333,7 +333,7 @@ replace ( github.com/libopenstorage/autopilot-api => github.com/libopenstorage/autopilot-api v0.6.1-0.20210301232050-ca2633c6e114 github.com/libopenstorage/openstorage => github.com/libopenstorage/openstorage v0.0.0-20230511212757-41751b27d69f github.com/libopenstorage/secrets => github.com/libopenstorage/secrets v0.0.0-20220413195519-57d1c446c5e9 - github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20231025110112-14cdb50c205e + github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20231123040148-68149a57dc24 github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20230930052008-186afa1e82c4 github.com/portworx/torpedo => github.com/portworx/torpedo v0.0.0-20231013194137-84e2957806f0 gopkg.in/fsnotify.v1 v1.4.7 => github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index a56f5495fe..8e2c0357c1 100644 --- a/go.sum +++ b/go.sum @@ -2894,8 +2894,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polyfloyd/go-errorlint v1.0.5/go.mod h1:APVvOesVSAnne5SClsPxPdfvZTVDojXh1/G3qb5wjGI= github.com/portworx/dcos-secrets v0.0.0-20180616013705-8e8ec3f66611/go.mod h1:4hklRW/4DQpLqkcXcjtNprbH2tz/sJaNtqinfPWl/LA= -github.com/portworx/kdmp v0.4.1-0.20231025110112-14cdb50c205e h1:qmG0TUupl1PWqLTlMg58wsMQYvcNgU+GPwV2UFny62k= -github.com/portworx/kdmp v0.4.1-0.20231025110112-14cdb50c205e/go.mod h1:mcZrNtJqqVH9iywX905p5EoycJLb/dB77HFtKjn/VLE= +github.com/portworx/kdmp v0.4.1-0.20231123040148-68149a57dc24 h1:phnNwa/LAR8Bed5T9Ji9m8l9mHdO500kpd4obfG4YgE= +github.com/portworx/kdmp v0.4.1-0.20231123040148-68149a57dc24/go.mod h1:mMZUDNoAfslBw6DxEXKN2KSEt14l/HIGtt1CzgOghp0= github.com/portworx/kvdb v0.0.0-20190105022415-cccaa09abfc9/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200723230726-2734b7f40194/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= diff --git a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go index 4d06e0dfc1..272ecc991f 100644 --- a/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go +++ b/vendor/github.com/portworx/kdmp/pkg/controllers/dataexport/reconcile.go @@ -62,6 +62,7 @@ const ( pxbackupAnnotationCreateByValue = "px-backup" backupObjectUIDKey = kdmpAnnotationPrefix + "backupobject-uid" pvcUIDKey = kdmpAnnotationPrefix + "pvc-uid" + kdmpStorageClassKey = kdmpAnnotationPrefix + "storage-class" volumeSnapShotCRDirectory = "csi-generic" snapDeleteAnnotation = "snapshotScheduledForDeletion" snapRestoreAnnotation = "snapshotScheduledForRestore" @@ -76,6 +77,7 @@ const ( defaultTimeout = 1 * time.Minute progressCheckInterval = 5 * time.Second compressionKey = "KDMP_COMPRESSION" + excludeFileListKey = "KDMP_EXCLUDE_FILE_LIST" backupPath = "KDMP_BACKUP_PATH" ) @@ -261,6 +263,8 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er var compressionType string var podDataPath string + var excludeFileList string + pvcStorageClass := dataExport.Labels[kdmpStorageClassKey] var backupLocation *storkapi.BackupLocation var data updateDataExportDetail if driverName != drivers.Rsync { @@ -273,11 +277,29 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er kdmpData, err := core.Instance().GetConfigMap(utils.KdmpConfigmapName, utils.KdmpConfigmapNamespace) if err != nil { logrus.Errorf("failed reading config map %v: %v", utils.KdmpConfigmapName, err) - logrus.Warnf("default to %s compression", utils.DefaultCompresion) - compressionType = utils.DefaultCompresion - } else { - compressionType = kdmpData.Data[compressionKey] - podDataPath = kdmpData.Data[backupPath] + if err != nil { + msg := fmt.Sprintf("Failed in parsing the excludeFileList configmap parameter from configmap [%v/%v]: %v", utils.KdmpConfigmapNamespace, utils.KdmpConfigmapName, err) + logrus.Errorf(msg) + data := updateDataExportDetail{ + status: kdmpapi.DataExportStatusFailed, + reason: msg, + } + return false, c.updateStatus(dataExport, data) + } + } + compressionType = kdmpData.Data[compressionKey] + podDataPath = kdmpData.Data[backupPath] + if len(kdmpData.Data[excludeFileListKey]) != 0 { + excludeFileList, err = parseExcludeFileListKey(pvcStorageClass, kdmpData.Data[excludeFileListKey]) + if err != nil { + msg := fmt.Sprintf("Failed in parsing the excludeFileList configmap parameter from configmap [%v/%v]", utils.KdmpConfigmapNamespace, utils.KdmpConfigmapName) + logrus.Errorf(msg) + data := updateDataExportDetail{ + status: kdmpapi.DataExportStatusFailed, + reason: msg, + } + return false, c.updateStatus(dataExport, data) + } } blName := dataExport.Spec.Destination.Name blNamespace := dataExport.Spec.Destination.Namespace @@ -287,7 +309,7 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er blNamespace = vb.Spec.BackupLocation.Namespace } - backupLocation, err := readBackupLocation(blName, blNamespace, "") + backupLocation, err = readBackupLocation(blName, blNamespace, "") if err != nil { msg := fmt.Sprintf("reading of backuplocation [%v/%v] failed: %v", blNamespace, blName, err) logrus.Errorf(msg) @@ -308,6 +330,7 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er driver, srcPVCName, compressionType, + excludeFileList, dataExport, podDataPath, utils.KdmpConfigmapName, @@ -503,6 +526,35 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er return false, nil } +// If pvc storageclass and the configured storageclass matches, extract the configured ignore file list and return it. +// For code reference, adding the sample of the configmap parameter. +// +// KDMP_EXCLUDE_FILE_LIST: | +// px-db=dir1,file1,dir2 +// mysql=dir1,file1,dir2 +// +// After we read, the parameter value comes as follow: +// "px-db=dir1,file1,dir2\nmysql=dir1,file1,dir2\n" +func parseExcludeFileListKey(pvcStorageClass string, excludeFileListValue string) (string, error) { + // trim the ending "\n" character, if it present + excludeFileListValue = strings.TrimSuffix(string(excludeFileListValue), "\n") + storageClassList := strings.Split(excludeFileListValue, "\n") + var excludeFileList string + for _, storageClass := range storageClassList { + equalSignSplit := strings.Split(storageClass, "=") + if len(equalSignSplit) != 2 { + return "", fmt.Errorf("invalid exclude file list in the configmap parameter") + } + // if the PVC storageclass and configure storageclass are same, extract the configured ignore file list + if pvcStorageClass == equalSignSplit[0] { + excludeFileList = equalSignSplit[1] + } + + } + logrus.Infof("parseExcludeFileListKey: configured excludeFileList - %v", excludeFileList) + return excludeFileList, nil +} + func appendPodLogToStork(jobName string, namespace string) { // Get job and check whether it has live pod attaced to it job, err := batch.Instance().GetJob(jobName, namespace) @@ -565,7 +617,7 @@ func (c *Controller) createJobCredCertSecrets( // Check if the above env is present and read the certs file contents and // secret for the job pod for kopia to access the same - err := createCertificateSecret(utils.GetCertSecretName(dataExport.Name), namespace, dataExport.Labels) + err := createCertificateSecret(utils.GetCertSecretName(dataExport.Name), namespace, blName, blNamespace, dataExport.Labels) if err != nil { msg := fmt.Sprintf("error in creating certificate secret[%v/%v]: %v", namespace, dataExport.Name, err) logrus.Errorf(msg) @@ -1752,6 +1804,7 @@ func startTransferJob( drv drivers.Interface, srcPVCName string, compressionType string, + excludeFileList string, dataExport *kdmpapi.DataExport, podDataPath string, jobConfigMap string, @@ -1812,6 +1865,7 @@ func startTransferJob( drivers.WithCertSecretName(utils.GetCertSecretName(dataExport.GetName())), drivers.WithCertSecretNamespace(dataExport.Spec.Source.Namespace), drivers.WithCompressionType(compressionType), + drivers.WithExcludeFileList(excludeFileList), drivers.WithPodDatapathType(podDataPath), drivers.WithJobConfigMap(jobConfigMap), drivers.WithJobConfigMapNs(jobConfigMapNs), @@ -2156,7 +2210,15 @@ func createAzureSecret(secretName string, backupLocation *storkapi.BackupLocatio return err } -func createCertificateSecret(secretName, namespace string, labels map[string]string) error { +func createCertificateSecret(secretName, namespace, blName, blNamespace string, labels map[string]string) error { + backupLocation, err := readBackupLocation(blName, blNamespace, "") + if err != nil { + return err + } + // when DisableSSL is true, dont create job cert secrets. + if backupLocation.Location.Type == storkapi.BackupLocationS3 && backupLocation.Location.S3Config.DisableSSL { + return nil + } drivers.CertFilePath = os.Getenv(drivers.CertDirPath) if drivers.CertFilePath != "" { certificateData, err := os.ReadFile(filepath.Join(drivers.CertFilePath, drivers.CertFileName)) diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go index 800201cb38..1c29b69288 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/kopiabackup/kopiabackup.go @@ -292,6 +292,13 @@ func jobFor( splitCmd = append(splitCmd, "--compression", jobOption.Compression) cmd = strings.Join(splitCmd, " ") } + + if jobOption.ExcludeFileList != "" { + splitCmd := strings.Split(cmd, " ") + splitCmd = append(splitCmd, "--exclude-file-list", jobOption.ExcludeFileList) + cmd = strings.Join(splitCmd, " ") + } + kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage, jobOption.KopiaImageExecutorSource, jobOption.KopiaImageExecutorSourceNs, diff --git a/vendor/github.com/portworx/kdmp/pkg/drivers/options.go b/vendor/github.com/portworx/kdmp/pkg/drivers/options.go index ec7ed018a7..61d597c866 100644 --- a/vendor/github.com/portworx/kdmp/pkg/drivers/options.go +++ b/vendor/github.com/portworx/kdmp/pkg/drivers/options.go @@ -37,6 +37,7 @@ type JobOpts struct { MaintenanceType string RepoPVCName string Compression string + ExcludeFileList string PodDataPath string // JobConfigMap holds any config needs to be provided to job // from the caller. Eg: executor image name, secret, etc.. @@ -456,6 +457,14 @@ func WithCompressionType(compressionType string) JobOption { } } +// WithExcludeFileList is job parameter. +func WithExcludeFileList(excludeFileList string) JobOption { + return func(opts *JobOpts) error { + opts.ExcludeFileList = excludeFileList + return nil + } +} + // WithPodDatapathType is job parameter. func WithPodDatapathType(podDataPath string) JobOption { return func(opts *JobOpts) error { diff --git a/vendor/github.com/portworx/kdmp/pkg/executor/common.go b/vendor/github.com/portworx/kdmp/pkg/executor/common.go index 41f3a6845f..7ff5ab76b2 100644 --- a/vendor/github.com/portworx/kdmp/pkg/executor/common.go +++ b/vendor/github.com/portworx/kdmp/pkg/executor/common.go @@ -37,7 +37,7 @@ const ( secretAccessKeyPath = "/etc/cred-secret/secretAccessKey" bucketPath = "/etc/cred-secret/path" endpointPath = "/etc/cred-secret/endpoint" - sseTypePath = "/etc/cred-secret/sse" + sseTypePath = "/etc/cred-secret/sse" passwordPath = "/etc/cred-secret/password" regionPath = "/etc/cred-secret/region" disableSslPath = "/etc/cred-secret/disablessl" @@ -91,7 +91,7 @@ type S3Config struct { // Region will be defaulted to us-east-1 if not provided Region string DisableSSL bool - SseType string + SseType string } // AzureConfig specifies the config required to connect to Azure Blob Storage diff --git a/vendor/modules.txt b/vendor/modules.txt index dd23eca038..58b46a3d1e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1041,8 +1041,8 @@ github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 ## explicit github.com/pmezard/go-difflib/difflib -# github.com/portworx/kdmp v0.4.1-0.20230927104323-84564ac8b457 => github.com/portworx/kdmp v0.4.1-0.20231025110112-14cdb50c205e -## explicit; go 1.20 +# github.com/portworx/kdmp v0.4.1-0.20231123040148-68149a57dc24 => github.com/portworx/kdmp v0.4.1-0.20231123040148-68149a57dc24 +## explicit; go 1.21 github.com/portworx/kdmp/pkg/apis/kdmp github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1 github.com/portworx/kdmp/pkg/client/clientset/versioned @@ -2479,7 +2479,7 @@ sigs.k8s.io/yaml # github.com/libopenstorage/autopilot-api => github.com/libopenstorage/autopilot-api v0.6.1-0.20210301232050-ca2633c6e114 # github.com/libopenstorage/openstorage => github.com/libopenstorage/openstorage v0.0.0-20230511212757-41751b27d69f # github.com/libopenstorage/secrets => github.com/libopenstorage/secrets v0.0.0-20220413195519-57d1c446c5e9 -# github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20231025110112-14cdb50c205e +# github.com/portworx/kdmp => github.com/portworx/kdmp v0.4.1-0.20231123040148-68149a57dc24 # github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20230930052008-186afa1e82c4 # github.com/portworx/torpedo => github.com/portworx/torpedo v0.0.0-20231013194137-84e2957806f0 # gopkg.in/fsnotify.v1 v1.4.7 => github.com/fsnotify/fsnotify v1.4.7