Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add an option to download from remote locations #122

Merged
merged 2 commits into from
May 6, 2024
Merged

Add an option to download from remote locations #122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0fdabee
Add an option to download from remote locations
orgrim May 5, 2024
4b39719
Add an option to list files from remote locations
orgrim May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ options of the PostgreSQL instance.
* Purge based on age and number of dumps to keep
* Dump from a hot standby by pausing replication replay
* Encrypt and decrypt dumps and other files
* Upload dumps to S3, GCS, Azure or a remote host with SFTP
* Upload and download dumps to S3, GCS, Azure or a remote host with SFTP

## Install

Expand Down Expand Up @@ -213,6 +213,23 @@ on the remote location as the local directory.
When files are encrypted and their unencrypted source is kept, only encrypted
files are uploaded.

### Downloading from remote locations

Previously uploaded files can be downloaded using the `--download` option with
a value different than `none`, similarly to `--upload`. The options to setup
the remote access are the same as `--upload`.

It is possible to only list remote files with `--list-remote` with a value
different than `none`, similarly to `--upload` and `--download`.

When listing or downloading files, dumps are not performed. Arguments on the
commandline (database names when dumping) are used as shell globs to
select/filter files.

If `--download` is used at the same time as `--decrypt`, files are downloaded
first, then files matching globs are decrypted.


## Restoring files

The following files are created:
Expand Down
54 changes: 37 additions & 17 deletions config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ type options struct {
DumpOnly bool

Upload string // values are none, s3, sftp, gcs
Download string // values are none, s3, sftp, gcs
ListRemote string // values are none, s3, sftp, gcs
PurgeRemote bool
S3Region string
S3Bucket string
Expand Down Expand Up @@ -132,6 +134,8 @@ func defaultOptions() options {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
}
}
Expand Down Expand Up @@ -284,6 +288,8 @@ func parseCli(args []string) (options, []string, error) {
pflag.StringVar(&opts.CipherPrivateKey, "cipher-private-key", "", "AGE private key for decryption; in Bech32 encoding starting with 'AGE-SECRET-KEY-1'\n")

pflag.StringVar(&opts.Upload, "upload", "none", "upload produced files to target (s3, gcs,..) use \"none\" to override\nconfiguration file and disable upload")
pflag.StringVar(&opts.Download, "download", "none", "download files from target (s3, gcs,..) instead of dumping. DBNAMEs become\nglobs to select files")
pflag.StringVar(&opts.ListRemote, "list-remote", "none", "list the remote files on s3, gcs, sftp, azure instead of dumping. DBNAMEs become\nglobs to select files")
purgeRemote := pflag.String("purge-remote", "no", "purge the file on remote location after upload, with the same rules\nas the local directory")

pflag.StringVar(&opts.S3Region, "s3-region", "", "S3 region")
Expand Down Expand Up @@ -438,35 +444,45 @@ func parseCli(args []string) (options, []string, error) {
}
}

// Validate upload option
// Validate upload and download options
stores := []string{"none", "s3", "sftp", "gcs", "azure"}
if err := validateEnum(opts.Upload, stores); err != nil {
return opts, changed, fmt.Errorf("invalid value for --upload: %s", err)
}

if err := validateEnum(opts.Download, stores); err != nil {
return opts, changed, fmt.Errorf("invalid value for --download: %s", err)
}

if err := validateEnum(opts.ListRemote, stores); err != nil {
return opts, changed, fmt.Errorf("invalid value for --list-remote: %s", err)
}

opts.PurgeRemote, err = validateYesNoOption(*purgeRemote)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --purge-remote: %s", err)
}

switch opts.Upload {
case "s3":
// Validate S3 options
opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err)
}
for _, o := range []string{opts.Upload, opts.Download, opts.ListRemote} {
switch o {
case "s3":
// Validate S3 options
opts.S3ForcePath, err = validateYesNoOption(*S3ForcePath)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-force-path: %s", err)
}

S3WithTLS, err := validateYesNoOption(*S3UseTLS)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err)
}
opts.S3DisableTLS = !S3WithTLS
S3WithTLS, err := validateYesNoOption(*S3UseTLS)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --s3-tls: %s", err)
}
opts.S3DisableTLS = !S3WithTLS

case "sftp":
opts.SFTPIgnoreKnownHosts, err = validateYesNoOption(*SFTPIgnoreHostKey)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --sftp-ignore-hostkey: %s", err)
case "sftp":
opts.SFTPIgnoreKnownHosts, err = validateYesNoOption(*SFTPIgnoreHostKey)
if err != nil {
return opts, changed, fmt.Errorf("invalid value for --sftp-ignore-hostkey: %s", err)
}
}
}

Expand Down Expand Up @@ -816,6 +832,10 @@ func mergeCliAndConfigOptions(cliOpts options, configOpts options, onCli []strin

case "upload":
opts.Upload = cliOpts.Upload
case "download":
opts.Download = cliOpts.Download
case "list-remote":
opts.ListRemote = cliOpts.ListRemote
case "purge-remote":
opts.PurgeRemote = cliOpts.PurgeRemote

Expand Down
55 changes: 55 additions & 0 deletions config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,8 @@ func TestDefaultOptions(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
}

Expand Down Expand Up @@ -238,6 +240,8 @@ func TestParseCli(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -262,6 +266,8 @@ func TestParseCli(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand Down Expand Up @@ -311,13 +317,42 @@ func TestParseCli(t *testing.T) {
CipherPassphrase: "testpass",
WithRolePasswords: true,
Upload: "wrong",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
false,
"invalid value for --upload: value not found in [none s3 sftp gcs azure]",
"",
},
{
[]string{"--download", "wrong"},
options{
Directory: "/var/backups/postgresql",
Format: 'c',
DirJobs: 1,
CompressLevel: -1,
Jobs: 1,
PauseTimeout: 3600,
PurgeInterval: -30 * 24 * time.Hour,
PurgeKeep: 0,
SumAlgo: "none",
CfgFile: "/etc/pg_back/pg_back.conf",
TimeFormat: timeFormat,
Encrypt: true,
CipherPassphrase: "testpass",
WithRolePasswords: true,
Upload: "none",
Download: "wrong",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
false,
"invalid value for --download: value not found in [none s3 sftp gcs azure]",
"",
},
{
[]string{"--decrypt", "--encrypt"},
defaults,
Expand All @@ -344,6 +379,8 @@ func TestParseCli(t *testing.T) {
CipherPassphrase: "mypass",
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -369,6 +406,8 @@ func TestParseCli(t *testing.T) {
CipherPrivateKey: "mykey",
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand All @@ -394,6 +433,8 @@ func TestParseCli(t *testing.T) {
CipherPublicKey: "fakepubkey",
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
false,
Expand Down Expand Up @@ -498,6 +539,8 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -519,6 +562,8 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -539,6 +584,8 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand All @@ -559,6 +606,8 @@ func TestLoadConfigurationFile(t *testing.T) {
TimeFormat: "2006-01-02_15-04-05",
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -608,6 +657,8 @@ func TestLoadConfigurationFile(t *testing.T) {
}},
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -649,6 +700,8 @@ func TestLoadConfigurationFile(t *testing.T) {
}},
WithRolePasswords: false,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
},
},
Expand Down Expand Up @@ -715,6 +768,8 @@ func TestMergeCliAndConfigoptions(t *testing.T) {
TimeFormat: timeFormat,
WithRolePasswords: true,
Upload: "none",
Download: "none",
ListRemote: "none",
AzureEndpoint: "blob.core.windows.net",
}

Expand Down
Loading
Loading