Skip to content

Commit

Permalink
Add Snapshot Restore command (#947)
Browse files Browse the repository at this point in the history
Snapshot Restore generates commands to restore dumped snapshots to a Sourcegraph instance
  • Loading branch information
jac authored Mar 7, 2023
1 parent 7875b7d commit 37ff66f
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 37 deletions.
57 changes: 25 additions & 32 deletions cmd/src/snapshot_databases.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"flag"
"fmt"
"os"
"os/exec"
"strings"

"github.com/sourcegraph/sourcegraph/lib/errors"
Expand All @@ -18,7 +19,7 @@ func init() {
Note that these commands are intended for use as reference - you may need to adjust the commands for your deployment.
USAGE
src [-v] snapshot databases <pg_dump|docker|kubectl> [--targets=<docker|k8s|"targets.yaml">]
src [-v] snapshot databases [--targets=<docker|k8s|"targets.yaml">] [--run] <pg_dump|docker|kubectl>
TARGETS FILES
Predefined targets are available based on default Sourcegraph configurations ('docker', 'k8s').
Expand All @@ -38,6 +39,7 @@ TARGETS FILES
`
flagSet := flag.NewFlagSet("databases", flag.ExitOnError)
targetsKeyFlag := flagSet.String("targets", "auto", "predefined targets ('docker' or 'k8s'), or a custom targets.yaml file")
run := flagSet.Bool("run", false, "Automatically run the commands")

snapshotCommands = append(snapshotCommands, &command{
flagSet: flagSet,
Expand All @@ -47,35 +49,13 @@ TARGETS FILES
}
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})

var builder string
if len(args) > 0 {
builder = args[0]
}
builder := flagSet.Arg(0)

targetKey := "docker"
var commandBuilder pgdump.CommandBuilder
switch builder {
case "pg_dump", "":
targetKey = "local"
commandBuilder = func(t pgdump.Target) (string, error) {
cmd := pgdump.Command(t)
if t.Target != "" {
return fmt.Sprintf("%s --host=%s", cmd, t.Target), nil
}
return cmd, nil
}
case "docker":
commandBuilder = func(t pgdump.Target) (string, error) {
return fmt.Sprintf("docker exec -it %s sh -c '%s'", t.Target, pgdump.Command(t)), nil
}
case "kubectl":
targetKey = "k8s"
commandBuilder = func(t pgdump.Target) (string, error) {
return fmt.Sprintf("kubectl exec -it %s -- bash -c '%s'", t.Target, pgdump.Command(t)), nil
}
default:
commandBuilder, targetKey := pgdump.Builder(builder, pgdump.DumpCommand)
if targetKey == "" {
return errors.Newf("unknown or invalid template type %q", builder)
}

if *targetsKeyFlag != "auto" {
targetKey = *targetsKeyFlag
}
Expand All @@ -94,19 +74,32 @@ TARGETS FILES
out.WriteLine(output.Emojif(output.EmojiInfo, "Using predefined targets for %s environments", targetKey))
}

commands, err := pgdump.BuildCommands(srcSnapshotDir, commandBuilder, targets)
commands, err := pgdump.BuildCommands(srcSnapshotDir, commandBuilder, targets, true)
if err != nil {
return errors.Wrap(err, "failed to build commands")
}

_ = os.MkdirAll(srcSnapshotDir, os.ModePerm)

b := out.Block(output.Emoji(output.EmojiSuccess, "Run these commands to generate the required database dumps:"))
b.Write("\n" + strings.Join(commands, "\n"))
b.Close()
if *run {
for _, c := range commands {
out.WriteLine(output.Emojif(output.EmojiInfo, "Running command: %q", c))
command := exec.Command("bash", "-c", c)
output, err := command.CombinedOutput()
out.Write(string(output))
if err != nil {
return errors.Wrapf(err, "failed to run command: %q", c)
}
}

out.WriteLine(output.Styledf(output.StyleSuggestion, "Note that you may need to do some additional setup, such as authentication, beforehand."))
out.WriteLine(output.Emoji(output.EmojiSuccess, "Successfully completed dump commands"))
} else {
b := out.Block(output.Emoji(output.EmojiSuccess, "Run these commands to generate the required database dumps:"))
b.Write("\n" + strings.Join(commands, "\n"))
b.Close()

out.WriteLine(output.Styledf(output.StyleSuggestion, "Note that you may need to do some additional setup, such as authentication, beforehand."))
}
return nil
},
usageFunc: func() { fmt.Fprint(flag.CommandLine.Output(), usage) },
Expand Down
108 changes: 108 additions & 0 deletions cmd/src/snapshot_restore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package main

import (
"flag"
"fmt"
"os"
"os/exec"
"strings"

"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/output"
"gopkg.in/yaml.v3"

"github.com/sourcegraph/src-cli/internal/pgdump"
)

func init() {
usage := `'src snapshot restore' restores a Sourcegraph instance using Sourcegraph database dumps.
Note that these commands are intended for use as reference - you may need to adjust the commands for your deployment.
USAGE
src [-v] snapshot restore [--targets<docker|k8s|"targets.yaml">] [--run] <pg_dump|docker|kubectl>
TARGETS FILES
Predefined targets are available based on default Sourcegraph configurations ('docker', 'k8s').
Custom targets configuration can be provided in YAML format with '--targets=target.yaml', e.g.
primary:
target: ... # the DSN of the database deployment, e.g. in docker, the name of the database container
dbname: ... # name of database
username: ... # username for database access
password: ... # password for database access - only include password if it is non-sensitive
codeintel:
# same as above
codeinsights:
# same as above
See the pgdump.Targets type for more details.
`

flagSet := flag.NewFlagSet("restore", flag.ExitOnError)
targetsKeyFlag := flagSet.String("targets", "auto", "predefined targets ('docker' or 'k8s'), or a custom targets.yaml file")
run := flagSet.Bool("run", false, "Automatically run the commands")

snapshotCommands = append(snapshotCommands, &command{
flagSet: flagSet,
handler: func(args []string) error {
if err := flagSet.Parse(args); err != nil {
return err
}
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})

builder := flagSet.Arg(0)

commandBuilder, targetKey := pgdump.Builder(builder, pgdump.RestoreCommand)
if targetKey == "" {
return errors.Newf("unknown or invalid template type %q", builder)
}

if *targetsKeyFlag != "auto" {
targetKey = *targetsKeyFlag
}

targets, ok := predefinedDatabaseDumpTargets[targetKey]
if !ok {
out.WriteLine(output.Emojif(output.EmojiInfo, "Using targets defined in targets file %q", targetKey))
f, err := os.Open(targetKey)
if err != nil {
return errors.Wrapf(err, "invalid targets file %q", targetKey)
}
if err := yaml.NewDecoder(f).Decode(&targets); err != nil {
return errors.Wrapf(err, "invalid targets file %q", targetKey)
}
} else {
out.WriteLine(output.Emojif(output.EmojiInfo, "Using predefined targets for %s environments", targetKey))
}

commands, err := pgdump.BuildCommands(srcSnapshotDir, commandBuilder, targets, false)
if err != nil {
return errors.Wrap(err, "failed to build commands")
}
if *run {
for _, c := range commands {
out.WriteLine(output.Emojif(output.EmojiInfo, "Running command: %q", c))
command := exec.Command("bash", "-c", c)
output, err := command.CombinedOutput()
out.Write(string(output))
if err != nil {
return errors.Wrapf(err, "failed to run command: %q", c)
}
}

out.WriteLine(output.Emoji(output.EmojiSuccess, "Successfully completed restore commands"))
out.WriteLine(output.Styledf(output.StyleSuggestion, "It may be necessary to restart your Sourcegraph instance after restoring"))
} else {
b := out.Block(output.Emoji(output.EmojiSuccess, "Run these commands to restore the databases:"))
b.Write("\n" + strings.Join(commands, "\n"))
b.Close()

out.WriteLine(output.Styledf(output.StyleSuggestion, "Note that you may need to do some additional setup, such as authentication, beforehand."))
}

return nil
},

usageFunc: func() { fmt.Fprint(flag.CommandLine.Output(), usage) },
})
}
56 changes: 51 additions & 5 deletions internal/pgdump/pgdump.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,19 @@ type Target struct {
Password string `yaml:"password"`
}

// Command generates a pg_dump command that can be used for on-prem-to-Cloud migrations.
func Command(t Target) string {
dump := fmt.Sprintf("pg_dump --no-owner --format=p --no-acl --username=%s --dbname=%s",
// RestoreCommand generates a psql command that can be used for migrations.
func RestoreCommand(t Target) string {
dump := fmt.Sprintf("psql --username=%s --dbname=%s 1>/dev/null",
t.Username, t.DBName)
if t.Password == "" {
return dump
}
return fmt.Sprintf("PGPASSWORD=%s %s", t.Password, dump)
}

// DumpCommand generates a pg_dump command that can be used for on-prem-to-Cloud migrations.
func DumpCommand(t Target) string {
dump := fmt.Sprintf("pg_dump --no-owner --format=p --no-acl --clean --if-exists --username=%s --dbname=%s",
t.Username, t.DBName)
if t.Password == "" {
return dump
Expand Down Expand Up @@ -61,17 +71,53 @@ func Outputs(dir string, targets Targets) []Output {
}

type CommandBuilder func(Target) (string, error)
type PGCommand func(Target) string

// Builder generates the CommandBuilder and targetKey for a given builder and PGCommand
func Builder(builder string, command PGCommand) (commandBuilder CommandBuilder, targetKey string) {
switch builder {
case "pg_dump", "":
targetKey = "local"
commandBuilder = func(t Target) (string, error) {
cmd := command(t)
if t.Target != "" {
return fmt.Sprintf("%s --host=%s", cmd, t.Target), nil
}
return cmd, nil
}
case "docker":
targetKey = "docker"
commandBuilder = func(t Target) (string, error) {
return fmt.Sprintf("docker exec -i %s sh -c '%s'", t.Target, command(t)), nil
}
case "kubectl":
targetKey = "k8s"
commandBuilder = func(t Target) (string, error) {
return fmt.Sprintf("kubectl exec -i %s -- bash -c '%s'", t.Target, command(t)), nil
}
default:
return commandBuilder, targetKey
}
return commandBuilder, targetKey
}

// BuildCommands generates commands that output Postgres dumps and sends them to predefined
// files for each target database.
func BuildCommands(outDir string, commandBuilder CommandBuilder, targets Targets) ([]string, error) {
func BuildCommands(outDir string, commandBuilder CommandBuilder, targets Targets, dump bool) ([]string, error) {
var commands []string
for _, t := range Outputs(outDir, targets) {
c, err := commandBuilder(t.Target)
if err != nil {
return nil, errors.Wrapf(err, "generating command for %q", t.Output)
}
commands = append(commands, fmt.Sprintf("%s > %s", c, t.Output))

if dump {
// When dumping use output redirection to dump command stdout to target file
commands = append(commands, fmt.Sprintf("%s > %s", c, t.Output))
} else {
// When restoring use input redirection to pass target file to command stdin
commands = append(commands, fmt.Sprintf("%s < %s", c, t.Output))
}
}
return commands, nil
}

0 comments on commit 37ff66f

Please sign in to comment.