From e7a1e427cec53cd38602ca91d298d7e060488023 Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Fri, 25 Nov 2022 16:19:27 -0800 Subject: [PATCH 1/5] snapshot: add pg_dump command builder with 'snapshot databases' --- cmd/src/snapshot_databases.go | 151 ++++++++++++++++++++++++++++++++++ internal/pgdump/pgdump.go | 64 ++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 cmd/src/snapshot_databases.go create mode 100644 internal/pgdump/pgdump.go diff --git a/cmd/src/snapshot_databases.go b/cmd/src/snapshot_databases.go new file mode 100644 index 0000000000..354eb61678 --- /dev/null +++ b/cmd/src/snapshot_databases.go @@ -0,0 +1,151 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/lib/output" + "gopkg.in/yaml.v3" + + "github.com/sourcegraph/src-cli/internal/pgdump" +) + +func init() { + usage := `'src snapshot databases' generates commands to export Sourcegraph database dumps. +Note that these commands are intended for use as reference - you may need to adjust the commands for your deployment. + +USAGE + src [-v] snapshot databases [--targets=] + +TARGETS FILES + Predefined targets are available based on default Sourcegraph configurations ('docker', 'k8s'). + Custom targets configuration can be provided in YAML format with '--targets=target.yaml', e.g. + + primary: + entity: ... + dbname: ... + username: ... + password: ... + codeintel: + # same as above + codeinsights: + # same as above +` + flagSet := flag.NewFlagSet("databases", flag.ExitOnError) + targetsKeyFlag := flagSet.String("targets", "auto", "predefined targets ('docker' or 'k8s'), or a custom targets.yaml file") + + snapshotCommands = append(snapshotCommands, &command{ + flagSet: flagSet, + handler: func(args []string) error { + if err := flagSet.Parse(args); err != nil { + return err + } + out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose}) + + var builder string + if len(args) > 0 { + builder = args[0] + } + + targetKey := "docker" + var commandBuilder pgdump.CommandBuilder + switch builder { + case "pg_dump", "": + commandBuilder = func(t pgdump.Target) (string, error) { + return pgdump.Command(t), nil + } + case "docker": + commandBuilder = func(t pgdump.Target) (string, error) { + // docker exec -it pgsql sh -c 'pg_dump --no-owner --format=p --no-acl --username=sg --dbname=sg' + return fmt.Sprintf("docker exec -it %s sh -c '%s'", t.Target, pgdump.Command(t)), nil + } + case "kubectl": + targetKey = "k8s" + commandBuilder = func(t pgdump.Target) (string, error) { + // docker exec -it pgsql sh -c 'pg_dump --no-owner --format=p --no-acl --username=sg --dbname=sg' + return fmt.Sprintf("kubectl exec -it %s -- bash -c '%s'", t.Target, pgdump.Command(t)), nil + } + default: + return errors.Newf("unknown or invalid template type %q", builder) + } + if *targetsKeyFlag != "auto" { + targetKey = *targetsKeyFlag + } + + targets, ok := predefinedDatabaseDumpTargets[targetKey] + if !ok { + out.WriteLine(output.Emojif(output.EmojiInfo, "Using targets defined in targets file %q", targetKey)) + f, err := os.Open(targetKey) + if err != nil { + return errors.Wrapf(err, "invalid targets file %q", targetKey) + } + if err := yaml.NewDecoder(f).Decode(&targets); err != nil { + return errors.Wrapf(err, "invalid targets file %q", targetKey) + } + } else { + out.WriteLine(output.Emojif(output.EmojiInfo, "Using predefined targets for %s environments", targetKey)) + } + + commands, err := pgdump.BuildCommands(commandBuilder, targets) + if err != nil { + return errors.Wrap(err, "failed to build commands") + } + + b := out.Block(output.Emoji(output.EmojiSuccess, "Commands generated - run them all to generate required database dumps:")) + b.Write("\n" + strings.Join(commands, "\n")) + b.Close() + + out.WriteLine(output.Styledf(output.StyleSuggestion, "Note that you may need to do some additional setup, such as authentication, beforehand.")) + + return nil + }, + usageFunc: func() { fmt.Fprint(flag.CommandLine.Output(), usage) }, + }) +} + +// predefinedDatabaseDumpTargets is based on default Sourcegraph configurations. +var predefinedDatabaseDumpTargets = map[string]pgdump.Targets{ + "docker": { // based on deploy-sourcegraph-managed + Primary: pgdump.Target{ + Target: "pgsql", + DBName: "sg", + Username: "sg", + Password: "sg", + }, + CodeIntel: pgdump.Target{ + Target: "codeintel-db", + DBName: "sg", + Username: "sg", + Password: "sg", + }, + CodeInsights: pgdump.Target{ + Target: "codeinsights-db", + DBName: "postgres", + Username: "postgres", + Password: "password", + }, + }, + "k8s": { // based on deploy-sourcegraph-helm + Primary: pgdump.Target{ + Target: "statefulset/pgsql", + DBName: "sg", + Username: "sg", + Password: "sg", + }, + CodeIntel: pgdump.Target{ + Target: "statefulset/codeintel-db", + DBName: "sg", + Username: "sg", + Password: "sg", + }, + CodeInsights: pgdump.Target{ + Target: "statefulset/codeinsights-db", + DBName: "postgres", + Username: "postgres", + Password: "password", + }, + }, +} diff --git a/internal/pgdump/pgdump.go b/internal/pgdump/pgdump.go new file mode 100644 index 0000000000..ccbb9a0c6f --- /dev/null +++ b/internal/pgdump/pgdump.go @@ -0,0 +1,64 @@ +package pgdump + +import ( + "fmt" + + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +// Targets represents configuration for each of Sourcegraph's databases. +type Targets struct { + Primary Target `yaml:"primary"` + CodeIntel Target `yaml:"codeintel"` + CodeInsights Target `yaml:"codeinsights"` +} + +// Target represents a database for pg_dump to export. +type Target struct { + // e.g. container, deployment, statefulset, etc. + Target string `yaml:"entity"` + + DBName string `yaml:"dbname"` + Username string `yaml:"username"` + + // Only include password if non-sensitive + Password string `yaml:"password"` +} + +// Command generates a pg_dump command that can be used for on-prem-to-Cloud migrations. +func Command(t Target) string { + dump := fmt.Sprintf("pg_dump --no-owner --format=p --no-acl --username=%s --dbname=%s", + t.Username, t.DBName) + if t.Password == "" { + return dump + } + return fmt.Sprintf("PGPASSWORD=%s %s", t.Password, dump) +} + +type CommandBuilder func(Target) (string, error) + +// BuildCommands generates commands that output Postgres dumps and sends them to predefined +// files for each target database. +func BuildCommands(commandBuilder CommandBuilder, targets Targets) ([]string, error) { + var commands []string + for _, t := range []struct { + Output string + Target Target + }{{ + Output: "primary.sql", + Target: targets.Primary, + }, { + Output: "codeintel.sql", + Target: targets.CodeIntel, + }, { + Output: "codeinsights.sql", + Target: targets.CodeInsights, + }} { + c, err := commandBuilder(t.Target) + if err != nil { + return nil, errors.Wrapf(err, "generating command for %q", t.Output) + } + commands = append(commands, fmt.Sprintf("%s > %s", c, t.Output)) + } + return commands, nil +} From f9fa6cb5b72f9290f11befdd55075451e6252722 Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Fri, 25 Nov 2022 16:30:37 -0800 Subject: [PATCH 2/5] remove bad docstrings --- cmd/src/snapshot_databases.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/src/snapshot_databases.go b/cmd/src/snapshot_databases.go index 354eb61678..984bfa39fd 100644 --- a/cmd/src/snapshot_databases.go +++ b/cmd/src/snapshot_databases.go @@ -59,13 +59,11 @@ TARGETS FILES } case "docker": commandBuilder = func(t pgdump.Target) (string, error) { - // docker exec -it pgsql sh -c 'pg_dump --no-owner --format=p --no-acl --username=sg --dbname=sg' return fmt.Sprintf("docker exec -it %s sh -c '%s'", t.Target, pgdump.Command(t)), nil } case "kubectl": targetKey = "k8s" commandBuilder = func(t pgdump.Target) (string, error) { - // docker exec -it pgsql sh -c 'pg_dump --no-owner --format=p --no-acl --username=sg --dbname=sg' return fmt.Sprintf("kubectl exec -it %s -- bash -c '%s'", t.Target, pgdump.Command(t)), nil } default: From 68e88e68d12dd15bda4e1a459b58881e06085114 Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Mon, 28 Nov 2022 10:43:47 -0800 Subject: [PATCH 3/5] Clarify target Co-authored-by: Michael Lin --- internal/pgdump/pgdump.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/pgdump/pgdump.go b/internal/pgdump/pgdump.go index ccbb9a0c6f..7627acfe5f 100644 --- a/internal/pgdump/pgdump.go +++ b/internal/pgdump/pgdump.go @@ -15,8 +15,11 @@ type Targets struct { // Target represents a database for pg_dump to export. type Target struct { - // e.g. container, deployment, statefulset, etc. - Target string `yaml:"entity"` + // Target is the DSN of the database deployment: + // + // - in docker, the name of the database container, e.g. pgsql, codeintel-db, codeinsights-db + // - in k8s, the name of the deployment or statefulset, e.g. deploy/pgsql, sts/pgsql + Target string `yaml:"target"` DBName string `yaml:"dbname"` Username string `yaml:"username"` From 6a01d315f9eb9d507ee31d02c7e9e70604f3b816 Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Mon, 28 Nov 2022 10:51:59 -0800 Subject: [PATCH 4/5] make target represent database host in template pg_dump --- cmd/src/snapshot_databases.go | 24 +++++++++++++++++++++++- internal/pgdump/pgdump.go | 1 + 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cmd/src/snapshot_databases.go b/cmd/src/snapshot_databases.go index 984bfa39fd..014841ac9c 100644 --- a/cmd/src/snapshot_databases.go +++ b/cmd/src/snapshot_databases.go @@ -54,8 +54,13 @@ TARGETS FILES var commandBuilder pgdump.CommandBuilder switch builder { case "pg_dump", "": + targetKey = "local" commandBuilder = func(t pgdump.Target) (string, error) { - return pgdump.Command(t), nil + cmd := pgdump.Command(t) + if t.Target != "" { + return fmt.Sprintf("%s --host=%s", cmd, t.Target), nil + } + return cmd, nil } case "docker": commandBuilder = func(t pgdump.Target) (string, error) { @@ -106,6 +111,23 @@ TARGETS FILES // predefinedDatabaseDumpTargets is based on default Sourcegraph configurations. var predefinedDatabaseDumpTargets = map[string]pgdump.Targets{ + "local": { + Primary: pgdump.Target{ + DBName: "sg", + Username: "sg", + Password: "sg", + }, + CodeIntel: pgdump.Target{ + DBName: "sg", + Username: "sg", + Password: "sg", + }, + CodeInsights: pgdump.Target{ + DBName: "postgres", + Username: "postgres", + Password: "password", + }, + }, "docker": { // based on deploy-sourcegraph-managed Primary: pgdump.Target{ Target: "pgsql", diff --git a/internal/pgdump/pgdump.go b/internal/pgdump/pgdump.go index 7627acfe5f..60e9bddd58 100644 --- a/internal/pgdump/pgdump.go +++ b/internal/pgdump/pgdump.go @@ -19,6 +19,7 @@ type Target struct { // // - in docker, the name of the database container, e.g. pgsql, codeintel-db, codeinsights-db // - in k8s, the name of the deployment or statefulset, e.g. deploy/pgsql, sts/pgsql + // - in plain pg_dump, the server host or socket directory Target string `yaml:"target"` DBName string `yaml:"dbname"` From 1af05275e9010a828be9b07d758446d1d6161feb Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Mon, 28 Nov 2022 13:35:58 -0800 Subject: [PATCH 5/5] improve cmd docs --- cmd/src/snapshot_databases.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/src/snapshot_databases.go b/cmd/src/snapshot_databases.go index 014841ac9c..1a0a618315 100644 --- a/cmd/src/snapshot_databases.go +++ b/cmd/src/snapshot_databases.go @@ -25,14 +25,16 @@ TARGETS FILES Custom targets configuration can be provided in YAML format with '--targets=target.yaml', e.g. primary: - entity: ... - dbname: ... - username: ... - password: ... + target: ... # the DSN of the database deployment, e.g. in docker, the name of the database container + dbname: ... # name of database + username: ... # username for database access + password: ... # password for database access - only include password if it is non-sensitive codeintel: # same as above codeinsights: # same as above + + See the pgdump.Targets type for more details. ` flagSet := flag.NewFlagSet("databases", flag.ExitOnError) targetsKeyFlag := flagSet.String("targets", "auto", "predefined targets ('docker' or 'k8s'), or a custom targets.yaml file")