diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..aa17bf2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,66 @@ +version: '3.0' + +services: + pg_without_label: + image: postgres:latest + environment: + POSTGRES_PASSWORD: test + POSTGRES_DB: test + + pg_dump_disabled: + image: postgres:latest + environment: + POSTGRES_PASSWORD: test + POSTGRES_DB: test + labels: + - 'go.dumper.enabled=false' + - 'go.dumper.databaseUser=postgres' + - 'go.dumper.databaseType=pg' + + pg_dump_1: + image: postgres:latest + environment: + POSTGRES_PASSWORD: test + POSTGRES_DB: test + labels: + - 'go-dumper.enabled=true' + - 'go-dumper.user=postgres' + - 'go-dumper.type=pg' + + pg_dump_2: + image: postgres:latest + environment: + POSTGRES_PASSWORD: test + POSTGRES_DB: test + labels: + - 'go.dumper.enabled=true' + - 'go.dumper.databaseUser=postgres' + - 'go.dumper.databaseType=pg' + + mysql_dump: + image: mysql:latest + command: --default-authentication-plugin=mysql_native_password + environment: + MYSQL_DATABASE: 'db' + MYSQL_USER: 'user' + MYSQL_ROOT_PASSWORD: root + labels: + - 'go.dumper.enabled=true' + - 'go.dumper.databaseUser=root' + - 'go.dumper.databasePassword=root' + - 'go.dumper.databaseName=db' + - 'go.dumper.databaseType=mysql' + + minio: + image: minio/minio + entrypoint: sh + command: -c 'mkdir -p ./data/s3/default && minio server ./data/s3 -console-address ":9001"' + environment: + - MINIO_REGION=eu-west-3 + - MINIO_ACCESS_KEY=2OKVH3X1NLIGU9RA4VER + - MINIO_SECRET_KEY=RSrrhXZ6LTejtHyhgZfilWO3On1ltOU8YI1B4Act + ports: + - 9000:9000 + - 9001:9001 + volumes: + - ./minio:/data/s3 diff --git a/dumper/common.go b/dumper/common.go new file mode 100644 index 0000000..50376ca --- /dev/null +++ b/dumper/common.go @@ -0,0 +1,40 @@ +package dumper + +import "github.com/docker/docker/api/types" + +func ExtractDumpOptionsFromLabels(labels map[string]string) DumpOptions { + return DumpOptions{ + Enabled: labels["go-dumper.enabled"] == "true", + User: labels["go-dumper.user"], + Type: labels["go-dumper.type"], + } +} + +func BuildContainerDumpCommandArgs(containerId string, dumpOptions DumpOptions) []string { + args := []string{"exec", containerId} + + if dumpOptions.Type == PG { + args = append(args, + "pg_dumpall", + "-U", dumpOptions.User, + "--if-exists", + "-c", + ) + } + + return args +} + +func FindContainersByTypes(containers []types.Container, containerType string) []types.Container { + containerToDump := make([]types.Container, 0) + + for _, container := range containers { + containerDumpConfig := ExtractDumpOptionsFromLabels(container.Labels) + + if containerDumpConfig.Enabled && containerDumpConfig.Type == containerType { + containerToDump = append(containerToDump, container) + } + } + + return containerToDump +} diff --git a/dumper/common_test.go b/dumper/common_test.go new file mode 100644 index 0000000..0ffd249 --- /dev/null +++ b/dumper/common_test.go @@ -0,0 +1,43 @@ +package dumper_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/Thomascogez/dump-and-dumper/dumper" +) + +func TestExtractDumpConfigFromLabels(t *testing.T) { + + labels := make(map[string]string) + + labels["go-dumper.enabled"] = "true" + labels["go-dumper.user"] = "user" + labels["go-dumper.type"] = "pg" + + dumpConfig := dumper.ExtractDumpOptionsFromLabels(labels) + + if !dumpConfig.Enabled || dumpConfig.Type != "pg" || dumpConfig.User != "user" { + t.Fail() + } +} + +func TestBuildContainerDumpCommandArgs(t *testing.T) { + containerDumpConfig := dumper.DumpOptions{ + Enabled: true, + User: "postgres", + Type: "pg", + } + + testContainerId := "containerId" + + pgContainerDumpArgs := dumper.BuildContainerDumpCommandArgs(testContainerId, containerDumpConfig) + + argsString := strings.Join(pgContainerDumpArgs[:], " ") + + if argsString != fmt.Sprintf("exec %s pg_dumpall -U %s --if-exists -c", testContainerId, containerDumpConfig.User) { + t.Fail() + } + +} diff --git a/dumper/pgDumper.go b/dumper/pgDumper.go new file mode 100644 index 0000000..7bacc9c --- /dev/null +++ b/dumper/pgDumper.go @@ -0,0 +1,39 @@ +package dumper + +import ( + "os" + "os/exec" + "sync" + + "github.com/Thomascogez/dump-and-dumper/helpers" + "github.com/docker/docker/api/types" +) + +type PgDumper struct{} + +func (pgDumper PgDumper) Dump(containers []types.Container) { + + var wg sync.WaitGroup + + for _, container := range containers { + wg.Add(1) + + go func(container types.Container) { + dumpConfig := ExtractDumpOptionsFromLabels(container.Labels) + dumpCommandArgs := BuildContainerDumpCommandArgs(container.ID, dumpConfig) + + tempDumpFile, tempDumpFolderPath, tempDumpFileName := helpers.CreateTempDumpFile() + + dumpCommand := exec.Command("docker", dumpCommandArgs...) + dumpCommand.Stdout = tempDumpFile + dumpCommand.Run() + + println(tempDumpFileName) + tempDumpFile.Close() + os.RemoveAll(tempDumpFolderPath) + wg.Done() + }(container) + } + + wg.Wait() +} diff --git a/dumper/types.go b/dumper/types.go new file mode 100644 index 0000000..a9facc4 --- /dev/null +++ b/dumper/types.go @@ -0,0 +1,17 @@ +package dumper + +import "github.com/docker/docker/api/types" + +type Dumper interface { + Dump(container *types.Container) +} + +const ( + PG = "pg" +) + +type DumpOptions struct { + Enabled bool + User string + Type string +} diff --git a/go.sum b/go.sum index bf16179..1c479c0 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -54,6 +55,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/helpers/common.go b/helpers/common.go new file mode 100644 index 0000000..19043ad --- /dev/null +++ b/helpers/common.go @@ -0,0 +1,7 @@ +package helpers + +func CheckError(err error) { + if err != nil { + panic(err.Error()) + } +} diff --git a/helpers/file.go b/helpers/file.go new file mode 100644 index 0000000..02eff8d --- /dev/null +++ b/helpers/file.go @@ -0,0 +1,31 @@ +package helpers + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "time" +) + +func FileNameFromCurrentTimestamp(fileExtension string) string { + return fmt.Sprintf("%d.%s", time.Now().UnixMilli(), fileExtension) +} + +func CreateTempDumpFile() (*os.File, string, string) { + cwd, err := os.Getwd() + CheckError(err) + + tempDumpFolderPath, err := ioutil.TempDir(cwd, "dump-") + + tempDumpFileName := FileNameFromCurrentTimestamp("sql") + tempDumpFilePath := path.Join(tempDumpFolderPath, tempDumpFileName) + tempDumpFile, err := os.Create(tempDumpFilePath) + + if err != nil { + os.RemoveAll(tempDumpFolderPath) + CheckError(err) + } + + return tempDumpFile, tempDumpFolderPath, tempDumpFileName +} diff --git a/main.go b/main.go index 7905807..e439e6f 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,25 @@ package main +import ( + "context" + + "github.com/Thomascogez/dump-and-dumper/dumper" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + func main() { + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + panic(err) + } + + containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{}) + if err != nil { + panic(err) + } + + pgContainersToDump := dumper.FindContainersByTypes(containers, dumper.PG) + dumper.PgDumper{}.Dump(pgContainersToDump) }