fix: switch to ecr for faster image pull (#364)

* fix: switch to ecr for faster image pull * fix: use mirrored 3rd party image * fix: docker run config * fix: progress bar should not go backwards * chore: configure viper in integration tests
supabase · Aug 15, 2022 · 7032d44 · 7032d44
1 parent 33cf0d2
commit 7032d44
Show file tree

Hide file tree

Showing 8 changed files with 76 additions and 105 deletions.
diff --git a/internal/db/remote/changes/changes.go b/internal/db/remote/changes/changes.go
@@ -99,25 +99,19 @@ func run(p utils.Program, ctx context.Context, username, password, database stri
 
 	// Pull images.
 	{
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.DbImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.DbImage,
-				types.ImagePullOptions{},
-			)
+		dbImage := utils.GetRegistryImageUrl(utils.DbImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, dbImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, dbImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.DifferImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.DifferImage,
-				types.ImagePullOptions{},
-			)
+		diffImage := utils.GetRegistryImageUrl(utils.DifferImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, diffImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, diffImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}

diff --git a/internal/db/remote/commit/commit.go b/internal/db/remote/commit/commit.go
@@ -115,25 +115,19 @@ func run(p utils.Program, ctx context.Context, username, password, database stri
 
 	// Pull images.
 	{
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.DbImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.DbImage,
-				types.ImagePullOptions{},
-			)
+		dbImage := utils.GetRegistryImageUrl(utils.DbImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, dbImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, dbImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.DifferImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.DifferImage,
-				types.ImagePullOptions{},
-			)
+		diffImage := utils.GetRegistryImageUrl(utils.DifferImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, diffImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, diffImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}

diff --git a/internal/gen/types/typescript/typescript.go b/internal/gen/types/typescript/typescript.go
@@ -75,13 +75,10 @@ func Run(useLocal bool, dbUrl string) error {
 
 		defer utils.DockerRemoveAll()
 
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.PgmetaImage); err != nil {
+		metaImage := utils.GetRegistryImageUrl(utils.PgmetaImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, metaImage); err != nil {
 			fmt.Fprintln(os.Stderr, "Downloading type generator...")
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.PgmetaImage,
-				types.ImagePullOptions{},
-			)
+			out, err := utils.Docker.ImagePull(ctx, metaImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}

diff --git a/internal/start/start.go b/internal/start/start.go
@@ -116,142 +116,109 @@ func run(p utils.Program) error {
 
 	// Pull images.
 	{
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.DbImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.DbImage,
-				types.ImagePullOptions{},
-			)
+		dbImage := utils.GetRegistryImageUrl(utils.DbImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, dbImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, dbImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.KongImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.KongImage,
-				types.ImagePullOptions{},
-			)
+		kongImage := utils.GetRegistryImageUrl(utils.KongImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, kongImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, kongImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.GotrueImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.GotrueImage,
-				types.ImagePullOptions{},
-			)
+		gotrueImage := utils.GetRegistryImageUrl(utils.GotrueImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, gotrueImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, gotrueImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.InbucketImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.InbucketImage,
-				types.ImagePullOptions{},
-			)
+		inbucketImage := utils.GetRegistryImageUrl(utils.InbucketImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, inbucketImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, inbucketImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.RealtimeImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.RealtimeImage,
-				types.ImagePullOptions{},
-			)
+		realtimeImage := utils.GetRegistryImageUrl(utils.RealtimeImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, realtimeImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, realtimeImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.PostgrestImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.PostgrestImage,
-				types.ImagePullOptions{},
-			)
+		restImage := utils.GetRegistryImageUrl(utils.PostgrestImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, restImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, restImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.StorageImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.StorageImage,
-				types.ImagePullOptions{},
-			)
+		storageImage := utils.GetRegistryImageUrl(utils.StorageImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, storageImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, storageImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.DifferImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.DifferImage,
-				types.ImagePullOptions{},
-			)
+		diffImage := utils.GetRegistryImageUrl(utils.DifferImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, diffImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, diffImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.PgmetaImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.PgmetaImage,
-				types.ImagePullOptions{},
-			)
+		metaImage := utils.GetRegistryImageUrl(utils.PgmetaImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, metaImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, metaImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.StudioImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.StudioImage,
-				types.ImagePullOptions{},
-			)
+		studioImage := utils.GetRegistryImageUrl(utils.StudioImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, studioImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, studioImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}
 			if err := utils.ProcessPullOutput(out, p); err != nil {
 				return err
 			}
 		}
-		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, "docker.io/"+utils.DenoRelayImage); err != nil {
-			out, err := utils.Docker.ImagePull(
-				ctx,
-				"docker.io/"+utils.DenoRelayImage,
-				types.ImagePullOptions{},
-			)
+		denoImage := utils.GetRegistryImageUrl(utils.DenoRelayImage)
+		if _, _, err := utils.Docker.ImageInspectWithRaw(ctx, denoImage); err != nil {
+			out, err := utils.Docker.ImagePull(ctx, denoImage, types.ImagePullOptions{})
 			if err != nil {
 				return err
 			}

diff --git a/internal/utils/api.go b/internal/utils/api.go
@@ -2,10 +2,10 @@ package utils
 
 import (
 	"log"
-	"os"
 	"sync"
 
 	"github.com/deepmap/oapi-codegen/pkg/securityprovider"
+	"github.com/spf13/viper"
 	supabase "github.com/supabase/cli/pkg/api"
 )
 
@@ -51,7 +51,7 @@ var RegionMap = map[string]string{
 }
 
 func GetSupabaseAPIHost() string {
-	apiHost := os.Getenv("SUPABASE_INTERNAL_API_HOST")
+	apiHost := viper.GetString("INTERNAL_API_HOST")
 	if apiHost == "" {
 		apiHost = "https://api.supabase.io"
 	}

diff --git a/internal/utils/container_output.go b/internal/utils/container_output.go
@@ -41,16 +41,14 @@ func ProcessPullOutput(out io.ReadCloser, p Program) error {
 				total:   progress.Progress.Total,
 			}
 
-			var sumCurrent, sumTotal int64
+			var overallProgress float64
 			for _, percentage := range downloads {
-				sumCurrent += percentage.current
-				sumTotal += percentage.total
+				if percentage.total > 0 {
+					progress := float64(percentage.current) / float64(percentage.total)
+					overallProgress += progress / float64(len(downloads))
+				}
 			}
 
-			var overallProgress float64
-			if sumTotal != 0 {
-				overallProgress = float64(sumCurrent) / float64(sumTotal)
-			}
 			p.Send(ProgressMsg(&overallProgress))
 		}
 	}

diff --git a/internal/utils/docker.go b/internal/utils/docker.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/docker/docker/api/types"
@@ -70,6 +71,7 @@ func DockerRun(
 	config *container.Config,
 	hostConfig *container.HostConfig,
 ) (io.Reader, error) {
+	config.Image = GetRegistryImageUrl(config.Image)
 	container, err := Docker.ContainerCreate(ctx, config, hostConfig, nil, nil, name)
 	if err != nil {
 		return nil, err
@@ -143,6 +145,20 @@ func DockerAddFile(ctx context.Context, container string, fileName string, conte
 	return nil
 }
 
+func GetRegistryImageUrl(imageName string) string {
+	const ecr = "public.ecr.aws/t3w2s2c9"
+	registry := viper.GetString("INTERNAL_IMAGE_REGISTRY")
+	if registry == "" {
+		// Defaults to Supabase public ECR for faster image pull
+		registry = ecr
+	}
+	if registry == ecr {
+		parts := strings.Split(imageName, "/")
+		imageName = parts[len(parts)-1]
+	}
+	return registry + "/" + imageName
+}
+
 func DockerPullImageIfNotCached(ctx context.Context, imageName string) error {
 	imageUrl := "docker.io/" + imageName
 	if _, _, err := Docker.ImageInspectWithRaw(ctx, imageUrl); err == nil {

diff --git a/test/main_test.go b/test/main_test.go
@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/docker/docker/client"
+	"github.com/spf13/viper"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/test/mocks/docker"
 	"github.com/supabase/cli/test/mocks/supabase"
@@ -42,6 +43,10 @@ func TestMain(m *testing.M) {
 	}
 	os.Setenv("SUPABASE_INTERNAL_API_HOST", "http://127.0.0.1"+SupabasePort)
 
+	// Configure viper env loader
+	viper.SetEnvPrefix("SUPABASE")
+	viper.AutomaticEnv()
+
 	// run tests
 	exitVal := m.Run()