Kelp UI: improve ccxt download to show progress for better visibility (…

…#408)

* 1 - add github.com/cavaliercoder/grab as a dependency

* 2 - DownloadFileWithGrab and usage

* 3 - updated DownloadFileWithGrab to provide download progress details

* 4 - update error message on download failure

* 5 - accommodate a possible nil resp.HTTPResponse

* 6 - set req.NoResume = true so it does a fresh download to avoid corrupted files

* 7 - timer for calculating download speed should start immediately after respose received

* 8 - invokeUpdateHandler on download finished as well

cmd/server_amd64.go

@@ -53,6 +53,7 @@ const pingPlaceholder = "PING_URL"
 const sleepNumSecondsBeforeReadyString = 1
 const readyPlaceholder = "READY_STRING"
 const readyStringIndicator = "Serving frontend and API server on HTTP port"
+const downloadCcxtUpdateIntervalLogMillis = 1000
 
 type serverInputs struct {
 	port              *uint16
@@ -373,8 +374,30 @@ func downloadCcxtBinary(kos *kelpos.KelpOS, ccxtDirPath *kelpos.OSPath, ccxtZipD
 		return nil
 	}
 	downloadURL := fmt.Sprintf("%s/%s", ccxtDownloadBaseURL, filenameWithExt)
-	log.Printf("download ccxt from %s to location: %s", downloadURL, ccxtZipDownloadPath.AsString())
-	networking.DownloadFile(downloadURL, ccxtZipDownloadPath.Native())
+	log.Printf("download ccxt from %s to location: %s ...", downloadURL, ccxtZipDownloadPath.AsString())
+	e = networking.DownloadFileWithGrab(
+		downloadURL,
+		ccxtZipDownloadPath.Native(),
+		downloadCcxtUpdateIntervalLogMillis,
+		func(statusCode int, statusString string) {
+			log.Printf("  response_status = %s, code = %d\n", statusString, statusCode)
+		},
+		func(completedBytes float64, sizeBytes float64, speedBytesPerSec float64) {
+			log.Printf("  downloaded %.2f / %.2f MB (%.2f%%) at an average speed of %.2f MB/sec\n",
+				completedBytes,
+				sizeBytes,
+				100*(float64(completedBytes)/float64(sizeBytes)),
+				speedBytesPerSec,
+			)
+		},
+		func(filename string) {
+			log.Printf("  done\n")
+			log.Printf("... downloaded file from URL '%s' to destination '%s'\n", downloadURL, filename)
+		},
+	)
+	if e != nil {
+		return fmt.Errorf("could not download ccxt from '%s' to location '%s': %s", downloadURL, ccxtZipDownloadPath.AsString(), e)
+	}
 	return nil
 }
 

glide.yaml

@@ -1,5 +1,7 @@
 package: github.com/stellar/kelp
 import:
+- package: github.com/cavaliercoder/grab
+  version: 9f014744ee41e6bca139fe07601e65ac8b9c5109
 - package: github.com/Beldur/kraken-go-api-client
   version: 8d8ccfe4cc60d1703ffe596c889b887a318a884c
 - package: github.com/pkg/errors

support/networking/network.go

@@ -9,6 +9,9 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
+
+	"github.com/cavaliercoder/grab"
 )
 
 // JSONRequestDynamicHeaders submits an HTTP web request and parses the response into the responseData object as JSON
@@ -128,3 +131,63 @@ func DownloadFile(url string, filepath string) error {
 	}
 	return nil
 }
+
+// DownloadFileWithGrab is a download function that uses the grab third-party library
+func DownloadFileWithGrab(
+	url string,
+	filepath string,
+	updateIntervalMillis int,
+	statusCodeHandler func(statusCode int, statusString string),
+	updateHandler func(completedMB float64, sizeMB float64, speedMBPerSec float64),
+	finishHandler func(filename string),
+) error {
+	// create client
+	client := grab.NewClient()
+	req, e := grab.NewRequest(filepath, url)
+	if e != nil {
+		return fmt.Errorf("could not make new grab request: %s", e)
+	}
+	req.NoResume = true
+
+	// start download
+	resp := client.Do(req)
+	if resp.HTTPResponse != nil {
+		statusCodeHandler(resp.HTTPResponse.StatusCode, resp.HTTPResponse.Status)
+	} else {
+		statusCodeHandler(-1, "nil resp.HTTPResponse")
+	}
+	tic := time.Now().UnixNano()
+
+	// start UI loop
+	t := time.NewTicker(time.Duration(updateIntervalMillis) * time.Millisecond)
+	defer t.Stop()
+
+	invokeUpdateHandler := func() {
+		toc := time.Now().UnixNano()
+		timeElapsedSec := float64(toc-tic) / float64(time.Second)
+		mbCompleted := float64(resp.BytesComplete()) / 1024 / 1024
+		speedMBPerSec := float64(mbCompleted) / float64(timeElapsedSec)
+		sizeMB := float64(resp.Size()) / 1024 / 2014
+		updateHandler(mbCompleted, sizeMB, speedMBPerSec)
+	}
+Loop:
+	for {
+		select {
+		case <-t.C:
+			invokeUpdateHandler()
+
+		case <-resp.Done:
+			// download is complete
+			invokeUpdateHandler()
+			break Loop
+		}
+	}
+
+	// check for errors
+	if e = resp.Err(); e != nil {
+		return fmt.Errorf("error while downloading: %s", e)
+	}
+
+	finishHandler(resp.Filename)
+	return nil
+}