[cpackget] Encoded progress, when tool is called from other tools

cmd/commands/checksum.go

@@ -56,7 +56,7 @@ might be supported. The used function will be prefixed to the ".checksum" extens
 
 By default the checksum file will be created in the same directory as the provided pack.`,
 	Args:              cobra.ExactArgs(1),
-	PersistentPreRunE: configureInstallerVerbose,
+	PersistentPreRunE: configureInstallerGlobalCmd,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return cryptography.GenerateChecksum(args[0], checksumCreateCmdFlags.outputDir, checksumCreateCmdFlags.hashAlgorithm)
 	},
@@ -75,7 +75,7 @@ The used hash function is inferred from the checksum filename, and if any of the
 computed doesn't match the one provided in the checksum file an error will be thrown.
 If the .checksum file is in another directory, specify it with the -p/--path flag`,
 	Args:              cobra.ExactArgs(1),
-	PersistentPreRunE: configureInstallerVerbose,
+	PersistentPreRunE: configureInstallerGlobalCmd,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if checksumVerifyCmdFlags.checksumPath != "" {
 			return cryptography.VerifyChecksum(args[0], checksumVerifyCmdFlags.checksumPath)

cmd/commands/root.go

@@ -42,7 +42,7 @@ const defaultPublicIndex = "https://www.keil.com/pack/index.pidx"
 
 var viper *viperType.Viper
 
-func configureInstallerVerbose(cmd *cobra.Command, args []string) error {
+func configureInstallerGlobalCmd(cmd *cobra.Command, args []string) error {
 	verbosiness := viper.GetBool("verbose")
 	quiet := viper.GetBool("quiet")
 	if quiet && verbosiness {
@@ -60,12 +60,15 @@ func configureInstallerVerbose(cmd *cobra.Command, args []string) error {
 		log.SetLevel(log.DebugLevel)
 	}
 
+	encodedProgress := viper.GetBool("encoded-progress")
+	utils.SetEncodedProgress(encodedProgress)
+
 	return nil
 }
 
 // configureInstaller configures cpackget installer for adding or removing pack/pdsc
 func configureInstaller(cmd *cobra.Command, args []string) error {
-	err := configureInstallerVerbose(cmd, args)
+	err := configureInstallerGlobalCmd(cmd, args)
 	if err != nil {
 		return err
 	}
@@ -173,6 +176,7 @@ func NewCli() *cobra.Command {
 	rootCmd.Flags().BoolVarP(&flags.version, "version", "V", false, "Prints the version number of cpackget and exit")
 	rootCmd.PersistentFlags().BoolP("quiet", "q", false, "Run cpackget silently, printing only error messages")
 	rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Sets verboseness level: None (Errors + Info + Warnings), -v (all + Debugging). Specify \"-q\" for no messages")
+	rootCmd.PersistentFlags().BoolP("encoded-progress", "E", false, "Reports encoded progress for files and download when used by other tools")
 	rootCmd.PersistentFlags().StringP("pack-root", "R", defaultPackRoot, "Specifies pack root folder. Defaults to CMSIS_PACK_ROOT environment variable")
 	rootCmd.PersistentFlags().UintP("concurrent-downloads", "C", 5, "Number of concurrent batch downloads. Set to 0 to disable concurrency")
 	rootCmd.PersistentFlags().UintP("timeout", "T", 0, "Set maximum duration (in seconds) of a download. Disabled by default")
@@ -181,6 +185,7 @@ func NewCli() *cobra.Command {
 	_ = viper.BindPFlag("pack-root", rootCmd.PersistentFlags().Lookup("pack-root"))
 	_ = viper.BindPFlag("verbose", rootCmd.PersistentFlags().Lookup("verbose"))
 	_ = viper.BindPFlag("quiet", rootCmd.PersistentFlags().Lookup("quiet"))
+	_ = viper.BindPFlag("encoded-progress", rootCmd.PersistentFlags().Lookup("encoded-progress"))
 
 	for _, cmd := range AllCommands {
 		rootCmd.AddCommand(cmd)

cmd/commands/signature.go

@@ -97,7 +97,7 @@ The referenced pack must be in its original/compressed form (.pack), and be pres
 
   $ cpackget signature-create Vendor.Pack.1.2.3.pack -k private.key -c certificate.pem`,
 	Args:              cobra.ExactArgs(1),
-	PersistentPreRunE: configureInstallerVerbose,
+	PersistentPreRunE: configureInstallerGlobalCmd,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if signatureCreateflags.keyPath == "" {
 			if !signatureCreateflags.certOnly {
@@ -154,7 +154,7 @@ The referenced pack must be in its original/compressed form (.pack), and be pres
 
   $ cpackget signature-verify Vendor.Pack.1.2.3.pack.signed`,
 	Args:              cobra.ExactArgs(1),
-	PersistentPreRunE: configureInstallerVerbose,
+	PersistentPreRunE: configureInstallerGlobalCmd,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if signatureVerifyflags.export && (signatureVerifyflags.skipCertValidation || signatureVerifyflags.skipInfo) {
 			log.Error("-e/--export does not need any other flags")

cmd/installer/pack.go

@@ -297,13 +297,23 @@ func (p *PackType) install(installation *PacksInstallationType, checkEula bool)
 	// as it cleans the stdout buffer
 	interactiveTerminal := utils.IsTerminalInteractive()
 	var progress *progressbar.ProgressBar
-	if interactiveTerminal && log.GetLevel() != log.ErrorLevel {
-		progress = progressbar.Default(int64(len(p.zipReader.File)), "I:")
+	var encodedProgress *utils.EncodedProgress
+
+	if utils.GetEncodedProgress() {
+		encodedProgress = utils.NewEncodedProgress(int64(len(p.zipReader.File)), 0, p.path)
+	} else {
+		if interactiveTerminal && log.GetLevel() != log.ErrorLevel {
+			progress = progressbar.Default(int64(len(p.zipReader.File)), "I:")
+		}
 	}
 
 	for _, file := range p.zipReader.File {
 		if interactiveTerminal && log.GetLevel() != log.ErrorLevel {
-			_ = progress.Add64(1)
+			if utils.GetEncodedProgress() {
+				_ = encodedProgress.Add(1)
+			} else {
+				_ = progress.Add64(1)
+			}
 		}
 		err = utils.SecureInflateFile(file, packHomeDir, p.Subfolder)
 		if err != nil {

cmd/utils/encodedProgress.go

@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/* Copyright Contributors to the cpackget project. */
+
+package utils
+
+import (
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type EncodedProgress struct {
+	mu             sync.Mutex
+	total          int64
+	current        int
+	currentPercent int
+	instanceNo     int
+	name           string
+}
+
+func NewEncodedProgress(max int64, instNo int, filename string) *EncodedProgress {
+	return &EncodedProgress{
+		total:      max,
+		instanceNo: instNo,
+		name:       filename,
+	}
+}
+
+func (p *EncodedProgress) Add(count int) int {
+	p.mu.Lock()
+	newCount := count
+	p.current += newCount
+	p.Print()
+	p.mu.Unlock()
+	return newCount
+}
+
+func (p *EncodedProgress) Write(bs []byte) (int, error) {
+	p.mu.Lock()
+	newCount := len(bs)
+	p.current += newCount
+	p.Print()
+	p.mu.Unlock()
+	return newCount, nil
+}
+
+/* Encodes information to show progress when called by GUI or other tools
+ * I: Instance number (always counts up), connected to the filename
+ * F: Filename currently processed
+ * T: Total bytes of file or numbers of files
+ * P: Currently processed percentage
+ * C: Currently processed bytes or numbers of files
+ */
+func (p *EncodedProgress) Print() {
+	newPercent := int(float64(p.current) / float64(p.total) * 100)
+	if p.currentPercent != newPercent {
+		if p.currentPercent == 0 {
+			log.Infof("[I%d:F%s,T%d,P%d]", p.instanceNo, p.name, p.total, newPercent)
+		} else {
+			log.Infof("[I%d:P%d,C%d]", p.instanceNo, newPercent, p.current)
+		}
+		p.currentPercent = newPercent
+	}
+}

cmd/utils/encodedProgress_test.go

@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/* Copyright Contributors to the cpackget project. */
+
+package utils_test
+
+import (
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/open-cmsis-pack/cpackget/cmd/utils"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+var gText string
+
+// LogCapturer reroutes testing.T log output
+type LogCapturer interface {
+	Release()
+}
+
+type logCapturer struct {
+	*testing.T
+	origOut io.Writer
+}
+
+func (tl logCapturer) Write(p []byte) (n int, err error) {
+	//tl.Logf((string)(p))
+	gText += string(p)
+	return len(p), nil
+}
+
+func (tl logCapturer) Release() {
+	logrus.SetOutput(tl.origOut)
+}
+
+// CaptureLog redirects logrus output to testing.Log
+func CaptureLog(t *testing.T) LogCapturer {
+	lc := logCapturer{T: t, origOut: logrus.StandardLogger().Out}
+	if !testing.Verbose() {
+		logrus.SetOutput(lc)
+	}
+	gText = ""
+	return &lc
+}
+
+func Add() {
+	maxCnt := int64(10)
+	progressWriter := utils.NewEncodedProgress(maxCnt, 0, "Test progressbar")
+
+	for i := int64(0); i < maxCnt; i++ {
+		progressWriter.Add(1)
+	}
+
+}
+
+func TestEncodedProgres(t *testing.T) {
+	assert := assert.New(t)
+
+	t.Run("test encoded progress", func(t *testing.T) {
+		Log := CaptureLog(t)
+		defer Log.Release()
+
+		length := int64(10)
+		instCnt := 0
+		fileBase := "Testing"
+		progressWriter := utils.NewEncodedProgress(length, instCnt, fileBase)
+
+		for i := int64(0); i < length; i++ {
+			progressWriter.Add(1)
+		}
+
+		assert.True(gText == "I: [I0:FTesting,T10,P10]\nI: [I0:P20,C2]\nI: [I0:P30,C3]\nI: [I0:P40,C4]\nI: [I0:P50,C5]\nI: [I0:P60,C6]\nI: [I0:P70,C7]\nI: [I0:P80,C8]\nI: [I0:P90,C9]\nI: [I0:P100,C10]\n")
+	})
+
+	t.Run("test encoded progress with write interface", func(t *testing.T) {
+		Log := CaptureLog(t)
+		defer Log.Release()
+
+		text := "ProgressWriter: Write interface"
+		length := int64(len(text))
+		instCnt := 0
+		fileBase := "Testing"
+		progressWriter := utils.NewEncodedProgress(length, instCnt, fileBase)
+
+		fmt.Fprint(progressWriter, text)
+
+		assert.True(gText == "I: [I0:FTesting,T31,P100]\n")
+	})
+
+}

cmd/utils/utils.go

@@ -9,6 +9,7 @@ import (
 	"encoding/base64"
 	"encoding/xml"
 	"errors"
+	"fmt"
 	"io"
 	"io/fs"
 	"math"
@@ -29,10 +30,22 @@ import (
 	"golang.org/x/net/html/charset"
 )
 
+var gEncodedProgress = false
+
+func SetEncodedProgress(encodedProgress bool) {
+	gEncodedProgress = encodedProgress
+}
+
+func GetEncodedProgress() bool {
+	return gEncodedProgress
+}
+
 // CacheDir is used for cpackget to temporarily host downloaded pack files
 // before moving it to CMSIS_PACK_ROOT
 var CacheDir string
 
+var instCnt = 0
+
 var HTTPClient *http.Client
 
 type TimeoutTransport struct {
@@ -139,15 +152,22 @@ func DownloadFile(URL string, timeout int) (string, error) {
 	log.Infof("Downloading %s...", fileBase)
 	writers := []io.Writer{out}
 	if log.GetLevel() != log.ErrorLevel {
-		if IsTerminalInteractive() {
-			length := resp.ContentLength
-			progressWriter := progressbar.DefaultBytes(length, "I:")
+		length := resp.ContentLength
+		if GetEncodedProgress() {
+			progressWriter := NewEncodedProgress(length, instCnt, fileBase)
 			writers = append(writers, progressWriter)
+			instCnt++
+		} else {
+			if IsTerminalInteractive() {
+				progressWriter := progressbar.DefaultBytes(length, "I:")
+				writers = append(writers, progressWriter)
+			}
 		}
 	}
 
 	// Download file in smaller bits straight to a local file
 	written, err := SecureCopy(io.MultiWriter(writers...), resp.Body)
+	fmt.Printf("\n")
 	log.Debugf("Downloaded %d bytes", written)
 
 	if err != nil {