Support multipart upload

artifactory/commands/generic/upload.go

@@ -200,7 +200,7 @@ func (uc *UploadCommand) upload() (err error) {
 func getMinChecksumDeploySize() (int64, error) {
 	minChecksumDeploySize := os.Getenv("JFROG_CLI_MIN_CHECKSUM_DEPLOY_SIZE_KB")
 	if minChecksumDeploySize == "" {
-		return 10240, nil
+		return services.DefaultMinChecksumDeploy, nil
 	}
 	minSize, err := strconv.ParseInt(minChecksumDeploySize, 10, 64)
 	err = errorutils.CheckError(err)
@@ -218,6 +218,8 @@ func getUploadParams(f *spec.File, configuration *utils.UploadConfiguration, bui
 	}
 	uploadParams.Deb = configuration.Deb
 	uploadParams.MinChecksumDeploy = configuration.MinChecksumDeploySize
+	uploadParams.MinSplitSize = configuration.MinSplitSizeMB * rtServicesUtils.SizeMiB
+	uploadParams.SplitCount = configuration.SplitCount
 	uploadParams.AddVcsProps = addVcsProps
 	uploadParams.BuildProps = buildProps
 	uploadParams.Archive = f.Archive

artifactory/commands/transferfiles/state/timeestimation.go

@@ -6,13 +6,13 @@ import (
 
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
 
+	"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 )
 
 const (
 	milliSecsInSecond               = 1000
-	bytesInMB                       = 1024 * 1024
-	bytesPerMilliSecToMBPerSec      = float64(milliSecsInSecond) / float64(bytesInMB)
+	bytesPerMilliSecToMBPerSec      = float64(milliSecsInSecond) / float64(utils.SizeMiB)
 	minTransferTimeToShowEstimation = time.Minute * 5
 )
 

artifactory/commands/transferfiles/state/timeestimation_test.go

@@ -5,6 +5,8 @@ import (
 	"time"
 
 	"github.com/jfrog/build-info-go/utils"
+	rtServicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
+
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
@@ -41,9 +43,9 @@ func TestGetSpeed(t *testing.T) {
 	// Chunk 1: one of the files is checksum-deployed
 	chunkStatus1 := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
-			createFileUploadStatusResponse(repo1Key, 10*bytesInMB, false, api.Success),
-			createFileUploadStatusResponse(repo1Key, 15*bytesInMB, false, api.Success),
-			createFileUploadStatusResponse(repo1Key, 8*bytesInMB, true, api.Success),
+			createFileUploadStatusResponse(repo1Key, 10*rtServicesUtils.SizeMiB, false, api.Success),
+			createFileUploadStatusResponse(repo1Key, 15*rtServicesUtils.SizeMiB, false, api.Success),
+			createFileUploadStatusResponse(repo1Key, 8*rtServicesUtils.SizeMiB, true, api.Success),
 		},
 	}
 	addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
@@ -54,8 +56,8 @@ func TestGetSpeed(t *testing.T) {
 	// Chunk 2: the upload of one of the files failed and the files are not included in the repository's total size (includedInTotalSize == false)
 	chunkStatus2 := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
-			createFileUploadStatusResponse(repo1Key, 21.25*bytesInMB, false, api.Success),
-			createFileUploadStatusResponse(repo1Key, 6*bytesInMB, false, api.Fail),
+			createFileUploadStatusResponse(repo1Key, int64(21.25*float64(rtServicesUtils.SizeMiB)), false, api.Success),
+			createFileUploadStatusResponse(repo1Key, 6*rtServicesUtils.SizeMiB, false, api.Fail),
 		},
 	}
 	addChunkStatus(t, timeEstMng, chunkStatus2, 2, false, 5*milliSecsInSecond)
@@ -85,7 +87,7 @@ func TestGetEstimatedRemainingTimeStringNotAvailableYet(t *testing.T) {
 	// Chunk 1: one of the files is checksum-deployed
 	chunkStatus1 := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
-			createFileUploadStatusResponse(repo1Key, 8*bytesInMB, true, api.Success),
+			createFileUploadStatusResponse(repo1Key, 8*rtServicesUtils.SizeMiB, true, api.Success),
 		},
 	}
 	assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString())
@@ -119,8 +121,8 @@ func newDefaultTimeEstimationManager(t *testing.T, buildInfoRepos bool) *TimeEst
 	assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, buildInfoRepos, true))
 	assert.NoError(t, stateManager.SetRepoState(repo2Key, 0, 0, buildInfoRepos, true))
 
-	assert.NoError(t, stateManager.IncTransferredSizeAndFilesPhase1(0, 100*bytesInMB))
-	stateManager.OverallTransfer.TotalSizeBytes = 600 * bytesInMB
+	assert.NoError(t, stateManager.IncTransferredSizeAndFilesPhase1(0, 100*rtServicesUtils.SizeMiB))
+	stateManager.OverallTransfer.TotalSizeBytes = 600 * rtServicesUtils.SizeMiB
 	return &TimeEstimationManager{stateManager: stateManager}
 }
 
@@ -149,7 +151,7 @@ func TestAddingToFullLastSpeedsSlice(t *testing.T) {
 // Adds a chunk with one non checksum-deployed file and calculates and returns the chunk speed.
 func addOneFileChunk(t *testing.T, timeEstMng *TimeEstimationManager, workingThreads, chunkDurationMilli, chunkSizeMb int) float64 {
 	chunkDuration := int64(chunkDurationMilli * milliSecsInSecond)
-	chunkSize := int64(chunkSizeMb * bytesInMB)
+	chunkSize := int64(chunkSizeMb) * rtServicesUtils.SizeMiB
 	chunkStatus := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
 			createFileUploadStatusResponse(repo1Key, chunkSize, false, api.Success),
@@ -174,17 +176,17 @@ func TestTransferredSizeInState(t *testing.T) {
 	// Add a chunk of repo1 with multiple successful files, which are included in total.
 	chunkStatus1 := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
-			createFileUploadStatusResponse(repo1Key, 10*bytesInMB, false, api.Success),
+			createFileUploadStatusResponse(repo1Key, 10*rtServicesUtils.SizeMiB, false, api.Success),
 			// Checksum-deploy should not affect the update size.
-			createFileUploadStatusResponse(repo1Key, 15*bytesInMB, true, api.Success),
+			createFileUploadStatusResponse(repo1Key, 15*rtServicesUtils.SizeMiB, true, api.Success),
 		},
 	}
 	addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
 
 	// Add another chunk of repo1 which is not included in total. Expected not to be included in update.
 	chunkStatus2 := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
-			createFileUploadStatusResponse(repo1Key, 21*bytesInMB, false, api.Success),
+			createFileUploadStatusResponse(repo1Key, 21*rtServicesUtils.SizeMiB, false, api.Success),
 		},
 	}
 	addChunkStatus(t, timeEstMng, chunkStatus2, 3, false, 10*milliSecsInSecond)
@@ -195,8 +197,8 @@ func TestTransferredSizeInState(t *testing.T) {
 	// Add a chunk of repo2 which is included in total. The failed file should be ignored.
 	chunkStatus3 := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
-			createFileUploadStatusResponse(repo2Key, 13*bytesInMB, false, api.Success),
-			createFileUploadStatusResponse(repo2Key, 133*bytesInMB, false, api.Fail),
+			createFileUploadStatusResponse(repo2Key, 13*rtServicesUtils.SizeMiB, false, api.Success),
+			createFileUploadStatusResponse(repo2Key, 133*rtServicesUtils.SizeMiB, false, api.Fail),
 		},
 	}
 	addChunkStatus(t, timeEstMng, chunkStatus3, 3, true, 10*milliSecsInSecond)
@@ -205,7 +207,7 @@ func TestTransferredSizeInState(t *testing.T) {
 	// Add one more chunk of repo2.
 	chunkStatus4 := api.ChunkStatus{
 		Files: []api.FileUploadStatusResponse{
-			createFileUploadStatusResponse(repo2Key, 9*bytesInMB, false, api.Success),
+			createFileUploadStatusResponse(repo2Key, 9*rtServicesUtils.SizeMiB, false, api.Success),
 		},
 	}
 	addChunkStatus(t, timeEstMng, chunkStatus4, 3, true, 10*milliSecsInSecond)

artifactory/utils/storageinfo.go

@@ -4,27 +4,23 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/jfrog/gofrog/datastructures"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-client-go/artifactory"
 	"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
 	clientUtils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/httputils"
-	"strconv"
-	"strings"
-	"time"
 )
 
 const (
 	serviceManagerRetriesPerRequest                  = 3
 	serviceManagerRetriesWaitPerRequestMilliSecs int = 1000
 	storageInfoRepoMissingError                      = "one or more of the requested repositories were not found"
-
-	bytesInKB int64 = 1024
-	bytesInMB       = 1024 * bytesInKB
-	bytesInGB       = 1024 * bytesInMB
-	bytesInTB       = 1024 * bytesInGB
 )
 
 var getRepoSummaryPollingTimeout = 10 * time.Minute
@@ -171,36 +167,36 @@ func convertStorageSizeStringToBytes(sizeStr string) (int64, error) {
 	case "bytes":
 		sizeInBytes = sizeInUnit
 	case "KB":
-		sizeInBytes = sizeInUnit * float64(bytesInKB)
+		sizeInBytes = sizeInUnit * float64(utils.SizeKib)
 	case "MB":
-		sizeInBytes = sizeInUnit * float64(bytesInMB)
+		sizeInBytes = sizeInUnit * float64(utils.SizeMiB)
 	case "GB":
-		sizeInBytes = sizeInUnit * float64(bytesInGB)
+		sizeInBytes = sizeInUnit * float64(utils.SizeGiB)
 	case "TB":
-		sizeInBytes = sizeInUnit * float64(bytesInTB)
+		sizeInBytes = sizeInUnit * float64(utils.SizeTiB)
 	default:
 		return 0, errorutils.CheckErrorf("could not parse size string '%s'", sizeStr)
 	}
 	return int64(sizeInBytes), nil
 }
 
 func ConvertIntToStorageSizeString(num int64) string {
-	if num > bytesInTB {
-		newNum := float64(num) / float64(bytesInTB)
+	if num > utils.SizeTiB {
+		newNum := float64(num) / float64(utils.SizeTiB)
 		stringNum := fmt.Sprintf("%.1f", newNum)
 		return stringNum + "TB"
 	}
-	if num > bytesInGB {
-		newNum := float64(num) / float64(bytesInGB)
+	if num > utils.SizeGiB {
+		newNum := float64(num) / float64(utils.SizeGiB)
 		stringNum := fmt.Sprintf("%.1f", newNum)
 		return stringNum + "GB"
 	}
-	if num > bytesInMB {
-		newNum := float64(num) / float64(bytesInMB)
+	if num > utils.SizeMiB {
+		newNum := float64(num) / float64(utils.SizeMiB)
 		stringNum := fmt.Sprintf("%.1f", newNum)
 		return stringNum + "MB"
 	}
-	newNum := float64(num) / float64(bytesInKB)
+	newNum := float64(num) / float64(utils.SizeKib)
 	stringNum := fmt.Sprintf("%.1f", newNum)
 	return stringNum + "KB"
 }

artifactory/utils/storageinfo_test.go

@@ -69,12 +69,12 @@ func TestConvertStorageSizeStringToBytes(t *testing.T) {
 		expectedSizeBeforeConversion float64
 	}{
 		{"bytes", "2.22 bytes", false, 2.22},
-		{"KB", "3.333 KB", false, 3.333 * float64(bytesInKB)},
-		{"KB with comma", "1,004.64 KB", false, 1004.64 * float64(bytesInKB)},
-		{"MB", "4.4444 MB", false, 4.4444 * float64(bytesInMB)},
-		{"GB", "5.55555 GB", false, 5.55555 * float64(bytesInGB)},
-		{"TB", "6.666666 TB", false, 6.666666 * float64(bytesInTB)},
-		{"int", "7 KB", false, 7 * float64(bytesInKB)},
+		{"KB", "3.333 KB", false, 3.333 * float64(clientUtils.SizeKib)},
+		{"KB with comma", "1,004.64 KB", false, 1004.64 * float64(clientUtils.SizeKib)},
+		{"MB", "4.4444 MB", false, 4.4444 * float64(clientUtils.SizeMiB)},
+		{"GB", "5.55555 GB", false, 5.55555 * float64(clientUtils.SizeGiB)},
+		{"TB", "6.666666 TB", false, 6.666666 * float64(clientUtils.SizeTiB)},
+		{"int", "7 KB", false, 7 * float64(clientUtils.SizeKib)},
 		{"size missing", "8", true, -1},
 		{"unexpected size", "8 kb", true, -1},
 		{"too many separators", "8 K B", true, -1},

artifactory/utils/upload.go

@@ -15,4 +15,6 @@ type UploadConfiguration struct {
 	Threads               int
 	MinChecksumDeploySize int64
 	ExplodeArchive        bool
+	SplitCount            int
+	MinSplitSizeMB        int64
 }

common/progressbar/filesprogressbar.go

@@ -1,14 +1,15 @@
 package progressbar
 
 import (
-	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"net/url"
 	"os"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+
 	"github.com/jfrog/jfrog-cli-core/v2/common/commands"
 	corelog "github.com/jfrog/jfrog-cli-core/v2/utils/log"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
@@ -19,8 +20,6 @@ import (
 	"github.com/vbauerster/mpb/v7/decor"
 )
 
-var terminalWidth int
-
 type filesProgressBarManager struct {
 	// A list of progress bar objects.
 	bars []progressBar
@@ -74,7 +73,7 @@ func (p *filesProgressBarManager) NewProgressReader(total int64, label, path str
 		mpb.BarRemoveOnComplete(),
 		mpb.AppendDecorators(
 			// Extra chars length is the max length of the KibiByteCounter
-			decor.Name(buildProgressDescription(label, path, 17)),
+			decor.Name(buildProgressDescription(label, path, progressbar.GetTerminalWidth(), 17)),
 			decor.CountersKibiByte("%3.1f/%3.1f"),
 		),
 	)
@@ -87,49 +86,46 @@ func (p *filesProgressBarManager) NewProgressReader(total int64, label, path str
 	return &readerProgressBar
 }
 
-// Changes progress indicator state and acts accordingly.
-func (p *filesProgressBarManager) SetProgressState(id int, state string) {
-	if state == "Merging" {
-		p.addNewMergingSpinner(id)
-	}
-}
-
 // Initializes a new progress bar, that replaces the progress bar with the given replacedBarId
-func (p *filesProgressBarManager) addNewMergingSpinner(replacedBarId int) {
-	// Write Lock when appending a new bar to the slice
+func (p *filesProgressBarManager) SetMergingState(replacedBarId int, useSpinner bool) (bar ioUtils.Progress) { // Write Lock when appending a new bar to the slice
 	p.barsRWMutex.Lock()
 	defer p.barsRWMutex.Unlock()
 	replacedBar := p.bars[replacedBarId-1].getProgressBarUnit()
 	p.bars[replacedBarId-1].Abort()
-	newBar := p.container.New(1,
-		mpb.SpinnerStyle(createSpinnerFramesArray()...).PositionLeft(),
+	newBar := p.container.New(100,
+		getMergingProgress(useSpinner),
+		mpb.BarRemoveOnComplete(),
 		mpb.AppendDecorators(
-			decor.Name(buildProgressDescription("  Merging  ", replacedBar.description, 0)),
+			decor.Name(buildProgressDescription("  Merging  ", replacedBar.description, progressbar.GetTerminalWidth(), 0)),
 		),
 	)
 	// Bar replacement is a simple spinner and thus does not implement any read functionality
 	unit := &progressBarUnit{bar: newBar, description: replacedBar.description}
 	progressBar := SimpleProgressBar{progressBarUnit: unit, Id: replacedBarId}
 	p.bars[replacedBarId-1] = &progressBar
+	return &progressBar
+}
+
+func getMergingProgress(useSpinner bool) mpb.BarFillerBuilder {
+	if useSpinner {
+		return mpb.SpinnerStyle(createSpinnerFramesArray()...).PositionLeft()
+	}
+	return mpb.BarStyle().Lbound("|").Filler("🟩").Tip("🟩").Padding("⬛").Refiller("").Rbound("|")
 }
 
-func buildProgressDescription(label, path string, extraCharsLen int) string {
+func buildProgressDescription(label, path string, terminalWidth, extraCharsLen int) string {
 	separator := " | "
 	// Max line length after decreasing bar width (*2 in case unicode chars with double width are used) and the extra chars
 	descMaxLength := terminalWidth - (progressbar.ProgressBarWidth*2 + extraCharsLen)
 	return buildDescByLimits(descMaxLength, " "+label+separator, shortenUrl(path), separator)
 }
 
 func shortenUrl(path string) string {
-	if _, err := url.ParseRequestURI(path); err != nil {
-		return path
-	}
-
-	semicolonIndex := strings.Index(path, ";")
-	if semicolonIndex == -1 {
+	parsedUrl, err := url.ParseRequestURI(path)
+	if err != nil {
 		return path
 	}
-	return path[:semicolonIndex]
+	return strings.TrimPrefix(parsedUrl.Path, "/artifactory")
 }
 
 func buildDescByLimits(descMaxLength int, prefix, path, suffix string) string {

common/progressbar/filesprogressbar_test.go

@@ -1,22 +1,19 @@
 package progressbar
 
 import (
-	"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
 	"testing"
+
+	"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
+	"github.com/stretchr/testify/assert"
 )
 
+const terminalWidth = 100
+
 func TestBuildProgressDescription(t *testing.T) {
 	// Set an arbitrary terminal width
-	terminalWidth = 100
-	tests := getTestCases()
-	for _, test := range tests {
+	for _, test := range getTestCases() {
 		t.Run(test.name, func(t *testing.T) {
-			desc := buildProgressDescription(test.prefix, test.path, test.extraCharsLen)
-
-			// Validate result
-			if desc != test.expectedDesc {
-				t.Errorf("Expected value of: \"%s\", got: \"%s\".", test.expectedDesc, desc)
-			}
+			assert.Equal(t, test.expectedDesc, buildProgressDescription(test.prefix, test.path, terminalWidth, test.extraCharsLen))
 		})
 	}
 }

common/progressbar/readerprogressbar.go

@@ -20,6 +20,10 @@ func (p *ReaderProgressBar) ActionWithProgress(reader io.Reader) (results io.Rea
 	return p.readWithProgress(reader)
 }
 
+func (p *ReaderProgressBar) SetProgress(progress int64) {
+	p.bar.SetCurrent(progress)
+}
+
 // Abort aborts a progress indicator. Called on both successful and unsuccessful operations
 func (p *ReaderProgressBar) Abort() {
 	close(p.incrChannel)