Upgrade jfrog-cli-core to 2.45.2

.github/workflows/analysis.yml

@@ -36,7 +36,8 @@ jobs:
         with:
           go-version: 1.20.x
 
+      # Temporarily set version 2.18.0 to workaround https://github.com/securego/gosec/issues/1046
       - name: Run Gosec Security Scanner
-        uses: securego/gosec@master
+        uses: securego/gosec@v2.18.0
         with:
           args: -exclude G204,G301,G302,G304,G306 -tests -exclude-dir \.*test\.* ./...

artifactory/commands/transferfiles/delayedartifactshandler.go

@@ -174,6 +174,13 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon
 			return err
 		}
 
+		if base.progressBar != nil {
+			base.progressBar.changeNumberOfDelayedFiles(-1 * len(delayedArtifactsFile.DelayedArtifacts))
+		}
+		if err = base.stateManager.ChangeDelayedFilesCountBy(uint64(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
+			log.Warn("Couldn't decrease the delayed files counter", err.Error())
+		}
+
 		// Remove the file, so it won't be consumed again.
 		if err = os.Remove(filePath); err != nil {
 			return errorutils.CheckError(err)
@@ -203,6 +210,23 @@ func getDelayFiles(repoKeys []string) (filesPaths []string, err error) {
 	return getErrorOrDelayFiles(repoKeys, getJfrogTransferRepoDelaysDir)
 }
 
+func getDelayedFilesCount(repoKeys []string) (int, error) {
+	files, err := getDelayFiles(repoKeys)
+	if err != nil {
+		return -1, err
+	}
+
+	count := 0
+	for _, file := range files {
+		delayedFiles, err := readDelayFile(file)
+		if err != nil {
+			return -1, err
+		}
+		count += len(delayedFiles.DelayedArtifacts)
+	}
+	return count, nil
+}
+
 const (
 	maven  = "Maven"
 	gradle = "Gradle"
@@ -256,6 +280,12 @@ func (delayHelper delayUploadHelper) delayUploadIfNecessary(phase phaseBase, fil
 		if shouldDelay(file.Name) {
 			delayed = true
 			delayHelper.delayedArtifactsChannelMng.add(file)
+			if phase.progressBar != nil {
+				phase.progressBar.changeNumberOfDelayedFiles(1)
+			}
+			if err := phase.stateManager.ChangeDelayedFilesCountBy(1, true); err != nil {
+				log.Warn("Couldn't increase the delayed files counter", err.Error())
+			}
 		}
 	}
 	return

artifactory/commands/transferfiles/fulltransfer.go

@@ -131,6 +131,14 @@ func (m *fullTransferPhase) transferFolder(params folderParams, logMsgPrefix str
 	uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) (err error) {
 	log.Debug(logMsgPrefix+"Handling folder:", path.Join(m.repoKey, params.relativePath))
 
+	// Increment progress number of folders
+	if m.progressBar != nil {
+		m.progressBar.incNumberOfVisitedFolders()
+	}
+	if err = m.stateManager.IncVisitedFolders(); err != nil {
+		return
+	}
+
 	// Get the directory's node from the snapshot manager, and use information from previous transfer attempts if such exist.
 	node, done, err := m.getAndHandleDirectoryNode(params, logMsgPrefix)
 	if err != nil || done {

artifactory/commands/transferfiles/phase.go

@@ -79,8 +79,8 @@ func (pb *phaseBase) StopGracefully() {
 		pb.progressBar.StopGracefully()
 	}
 	if pb.pcDetails != nil {
-		pb.pcDetails.chunkBuilderProducerConsumer.Cancel()
-		pb.pcDetails.chunkUploaderProducerConsumer.Cancel()
+		pb.pcDetails.chunkBuilderProducerConsumer.Cancel(true)
+		pb.pcDetails.chunkUploaderProducerConsumer.Cancel(true)
 	}
 }
 

artifactory/commands/transferfiles/state/runstatus.go

@@ -33,10 +33,12 @@ type TransferRunStatus struct {
 	Version        int    `json:"version,omitempty"`
 	CurrentRepoKey string `json:"current_repo,omitempty"`
 	// True if currently transferring a build info repository.
-	BuildInfoRepo         bool `json:"build_info_repo,omitempty"`
-	CurrentRepoPhase      int  `json:"current_repo_phase,omitempty"`
-	WorkingThreads        int  `json:"working_threads,omitempty"`
-	TransferFailures      uint `json:"transfer_failures,omitempty"`
+	BuildInfoRepo         bool   `json:"build_info_repo,omitempty"`
+	CurrentRepoPhase      int    `json:"current_repo_phase,omitempty"`
+	WorkingThreads        int    `json:"working_threads,omitempty"`
+	VisitedFolders        uint64 `json:"visited_folders,omitempty"`
+	DelayedFiles          uint64 `json:"delayed_files,omitempty"`
+	TransferFailures      uint   `json:"transfer_failures,omitempty"`
 	TimeEstimationManager `json:"time_estimation,omitempty"`
 	StaleChunks           []StaleChunks `json:"stale_chunks,omitempty"`
 }

artifactory/commands/transferfiles/state/statemanager.go

@@ -67,7 +67,7 @@ func (ts *TransferStateManager) UnlockTransferStateManager() error {
 // buildInfoRepo  - True if build info repository
 // reset          - Delete the repository's previous transfer info
 func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, totalFiles int64, buildInfoRepo, reset bool) error {
-	err := ts.TransferState.Action(func(state *TransferState) error {
+	err := ts.Action(func(*TransferState) error {
 		transferState, repoTransferSnapshot, err := getTransferStateAndSnapshot(repoKey, reset)
 		if err != nil {
 			return err
@@ -82,9 +82,10 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
 	if err != nil {
 		return err
 	}
-	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
+	return ts.action(func(transferRunStatus *TransferRunStatus) error {
 		transferRunStatus.CurrentRepoKey = repoKey
 		transferRunStatus.BuildInfoRepo = buildInfoRepo
+		transferRunStatus.VisitedFolders = 0
 
 		transferRunStatus.OverallTransfer.TransferredUnits += ts.CurrentRepo.Phase1Info.TransferredUnits
 		transferRunStatus.OverallTransfer.TransferredSizeBytes += ts.CurrentRepo.Phase1Info.TransferredSizeBytes
@@ -263,6 +264,24 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er
 	})
 }
 
+func (ts *TransferStateManager) IncVisitedFolders() error {
+	return ts.action(func(transferRunStatus *TransferRunStatus) error {
+		transferRunStatus.VisitedFolders++
+		return nil
+	})
+}
+
+func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint64, increase bool) error {
+	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
+		if increase {
+			transferRunStatus.DelayedFiles += count
+		} else {
+			transferRunStatus.DelayedFiles -= count
+		}
+		return nil
+	})
+}
+
 func (ts *TransferStateManager) ChangeTransferFailureCountBy(count uint, increase bool) error {
 	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
 		if increase {

artifactory/commands/transferfiles/state/statemanager_test.go

@@ -146,6 +146,48 @@ func TestReposOverallBiFiles(t *testing.T) {
 	assert.Equal(t, int64(8), stateManager.OverallBiFiles.TransferredUnits)
 }
 
+func TestChangeDelayedFilesCountBy(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase delayed files count
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(2, true))
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(4, true))
+	assert.Equal(t, uint64(6), stateManager.DelayedFiles)
+
+	// Decrease delayed files count
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(3, false))
+	assert.Equal(t, uint64(3), stateManager.DelayedFiles)
+}
+
+func TestVisitedFolders(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase visited folders count
+	assert.NoError(t, stateManager.IncVisitedFolders())
+	assert.NoError(t, stateManager.IncVisitedFolders())
+	assert.Equal(t, uint64(2), stateManager.VisitedFolders)
+
+	// Set repository state and ensure the visited folders became 0
+	assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, true, true))
+	assert.Zero(t, stateManager.VisitedFolders)
+}
+
+func TestChangeTransferFailureCountBy(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase failures count
+	assert.NoError(t, stateManager.ChangeTransferFailureCountBy(2, true))
+	assert.NoError(t, stateManager.ChangeTransferFailureCountBy(4, true))
+	assert.Equal(t, uint(6), stateManager.TransferFailures)
+
+	// Decrease failures count
+	assert.NoError(t, stateManager.ChangeTransferFailureCountBy(3, false))
+	assert.Equal(t, uint(3), stateManager.TransferFailures)
+}
+
 func assertReposTransferredSize(t *testing.T, stateManager *TransferStateManager, expectedSize int64, repoKeys ...string) {
 	totalTransferredSize, err := stateManager.GetReposTransferredSizeBytes(repoKeys...)
 	assert.NoError(t, err)

artifactory/commands/transferfiles/status.go

@@ -10,6 +10,7 @@ import (
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
@@ -80,7 +81,7 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings.
 	addString(output, "⌛", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1)
 	failureTxt := strconv.FormatUint(uint64(stateManager.TransferFailures), 10)
 	if stateManager.TransferFailures > 0 {
-		failureTxt += " (" + "In Phase 3 and in subsequent executions, we'll retry transferring the failed files." + ")"
+		failureTxt += " (" + progressbar.RetryFailureContentNote + ")"
 	}
 	addString(output, "❌", "Transfer failures", failureTxt, 2)
 }
@@ -94,20 +95,28 @@ func calcPercentageInt64(transferred, total int64) string {
 
 func setRepositoryStatus(stateManager *state.TransferStateManager, output *strings.Builder) {
 	addTitle(output, "Current Repository Status")
-	addString(output, "🏷 ", "Name", stateManager.CurrentRepoKey, 2)
+	addString(output, "🏷 ", "Name", stateManager.CurrentRepoKey, 3)
 	currentRepo := stateManager.CurrentRepo
 	switch stateManager.CurrentRepoPhase {
 	case api.Phase1, api.Phase3:
 		if stateManager.CurrentRepoPhase == api.Phase1 {
-			addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 2)
+			addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 3)
 		} else {
-			addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 2)
+			addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 3)
 		}
-		addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 2)
-		addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 2)
+		addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 3)
+		addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 3)
 	case api.Phase2:
-		addString(output, "🔢", "Phase", "Transferring newly created and modified files (2/3)", 2)
+		addString(output, "🔢", "Phase", "Transferring newly created and modified files (2/3)", 3)
 	}
+	if stateManager.CurrentRepoPhase == api.Phase1 {
+		addString(output, "📁", "Visited folders", strconv.FormatUint(stateManager.VisitedFolders, 10), 2)
+	}
+	delayedTxt := strconv.FormatUint(stateManager.DelayedFiles, 10)
+	if stateManager.DelayedFiles > 0 {
+		delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")"
+	}
+	addString(output, "✋", "Delayed files", delayedTxt, 2)
 }
 
 func addStaleChunks(stateManager *state.TransferStateManager, output *strings.Builder) {

artifactory/commands/transferfiles/status_test.go

@@ -69,14 +69,16 @@ func TestShowStatus(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		0.011 MB/s")
 	assert.Contains(t, results, "Estimated time remaining:	Less than a minute")
-	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files.)")
+	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)")
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
-	assert.Contains(t, results, "Name:		repo1")
-	assert.Contains(t, results, "Phase:		Transferring all files in the repository (1/3)")
-	assert.Contains(t, results, "Storage:		4.9 KiB / 9.8 KiB (50.0%)")
-	assert.Contains(t, results, "Files:		500 / 10000 (5.0%)")
+	assert.Contains(t, results, "Name:			repo1")
+	assert.Contains(t, results, "Phase:			Transferring all files in the repository (1/3)")
+	assert.Contains(t, results, "Visited folders:		15")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.Contains(t, results, "Storage:			4.9 KiB / 9.8 KiB (50.0%)")
+	assert.Contains(t, results, "Files:			500 / 10000 (5.0%)")
 }
 
 func TestShowStatusDiffPhase(t *testing.T) {
@@ -99,14 +101,16 @@ func TestShowStatusDiffPhase(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		0.011 MB/s")
 	assert.Contains(t, results, "Estimated time remaining:	Not available in this phase")
-	assert.Contains(t, results, "Transfer failures:		223")
+	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)")
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
-	assert.Contains(t, results, "Name:		repo1")
-	assert.Contains(t, results, "Phase:		Transferring newly created and modified files (2/3)")
-	assert.NotContains(t, results, "Storage:		4.9 KiB / 9.8 KiB (50.0%)")
-	assert.NotContains(t, results, "Files:		500 / 10000 (5.0%)")
+	assert.Contains(t, results, "Name:			repo1")
+	assert.Contains(t, results, "Phase:			Transferring newly created and modified files (2/3)")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.NotContains(t, results, "Visited folders")
+	assert.NotContains(t, results, "Storage:			4.9 KiB / 9.8 KiB (50.0%)")
+	assert.NotContains(t, results, "Files:			500 / 10000 (5.0%)")
 }
 
 func TestShowBuildInfoRepo(t *testing.T) {
@@ -133,10 +137,12 @@ func TestShowBuildInfoRepo(t *testing.T) {
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
-	assert.Contains(t, results, "Name:		repo1")
-	assert.Contains(t, results, "Phase:		Retrying transfer failures (3/3)")
-	assert.Contains(t, results, "Storage:		4.9 KiB / 9.8 KiB (50.0%)")
-	assert.Contains(t, results, "Files:		500 / 10000 (5.0%)")
+	assert.Contains(t, results, "Name:			repo1")
+	assert.Contains(t, results, "Phase:			Retrying transfer failures (3/3)")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.Contains(t, results, "Storage:			4.9 KiB / 9.8 KiB (50.0%)")
+	assert.Contains(t, results, "Files:			500 / 10000 (5.0%)")
+	assert.NotContains(t, results, "Visited folders")
 }
 
 func TestShowStaleChunks(t *testing.T) {
@@ -174,11 +180,13 @@ func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks
 	stateManager.TotalRepositories.TotalUnits = 1111
 	stateManager.TotalRepositories.TransferredUnits = 15
 	stateManager.WorkingThreads = 16
+	stateManager.VisitedFolders = 15
+	stateManager.DelayedFiles = 20
 	stateManager.TransferFailures = 223
 
-	stateManager.TimeEstimationManager.LastSpeeds = []float64{12}
-	stateManager.TimeEstimationManager.LastSpeedsSum = 12
-	stateManager.TimeEstimationManager.SpeedsAverage = 12
+	stateManager.LastSpeeds = []float64{12}
+	stateManager.LastSpeedsSum = 12
+	stateManager.SpeedsAverage = 12
 
 	if staleChunks {
 		stateManager.StaleChunks = append(stateManager.StaleChunks, state.StaleChunks{

artifactory/commands/transferfiles/transfer.go

@@ -228,8 +228,16 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui
 			return e
 		}
 		tdc.stateManager.TransferFailures = uint(numberInitialErrors)
+
+		numberInitialDelays, e := getDelayedFilesCount(allSourceLocalRepos)
+		if e != nil {
+			return e
+		}
+		tdc.stateManager.DelayedFiles = uint64(numberInitialDelays)
 	} else {
+		tdc.stateManager.VisitedFolders = 0
 		tdc.stateManager.TransferFailures = 0
+		tdc.stateManager.DelayedFiles = 0
 	}
 	return nil
 }