Merge branch 'dev'

artifactory/commands/transferfiles/localgeneratedfilter.go

@@ -160,5 +160,8 @@ func (lg *locallyGeneratedFilter) getNonLocallyGeneratedResults(aqlResultItems [
 }
 
 func getPathInRepo(aqlResultItem *utils.ResultItem) string {
+	if aqlResultItem.Path == "." {
+		return aqlResultItem.Name
+	}
 	return aqlResultItem.Path + "/" + aqlResultItem.Name
 }

artifactory/commands/transferfiles/localgeneratedfilter_test.go

@@ -22,6 +22,7 @@ var filterLocallyGeneratedCases = []struct {
 	{paths: []utils.ResultItem{{Path: "a", Name: "b"}}, returnedPath: []string{"a/b"}},
 	{paths: []utils.ResultItem{{Path: "a", Name: "b"}, {Path: "a/b", Name: "e"}}, returnedPath: []string{"a/b/e"}},
 	{paths: []utils.ResultItem{{Path: "a", Name: "b"}, {Path: "a/b", Name: "e"}}, returnedPath: []string{"a/b", "a/b/e"}},
+	{paths: []utils.ResultItem{{Path: ".", Name: "a"}, {Path: "b", Name: "c"}}, returnedPath: []string{"b/c"}},
 }
 
 func TestFilterLocallyGenerated(t *testing.T) {
@@ -99,3 +100,20 @@ func TestFilterLocallyGeneratedEnabled(t *testing.T) {
 	assert.False(t, NewLocallyGenerated(context.Background(), servicesManager, "7.54.5").IsEnabled())
 	assert.False(t, NewLocallyGenerated(context.Background(), servicesManager, "6.0.0").IsEnabled())
 }
+
+var getPathInRepoCases = []struct {
+	aqlResultItem      utils.ResultItem
+	expectedPathInRepo string
+}{
+	{aqlResultItem: utils.ResultItem{Path: ".", Name: "a"}, expectedPathInRepo: "a"},
+	{aqlResultItem: utils.ResultItem{Path: "a", Name: "b"}, expectedPathInRepo: "a/b"},
+}
+
+func TestGetPathInRepo(t *testing.T) {
+	for _, testCase := range getPathInRepoCases {
+		t.Run("", func(t *testing.T) {
+			aqlResultsItem := testCase.aqlResultItem
+			assert.Equal(t, testCase.expectedPathInRepo, getPathInRepo(&aqlResultsItem))
+		})
+	}
+}

artifactory/commands/transferfiles/manager.go

@@ -238,8 +238,13 @@ func runProducerConsumers(pcWrapper *producerConsumerWrapper) (executionErr erro
 	// Run() is a blocking method, so once all chunk builders are idle, the tasks queue closes and Run() stops running.
 	pcWrapper.chunkBuilderProducerConsumer.Run()
 	if pcWrapper.chunkUploaderProducerConsumer.IsStarted() {
+		// There might be a moment when the chunk uploader has no upload tasks.
+		// This circumstance might lead to setting the finish notification before completing all file uploads.
+		// To address this, we reset the finish notification to ensure no remaining upload tasks after the next finish notification.
+		pcWrapper.chunkUploaderProducerConsumer.ResetFinishNotificationIfActive()
 		// Wait till notified that the uploader finished its tasks, and it will not receive new tasks from the builder.
 		<-pcWrapper.chunkUploaderProducerConsumer.GetFinishedNotification()
+		log.Debug("Chunk uploaded producer consumer has completed all tasks. All files relevant to this phase have all been uploaded.")
 	}
 	// Close the tasks queue with Done().
 	pcWrapper.chunkUploaderProducerConsumer.Done()

artifactory/commands/transferfiles/manager_test.go

@@ -0,0 +1,37 @@
+package transferfiles
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRunProducerConsumers(t *testing.T) {
+	// Create the producer-consumers
+	producerConsumerWrapper := newProducerConsumerWrapper()
+
+	// Add 10 tasks for the chunkBuilderProducerConsumer. Each task provides a task to the chunkUploaderProducerConsumer.
+	for i := 0; i < 10; i++ {
+		_, err := producerConsumerWrapper.chunkBuilderProducerConsumer.AddTask(func(int) error {
+			time.Sleep(time.Millisecond * 100)
+			_, err := producerConsumerWrapper.chunkUploaderProducerConsumer.AddTask(
+				func(int) error {
+					time.Sleep(time.Millisecond)
+					return nil
+				},
+			)
+			assert.NoError(t, err)
+			return nil
+		})
+		assert.NoError(t, err)
+	}
+
+	// Run the producer-consumers
+	err := runProducerConsumers(&producerConsumerWrapper)
+	assert.NoError(t, err)
+
+	// Assert no active treads left in the producer-consumers
+	assert.Zero(t, producerConsumerWrapper.chunkBuilderProducerConsumer.ActiveThreads())
+	assert.Zero(t, producerConsumerWrapper.chunkUploaderProducerConsumer.ActiveThreads())
+}

go.mod

@@ -14,9 +14,9 @@ require (
 	github.com/google/uuid v1.5.0
 	github.com/gookit/color v1.5.4
 	github.com/jfrog/build-info-go v1.9.18
-	github.com/jfrog/gofrog v1.3.2
+	github.com/jfrog/gofrog v1.4.0
 	github.com/jfrog/jfrog-apps-config v1.0.1
-	github.com/jfrog/jfrog-client-go v1.35.3
+	github.com/jfrog/jfrog-client-go v1.35.4
 	github.com/magiconair/properties v1.8.7
 	github.com/manifoldco/promptui v0.9.0
 	github.com/owenrumney/go-sarif/v2 v2.3.0
@@ -103,4 +103,4 @@ require (
 
 // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231220102935-c8776c613ad8
 
-// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.3.2-0.20231130091721-6d742be8bc7a
+// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.3.3-0.20231223133729-ef57bd08cedc

go.sum

@@ -87,12 +87,12 @@ github.com/jedib0t/go-pretty/v6 v6.4.0 h1:YlI/2zYDrweA4MThiYMKtGRfT+2qZOO65ulej8
 github.com/jedib0t/go-pretty/v6 v6.4.0/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI=
 github.com/jfrog/build-info-go v1.9.18 h1:0RKeZtNWZjONX5j94aIPQSKMi1whP2evmGQymYF+5mA=
 github.com/jfrog/build-info-go v1.9.18/go.mod h1:/5VZXH2Ud0IK3cOFwPykjwPOaEcHhzzbjnRiou+YKpM=
-github.com/jfrog/gofrog v1.3.2 h1:TktKP+PdZdxjkYZxWWIq4DkTGSYtr9Slsy+egZpEhUY=
-github.com/jfrog/gofrog v1.3.2/go.mod h1:AQo5Fq0G9nDEF6icH7MYQK0iohR4HuEAXl8jaxRuT6Q=
+github.com/jfrog/gofrog v1.4.0 h1:s7eysVnmIBfVheMs4LPU43MAlxwPa4K8u2N5h7kwzXA=
+github.com/jfrog/gofrog v1.4.0/go.mod h1:AQo5Fq0G9nDEF6icH7MYQK0iohR4HuEAXl8jaxRuT6Q=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
 github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
-github.com/jfrog/jfrog-client-go v1.35.3 h1:Kf4mErh1tlbHzKz3941+d9vpEsPM2clgdOaYOKfNQGI=
-github.com/jfrog/jfrog-client-go v1.35.3/go.mod h1:rXo6bjAe10V8nLMDYDl1n8sOM6mDRAGSZCo18Rs9kdM=
+github.com/jfrog/jfrog-client-go v1.35.4 h1:sBl+ELAa64pCA+SdpCs6Dtqrv7sZgfcf3T9R2V6lbpc=
+github.com/jfrog/jfrog-client-go v1.35.4/go.mod h1:3z/OO8M4dym7wOm5StrNq2gWIIzyMyE6CKPW2R291sw=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=

xray/commands/audit/sca/common_test.go

@@ -1,6 +1,7 @@
 package sca
 
 import (
+	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
 	"github.com/jfrog/jfrog-client-go/xray/services"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
 	"github.com/stretchr/testify/assert"
@@ -160,3 +161,65 @@ func TestBuildImpactPaths(t *testing.T) {
 	expectedImpactPaths = [][]services.ImpactPathNode{{{ComponentId: "dep1"}, {ComponentId: "dep2"}, {ComponentId: "dep3"}}}
 	reflect.DeepEqual(expectedImpactPaths, scanResult[0].Licenses[0].Components["dep3"].ImpactPaths)
 }
+
+func TestBuildXrayDependencyTree(t *testing.T) {
+	treeHelper := make(map[string][]string)
+	rootDep := []string{"topDep1", "topDep2", "topDep3"}
+	topDep1 := []string{"midDep1", "midDep2"}
+	topDep2 := []string{"midDep2", "midDep3"}
+	midDep1 := []string{"bottomDep1"}
+	midDep2 := []string{"bottomDep2", "bottomDep3"}
+	bottomDep3 := []string{"leafDep"}
+	treeHelper["rootDep"] = rootDep
+	treeHelper["topDep1"] = topDep1
+	treeHelper["topDep2"] = topDep2
+	treeHelper["midDep1"] = midDep1
+	treeHelper["midDep2"] = midDep2
+	treeHelper["bottomDep3"] = bottomDep3
+
+	expectedUniqueDeps := []string{"rootDep", "topDep1", "topDep2", "topDep3", "midDep1", "midDep2", "midDep3", "bottomDep1", "bottomDep2", "bottomDep3", "leafDep"}
+
+	// Constructing the expected tree Nodes
+	leafDepNode := &xrayUtils.GraphNode{Id: "leafDep", Nodes: []*xrayUtils.GraphNode{}}
+	bottomDep3Node := &xrayUtils.GraphNode{Id: "bottomDep3", Nodes: []*xrayUtils.GraphNode{}}
+	bottomDep2Node := &xrayUtils.GraphNode{Id: "bottomDep2", Nodes: []*xrayUtils.GraphNode{}}
+	bottomDep1Node := &xrayUtils.GraphNode{Id: "bottomDep1", Nodes: []*xrayUtils.GraphNode{}}
+	midDep3Node := &xrayUtils.GraphNode{Id: "midDep3", Nodes: []*xrayUtils.GraphNode{}}
+	midDep2Node := &xrayUtils.GraphNode{Id: "midDep2", Nodes: []*xrayUtils.GraphNode{}}
+	midDep1Node := &xrayUtils.GraphNode{Id: "midDep1", Nodes: []*xrayUtils.GraphNode{}}
+	topDep3Node := &xrayUtils.GraphNode{Id: "topDep3", Nodes: []*xrayUtils.GraphNode{}}
+	topDep2Node := &xrayUtils.GraphNode{Id: "topDep2", Nodes: []*xrayUtils.GraphNode{}}
+	topDep1Node := &xrayUtils.GraphNode{Id: "topDep1", Nodes: []*xrayUtils.GraphNode{}}
+	rootNode := &xrayUtils.GraphNode{Id: "rootDep", Nodes: []*xrayUtils.GraphNode{}}
+
+	// Setting children to parents
+	bottomDep3Node.Nodes = append(bottomDep3Node.Nodes, leafDepNode)
+	midDep2Node.Nodes = append(midDep2Node.Nodes, bottomDep3Node)
+	midDep2Node.Nodes = append(midDep2Node.Nodes, bottomDep2Node)
+	midDep1Node.Nodes = append(midDep1Node.Nodes, bottomDep1Node)
+	topDep2Node.Nodes = append(topDep2Node.Nodes, midDep3Node)
+	topDep2Node.Nodes = append(topDep2Node.Nodes, midDep2Node)
+	topDep1Node.Nodes = append(topDep1Node.Nodes, midDep2Node)
+	topDep1Node.Nodes = append(topDep1Node.Nodes, midDep1Node)
+	rootNode.Nodes = append(rootNode.Nodes, topDep1Node)
+	rootNode.Nodes = append(rootNode.Nodes, topDep2Node)
+	rootNode.Nodes = append(rootNode.Nodes, topDep3Node)
+
+	// Setting children to parents
+	leafDepNode.Parent = bottomDep3Node
+	bottomDep3Node.Parent = midDep2Node
+	bottomDep3Node.Parent = midDep2Node
+	bottomDep1Node.Parent = midDep1Node
+	midDep3Node.Parent = topDep2Node
+	midDep2Node.Parent = topDep2Node
+	midDep2Node.Parent = topDep1Node
+	midDep1Node.Parent = topDep1Node
+	topDep1Node.Parent = rootNode
+	topDep2Node.Parent = rootNode
+	topDep3Node.Parent = rootNode
+
+	tree, uniqueDeps := BuildXrayDependencyTree(treeHelper, "rootDep")
+
+	assert.ElementsMatch(t, expectedUniqueDeps, uniqueDeps)
+	assert.True(t, tests.CompareTree(tree, rootNode))
+}

xray/commands/audit/sca/java/deptreemanager.go

@@ -5,6 +5,7 @@ import (
 	"github.com/jfrog/gofrog/datastructures"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+	"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca"
 	xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
@@ -58,43 +59,24 @@ type depTreeNode struct {
 	Children []string `json:"children"`
 }
 
-// getGraphFromDepTree reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes.
+// Reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes.
 // It takes the output of the plugin's run (which is a byte representation of a list of paths of the output files, separated by newlines) as input.
 func getGraphFromDepTree(outputFilePaths string) (depsGraph []*xrayUtils.GraphNode, uniqueDeps []string, err error) {
 	modules, err := parseDepTreeFiles(outputFilePaths)
 	if err != nil {
 		return
 	}
-	uniqueDepsSet := datastructures.MakeSet[string]()
-	for _, moduleTree := range modules {
-		directDepId := GavPackageTypeIdentifier + moduleTree.Root
-		directDependency := &xrayUtils.GraphNode{
-			Id:    directDepId,
-			Nodes: []*xrayUtils.GraphNode{},
-		}
-		uniqueDepsSet.Add(directDepId)
-		populateDependencyTree(directDependency, moduleTree.Root, moduleTree, uniqueDepsSet)
-		depsGraph = append(depsGraph, directDependency)
-	}
-	uniqueDeps = uniqueDepsSet.ToSlice()
-	return
-}
 
-func populateDependencyTree(currNode *xrayUtils.GraphNode, currNodeId string, moduleTree *moduleDepTree, uniqueDepsSet *datastructures.Set[string]) {
-	if currNode.NodeHasLoop() {
-		return
-	}
-	for _, childId := range moduleTree.Nodes[currNodeId].Children {
-		childGav := GavPackageTypeIdentifier + childId
-		childNode := &xrayUtils.GraphNode{
-			Id:     childGav,
-			Nodes:  []*xrayUtils.GraphNode{},
-			Parent: currNode,
+	allModulesUniqueDeps := datastructures.MakeSet[string]()
+	for _, module := range modules {
+		moduleTree, moduleUniqueDeps := getModuleTreeAndDependencies(module)
+		depsGraph = append(depsGraph, moduleTree)
+		for _, depToAdd := range moduleUniqueDeps {
+			allModulesUniqueDeps.Add(depToAdd)
 		}
-		uniqueDepsSet.Add(childGav)
-		populateDependencyTree(childNode, childId, moduleTree, uniqueDepsSet)
-		currNode.Nodes = append(currNode.Nodes, childNode)
 	}
+	uniqueDeps = allModulesUniqueDeps.ToSlice()
+	return
 }
 
 func parseDepTreeFiles(jsonFilePaths string) ([]*moduleDepTree, error) {
@@ -130,3 +112,21 @@ func getArtifactoryAuthFromServer(server *config.ServerDetails) (string, string,
 	}
 	return username, password, nil
 }
+
+// Returns a dependency tree and a flat list of the module's dependencies for the given module
+func getModuleTreeAndDependencies(module *moduleDepTree) (*xrayUtils.GraphNode, []string) {
+	moduleTreeMap := make(map[string][]string)
+	moduleDeps := module.Nodes
+	for depName, dependency := range moduleDeps {
+		dependencyId := GavPackageTypeIdentifier + depName
+		var childrenList []string
+		for _, childName := range dependency.Children {
+			childId := GavPackageTypeIdentifier + childName
+			childrenList = append(childrenList, childId)
+		}
+		if len(childrenList) > 0 {
+			moduleTreeMap[dependencyId] = childrenList
+		}
+	}
+	return sca.BuildXrayDependencyTree(moduleTreeMap, GavPackageTypeIdentifier+module.Root)
+}