diff --git a/artifactory/commands/transferfiles/localgeneratedfilter.go b/artifactory/commands/transferfiles/localgeneratedfilter.go index 5bb6560e3..eaac899b6 100644 --- a/artifactory/commands/transferfiles/localgeneratedfilter.go +++ b/artifactory/commands/transferfiles/localgeneratedfilter.go @@ -160,5 +160,8 @@ func (lg *locallyGeneratedFilter) getNonLocallyGeneratedResults(aqlResultItems [ } func getPathInRepo(aqlResultItem *utils.ResultItem) string { + if aqlResultItem.Path == "." { + return aqlResultItem.Name + } return aqlResultItem.Path + "/" + aqlResultItem.Name } diff --git a/artifactory/commands/transferfiles/localgeneratedfilter_test.go b/artifactory/commands/transferfiles/localgeneratedfilter_test.go index 63feef2f8..3b6e46a24 100644 --- a/artifactory/commands/transferfiles/localgeneratedfilter_test.go +++ b/artifactory/commands/transferfiles/localgeneratedfilter_test.go @@ -22,6 +22,7 @@ var filterLocallyGeneratedCases = []struct { {paths: []utils.ResultItem{{Path: "a", Name: "b"}}, returnedPath: []string{"a/b"}}, {paths: []utils.ResultItem{{Path: "a", Name: "b"}, {Path: "a/b", Name: "e"}}, returnedPath: []string{"a/b/e"}}, {paths: []utils.ResultItem{{Path: "a", Name: "b"}, {Path: "a/b", Name: "e"}}, returnedPath: []string{"a/b", "a/b/e"}}, + {paths: []utils.ResultItem{{Path: ".", Name: "a"}, {Path: "b", Name: "c"}}, returnedPath: []string{"b/c"}}, } func TestFilterLocallyGenerated(t *testing.T) { @@ -99,3 +100,20 @@ func TestFilterLocallyGeneratedEnabled(t *testing.T) { assert.False(t, NewLocallyGenerated(context.Background(), servicesManager, "7.54.5").IsEnabled()) assert.False(t, NewLocallyGenerated(context.Background(), servicesManager, "6.0.0").IsEnabled()) } + +var getPathInRepoCases = []struct { + aqlResultItem utils.ResultItem + expectedPathInRepo string +}{ + {aqlResultItem: utils.ResultItem{Path: ".", Name: "a"}, expectedPathInRepo: "a"}, + {aqlResultItem: utils.ResultItem{Path: "a", Name: "b"}, expectedPathInRepo: "a/b"}, +} + +func TestGetPathInRepo(t *testing.T) { + for _, testCase := range getPathInRepoCases { + t.Run("", func(t *testing.T) { + aqlResultsItem := testCase.aqlResultItem + assert.Equal(t, testCase.expectedPathInRepo, getPathInRepo(&aqlResultsItem)) + }) + } +} diff --git a/artifactory/commands/transferfiles/manager.go b/artifactory/commands/transferfiles/manager.go index 513331c49..1f981144d 100644 --- a/artifactory/commands/transferfiles/manager.go +++ b/artifactory/commands/transferfiles/manager.go @@ -238,8 +238,13 @@ func runProducerConsumers(pcWrapper *producerConsumerWrapper) (executionErr erro // Run() is a blocking method, so once all chunk builders are idle, the tasks queue closes and Run() stops running. pcWrapper.chunkBuilderProducerConsumer.Run() if pcWrapper.chunkUploaderProducerConsumer.IsStarted() { + // There might be a moment when the chunk uploader has no upload tasks. + // This circumstance might lead to setting the finish notification before completing all file uploads. + // To address this, we reset the finish notification to ensure no remaining upload tasks after the next finish notification. + pcWrapper.chunkUploaderProducerConsumer.ResetFinishNotificationIfActive() // Wait till notified that the uploader finished its tasks, and it will not receive new tasks from the builder. <-pcWrapper.chunkUploaderProducerConsumer.GetFinishedNotification() + log.Debug("Chunk uploaded producer consumer has completed all tasks. All files relevant to this phase have all been uploaded.") } // Close the tasks queue with Done(). pcWrapper.chunkUploaderProducerConsumer.Done() diff --git a/artifactory/commands/transferfiles/manager_test.go b/artifactory/commands/transferfiles/manager_test.go new file mode 100644 index 000000000..7e946eeff --- /dev/null +++ b/artifactory/commands/transferfiles/manager_test.go @@ -0,0 +1,37 @@ +package transferfiles + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRunProducerConsumers(t *testing.T) { + // Create the producer-consumers + producerConsumerWrapper := newProducerConsumerWrapper() + + // Add 10 tasks for the chunkBuilderProducerConsumer. Each task provides a task to the chunkUploaderProducerConsumer. + for i := 0; i < 10; i++ { + _, err := producerConsumerWrapper.chunkBuilderProducerConsumer.AddTask(func(int) error { + time.Sleep(time.Millisecond * 100) + _, err := producerConsumerWrapper.chunkUploaderProducerConsumer.AddTask( + func(int) error { + time.Sleep(time.Millisecond) + return nil + }, + ) + assert.NoError(t, err) + return nil + }) + assert.NoError(t, err) + } + + // Run the producer-consumers + err := runProducerConsumers(&producerConsumerWrapper) + assert.NoError(t, err) + + // Assert no active treads left in the producer-consumers + assert.Zero(t, producerConsumerWrapper.chunkBuilderProducerConsumer.ActiveThreads()) + assert.Zero(t, producerConsumerWrapper.chunkUploaderProducerConsumer.ActiveThreads()) +} diff --git a/go.mod b/go.mod index c59022b36..0213f6453 100644 --- a/go.mod +++ b/go.mod @@ -14,9 +14,9 @@ require ( github.com/google/uuid v1.5.0 github.com/gookit/color v1.5.4 github.com/jfrog/build-info-go v1.9.18 - github.com/jfrog/gofrog v1.3.2 + github.com/jfrog/gofrog v1.4.0 github.com/jfrog/jfrog-apps-config v1.0.1 - github.com/jfrog/jfrog-client-go v1.35.3 + github.com/jfrog/jfrog-client-go v1.35.4 github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 github.com/owenrumney/go-sarif/v2 v2.3.0 @@ -103,4 +103,4 @@ require ( // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231220102935-c8776c613ad8 -// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.3.2-0.20231130091721-6d742be8bc7a +// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.3.3-0.20231223133729-ef57bd08cedc diff --git a/go.sum b/go.sum index b1dda394f..57a8b3339 100644 --- a/go.sum +++ b/go.sum @@ -87,12 +87,12 @@ github.com/jedib0t/go-pretty/v6 v6.4.0 h1:YlI/2zYDrweA4MThiYMKtGRfT+2qZOO65ulej8 github.com/jedib0t/go-pretty/v6 v6.4.0/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI= github.com/jfrog/build-info-go v1.9.18 h1:0RKeZtNWZjONX5j94aIPQSKMi1whP2evmGQymYF+5mA= github.com/jfrog/build-info-go v1.9.18/go.mod h1:/5VZXH2Ud0IK3cOFwPykjwPOaEcHhzzbjnRiou+YKpM= -github.com/jfrog/gofrog v1.3.2 h1:TktKP+PdZdxjkYZxWWIq4DkTGSYtr9Slsy+egZpEhUY= -github.com/jfrog/gofrog v1.3.2/go.mod h1:AQo5Fq0G9nDEF6icH7MYQK0iohR4HuEAXl8jaxRuT6Q= +github.com/jfrog/gofrog v1.4.0 h1:s7eysVnmIBfVheMs4LPU43MAlxwPa4K8u2N5h7kwzXA= +github.com/jfrog/gofrog v1.4.0/go.mod h1:AQo5Fq0G9nDEF6icH7MYQK0iohR4HuEAXl8jaxRuT6Q= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-client-go v1.35.3 h1:Kf4mErh1tlbHzKz3941+d9vpEsPM2clgdOaYOKfNQGI= -github.com/jfrog/jfrog-client-go v1.35.3/go.mod h1:rXo6bjAe10V8nLMDYDl1n8sOM6mDRAGSZCo18Rs9kdM= +github.com/jfrog/jfrog-client-go v1.35.4 h1:sBl+ELAa64pCA+SdpCs6Dtqrv7sZgfcf3T9R2V6lbpc= +github.com/jfrog/jfrog-client-go v1.35.4/go.mod h1:3z/OO8M4dym7wOm5StrNq2gWIIzyMyE6CKPW2R291sw= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= diff --git a/xray/commands/audit/sca/common_test.go b/xray/commands/audit/sca/common_test.go index a0f751ee3..dd4bb1bab 100644 --- a/xray/commands/audit/sca/common_test.go +++ b/xray/commands/audit/sca/common_test.go @@ -1,6 +1,7 @@ package sca import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/tests" "github.com/jfrog/jfrog-client-go/xray/services" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "github.com/stretchr/testify/assert" @@ -160,3 +161,65 @@ func TestBuildImpactPaths(t *testing.T) { expectedImpactPaths = [][]services.ImpactPathNode{{{ComponentId: "dep1"}, {ComponentId: "dep2"}, {ComponentId: "dep3"}}} reflect.DeepEqual(expectedImpactPaths, scanResult[0].Licenses[0].Components["dep3"].ImpactPaths) } + +func TestBuildXrayDependencyTree(t *testing.T) { + treeHelper := make(map[string][]string) + rootDep := []string{"topDep1", "topDep2", "topDep3"} + topDep1 := []string{"midDep1", "midDep2"} + topDep2 := []string{"midDep2", "midDep3"} + midDep1 := []string{"bottomDep1"} + midDep2 := []string{"bottomDep2", "bottomDep3"} + bottomDep3 := []string{"leafDep"} + treeHelper["rootDep"] = rootDep + treeHelper["topDep1"] = topDep1 + treeHelper["topDep2"] = topDep2 + treeHelper["midDep1"] = midDep1 + treeHelper["midDep2"] = midDep2 + treeHelper["bottomDep3"] = bottomDep3 + + expectedUniqueDeps := []string{"rootDep", "topDep1", "topDep2", "topDep3", "midDep1", "midDep2", "midDep3", "bottomDep1", "bottomDep2", "bottomDep3", "leafDep"} + + // Constructing the expected tree Nodes + leafDepNode := &xrayUtils.GraphNode{Id: "leafDep", Nodes: []*xrayUtils.GraphNode{}} + bottomDep3Node := &xrayUtils.GraphNode{Id: "bottomDep3", Nodes: []*xrayUtils.GraphNode{}} + bottomDep2Node := &xrayUtils.GraphNode{Id: "bottomDep2", Nodes: []*xrayUtils.GraphNode{}} + bottomDep1Node := &xrayUtils.GraphNode{Id: "bottomDep1", Nodes: []*xrayUtils.GraphNode{}} + midDep3Node := &xrayUtils.GraphNode{Id: "midDep3", Nodes: []*xrayUtils.GraphNode{}} + midDep2Node := &xrayUtils.GraphNode{Id: "midDep2", Nodes: []*xrayUtils.GraphNode{}} + midDep1Node := &xrayUtils.GraphNode{Id: "midDep1", Nodes: []*xrayUtils.GraphNode{}} + topDep3Node := &xrayUtils.GraphNode{Id: "topDep3", Nodes: []*xrayUtils.GraphNode{}} + topDep2Node := &xrayUtils.GraphNode{Id: "topDep2", Nodes: []*xrayUtils.GraphNode{}} + topDep1Node := &xrayUtils.GraphNode{Id: "topDep1", Nodes: []*xrayUtils.GraphNode{}} + rootNode := &xrayUtils.GraphNode{Id: "rootDep", Nodes: []*xrayUtils.GraphNode{}} + + // Setting children to parents + bottomDep3Node.Nodes = append(bottomDep3Node.Nodes, leafDepNode) + midDep2Node.Nodes = append(midDep2Node.Nodes, bottomDep3Node) + midDep2Node.Nodes = append(midDep2Node.Nodes, bottomDep2Node) + midDep1Node.Nodes = append(midDep1Node.Nodes, bottomDep1Node) + topDep2Node.Nodes = append(topDep2Node.Nodes, midDep3Node) + topDep2Node.Nodes = append(topDep2Node.Nodes, midDep2Node) + topDep1Node.Nodes = append(topDep1Node.Nodes, midDep2Node) + topDep1Node.Nodes = append(topDep1Node.Nodes, midDep1Node) + rootNode.Nodes = append(rootNode.Nodes, topDep1Node) + rootNode.Nodes = append(rootNode.Nodes, topDep2Node) + rootNode.Nodes = append(rootNode.Nodes, topDep3Node) + + // Setting children to parents + leafDepNode.Parent = bottomDep3Node + bottomDep3Node.Parent = midDep2Node + bottomDep3Node.Parent = midDep2Node + bottomDep1Node.Parent = midDep1Node + midDep3Node.Parent = topDep2Node + midDep2Node.Parent = topDep2Node + midDep2Node.Parent = topDep1Node + midDep1Node.Parent = topDep1Node + topDep1Node.Parent = rootNode + topDep2Node.Parent = rootNode + topDep3Node.Parent = rootNode + + tree, uniqueDeps := BuildXrayDependencyTree(treeHelper, "rootDep") + + assert.ElementsMatch(t, expectedUniqueDeps, uniqueDeps) + assert.True(t, tests.CompareTree(tree, rootNode)) +} diff --git a/xray/commands/audit/sca/java/deptreemanager.go b/xray/commands/audit/sca/java/deptreemanager.go index e6add9fd2..53cab3b77 100644 --- a/xray/commands/audit/sca/java/deptreemanager.go +++ b/xray/commands/audit/sca/java/deptreemanager.go @@ -5,6 +5,7 @@ import ( "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca" xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" @@ -58,43 +59,24 @@ type depTreeNode struct { Children []string `json:"children"` } -// getGraphFromDepTree reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes. +// Reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes. // It takes the output of the plugin's run (which is a byte representation of a list of paths of the output files, separated by newlines) as input. func getGraphFromDepTree(outputFilePaths string) (depsGraph []*xrayUtils.GraphNode, uniqueDeps []string, err error) { modules, err := parseDepTreeFiles(outputFilePaths) if err != nil { return } - uniqueDepsSet := datastructures.MakeSet[string]() - for _, moduleTree := range modules { - directDepId := GavPackageTypeIdentifier + moduleTree.Root - directDependency := &xrayUtils.GraphNode{ - Id: directDepId, - Nodes: []*xrayUtils.GraphNode{}, - } - uniqueDepsSet.Add(directDepId) - populateDependencyTree(directDependency, moduleTree.Root, moduleTree, uniqueDepsSet) - depsGraph = append(depsGraph, directDependency) - } - uniqueDeps = uniqueDepsSet.ToSlice() - return -} -func populateDependencyTree(currNode *xrayUtils.GraphNode, currNodeId string, moduleTree *moduleDepTree, uniqueDepsSet *datastructures.Set[string]) { - if currNode.NodeHasLoop() { - return - } - for _, childId := range moduleTree.Nodes[currNodeId].Children { - childGav := GavPackageTypeIdentifier + childId - childNode := &xrayUtils.GraphNode{ - Id: childGav, - Nodes: []*xrayUtils.GraphNode{}, - Parent: currNode, + allModulesUniqueDeps := datastructures.MakeSet[string]() + for _, module := range modules { + moduleTree, moduleUniqueDeps := getModuleTreeAndDependencies(module) + depsGraph = append(depsGraph, moduleTree) + for _, depToAdd := range moduleUniqueDeps { + allModulesUniqueDeps.Add(depToAdd) } - uniqueDepsSet.Add(childGav) - populateDependencyTree(childNode, childId, moduleTree, uniqueDepsSet) - currNode.Nodes = append(currNode.Nodes, childNode) } + uniqueDeps = allModulesUniqueDeps.ToSlice() + return } func parseDepTreeFiles(jsonFilePaths string) ([]*moduleDepTree, error) { @@ -130,3 +112,21 @@ func getArtifactoryAuthFromServer(server *config.ServerDetails) (string, string, } return username, password, nil } + +// Returns a dependency tree and a flat list of the module's dependencies for the given module +func getModuleTreeAndDependencies(module *moduleDepTree) (*xrayUtils.GraphNode, []string) { + moduleTreeMap := make(map[string][]string) + moduleDeps := module.Nodes + for depName, dependency := range moduleDeps { + dependencyId := GavPackageTypeIdentifier + depName + var childrenList []string + for _, childName := range dependency.Children { + childId := GavPackageTypeIdentifier + childName + childrenList = append(childrenList, childId) + } + if len(childrenList) > 0 { + moduleTreeMap[dependencyId] = childrenList + } + } + return sca.BuildXrayDependencyTree(moduleTreeMap, GavPackageTypeIdentifier+module.Root) +}