Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
yahavi committed Jul 21, 2024
2 parents 41b7a22 + 82c0004 commit ed1bcb8
Show file tree
Hide file tree
Showing 30 changed files with 769 additions and 192 deletions.
69 changes: 66 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,8 @@
- [Add Builds to Indexing Configuration](#add-builds-to-indexing-configuration)
- [Request Graph Scan](#request-graph-scan)
- [Retrieve the Graph Scan Results](#retrieve-the-graph-scan-results)
- [Request Graph Enrich](#request-graph-enrich)
- [Retrieve the Graph Enrich Results](#retrieve-the-graph-enrich-results)
- [Generate Vulnerabilities Report](#generate-vulnerabilities-report)
- [Get Vulnerabilities Report Details](#get-vulnerabilities-report-details)
- [Get Vulnerabilities Report Content](#get-vulnerabilities-report-content)
Expand Down Expand Up @@ -222,7 +224,7 @@
- [Distribute Release Bundle](#distribute-release-bundle)
- [Delete Release Bundle Version](#delete-release-bundle-version)
- [Delete Release Bundle Version Promotion](#delete-release-bundle-version-promotion)
- [Export Release Bundle](#export-release-bundle)
- [Export Release Bundle Archive](#export-release-bundle-archive)
- [Import Release Bundle](#import-release-bundle)
- [Remote Delete Release Bundle](#remote-delete-release-bundle)
- [Lifecycle APIs](#lifecycle-apis)
Expand Down Expand Up @@ -453,7 +455,10 @@ params.ChecksumsCalcEnabled = false
targetProps := utils.NewProperties()
targetProps.AddProperty("key1", "val1")
params.TargetProps = targetProps

// When using the 'archive' option for upload, we can control the target path inside the uploaded archive using placeholders. This operation determines the TargetPathInArchive value.
TargetPathInArchive := "archive/path/"
// Size limit for files to be uploaded.
SizeLimit= &fspatterns.SizeThreshold{SizeInBytes: 10000, Condition: fspatterns.LessThan}
totalUploaded, totalFailed, err := rtManager.UploadFiles(params)
```

Expand Down Expand Up @@ -482,7 +487,9 @@ params.Retries = 5
params.SplitCount = 2
// MinSplitSize default value: 5120
params.MinSplitSize = 7168

// Optional fields to avoid AQL request
Sha256 = "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"
Size = 1000
totalDownloaded, totalFailed, err := rtManager.DownloadFiles(params)
```

Expand Down Expand Up @@ -2003,6 +2010,62 @@ scanId, err := xrayManager.ScanGraph(graphScanParams)
scanResults, err := xrayManager.GetScanGraphResults(scanId)
```
#### Request Graph Enrich
```go
graphImportParams := &XrayGraphImportParams{}
// Dependency tree. Each node must have a component identifier, see https://www.jfrog.com/confluence/display/JFROG/Xray+REST+API#XrayRESTAPI-ComponentIdentifiers.
graphScanParams.SBOMInput = "{
"bomFormat": "CycloneDX",
"specVersion": "1.4",
"serialNumber": "urn:uuid:3c94db59-0dbf-41cd-49e8-c4518ac2ef3c",
"version": 1,
"metadata": {
"timestamp": "2024-05-22T14:52:40Z",
"tools": [
{
"vendor": "JFrog Inc.",
"name": "Xray",
"version": "3.95.7"
}
],
"component": {
"type": "container",
"name": "jfrog/artifactory-pro:sha256",
"version": "2e774ffb112bcaef62804d97e6db3dc67b9169b440838b12ba12584cba2c5251"
}
},
"components": [
{
"bom-ref": "pkg:Oci/jfrog%2Fartifactory-pro:sha256@2e774ffb112bcaef62804d97e6db3dc67b9169b440838b12ba12584cba2c5251",
"type": "application",
"name": "jfrog/artifactory-pro:sha256",
"version": "2e774ffb112bcaef62804d97e6db3dc67b9169b440838b12ba12584cba2c5251",
"hashes": [
{
"alg": "SHA-256",
"content": "2e774ffb112bcaef62804d97e6db3dc67b9169b440838b12ba12584cba2c5251"
}
],
"licenses": [],
"purl": "pkg:Oci/jfrog%2Fartifactory-pro:sha256@2e774ffb112bcaef62804d97e6db3dc67b9169b440838b12ba12584cba2c5251"
}
],
"dependencies": []
}
"
scanId, err := xrayManager.ImportGraph(graphImportParams)
```
#### Retrieve the Graph Enrich Results
```go
// scanId should be received from xrayManager.ImportGraph(graphImportParams) request.
enrichResults, err := xrayManager.GetImportGraphResults(scanId)
```
#### Generate Vulnerabilities Report
```go
Expand Down
4 changes: 2 additions & 2 deletions artifactory/services/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func removeNotToBeDeletedDirs(specFile *utils.CommonParams, ds *DeleteService, d
if err != nil {
return nil, err
}
bufferFiles, err := utils.FilterCandidateToBeDeleted(deleteCandidates, resultWriter, "folder")
bufferFiles, err := utils.FilterCandidateToBeDeleted(deleteCandidates, resultWriter, utils.Folder)
if len(bufferFiles) > 0 {
defer func() {
for _, file := range bufferFiles {
Expand Down Expand Up @@ -256,7 +256,7 @@ func getSortedArtifactsToNotDelete(specFile *utils.CommonParams, ds *DeleteServi
// 1. Go sorts strings differently from Artifactory's database, when the strings include special chars, such as dashes.
// 2. Artifactory sorts by database columns, so directories will be sorted differently than files,
// because the path and name cols have different values.
sortedResults, err := utils.FilterCandidateToBeDeleted(tempResults, resultWriter, "file")
sortedResults, err := utils.FilterCandidateToBeDeleted(tempResults, resultWriter, utils.File)
if err != nil {
return nil, err
}
Expand Down
97 changes: 77 additions & 20 deletions artifactory/services/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@ package services

import (
"errors"
"github.com/jfrog/build-info-go/entities"
biutils "github.com/jfrog/build-info-go/utils"
ioutils "github.com/jfrog/gofrog/io"
"github.com/jfrog/gofrog/version"
"net/http"
"os"
"path"
"path/filepath"
"sort"

biutils "github.com/jfrog/build-info-go/utils"
"github.com/jfrog/gofrog/version"

"github.com/jfrog/build-info-go/entities"
"strings"

"github.com/jfrog/jfrog-client-go/http/httpclient"

Expand Down Expand Up @@ -164,18 +164,30 @@ func (ds *DownloadService) prepareTasks(producer parallel.Runner, expectedChan c
errorsQueue.AddError(err)
}
}

var reader *content.ContentReader
// Create handler function for the current group.
fileHandlerFunc := ds.createFileHandlerFunc(downloadParams, successCounters)
// Search items.
log.Info("Searching items to download...")
switch downloadParams.GetSpecType() {
case utils.WILDCARD:
reader, err = ds.collectFilesUsingWildcardPattern(downloadParams)
case utils.BUILD:
reader, err = utils.SearchBySpecWithBuild(downloadParams.GetFile(), ds)
case utils.AQL:
reader, err = utils.SearchBySpecWithAql(downloadParams.GetFile(), ds, utils.SYMLINK)
// Check if we can avoid using AQL to get the file's info.
avoidAql, err := isFieldsProvidedToAvoidAql(downloadParams)
// Check for search errors.
if err != nil {
log.Error(err)
errorsQueue.AddError(err)
continue
}
if avoidAql {
reader, err = createResultsItemWithoutAql(downloadParams)
} else {
// Search items using AQL and get their details (size/checksum/etc.) from Artifactory.
switch downloadParams.GetSpecType() {
case utils.WILDCARD:
reader, err = utils.SearchBySpecWithPattern(downloadParams.GetFile(), ds, utils.SYMLINK)
case utils.BUILD:
reader, err = utils.SearchBySpecWithBuild(downloadParams.GetFile(), ds)
case utils.AQL:
reader, err = utils.SearchBySpecWithAql(downloadParams.GetFile(), ds, utils.SYMLINK)
}
}
// Check for search errors.
if err != nil {
Expand All @@ -197,8 +209,49 @@ func (ds *DownloadService) prepareTasks(producer parallel.Runner, expectedChan c
}()
}

func (ds *DownloadService) collectFilesUsingWildcardPattern(downloadParams DownloadParams) (*content.ContentReader, error) {
return utils.SearchBySpecWithPattern(downloadParams.GetFile(), ds, utils.SYMLINK)
func isFieldsProvidedToAvoidAql(downloadParams DownloadParams) (bool, error) {
if downloadParams.Sha256 != "" && downloadParams.Size != nil {
// If sha256 and size is provided, we can avoid using AQL to get the file's info.
return true, nil
} else if downloadParams.Sha256 == "" && downloadParams.Size == nil {
// If sha256 and size is missing, we can't avoid using AQL to get the file's info.
return false, nil
}
// If only one of the fields is provided, return an error.
return false, errors.New("both sha256 and size must be provided in order to avoid using AQL")
}

func createResultsItemWithoutAql(downloadParams DownloadParams) (*content.ContentReader, error) {
writer, err := content.NewContentWriter(content.DefaultKey, true, false)
if err != nil {
return nil, err
}
defer ioutils.Close(writer, &err)
repo, path, name, err := breakFileDownloadPathToParts(downloadParams.GetPattern())
if err != nil {
return nil, err
}
resultItem := &utils.ResultItem{
Type: string(utils.File),
Repo: repo,
Path: path,
Name: name,
Size: *downloadParams.Size,
Sha256: downloadParams.Sha256,
}
writer.Write(*resultItem)
return content.NewContentReader(writer.GetFilePath(), writer.GetArrayKey()), nil
}

func breakFileDownloadPathToParts(downloadPath string) (repo, path, name string, err error) {
if utils.IsWildcardPattern(downloadPath) {
return "", "", "", errorutils.CheckErrorf("downloading without AQL is not supported for the provided wildcard pattern: " + downloadPath)
}
parts := strings.Split(downloadPath, "/")
repo = parts[0]
path = strings.Join(parts[1:len(parts)-1], "/")
name = parts[len(parts)-1]
return
}

func (ds *DownloadService) produceTasks(reader *content.ContentReader, downloadParams DownloadParams, producer parallel.Runner, fileHandler fileHandlerFunc, errorsQueue *clientutils.ErrorsQueue) int {
Expand Down Expand Up @@ -247,7 +300,7 @@ func (ds *DownloadService) produceTasks(reader *content.ContentReader, downloadP
Target: downloadParams.GetTarget(),
Flat: flat,
}
if resultItem.Type != "folder" {
if resultItem.Type != string(utils.Folder) {
if len(ds.rbGpgValidationMap) != 0 {
// Gpg validation to the downloaded artifact
err = rbGpgValidate(ds.rbGpgValidationMap, downloadParams.GetBundle(), resultItem)
Expand Down Expand Up @@ -400,6 +453,7 @@ func (ds *DownloadService) downloadFile(downloadFileDetails *httpclient.Download
LocalFileName: downloadFileDetails.LocalFileName,
LocalPath: downloadFileDetails.LocalPath,
ExpectedSha1: downloadFileDetails.ExpectedSha1,
ExpectedSha256: downloadFileDetails.ExpectedSha256,
FileSize: downloadFileDetails.Size,
SplitCount: downloadParams.SplitCount,
Explode: downloadParams.Explode,
Expand Down Expand Up @@ -509,15 +563,15 @@ func (ds *DownloadService) createFileHandlerFunc(downloadParams DownloadParams,
return err
}
localPath, localFileName := fileutils.GetLocalPathAndFile(downloadData.Dependency.Name, downloadData.Dependency.Path, target, downloadData.Flat, placeholdersUsed)
if downloadData.Dependency.Type == "folder" {
if downloadData.Dependency.Type == string(utils.Folder) {
return createDir(localPath, localFileName, logMsgPrefix)
}
if err = removeIfSymlink(filepath.Join(localPath, localFileName)); err != nil {
return err
}
if downloadParams.IsSymlink() {
if isSymlink, e := ds.createSymlinkIfNeeded(ds.GetArtifactoryDetails().GetUrl(), localPath, localFileName, logMsgPrefix, downloadData, successCounters, threadId, downloadParams); isSymlink {
return e
if isSymlink, err := ds.createSymlinkIfNeeded(ds.GetArtifactoryDetails().GetUrl(), localPath, localFileName, logMsgPrefix, downloadData, successCounters, threadId, downloadParams); isSymlink {
return err
}
}
if err = ds.downloadFileIfNeeded(downloadPath, localPath, localFileName, logMsgPrefix, downloadData, downloadParams); err != nil {
Expand Down Expand Up @@ -592,6 +646,9 @@ type DownloadParams struct {
SplitCount int
PublicGpgKey string
SkipChecksum bool
// Optional fields to avoid AQL request
Sha256 string
Size *int64
}

func (ds *DownloadParams) IsFlat() bool {
Expand Down
38 changes: 38 additions & 0 deletions artifactory/services/download_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package services

import (
"github.com/stretchr/testify/assert"
"testing"
)

func TestBreakFileDownloadPathToParts(t *testing.T) {
testCases := []struct {
name string
downloadPath string
expectedRepo string
expectedPath string
expectedName string
expectError bool
}{
{"Single level path", "repo/file.txt", "repo", "", "file.txt", false},
{"Multi-level path", "repo/folder/subfolder/file.txt", "repo", "folder/subfolder", "file.txt", false},
{"Root level file", "repo/", "", "", "", true},
{"Empty path", "", "", "", "", true},
{"Invalid path", "file.txt", "", "", "", true},
{"Wildcard path", "repo/*.txt", "", "", "", true},
}

for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
repo, path, name, err := breakFileDownloadPathToParts(tt.downloadPath)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, tt.expectedRepo, repo)
assert.Equal(t, tt.expectedPath, path)
assert.Equal(t, tt.expectedName, name)
})
}
}
27 changes: 27 additions & 0 deletions artifactory/services/fspatterns/sizethreshold.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package fspatterns

// ThresholdCondition represents whether the threshold is for files above or below a specified size.
type ThresholdCondition int

const (
// GreaterThan is greater & equal
GreaterEqualThan ThresholdCondition = iota
// LessThan is only less
LessThan
)

type SizeThreshold struct {
SizeInBytes int64
Condition ThresholdCondition
}

func (st SizeThreshold) IsSizeWithinThreshold(actualSizeInBytes int64) bool {
switch st.Condition {
case GreaterEqualThan:
return actualSizeInBytes >= st.SizeInBytes
case LessThan:
return actualSizeInBytes < st.SizeInBytes
default:
return false
}
}
Loading

0 comments on commit ed1bcb8

Please sign in to comment.