Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/dev' into check-build-info-cha…
Browse files Browse the repository at this point in the history
…nges

# Conflicts:
#	go.mod
#	go.sum
  • Loading branch information
omerzi committed Dec 20, 2023
2 parents 1912d5c + 6a1417a commit a8555e4
Showing 23 changed files with 431 additions and 117 deletions.
12 changes: 8 additions & 4 deletions artifactory/services/fspatterns/utils.go
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ import (
)

// Return all the existing paths of the provided root path
func ListFiles(rootPath string, isRecursive, includeDirs, isSymlink bool, excludePathPattern string) ([]string, error) {
func ListFiles(rootPath string, isRecursive, includeDirs, excludeWithRelativePath, isSymlink bool, excludePathPattern string) ([]string, error) {
var paths []string
var err error
if isRecursive {
@@ -26,7 +26,11 @@ func ListFiles(rootPath string, isRecursive, includeDirs, isSymlink bool, exclud
if err != nil {
return paths, err
}
return filterFiles(paths, excludePathPattern)
var rootFilter string
if excludeWithRelativePath {
rootFilter = rootPath
}
return filterFiles(rootFilter, paths, excludePathPattern)
}

// Transform to regexp and prepare Exclude patterns to be used, exclusion patterns must be absolute paths.
@@ -49,13 +53,13 @@ func PrepareExcludePathPattern(exclusions []string, patternType utils.PatternTyp
return excludePathPattern
}

func filterFiles(files []string, excludePathPattern string) (filteredFiles []string, err error) {
func filterFiles(rootPath string, files []string, excludePathPattern string) (filteredFiles []string, err error) {
var excludedPath bool
for i := 0; i < len(files); i++ {
if files[i] == "." {
continue
}
excludedPath, err = isPathExcluded(files[i], excludePathPattern)
excludedPath, err = isPathExcluded(strings.TrimPrefix(files[i], rootPath), excludePathPattern)
if err != nil {
return
}
11 changes: 9 additions & 2 deletions artifactory/services/fspatterns/utils_test.go
Original file line number Diff line number Diff line change
@@ -12,12 +12,19 @@ func TestFilterFiles(t *testing.T) {
data := []struct {
files []string
ExcludePattern string
root string
result []string
}{
{[]string{"file1", filepath.Join("dir", "file1"), "file2.zip"}, "^*.zip$", []string{"file1", filepath.Join("dir", "file1")}},
{[]string{"file1", filepath.Join("dir", "file1"), "file2.zip"}, "^*.zip$", "", []string{"file1", filepath.Join("dir", "file1")}},
{[]string{
"file1",
"test.zip",
filepath.Join("test", "file1"),
filepath.Join("dir", "test", "should-be-filter"),
}, "(^.*test.*$)", "test", []string{"file1", "test.zip", filepath.Join("test", "file1")}},
}
for _, d := range data {
got, err := filterFiles(d.files, d.ExcludePattern)
got, err := filterFiles(d.root, d.files, d.ExcludePattern)
assert.NoError(t, err)
assert.Len(t, got, len(d.result))
assert.Contains(t, got, d.files[0])
1 change: 1 addition & 0 deletions artifactory/services/repository.go
Original file line number Diff line number Diff line change
@@ -77,6 +77,7 @@ type AdditionalRepositoryBaseParams struct {
PropertySets []string `json:"propertySets,omitempty"`
DownloadRedirect *bool `json:"downloadRedirect,omitempty"`
PriorityResolution *bool `json:"priorityResolution,omitempty"`
CdnRedirect *bool `json:"cdnRedirect,omitempty"`
}

type CargoRepositoryParams struct {
2 changes: 1 addition & 1 deletion artifactory/services/upload.go
Original file line number Diff line number Diff line change
@@ -306,7 +306,7 @@ func scanFilesByPattern(uploadParams UploadParams, rootPath string, progressMgr
if errorutils.CheckError(err) != nil {
return err
}
paths, err := fspatterns.ListFiles(rootPath, uploadParams.IsRecursive(), uploadParams.IsIncludeDirs(), uploadParams.IsSymlink(), excludePathPattern)
paths, err := fspatterns.ListFiles(rootPath, uploadParams.IsRecursive(), uploadParams.IsIncludeDirs(), false, uploadParams.IsSymlink(), excludePathPattern)
if err != nil {
return err
}
56 changes: 55 additions & 1 deletion artifactory/services/utils/aqlquerybuilder.go
Original file line number Diff line number Diff line change
@@ -2,14 +2,30 @@ package utils

import (
"fmt"
"golang.org/x/exp/slices"
"strconv"
"strings"
"unicode"

"golang.org/x/exp/slices"

"github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
)

const spaceEncoding = "%20"

var specialAqlCharacters = map[rune]string{
'/': "%2F",
'\\': "%5C",
'|': "%7C",
'*': "%2A",
'?': "%3F",
'\'': "%22",
':': "%3A",
';': "%3B",
'%': "%25",
}

// Returns an AQL body string to search file in Artifactory by pattern, according the specified arguments requirements.
func CreateAqlBodyForSpecWithPattern(params *CommonParams) (string, error) {
searchPattern := prepareSourceSearchPattern(params.Pattern, params.Target)
@@ -149,6 +165,44 @@ func CreateAqlQueryForPypi(repo, file string) string {
return fmt.Sprintf(itemsPart, repo, file, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_md5", "actual_sha1", "sha256"}))
}

// noinspection GoUnusedExportedFunction
func CreateAqlQueryForBuildInfoJson(project, buildName, buildNumber, timestamp string) string {
if project == "" {
project = "artifactory"
} else {
project = encodeForBuildInfoRepository(project)
}
itemsPart :=
`items.find({
"repo": "%s",
"path": {
"$match": "%s"
},
"name": {
"$match": "%s-%s.json"
}
})%s`
return fmt.Sprintf(itemsPart, project+"-build-info", encodeForBuildInfoRepository(buildName), encodeForBuildInfoRepository(buildNumber), timestamp, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_sha1", "actual_md5"}))
}

func encodeForBuildInfoRepository(value string) string {
results := ""
for _, char := range value {
if unicode.IsSpace(char) {
char = ' '
}
if encoding, exist := specialAqlCharacters[char]; exist {
results += encoding
} else {
results += string(char)
}
}
slashEncoding := specialAqlCharacters['/']
results = strings.ReplaceAll(results, slashEncoding+" ", slashEncoding+spaceEncoding)
results = strings.ReplaceAll(results, " "+slashEncoding, spaceEncoding+slashEncoding)
return results
}

func CreateAqlQueryForLatestCreated(repo, path string) string {
itemsPart :=
`items.find({` +
33 changes: 33 additions & 0 deletions artifactory/services/utils/aqlquerybuilder_test.go
Original file line number Diff line number Diff line change
@@ -220,3 +220,36 @@ func TestBuildKeyValQueryPart(t *testing.T) {
})
}
}

var encodeForBuildInfoRepositoryProvider = []struct {
value string
expectedEncoding string
}{
// Shouldn't encode
{"", ""},
{"a", "a"},
{"a b", "a b"},
{"a.b", "a.b"},
{"a&b", "a&b"},

// Should encode
{"a/b", "a%2Fb"},
{"a\\b", "a%5Cb"},
{"a:b", "a%3Ab"},
{"a|b", "a%7Cb"},
{"a*b", "a%2Ab"},
{"a?b", "a%3Fb"},
{"a / b", "a %20%2F%20 b"},

// Should convert whitespace to space
{"a\tb", "a b"},
{"a\nb", "a b"},
}

func TestEncodeForBuildInfoRepository(t *testing.T) {
for _, testCase := range encodeForBuildInfoRepositoryProvider {
t.Run(testCase.value, func(t *testing.T) {
assert.Equal(t, testCase.expectedEncoding, encodeForBuildInfoRepository(testCase.value))
})
}
}
6 changes: 6 additions & 0 deletions artifactory/services/utils/specutils.go
Original file line number Diff line number Diff line change
@@ -18,8 +18,14 @@ type Aql struct {
ItemsFind string `json:"items.find"`
}

type PathMapping struct {
Input string `json:"input"`
Output string `json:"output"`
}

type CommonParams struct {
Aql Aql
PathMapping PathMapping
Pattern string
Exclusions []string
Target string
12 changes: 9 additions & 3 deletions distribution/services/utils/distributionutils.go
Original file line number Diff line number Diff line change
@@ -34,17 +34,23 @@ func NewReleaseBundleParams(name, version string) ReleaseBundleParams {

func CreateBundleBody(releaseBundleParams ReleaseBundleParams, dryRun bool) (*ReleaseBundleBody, error) {
var bundleQueries []BundleQuery
var pathMappings []rtUtils.PathMapping

// Create release bundle queries
for _, specFile := range releaseBundleParams.SpecFiles {
// Create path mapping
if specFile.GetSpecType() == rtUtils.AQL {
pathMappings = distribution.CreatePathMappings(specFile.PathMapping.Input, specFile.PathMapping.Output)
} else {
pathMappings = distribution.CreatePathMappingsFromPatternAndTarget(specFile.Pattern, specFile.Target)
}

// Create AQL
aql, err := createAql(specFile)
if err != nil {
return nil, err
}

// Create path mapping
pathMappings := distribution.CreatePathMappings(specFile.Pattern, specFile.Target)

// Create added properties
addedProps := createAddedProps(specFile)

12 changes: 7 additions & 5 deletions distribution/services/utils/releasebundlebody.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package utils

import "github.com/jfrog/jfrog-client-go/utils/distribution"
import (
"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
)

// REST body for create and update a release bundle
type ReleaseBundleBody struct {
@@ -22,10 +24,10 @@ type BundleSpec struct {
}

type BundleQuery struct {
QueryName string `json:"query_name,omitempty"`
Aql string `json:"aql,omitempty"`
PathMappings []distribution.PathMapping `json:"mappings,omitempty"`
AddedProps []AddedProps `json:"added_props,omitempty"`
QueryName string `json:"query_name,omitempty"`
Aql string `json:"aql,omitempty"`
PathMappings []utils.PathMapping `json:"mappings,omitempty"`
AddedProps []AddedProps `json:"added_props,omitempty"`
}

type AddedProps struct {
25 changes: 12 additions & 13 deletions go.mod
Original file line number Diff line number Diff line change
@@ -6,24 +6,23 @@ require (
github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c
github.com/buger/jsonparser v1.1.1
github.com/forPelevin/gomoji v1.1.8
github.com/go-git/go-git/v5 v5.9.0
github.com/go-git/go-git/v5 v5.11.0
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/gookit/color v1.5.4
github.com/jfrog/build-info-go v1.9.13
github.com/jfrog/gofrog v1.3.1
github.com/jfrog/build-info-go v1.9.17
github.com/jfrog/gofrog v1.3.2
github.com/mholt/archiver/v3 v3.5.1
github.com/stretchr/testify v1.8.4
github.com/xanzy/ssh-agent v0.3.3
golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/term v0.13.0
golang.org/x/crypto v0.17.0
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
golang.org/x/term v0.15.0
)

require (
dario.cat/mergo v1.0.0 // indirect
github.com/CycloneDX/cyclonedx-go v0.7.2 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/andybalholm/brotli v1.0.1 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
@@ -46,18 +45,18 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect
github.com/skeema/knownhosts v1.2.1 // indirect
github.com/ulikunitz/xz v0.5.9 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/tools v0.13.0 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/tools v0.16.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

//replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go dev
replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231220102935-c8776c613ad8

// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev
Loading

0 comments on commit a8555e4

Please sign in to comment.