Support for Distribution path mapping with AQL (#2313)

distribution_test.go

@@ -445,7 +445,7 @@ func TestUpdateBundleProps(t *testing.T) {
 	cleanDistributionTest(t)
 }
 
-func TestBundlePathMapping(t *testing.T) {
+func TestBundlePathMappingFromPatternAndTarget(t *testing.T) {
 	initDistributionTest(t)
 
 	// Upload files
@@ -465,6 +465,28 @@ func TestBundlePathMapping(t *testing.T) {
 	cleanDistributionTest(t)
 }
 
+func TestBundlePathMappingFromPatternAndTargetUsingSpec(t *testing.T) {
+	initDistributionTest(t)
+
+	// Upload files
+	specFile, err := tests.CreateSpec(tests.DistributionUploadSpecB)
+	assert.NoError(t, err)
+	runRt(t, "u", "--spec="+specFile)
+
+	// Create and distribute release bundle with path mapping from <DistRepo1>/data/ to <DistRepo2>/target/
+	spec, err := tests.CreateSpec(tests.DistributionCreateWithPatternAndTarget)
+	assert.NoError(t, err)
+	runDs(t, "rbc", tests.BundleName, bundleVersion, "--sign", "--spec="+spec)
+	runDs(t, "rbd", tests.BundleName, bundleVersion, "--site=*", "--sync")
+
+	// Validate files are distributed to the target mapping
+	spec, err = tests.CreateSpec(tests.DistributionMappingDownload)
+	assert.NoError(t, err)
+	inttestutils.VerifyExistInArtifactory(tests.GetBundleMappingExpected(), spec, serverDetails, t)
+
+	cleanDistributionTest(t)
+}
+
 func TestBundlePathMappingUsingSpec(t *testing.T) {
 	initDistributionTest(t)
 

go.mod

@@ -131,8 +131,8 @@ require (
 
 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231126114158-b6719ec2da0e
 
-replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20231123122346-6325ab61a7b2
+replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20231127135302-daea501a3826
 
 // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20230418122323-2bf299dd6d27
 
-// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231101142932-422f20520a28
+replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231126134442-14887b84b87a

go.sum

@@ -243,10 +243,10 @@ github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk=
 github.com/jfrog/gofrog v1.3.1/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
 github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
-github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20231123122346-6325ab61a7b2 h1:BosujG/vEgXUEycZBxCLFZ58yRSJ20grvZ7sprU3kp0=
-github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20231123122346-6325ab61a7b2/go.mod h1:+jzmxaxiiJgzascARg5Ioi4o3pfFfewk8rE5Pbo8u9Q=
-github.com/jfrog/jfrog-client-go v1.34.5 h1:NYZrOHvT5D5BwwHdArIz5WnbP+DPbADjV/SPdN33bfQ=
-github.com/jfrog/jfrog-client-go v1.34.5/go.mod h1:0PVhP6xGvBBaUzOU9LKf5OYkke/gY2IFILHA++iabFM=
+github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20231127135302-daea501a3826 h1:N49B28/2uy2+Rm2djugMrQ0yvgruLqudsJRIqaLMAIc=
+github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20231127135302-daea501a3826/go.mod h1:gsB5hTyVwtwZDHvwcKxO6BnLgLbBVy+EcRTf/D5dNo8=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20231126134442-14887b84b87a h1:hZFfT7i+vwhDhJ7xf211XbS9uxjR83vC618di1Eo0wo=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20231126134442-14887b84b87a/go.mod h1:0PVhP6xGvBBaUzOU9LKf5OYkke/gY2IFILHA++iabFM=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jszwec/csvutil v1.8.0 h1:G7vS2LGdpZZDH1HmHeNbxOaJ/ZnJlpwGFvOkTkJzzNk=

schema/filespec-schema.json

@@ -42,6 +42,21 @@
           }
         }
       },
+      "pathMapping": {
+        "description": "If specified, determines release-bundle source and target paths for artifacts that had been fetched using aql.",
+        "properties": {
+          "input": {
+            "type": "string",
+            "description": "The input path for mapping.",
+            "examples": ["(.*)/old_folder/(.*)"]
+          },
+          "output": {
+            "type": "string",
+            "description": "The output path for mapping.",
+            "examples": ["$1/new_folder/$2"]
+          }
+        }
+      },
       "archive": {
         "type": "string",
         "enum": ["zip"],
@@ -227,6 +242,7 @@
           ]
         }
       },
+      "pathMapping": { "required": ["aql"] },
       "build": { "not": { "required": ["bundle", "limit", "offset"] } },
       "bundle": { "not": { "required": ["build", "limit", "offset"] } },
       "excludeArtifacts": { "required": ["build"] },

testdata/filespecs/dist_create_with_mapping.json

@@ -1,8 +1,20 @@
 {
   "files": [
     {
-      "pattern": "${DIST_REPO1}/data/(*)",
-      "target": "${DIST_REPO2}/target/{1}"
+      "aql": {
+        "items.find": {
+          "repo": "${DIST_REPO1}",
+          "path": "data",
+          "$or": [
+            {
+              "name": {
+                "$match": "*"
+              }
+            }
+          ]
+        }
+      },
+      "pathMapping": { "input":  "${DIST_REPO1}/data/(.*)", "output": "${DIST_REPO2}/target/$1" }
     }
   ]
-}
+}

testdata/filespecs/dist_create_with_pattern_and_target.json

@@ -0,0 +1,8 @@
+{
+  "files": [
+    {
+      "pattern": "${DIST_REPO1}/data/(*)",
+      "target": "${DIST_REPO2}/target/{1}"
+    }
+  ]
+}

testdata/go/projectmissingdependency/projectmissingdependecy.go

@@ -12,4 +12,4 @@ func execDependency() {
 func Exec() {
 	fmt.Println("Executing ")
 	execDependency()
-}
+}

utils/cliutils/utils.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/jfrog/gofrog/version"
+	"github.com/jfrog/jfrog-client-go/utils/log"
 	"io"
 	"net/http"
 	"os"
@@ -26,7 +27,6 @@ import (
 	clientutils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/content"
-	"github.com/jfrog/jfrog-client-go/utils/log"
 	"github.com/urfave/cli"
 )
 

utils/tests/consts.go

@@ -42,6 +42,7 @@ const (
 	DelSpecExclusions                                     = "delete_spec_exclusions.json"
 	DistributionCreateByAql                               = "dist_create_by_aql.json"
 	DistributionCreateWithMapping                         = "dist_create_with_mapping.json"
+	DistributionCreateWithPatternAndTarget                = "dist_create_with_pattern_and_target.json"
 	DistributionMappingDownload                           = "dist_mapping_download_spec.json"
 	DistributionRepoConfig1                               = "dist_repository_config1.json"
 	DistributionRepoConfig2                               = "dist_repository_config2.json"