diff --git a/datasetIngestor/syncDataToFileserver_unix.go b/datasetIngestor/syncDataToFileserver_unix.go index 268b01b..8baa50c 100644 --- a/datasetIngestor/syncDataToFileserver_unix.go +++ b/datasetIngestor/syncDataToFileserver_unix.go @@ -9,6 +9,8 @@ import ( "os" "os/exec" "strings" + version "github.com/mcuadros/go-version" + "regexp" ) // functionality needed for "de-central" data @@ -21,27 +23,54 @@ func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer // append trailing slash to sourceFolder to indicate that the *contents* of the folder should be copied // no special handling for blanks in sourceFolder needed here fullSourceFolderPath := sourceFolder + "/" - // check if filelisting given - // rsync can create only one level deep directory structure, here we need more, therefore mkdir -p - // This code is no longer needed, sine Edgar has a new rrsync wrapper which craetes the needed directory - // cmd := exec.Command("/usr/bin/ssh",RSYNCServer,"mkdir","-p",destFolder) - // // show rsync's output - // cmd.Stdout = os.Stdout - // cmd.Stderr = os.Stderr - // - // fmt.Printf("Running %v.\n", cmd.Args) - // cmd.Run() + + versionNumber, err := getRsyncVersion() + if err != nil { + log.Fatal("Error getting rsync version: ", err) + } + + rsyncCmd := buildRsyncCmd(versionNumber, absFileListing, fullSourceFolderPath, serverConnectString) + + // Show rsync's output + rsyncCmd.Stderr = os.Stderr + log.Printf("Running: %v.\n", rsyncCmd.Args) + err = rsyncCmd.Run() + return err +} - cmd := exec.Command("/usr/bin/rsync", "-e", "ssh -q", "-avxz", "--progress", "--msgs2stderr", fullSourceFolderPath, serverConnectString) - // // TODO: create folderstructure mkdir -p also for this case: - if absFileListing != "" { - cmd = exec.Command("/usr/bin/rsync", "-e", "ssh -q", "-avxzr", "--progress", "--msgs2stderr", "--files-from", absFileListing, fullSourceFolderPath, serverConnectString) +// Get rsync version +func getRsyncVersion() (string, error) { + cmd := exec.Command("/usr/bin/rsync", "--version") + output, err := cmd.Output() + if err != nil { + return "", err + } + version := string(output) + + // Use a regular expression to find the version number. + // It will match the first occurrence of a string in the format "x.y.z" in the `version` string, where "x", "y", and "z" are one or more digits. + re := regexp.MustCompile(`\d+\.\d+\.\d+`) + versionNumber := re.FindString(version) + if versionNumber == "" { + return "", fmt.Errorf("could not find version number in rsync version string: %s", version) } - // show rsync's output - // cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + + return versionNumber, nil +} - log.Printf("Running %v.\n", cmd.Args) - err = cmd.Run() - return err +// Check rsync version and adjust command accordingly +func buildRsyncCmd(versionNumber, absFileListing, fullSourceFolderPath, serverConnectString string) *exec.Cmd { + rsyncFlags := []string{"-e", "ssh", "-avxz", "--progress"} + if absFileListing != "" { + rsyncFlags = append([]string{"-r", "--files-from", absFileListing}, rsyncFlags...) + } + if version.Compare(versionNumber, "3.2.3", ">=") { + rsyncFlags = append(rsyncFlags, "--stderr=error") + // Full command: /usr/bin/rsync -e ssh -avxz --progress -r --files-from --stderr=error + } else { + rsyncFlags = append(rsyncFlags, "-q", "--msgs2stderr") + // Full command: /usr/bin/rsync -e ssh -avxz --progress -r --files-from -q --msgs2stderr + } + rsyncCmd := exec.Command("/usr/bin/rsync", append(rsyncFlags, fullSourceFolderPath, serverConnectString)...) + return rsyncCmd } diff --git a/datasetIngestor/syncDataToFileserver_unix_test.go b/datasetIngestor/syncDataToFileserver_unix_test.go new file mode 100644 index 0000000..72123c1 --- /dev/null +++ b/datasetIngestor/syncDataToFileserver_unix_test.go @@ -0,0 +1,78 @@ +// +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris + +package datasetIngestor + +import ( + "testing" + "regexp" + "strings" +) + +func TestGetRsyncVersion(t *testing.T) { + version, err := getRsyncVersion() + if err != nil { + t.Errorf("getRsyncVersion() returned an error: %v", err) + } + if version == "" { + t.Error("getRsyncVersion() returned an empty string") + } else { + match, _ := regexp.MatchString(`^\d{1,2}\.\d{1,2}\.\d{1,2}$`, version) + if !match { + t.Error("getRsyncVersion() returned wrong version string format: ", version) + } + } +} + +func TestBuildRsyncCmd(t *testing.T) { + tests := []struct { + name string + versionNumber string + absFileListing string + fullSourceFolder string + serverConnectStr string + expectedCmd string + }{ + { + name: "rsync version >= 3.2.3, absFileListing not empty", + versionNumber: "3.2.3", + absFileListing: "/path/to/file", + fullSourceFolder: "/source/folder", + serverConnectStr: "user@server:/dest/folder", + expectedCmd: "/usr/bin/rsync -r --files-from /path/to/file -e ssh -avxz --progress --stderr=error /source/folder user@server:/dest/folder", + }, + { + name: "rsync version < 3.2.3, absFileListing not empty", + versionNumber: "3.2.2", + absFileListing: "/path/to/file", + fullSourceFolder: "/source/folder", + serverConnectStr: "user@server:/dest/folder", + expectedCmd: "/usr/bin/rsync -r --files-from /path/to/file -e ssh -avxz --progress -q --msgs2stderr /source/folder user@server:/dest/folder", + }, + { + name: "rsync version >= 3.2.3, absFileListing empty", + versionNumber: "3.2.3", + absFileListing: "", + fullSourceFolder: "/source/folder", + serverConnectStr: "user@server:/dest/folder", + expectedCmd: "/usr/bin/rsync -e ssh -avxz --progress --stderr=error /source/folder user@server:/dest/folder", + }, + { + name: "rsync version < 3.2.3, absFileListing empty", + versionNumber: "3.2.2", + absFileListing: "", + fullSourceFolder: "/source/folder", + serverConnectStr: "user@server:/dest/folder", + expectedCmd: "/usr/bin/rsync -e ssh -avxz --progress -q --msgs2stderr /source/folder user@server:/dest/folder", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := buildRsyncCmd(tt.versionNumber, tt.absFileListing, tt.fullSourceFolder, tt.serverConnectStr) + cmdStr := strings.Join(cmd.Args, " ") + if cmdStr != tt.expectedCmd { + t.Errorf("Expected command: %s, got: %s", tt.expectedCmd, cmdStr) + } + }) + } +} diff --git a/datasetUtils/getAvailableDatasets.go b/datasetUtils/getAvailableDatasets.go index 87ad763..c1c43b7 100644 --- a/datasetUtils/getAvailableDatasets.go +++ b/datasetUtils/getAvailableDatasets.go @@ -5,6 +5,8 @@ import ( "log" "os/exec" "strings" + version "github.com/mcuadros/go-version" + "regexp" ) func GetAvailableDatasets(username string, RSYNCServer string, singleDatasetId string) []string { @@ -21,7 +23,21 @@ func GetAvailableDatasets(username string, RSYNCServer string, singleDatasetId s fmt.Printf("====== (only datasets highlighted in green will be retrieved)\n\n") fmt.Printf("====== If you can not find the dataset in this listing: may be you forgot\n") fmt.Printf("====== to start the necessary retrieve job from the the data catalog first ?\n\n") - cmd := exec.Command("rsync", "-e", "ssh -q", "--list-only", username+"@"+RSYNCServer+":retrieve/") + + // Get rsync version + versionNumber, err := getRsyncVersion() + if err != nil { + log.Fatal("Error getting rsync version: ", err) + } + + // Check rsync version and adjust command accordingly + var cmd *exec.Cmd + if version.Compare(versionNumber, "3.2.3", ">=") { + cmd = exec.Command("rsync", "-e", "ssh", "--list-only", username+"@"+RSYNCServer+":retrieve/") + } else { + cmd = exec.Command("rsync", "-e", "ssh -q", "--list-only", username+"@"+RSYNCServer+":retrieve/") + } + out, err := cmd.Output() if err != nil { log.Printf("Running %v.\n", cmd.Args) @@ -43,3 +59,22 @@ func GetAvailableDatasets(username string, RSYNCServer string, singleDatasetId s } return datasetList } + +// Get rsync version +func getRsyncVersion() (string, error) { + cmd := exec.Command("/usr/bin/rsync", "--version") + output, err := cmd.Output() + if err != nil { + return "", err + } + version := string(output) + + // Use a regular expression to find the version number + re := regexp.MustCompile(`\d+\.\d+\.\d+`) + versionNumber := re.FindString(version) + if versionNumber == "" { + return "", fmt.Errorf("could not find version number in rsync version string: %s", version) + } + + return versionNumber, nil +}