Check rsync version (#43)

* Check for rsync version * Use either `msgs2stderr` and `stderr=error` depending on the version * Unit tests for version-specific rsync commands * Adds github.com/mcuadros/go-version for comparing semantic versions
paulscherrerinstitute · Apr 25, 2024 · 59cde32 · 59cde32
1 parent 99d9ba1
commit 59cde32
Show file tree

Hide file tree

Showing 3 changed files with 163 additions and 21 deletions.
diff --git a/datasetIngestor/syncDataToFileserver_unix.go b/datasetIngestor/syncDataToFileserver_unix.go
@@ -9,6 +9,8 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	version "github.com/mcuadros/go-version"
+	"regexp"
 )
 
 // functionality needed for "de-central" data
@@ -21,27 +23,54 @@ func SyncDataToFileserver(datasetId string, user map[string]string, RSYNCServer
 	// append trailing slash to sourceFolder to indicate that the *contents* of the folder should be copied
 	// no special handling for blanks in sourceFolder needed here
 	fullSourceFolderPath := sourceFolder + "/"
-	// check if filelisting given
-	// rsync can create only one level deep directory structure, here we need more, therefore mkdir -p
-	// This code is no longer needed, sine Edgar has a new rrsync wrapper which craetes the needed directory
-	// cmd := exec.Command("/usr/bin/ssh",RSYNCServer,"mkdir","-p",destFolder)
-	// // show rsync's output
-	// cmd.Stdout = os.Stdout
-	// cmd.Stderr = os.Stderr
-	//
-	// fmt.Printf("Running %v.\n", cmd.Args)
-	// cmd.Run()
+
+	versionNumber, err := getRsyncVersion()
+	if err != nil {
+		log.Fatal("Error getting rsync version: ", err)
+	}
+
+	rsyncCmd := buildRsyncCmd(versionNumber, absFileListing, fullSourceFolderPath, serverConnectString)
+
+	// Show rsync's output	
+	rsyncCmd.Stderr = os.Stderr
+	log.Printf("Running: %v.\n", rsyncCmd.Args)
+	err = rsyncCmd.Run()
+	return err
+}
 
-	cmd := exec.Command("/usr/bin/rsync", "-e", "ssh -q", "-avxz", "--progress", "--msgs2stderr", fullSourceFolderPath, serverConnectString)
-	// // TODO: create folderstructure mkdir -p also for this case:
-	if absFileListing != "" {
-		cmd = exec.Command("/usr/bin/rsync", "-e", "ssh -q", "-avxzr", "--progress", "--msgs2stderr", "--files-from", absFileListing, fullSourceFolderPath, serverConnectString)
+// Get rsync version
+func getRsyncVersion() (string, error) {
+	cmd := exec.Command("/usr/bin/rsync", "--version")
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	version := string(output)
+
+	// Use a regular expression to find the version number.
+	// It will match the first occurrence of a string in the format "x.y.z" in the `version` string, where "x", "y", and "z" are one or more digits.
+	re := regexp.MustCompile(`\d+\.\d+\.\d+`)
+	versionNumber := re.FindString(version)
+	if versionNumber == "" {
+		return "", fmt.Errorf("could not find version number in rsync version string: %s", version)
 	}
-	// show rsync's output
-	// cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
+
+	return versionNumber, nil
+}
 
-	log.Printf("Running %v.\n", cmd.Args)
-	err = cmd.Run()
-	return err
+// Check rsync version and adjust command accordingly
+func buildRsyncCmd(versionNumber, absFileListing, fullSourceFolderPath, serverConnectString string) *exec.Cmd {
+	rsyncFlags := []string{"-e", "ssh", "-avxz", "--progress"}
+	if absFileListing != "" {
+		rsyncFlags = append([]string{"-r", "--files-from", absFileListing}, rsyncFlags...)
+	}
+	if version.Compare(versionNumber, "3.2.3", ">=") {
+		rsyncFlags = append(rsyncFlags, "--stderr=error")
+		// Full command: /usr/bin/rsync -e ssh -avxz --progress -r --files-from <absFileListing> --stderr=error <fullSourceFolderPath> <serverConnectString>
+	} else {
+		rsyncFlags = append(rsyncFlags, "-q", "--msgs2stderr")
+		// Full command: /usr/bin/rsync -e ssh -avxz --progress -r --files-from <absFileListing> -q --msgs2stderr <fullSourceFolderPath> <serverConnectString>
+	}
+	rsyncCmd := exec.Command("/usr/bin/rsync", append(rsyncFlags, fullSourceFolderPath, serverConnectString)...)
+	return rsyncCmd
 }
diff --git a/datasetIngestor/syncDataToFileserver_unix_test.go b/datasetIngestor/syncDataToFileserver_unix_test.go
@@ -0,0 +1,78 @@
+// +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris
+
+package datasetIngestor
+
+import (
+	"testing"
+	"regexp"
+	"strings"
+)
+
+func TestGetRsyncVersion(t *testing.T) {
+	version, err := getRsyncVersion()
+	if err != nil {
+		t.Errorf("getRsyncVersion() returned an error: %v", err)
+	}
+	if version == "" {
+		t.Error("getRsyncVersion() returned an empty string")
+	} else {
+		match, _ := regexp.MatchString(`^\d{1,2}\.\d{1,2}\.\d{1,2}$`, version)
+		if !match {
+			t.Error("getRsyncVersion() returned wrong version string format: ", version)
+		}
+	}
+}
+
+func TestBuildRsyncCmd(t *testing.T) {
+	tests := []struct {
+		name             string
+		versionNumber    string
+		absFileListing   string
+		fullSourceFolder string
+		serverConnectStr string
+		expectedCmd      string
+	}{
+		{
+			name:             "rsync version >= 3.2.3, absFileListing not empty",
+			versionNumber:    "3.2.3",
+			absFileListing:   "/path/to/file",
+			fullSourceFolder: "/source/folder",
+			serverConnectStr: "user@server:/dest/folder",
+			expectedCmd:      "/usr/bin/rsync -r --files-from /path/to/file -e ssh -avxz --progress --stderr=error /source/folder user@server:/dest/folder",
+		},
+		{
+			name:             "rsync version < 3.2.3, absFileListing not empty",
+			versionNumber:    "3.2.2",
+			absFileListing:   "/path/to/file",
+			fullSourceFolder: "/source/folder",
+			serverConnectStr: "user@server:/dest/folder",
+			expectedCmd:      "/usr/bin/rsync -r --files-from /path/to/file -e ssh -avxz --progress -q --msgs2stderr /source/folder user@server:/dest/folder",
+		},
+		{
+			name:             "rsync version >= 3.2.3, absFileListing empty",
+			versionNumber:    "3.2.3",
+			absFileListing:   "",
+			fullSourceFolder: "/source/folder",
+			serverConnectStr: "user@server:/dest/folder",
+			expectedCmd:      "/usr/bin/rsync -e ssh -avxz --progress --stderr=error /source/folder user@server:/dest/folder",
+		},
+		{
+			name:             "rsync version < 3.2.3, absFileListing empty",
+			versionNumber:    "3.2.2",
+			absFileListing:   "",
+			fullSourceFolder: "/source/folder",
+			serverConnectStr: "user@server:/dest/folder",
+			expectedCmd:      "/usr/bin/rsync -e ssh -avxz --progress -q --msgs2stderr /source/folder user@server:/dest/folder",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cmd := buildRsyncCmd(tt.versionNumber, tt.absFileListing, tt.fullSourceFolder, tt.serverConnectStr)
+			cmdStr := strings.Join(cmd.Args, " ")
+			if cmdStr != tt.expectedCmd {
+				t.Errorf("Expected command: %s, got: %s", tt.expectedCmd, cmdStr)
+			}
+		})
+	}
+}
diff --git a/datasetUtils/getAvailableDatasets.go b/datasetUtils/getAvailableDatasets.go
@@ -5,6 +5,8 @@ import (
 	"log"
 	"os/exec"
 	"strings"
+	version "github.com/mcuadros/go-version"
+	"regexp"
 )
 
 func GetAvailableDatasets(username string, RSYNCServer string, singleDatasetId string) []string {
@@ -21,7 +23,21 @@ func GetAvailableDatasets(username string, RSYNCServer string, singleDatasetId s
 		fmt.Printf("====== (only datasets highlighted in green will be retrieved)\n\n")
 		fmt.Printf("====== If you can not find the dataset in this listing: may be you forgot\n")
 		fmt.Printf("====== to start the necessary retrieve job from the the data catalog first ?\n\n")
-		cmd := exec.Command("rsync", "-e", "ssh -q", "--list-only", username+"@"+RSYNCServer+":retrieve/")
+
+		// Get rsync version
+		versionNumber, err := getRsyncVersion()
+		if err != nil {
+			log.Fatal("Error getting rsync version: ", err)
+		}
+
+		// Check rsync version and adjust command accordingly
+		var cmd *exec.Cmd
+		if version.Compare(versionNumber, "3.2.3", ">=") {
+			cmd = exec.Command("rsync", "-e", "ssh", "--list-only", username+"@"+RSYNCServer+":retrieve/")
+		} else {
+			cmd = exec.Command("rsync", "-e", "ssh -q", "--list-only", username+"@"+RSYNCServer+":retrieve/")
+		}
+
 		out, err := cmd.Output()
 		if err != nil {
 			log.Printf("Running %v.\n", cmd.Args)
@@ -43,3 +59,22 @@ func GetAvailableDatasets(username string, RSYNCServer string, singleDatasetId s
 	}
 	return datasetList
 }
+
+// Get rsync version
+func getRsyncVersion() (string, error) {
+	cmd := exec.Command("/usr/bin/rsync", "--version")
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	version := string(output)
+
+	// Use a regular expression to find the version number
+	re := regexp.MustCompile(`\d+\.\d+\.\d+`)
+	versionNumber := re.FindString(version)
+	if versionNumber == "" {
+		return "", fmt.Errorf("could not find version number in rsync version string: %s", version)
+	}
+
+	return versionNumber, nil
+}