From ffca2678b42fc364eda8589bdfb00ae1e8494e9c Mon Sep 17 00:00:00 2001 From: Tim Clifford Date: Wed, 27 Jul 2022 15:10:48 +0100 Subject: [PATCH 1/3] replacing pipe to gzip with an initial dump, then gzip it --- synchers/mariadb.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synchers/mariadb.go b/synchers/mariadb.go index da5c44f..8c17ceb 100644 --- a/synchers/mariadb.go +++ b/synchers/mariadb.go @@ -3,6 +3,7 @@ package synchers import ( "errors" "fmt" + "path/filepath" "reflect" "strconv" "strings" @@ -141,7 +142,7 @@ func (root MariadbSyncRoot) GetRemoteCommand(sourceEnvironment Environment) Sync } return SyncCommand{ - command: fmt.Sprintf("mysqldump {{ .dumpOptions }} -h{{ .hostname }} -u{{ .username }} -p{{ .password }} -P{{ .port }} {{ .tablesToIgnore }} {{ .database }} | gzip -c | cat > {{ .transferResource }}"), + command: fmt.Sprintf("mysqldump {{ .dumpOptions }} -h{{ .hostname }} -u{{ .username }} -p{{ .password }} -P{{ .port }} {{ .tablesToIgnore }} {{ .database }} > {{ .transferResource }} && gzip {{ .transferResource }}"), substitutions: map[string]interface{}{ "dumpOptions": "--max-allowed-packet=500M --quick --add-locks --no-autocommit --single-transaction", "hostname": m.DbHostname, @@ -150,7 +151,7 @@ func (root MariadbSyncRoot) GetRemoteCommand(sourceEnvironment Environment) Sync "port": m.DbPort, "tablesToIgnore": tablesWhoseDataToIgnore, "database": m.DbDatabase, - "transferResource": transferResource.Name, + "transferResource": strings.TrimSuffix(transferResource.Name, filepath.Ext(transferResource.Name)), }, } } From cbdd27da3743ae330a1562bb4fe53141ab3e45eb Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Thu, 28 Jul 2022 03:53:32 +1200 Subject: [PATCH 2/3] Adds explanatory variable --- synchers/mariadb.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synchers/mariadb.go b/synchers/mariadb.go index 8c17ceb..804b352 100644 --- a/synchers/mariadb.go +++ b/synchers/mariadb.go @@ -141,6 +141,9 @@ func (root MariadbSyncRoot) GetRemoteCommand(sourceEnvironment Environment) Sync tablesWhoseDataToIgnore += fmt.Sprintf("--ignore-table-data=%s.%s ", m.DbDatabase, s) } + //We remove the `.gz` from the transfer resource name for because we _first_ generate a plain `.sql` file + //and _then_ gzip it + resourceNameWithoutGz := strings.TrimSuffix(transferResource.Name, filepath.Ext(transferResource.Name)) return SyncCommand{ command: fmt.Sprintf("mysqldump {{ .dumpOptions }} -h{{ .hostname }} -u{{ .username }} -p{{ .password }} -P{{ .port }} {{ .tablesToIgnore }} {{ .database }} > {{ .transferResource }} && gzip {{ .transferResource }}"), substitutions: map[string]interface{}{ @@ -151,7 +154,7 @@ func (root MariadbSyncRoot) GetRemoteCommand(sourceEnvironment Environment) Sync "port": m.DbPort, "tablesToIgnore": tablesWhoseDataToIgnore, "database": m.DbDatabase, - "transferResource": strings.TrimSuffix(transferResource.Name, filepath.Ext(transferResource.Name)), + "transferResource": resourceNameWithoutGz, }, } } From ff639b161065c6eebf905cba836cf4f150d6d7ca Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Thu, 28 Jul 2022 04:26:21 +1200 Subject: [PATCH 3/3] Changes unzip for mariadb to use process substitution --- synchers/mariadb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synchers/mariadb.go b/synchers/mariadb.go index 804b352..0c9535a 100644 --- a/synchers/mariadb.go +++ b/synchers/mariadb.go @@ -165,7 +165,7 @@ func (m MariadbSyncRoot) GetLocalCommand(targetEnvironment Environment) SyncComm l = m.getEffectiveLocalDetails() } transferResource := m.GetTransferResource(targetEnvironment) - return generateSyncCommand("gunzip < {{ .transferResource }} | mysql -h{{ .hostname }} -u{{ .username }} -p{{ .password }} -P{{ .port }} {{ .database }}", + return generateSyncCommand("mysql -h{{ .hostname }} -u{{ .username }} -p{{ .password }} -P{{ .port }} {{ .database }} < <(gunzip < {{ .transferResource }})", map[string]interface{}{ "hostname": l.DbHostname, "username": l.DbUsername,