Skip to content

Commit

Permalink
services.mysqlBackup: apply suggested refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
@6543
6543 committed Jan 8, 2025
1 parent fc842d8 commit ceafafc
Show file tree
Hide file tree
Showing 2 changed files with 146 additions and 76 deletions.
97 changes: 43 additions & 54 deletions nixos/modules/services/backup/mysql-backup.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,40 +8,39 @@ let
cfg = config.services.mysqlBackup;
defaultUser = "mysqlbackup";

compressionPkg =
{
gzip = pkgs.gzip;
xz = pkgs.xz;
zstd = pkgs.zstd;
}
.${cfg.compressionAlg};

fileExt =
{
gzip = ".gz";
zstd = ".zst";
xz = ".xz";
}
.${cfg.compressionAlg};

validCompressionLevels = {
zstd = range: 1 <= range && range <= 19;
xz = range: 0 <= range && range <= 9;
gzip = range: 1 <= range && range <= 9;
compressionAlgs = {
gzip = rec {
pkg = pkgs.gzip;
ext = ".gz";
minLevel = 1;
maxLevel = 9;
cmd = compressionLevelFlag: "${pkg}/bin/gzip -c ${cfg.gzipOptions} ${compressionLevelFlag}";
validLevel = level: minLevel <= level && level <= maxLevel;
};
xz = rec {
pkg = pkgs.xz;
ext = ".xz";
minLevel = 0;
maxLevel = 9;
cmd = compressionLevelFlag: "${pkg}/bin/xz -z -c ${compressionLevelFlag} -";
validLevel = level: minLevel <= level && level <= maxLevel;
};
zstd = rec {
pkg = pkgs.zstd;
ext = ".zst";
minLevel = 1;
maxLevel = 19;
cmd = compressionLevelFlag: "${pkg}/bin/zstd ${compressionLevelFlag} -";
validLevel = level: minLevel <= level && level <= maxLevel;
};
};

compressionCmd =
let
compressionLevelFlag = lib.optionalString (cfg.compressionLevel != null) (
"-" + toString cfg.compressionLevel
);
in
{
gzip = "${pkgs.gzip}/bin/gzip -c ${cfg.gzipOptions} ${compressionLevelFlag}";
xz = "${pkgs.xz}/bin/xz -z -c ${compressionLevelFlag} -";
zstd = "${pkgs.zstd}/bin/zstd ${compressionLevelFlag} -";
}
.${cfg.compressionAlg};
compressionLevelFlag = lib.optionalString (cfg.compressionLevel != null) (
"-" + toString cfg.compressionLevel
);

selectedAlg = compressionAlgs.${cfg.compressionAlg};
compressionCmd = selectedAlg.cmd compressionLevelFlag;

backupScript = ''
set -o pipefail
Expand All @@ -54,7 +53,7 @@ let
'';

backupDatabaseScript = db: ''
dest="${cfg.location}/${db}${fileExt}"
dest="${cfg.location}/${db}${selectedAlg.ext}"
if ${pkgs.mariadb}/bin/mysqldump ${lib.optionalString cfg.singleTransaction "--single-transaction"} ${db} | ${compressionCmd} > $dest.tmp; then
mv $dest.tmp $dest
echo "Backed up to $dest"
Expand All @@ -80,11 +79,7 @@ in
};

compressionAlg = lib.mkOption {
type = lib.types.enum [
"gzip"
"zstd"
"xz"
];
type = lib.types.enum (lib.attrNames compressionAlgs);
default = "gzip";
description = ''
Compression algorithm to use for database dumps.
Expand All @@ -95,10 +90,13 @@ in
type = lib.types.nullOr lib.types.int;
default = null;
description = ''
Compression level to use for gzip, zstd or xz.
For gzip: 1-9 (note: if compression level is also specified in gzipOptions, the gzipOptions value will be overwritten)
For zstd: 1-19
For xz: 0-9
Compression level to use for ${lib.concatStringsSep ", " (lib.init (lib.attrNames compressionAlgs))} or ${lib.last (lib.attrNames compressionAlgs)}.
${lib.concatStringsSep "\n" (
lib.mapAttrsToList (
name: algo: " For ${name}: ${toString algo.minLevel}-${toString algo.maxLevel}"
) compressionAlgs
)}
(note: if compression level is also specified in gzipOptions, the gzipOptions value will be overwritten)
'';
};

Expand Down Expand Up @@ -150,17 +148,8 @@ in
# assert config to be correct
assertions = [
{
assertion =
cfg.compressionLevel == null || validCompressionLevels.${cfg.compressionAlg} cfg.compressionLevel;
message =
let
rangeMsg = {
"zstd" = "zstd compression level must be between 1 and 19";
"xz" = "xz compression level must be between 0 and 9";
"gzip" = "gzip compression level must be between 1 and 9";
};
in
rangeMsg.${cfg.compressionAlg};
assertion = cfg.compressionLevel == null || selectedAlg.validLevel cfg.compressionLevel;
message = "${cfg.compressionAlg} compression level must be between ${toString selectedAlg.minLevel} and ${toString selectedAlg.maxLevel}";
}
];

Expand All @@ -175,7 +164,7 @@ in
};

# add the compression tool to the system environment.
environment.systemPackages = [ compressionPkg ];
environment.systemPackages = [ selectedAlg.pkg ];

# ensure database user to be used to perform backup exist.
services.mysql.ensureUsers = [
Expand Down
125 changes: 103 additions & 22 deletions nixos/tests/mysql/mysql-backup.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,39 +41,120 @@ let
];
};
};

# Additional node to test compression configurations
compression =
{ pkgs, ... }:
{
services.mysql = {
inherit package;
enable = true;
initialDatabases = [
{
name = "testdb";
schema = ./testdb.sql;
}
];
};
};
};

testScript = ''
start_all()
# Delete backup file that may be left over from a previous test run.
# This is not needed on Hydra but useful for repeated local test runs.
master.execute("rm -f /var/backup/mysql/testdb.gz")
with subtest("Test default gzip backup"):
# Delete backup file that may be left over from a previous test run.
# This is not needed on Hydra but useful for repeated local test runs.
master.execute("rm -f /var/backup/mysql/testdb.gz")
# Need to have mysql started so that it can be populated with data.
master.wait_for_unit("mysql.service")
# Wait for testdb to be fully populated (5 rows).
master.wait_until_succeeds(
"mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
)
# Do a backup and wait for it to start
master.start_job("mysql-backup.service")
# wait for backup to fail, because of database 'doesnotexist'
master.wait_until_fails("systemctl is-active -q mysql-backup.service")
# Need to have mysql started so that it can be populated with data.
master.wait_for_unit("mysql.service")
# wait for backup file and check that data appears in backup
master.wait_for_file("/var/backup/mysql/testdb.gz")
master.succeed(
"${pkgs.gzip}/bin/zcat /var/backup/mysql/testdb.gz | grep hello"
)
# Wait for testdb to be fully populated (5 rows).
master.wait_until_succeeds(
"mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
)
# Check that a failed backup is logged
master.succeed(
"journalctl -u mysql-backup.service | grep 'fail.*doesnotexist' > /dev/null"
)
# Do a backup and wait for it to start
master.start_job("mysql-backup.service")
with subtest("Test compression algorithms"):
# Start MySQL on compression test node
compression.wait_for_unit("mysql.service")
compression.wait_until_succeeds(
"mysql -u root -D testdb -N -B -e 'select count(id) from tests' | grep -q 5"
)
# wait for backup to fail, because of database 'doesnotexist'
master.wait_until_fails("systemctl is-active -q mysql-backup.service")
# Test zstd compression
compression.succeed("""
nixos-rebuild switch -I nixos-config=<(echo '
{ ... }: {
imports = [ ${./mysql-backup.nix} ];
services.mysql.enable = true;
services.mysql.package = ${package};
services.mysqlBackup = {
enable = true;
databases = [ "testdb" ];
compressionAlg = "zstd";
compressionLevel = 15;
};
}
')
""")
compression.succeed("systemctl start mysql-backup.service")
compression.wait_for_file("/var/backup/mysql/testdb.zst")
compression.succeed("${pkgs.zstd}/bin/zstd -d -c /var/backup/mysql/testdb.zst | grep hello")
# wait for backup file and check that data appears in backup
master.wait_for_file("/var/backup/mysql/testdb.gz")
master.succeed(
"${pkgs.gzip}/bin/zcat /var/backup/mysql/testdb.gz | grep hello"
)
# Test xz compression
compression.succeed("""
nixos-rebuild switch -I nixos-config=<(echo '
{ ... }: {
imports = [ ${./mysql-backup.nix} ];
services.mysql.enable = true;
services.mysql.package = ${package};
services.mysqlBackup = {
enable = true;
databases = [ "testdb" ];
compressionAlg = "xz";
compressionLevel = 0;
};
}
')
""")
compression.succeed("systemctl start mysql-backup.service")
compression.wait_for_file("/var/backup/mysql/testdb.xz")
compression.succeed("${pkgs.xz}/bin/xz -d -c /var/backup/mysql/testdb.xz | grep hello")
# Check that a failed backup is logged
master.succeed(
"journalctl -u mysql-backup.service | grep 'fail.*doesnotexist' > /dev/null"
)
# Test invalid compression level
compression.fail("""
nixos-rebuild switch -I nixos-config=<(echo '
{ ... }: {
imports = [ ${./mysql-backup.nix} ];
services.mysql.enable = true;
services.mysql.package = ${package};
services.mysqlBackup = {
enable = true;
databases = [ "testdb" ];
compressionAlg = "gzip";
compressionLevel = 10;
};
}
')
""")
'';
};
in
Expand Down

0 comments on commit ceafafc

Please sign in to comment.