From 2fd26cffcc03a06bce35c22e64491bf69dae7d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Qu=C3=A9r=C3=A9?= Date: Fri, 14 Aug 2015 22:26:51 +0200 Subject: [PATCH] First commit --- AUTHORS | 7 ++ LICENSE | 22 +++++ Makefile | 26 ++++++ README.md | 18 +++- exemples/exemple_1.conf | 17 ++++ exemples/exemple_2.conf | 36 ++++++++ indi-backup | 147 +++++++++++++++++++++++++++++++ lib/storage.sh | 74 ++++++++++++++++ lib/storages/fs.sh | 49 +++++++++++ lib/storages/ssh.sh | 64 ++++++++++++++ lib/storages/swift.sh | 58 ++++++++++++ lib/task.sh | 82 +++++++++++++++++ lib/tasks/docker-mysql-dumper.sh | 59 +++++++++++++ lib/tasks/mysql.sh | 73 +++++++++++++++ lib/tasks/subtask.sh | 46 ++++++++++ lib/tasks/tarball-incremental.sh | 69 +++++++++++++++ lib/tasks/tarball.sh | 47 ++++++++++ lib/utils.sh | 52 +++++++++++ manpages/indi-backup.1 | 31 +++++++ 19 files changed, 975 insertions(+), 2 deletions(-) create mode 100644 AUTHORS create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 exemples/exemple_1.conf create mode 100644 exemples/exemple_2.conf create mode 100755 indi-backup create mode 100644 lib/storage.sh create mode 100644 lib/storages/fs.sh create mode 100644 lib/storages/ssh.sh create mode 100644 lib/storages/swift.sh create mode 100644 lib/task.sh create mode 100644 lib/tasks/docker-mysql-dumper.sh create mode 100644 lib/tasks/mysql.sh create mode 100644 lib/tasks/subtask.sh create mode 100644 lib/tasks/tarball-incremental.sh create mode 100644 lib/tasks/tarball.sh create mode 100644 lib/utils.sh create mode 100644 manpages/indi-backup.1 diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..42a57bc --- /dev/null +++ b/AUTHORS @@ -0,0 +1,7 @@ +## +# +# This project is developed and maintained by Indigen-Solution +# Website: http://www.indigen.com +# Contact: developer@indigen.com +# +## diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8235f80 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Indigen-Solution +See the AUTHORS file for details. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..03148f7 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +DESTDIR= +INSTALL_DIR=$(DESTDIR)/usr +BUILD_DIR=./build +NAME=indi-backup +LIB_DIR=./lib + +all: + mkdir -p $(BUILD_DIR) + sed -s 's/@@VERSION@@/$(shell git describe --tags --long)/' $(NAME) > $(BUILD_DIR)/$(NAME) + +install: all + mkdir -p $(INSTALL_DIR)/bin + mkdir -p $(INSTALL_DIR)/lib/$(NAME) + cp $(BUILD_DIR)/$(NAME) $(INSTALL_DIR)/bin + chmod 755 $(INSTALL_DIR)/$(NAME) + + cp -r $(LIB_DIR) $(INSTALL_DIR)/lib/$(NAME) + +uninstall: + rm -rf $(INSTALL_DIR)/bin/$(NAME) + rm -rf $(INSTALL_DIR)/lib/$(NAME) + +clean: + rm -rf $(BUILD_DIR) + +.PHONY: all install uninstall clean diff --git a/README.md b/README.md index cc2f394..d283cd3 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,21 @@ # indi-backup -A modular backup system write in bash with minimal dependency. +A modular backup system write in bash with minimal dependencies. + +### Installation + +#### Debian / Ubuntu +TODO + +#### Other linux distributions +``` +git clone https://github.com/indigen-solutions/indi-backup.git +cd indi-backup +make +sudo make install +``` + ### Documentation -Full documentation available [here](https://github.com/indigen-solutions/indi-backup/wiki). \ No newline at end of file +Full documentation is available [here](https://github.com/indigen-solutions/indi-backup/wiki). diff --git a/exemples/exemple_1.conf b/exemples/exemple_1.conf new file mode 100644 index 0000000..331dd60 --- /dev/null +++ b/exemples/exemple_1.conf @@ -0,0 +1,17 @@ +################################################################################ +# This file is a exemple of possible backup config for indi-backup +# It register a task "data" that create a tarball of /home and +# drop it in /backup +################################################################################ + +IB_TASKS="data" # Declare all available tasks +IB_STORAGES="disk" # Declare all available storages + +# Declare the "data" backup task +IB_TASK_data_TYPE="tarball" # The task is of type "tarball" +IB_TASK_data_FOLDER="/home" # List of folder that will be included in the tarball +IB_TASK_data_STORAGE="disk" # The name of the storage you want to use for this task + +# Declare the "disk" storage method +IB_STORAGE_disk_TYPE="fs" # The storage is of type "fs" +IB_STORAGE_disk_BASEPATH="/backup" # The file will be stored under /backup directory. diff --git a/exemples/exemple_2.conf b/exemples/exemple_2.conf new file mode 100644 index 0000000..f7a4ccd --- /dev/null +++ b/exemples/exemple_2.conf @@ -0,0 +1,36 @@ +################################################################################ +# This file is a exemple of possible backup config for indi-backup +# It register a task "web" that create an incremental tarball of /var/www, +# a task mysql that backup the mysql server and a task all that excute web and +# mysql. All the bakup are sent to a remote ssh server. +################################################################################ + +IB_TASKS="all web mysql" +IB_STORAGES="sshBackup" + +# Declare the web task +IB_TASK_web_TYPE="tarball-incremental" +IB_TASK_web_FOLDERS="/var/www" +IB_TASK_web_LIST_NAME="/root/backup-web.list" +IB_TASK_web_FILE_BASENAME="webdata" +IB_TASK_web_MASTER_FREQUENCY="weekly" +IB_TASK_web_MASTER_FREQUENCY_VALUE="1" +IB_TASK_web_STORAGE="sshBackup" + +# Declare the mysql task +IB_TASK_mysql_TYPE="mysql" +IB_TASK_mysql_USER="root" +IB_TASK_mysql_PASSWORD="password" +IB_TASK_mysql_DATABASES="my-database" +IB_TASK_mysql_STORAGE="sshBackup" + +# Declare the all task +IB_TASK_all_TYPE="subtask" +IB_TASK_all_SUBTASKS="web mysql" + +# Declare the "ssh-backup" storage +IB_STORAGE_sshBackup_TYPE="ssh" +IB_STORAGE_sshBackup_HOST="backup.exemple.com" +IB_STORAGE_sshBackup_USER="user" +IB_STORAGE_sshBackup_BASEPATH="/backup" +IB_STORAGE_sshBackup_SSHKEY="/root/.ssh/id_rsa" diff --git a/indi-backup b/indi-backup new file mode 100755 index 0000000..1f81a99 --- /dev/null +++ b/indi-backup @@ -0,0 +1,147 @@ +#!/bin/bash +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +################################################################################ +# This is the main indi-backup script +################################################################################ + +# +# Load libraries +# + +IB_LIB_DIR="/usr/lib/indi-backup" + +if [ ! -z $IB_DEVEL ] +then + IB_LIB_DIR="./lib" +fi + +source $IB_LIB_DIR/utils.sh + +source $IB_LIB_DIR/task.sh +source $IB_LIB_DIR/tasks/docker-mysql-dumper.sh +source $IB_LIB_DIR/tasks/mysql.sh +source $IB_LIB_DIR/tasks/subtask.sh +source $IB_LIB_DIR/tasks/tarball.sh +source $IB_LIB_DIR/tasks/tarball-incremental.sh + +source $IB_LIB_DIR/storage.sh +source $IB_LIB_DIR/storages/fs.sh +source $IB_LIB_DIR/storages/ssh.sh +source $IB_LIB_DIR/storages/swift.sh + + +# +# Set default variables +# +VERSION="@@VERSION@@" +CONFIG_FILE="/etc/indi-backup.conf" +TASKS="" +DATE=$(date --utc "+%Y%m%dT%H%M%SZ") + +IB_OPTIONS_LIST_TASKS="" +IB_OPTIONS_LIST_STORAGES="" + +# +# Parse command line arguments +# +while [[ $# > 0 ]] +do + key="$1" + case $key in + -c|--config) + CONFIG_FILE="$2" + shift + ;; + --list-tasks) + IB_OPTIONS_LIST_TASKS=1 + ;; + --list-storages) + IB_OPTIONS_LIST_STORAGES=1 + ;; + -h|--help) + ib_print_usage + exit 0 + ;; + --version) + echo "Version: $VERSION" + exit 0 + ;; + -*) + echo "Unknow options $1" + exit -1 + ;; + *) + TASKS="$TASKS $1" + ;; + esac + shift +done + +# +# Include config file and do some check +# +if [[ (! -f $CONFIG_FILE) || (! -r $CONFIG_FILE) ]] +then + echo "No valid config file found [$CONFIG_FILE]"; + exit -1; +fi + +source $CONFIG_FILE + +if [ -z "$IB_TASKS" ] +then + echo "Invalid IB_TASKS in config file" + exit -1; +fi + +if [ -z "$IB_STORAGES" ] +then + echo "Invalid IB_STORAGES in config file" + exit -1; +fi + +# +# Run options +# +if [ ! -z "$IB_OPTIONS_LIST_TASKS" ] +then + ib_task_list +fi + +if [ ! -z "$IB_OPTIONS_LIST_STORAGES" ] +then + ib_storage_list +fi + +# +# Run task +# +for task in $TASKS +do + ib_task_run $task; +done diff --git a/lib/storage.sh b/lib/storage.sh new file mode 100644 index 0000000..7b30685 --- /dev/null +++ b/lib/storage.sh @@ -0,0 +1,74 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a storage stream +# @param storageName The name of the storage stream to execute +# @param taskName The name of the task to execute +# @param itemName The name of the item you want to store +## +function ib_storage_run () { + local storageName="$1" + local taskName="$2" + local itemName="$3" + + for storage in $IB_STORAGES + do + if [[ "$storage" == "$storageName" ]] + then + local storageType=$(ib_get_conf_value "IB_STORAGE_${storageName}_TYPE") + case "$storageType" in + swift) + ib_storage_swift_run "$storageName" "$taskName" "$itemName" || return -1 + ;; + fs) + ib_storage_fs_run "$storageName" "$taskName" "$itemName" || return -1 + ;; + ssh) + ib_storage_ssh_run "$storageName" "$taskName" "$itemName" || return -1 + ;; + *) + echo "Unknow storage type [$storageType]" + return -1 + ;; + esac + return 0 + fi + done + echo "No storage [$storageName] found" +} + +## +# This method list all the registered storage in the configuration file. +## +function ib_storage_list() { + for storageName in $IB_STORAGES + do + local type=$(ib_get_conf_value "IB_STORAGE_${storageName}_TYPE") + echo " * ${storageName}" + echo " - Type: ${type}" + done +} diff --git a/lib/storages/fs.sh b/lib/storages/fs.sh new file mode 100644 index 0000000..b3077e7 --- /dev/null +++ b/lib/storages/fs.sh @@ -0,0 +1,49 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a storage stream of type fs that store item on the +# filesystem +# @param storageName The name of the storage stream to execute +# @param taskName The name of the task to execute +# @param itemName The name of the item you want to store +## +function ib_storage_fs_run() { + local storageName="$1" + local taskName="$2" + local itemName="$3" + local fsBasePath=$(ib_get_conf_value "IB_STORAGE_${storageName}_BASEPATH") + + if [ -z "$fsBasePath" ] + then + echo "No valid IB_STORAGE_${storageName}_BASEPATH found" + return -1 + fi + + local fileName="${fsBasePath}/${taskName}/${DATE}/${itemName}" + mkdir -p $(dirname "$fileName") + cat > "${fileName}" +} diff --git a/lib/storages/ssh.sh b/lib/storages/ssh.sh new file mode 100644 index 0000000..4cefc5d --- /dev/null +++ b/lib/storages/ssh.sh @@ -0,0 +1,64 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a storage stream of type ssh that store item on a +# remote ssh server +# @param storageName The name of the storage stream to execute +# @param taskName The name of the task to execute +# @param itemName The name of the item you want to store +## +function ib_storage_ssh_run() { + local storageName="$1" + local taskName="$2" + local itemName="$3" + local sshUser=$(ib_get_conf_value "IB_STORAGE_${storageName}_USER") + local sshHost=$(ib_get_conf_value "IB_STORAGE_${storageName}_HOST") + local sshPort=$(ib_get_conf_value "IB_STORAGE_${storageName}_PORT") + local sshBasePath=$(ib_get_conf_value "IB_STORAGE_${storageName}_BASEPATH") + local sshKey=$(ib_get_conf_value "IB_STORAGE_${storageName}_SSHKEY") + + if [ -z "$sshUser" ]; then echo "No valid IB_STORAGE_${storageName}_USER found"; return -1; fi + if [ -z "$sshHost" ]; then echo "No valid IB_STORAGE_${storageName}_HOST found"; return -1; fi + if [ -z "$sshBasePath" ]; then echo "No valid IB_STORAGE_${storageName}_BASEPATH found"; return -1; fi + + if [ -z "$sshPort" ] + then + sshPort=22 + fi + + if [[ (! -z "$sshKey") && ( ! -r "$sshKey" )]] + then + echo "Can'r read ssh key $sshKey"; + elif [ ! -z $sshKey ] + then + sshKey="-i ${sshKey}" + fi + + local fileName="${sshBasePath}/${taskName}/${DATE}/${itemName}" + ssh "${sshUser}@${sshHost}" $sshKey -p "${sshPort}" \ + "mkdir -p \"\$(dirname '${fileName}')\" ; cat > '${fileName}'" +} diff --git a/lib/storages/swift.sh b/lib/storages/swift.sh new file mode 100644 index 0000000..8ac4c6a --- /dev/null +++ b/lib/storages/swift.sh @@ -0,0 +1,58 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a storage stream of type swift that store item on a +# swift server +# @param storageName The name of the storage stream to execute +# @param taskName The name of the task to execute +# @param itemName The name of the item you want to store +## +function ib_storage_swift_run () { + local storageName="$1" + local taskName="$2" + local itemName="$3" + local swiftAuthUrl=$(ib_get_conf_value "IB_STORAGE_${storageName}_AUTHURL") + local swiftUser=$(ib_get_conf_value "IB_STORAGE_${storageName}_USER") + local swiftPassword=$(ib_get_conf_value "IB_STORAGE_${storageName}_PASSWORD") + local swiftBasePath=$(ib_get_conf_value "IB_STORAGE_${storageName}_BASEPATH") + local swiftContainer=$(ib_get_conf_value "IB_STORAGE_${storageName}_CONTAINER") + local swiftSplitSize=$(ib_get_conf_value "IB_STORAGE_${storageName}_SPLIT_SIZE") + + if [ -z "$swiftAuthUrl" ]; then echo "No valid SWIFT_AUTHURL found"; return -1; fi + if [ -z "$swiftUser" ]; then echo "No valid SWIFT_USER found"; return -1; fi + if [ -z "$swiftPassword" ]; then echo "No valid SWIFT_PASSWORD found"; return -1; fi + if [ -z "$swiftBasePath" ]; then echo "No valid SWIFT_BASEPATH found"; return -1; fi + if [ -z "$swiftContainer" ]; then echo "No valid SWIFT_CONTAINER found"; return -1; fi + + if [ -z "$swiftSplitSize" ] + then + swiftSplitSize="2G" + fi + + stdin2swift -a "$swiftAuthUrl" -u "$swiftUser" -p "$swiftPassword" -s "$swiftSplitSize" \ + "$swiftContainer" "${swiftBasePath}/${taskName}/${DATE}/${itemName}" +} diff --git a/lib/task.sh b/lib/task.sh new file mode 100644 index 0000000..a8591c5 --- /dev/null +++ b/lib/task.sh @@ -0,0 +1,82 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a task decalred in the config file +# @param taskName The name of the task to execute +# @description This method loop through task declared in IB_TASKS and if it +# find task passed as parameter it call the right method according to the task +# type. +## +function ib_task_run() { + local taskName="$1" + + echo "Starting task ${taskName}" + for task in $IB_TASKS + do + if [[ "$task" == "$taskName" ]] + then + local taskType=$(ib_get_conf_value "IB_TASK_${taskName}_TYPE") + case "$taskType" in + subtask) + ib_task_subtask_run "$taskName" || return -1 + ;; + mysql) + ib_task_mysql_run "$taskName" || return -1 + ;; + tarball) + ib_task_tarball_run "$taskName" || return -1 + ;; + tarball-incremental) + ib_task_tarball-incremental_run "$taskName" || return -1 + ;; + docker-mysql-dumper) + ib_task_docker-mysql-dumper_run "$taskName" || return -1 + ;; + *) + echo "Unknow task type [$taskType]" + return -1 + ;; + esac + echo "Task ${taskName} success" + return 0 + fi + done + echo "No task [$taskName] found" + return -1; +} + +## +# This method list all the registered tasks in the configuration file. +## +function ib_task_list() { + for taskName in $IB_TASKS + do + local type=$(ib_get_conf_value "IB_TASK_${taskName}_TYPE") + echo " * ${taskName}" + echo " - Type: ${type}" + done +} diff --git a/lib/tasks/docker-mysql-dumper.sh b/lib/tasks/docker-mysql-dumper.sh new file mode 100644 index 0000000..0e2a141 --- /dev/null +++ b/lib/tasks/docker-mysql-dumper.sh @@ -0,0 +1,59 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a task of type "docker-mysql-dumper" +# @param taskName The name of the task to execute +## +function ib_task_docker-mysql-dumper_run() { + local taskName="$1"; + local dockerSocket=$(ib_get_conf_value "IB_TASK_${taskName}_DOCKER_SOCKET") + local storageName=$(ib_get_conf_value "IB_TASK_${taskName}_STORAGE") + local tmpDir=$(ib_get_conf_value "IB_TASK_${taskName}_TMP_DIR") + + if [ -z "$dockerSocket" ] + then + dockerSocket="/var/run/docker.sock" + fi + + if [ -z "$tmpDir" ] + then + tmpDir="/tmp" + fi + + if [ -z "$storageName" ]; then echo "No valid IB_TASK_${taskName}_STORAGE found"; return -1; fi + + local tempDir=$(mktemp -qd --tmpdir="$tmpDir") + + docker run -it --rm -v "${dockerSocket}:/var/run/docker.sock" -v "${tempDir}:/mnt" indigen/mysql-dumper backups + + for file in $(ls "$tempDir/backups") + do + cat "$tempDir/backups/$file" | \ + ib_storage_run "$storageName" "$taskName" "${DATE}-${file}" + done + rm -rf "$tempDir" +} diff --git a/lib/tasks/mysql.sh b/lib/tasks/mysql.sh new file mode 100644 index 0000000..81e59d8 --- /dev/null +++ b/lib/tasks/mysql.sh @@ -0,0 +1,73 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a task of type "mysql" +# @param taskName The name of the task to execute +## +function ib_task_mysql_run() { + local taskName="$1"; + local mysqlHost=$(ib_get_conf_value "IB_TASK_${taskName}_HOST") + local mysqlPort=$(ib_get_conf_value "IB_TASK_${taskName}_PORT") + local mysqlUser=$(ib_get_conf_value "IB_TASK_${taskName}_USER") + local mysqlPassword=$(ib_get_conf_value "IB_TASK_${taskName}_PASSWORD") + local mysqlDumpOpts=$(ib_get_conf_value "IB_TASK_${taskName}_DUMP_OPTS") + local mysqlDatabases=$(ib_get_conf_value "IB_TASK_${taskName}_DATABASES") + local storageName=$(ib_get_conf_value "IB_TASK_${taskName}_STORAGE") + + if [ -z "$mysqlHost" ];then mysqlHost="localhost"; fi + if [ -z "$mysqlPort" ];then mysqlPort="3306"; fi + if [ -z "$mysqlUser" ];then mysqlPort="root"; fi + if [ -z "$mysqlPassword" ];then mysqlPassword=""; fi + if [ -z "$mysqlDumpOpts" ];then mysqlDumpOpts="--opt"; fi + if [ -z "$mysqlDatabases" ];then mysqlDatabases="__ALL__"; fi + + if [ -z "$storageName" ]; then echo "No valid IB_TASK_${taskName}_STORAGE found"; return -1; fi + + if [[ "$mysqlDatabases" == "__ALL__" ]] + then + mysqlDatabases=$(mysql --batch --raw --skip-pager --skip-column-names -h "$mysqlHost" \ + --port "$mysqlPort" -u "$mysqlUser" "-p$mysqlPassword" \ + --execute='SHOW DATABASES') + if [ -z "$mysqlDatabases" ] + then + echo "Can't list mysql databases on server [$mysqlHost]" + return -1 + fi + fi + + for database in $mysqlDatabases + do + if [[ ($database == "information_schema") || ($database == "performance_schema") ]] + then + continue; + fi + mysqldump -h "$mysqlHost" --port "$mysqlPort" -u "$mysqlUser" "-p$mysqlPassword" \ + $mysqlDumpOpts "$database" | gzip | \ + ib_storage_run "$storageName" "$taskName" "${database}-${DATE}.sql.gz" + done + return 0; +} diff --git a/lib/tasks/subtask.sh b/lib/tasks/subtask.sh new file mode 100644 index 0000000..57f29a8 --- /dev/null +++ b/lib/tasks/subtask.sh @@ -0,0 +1,46 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a task of type "subtask" +# @param taskName The name of the task to execute +## +function ib_task_subtask_run() { + local taskName="$1"; + local subtasks=$(ib_get_conf_value "IB_TASK_${taskName}_SUBTASKS") + + if [ -z "$subtasks" ]; then echo "No valid IB_TASK_${taskName}_SUBTASKS found"; return -1; fi + + for subtask in $subtasks + do + if ! ib_task_run $subtask + then + return -1; + fi + done + return 0; +} + diff --git a/lib/tasks/tarball-incremental.sh b/lib/tasks/tarball-incremental.sh new file mode 100644 index 0000000..26e2b30 --- /dev/null +++ b/lib/tasks/tarball-incremental.sh @@ -0,0 +1,69 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +## +# This method execute a task of type "tarball-incremental" +# @param taskName The name of the task to execute +## +function ib_task_tarball-incremental_run() { + local taskName="$1" + local folders=$(ib_get_conf_value "IB_TASK_${taskName}_FOLDERS") + local listFile=$(ib_get_conf_value "IB_TASK_${taskName}_LIST_FILE") + local fileBaseName=$(ib_get_conf_value "IB_TASK_${taskName}_FILE_BASENAME") + local storageName=$(ib_get_conf_value "IB_TASK_${taskName}_STORAGE") + local masterFrequency=$(ib_get_conf_value "IB_TASK_data2_MASTER_FREQUENCY") + local masterFrequencyValue=$(ib_get_conf_value "IB_TASK_data2_MASTER_FREQUENCY_VALUE") + + if [ -z "$storageName" ]; then echo "No valid IB_TASK_${taskName}_STORAGE found"; return -1; fi + if [ -z "$listFile" ]; then echo "No valid IB_TASK_${taskName}_LIST_FILE found"; return -1; fi + if [ -z "$folders" ]; then echo "No valid IB_TASK_${taskName}_FOLDERS found"; return -1; fi + + if [ -z "$fileBaseName" ] + then + fileBaseName="backup" + fi + + if [ -z "$masterFrequency" ] + then + masterFrequency="weekly" + masterFrequencyValue=1 + fi + + if [ -z "$masterFrequencyValue" ] + then + masterFrequencyValue=1 + fi + + echo $masterFrequency $masterFrequencyValue + if [[ (( "$masterFrequency" == "weekly" ) && ( $(date "+%u") -eq "$masterFrequencyValue" )) || \ + (( "$masterFrequency" == "monthly" ) && ( $(date "+%d") -eq "$masterFrequencyValue" )) ]] + then + rm -f "$listFile" + fi + + tar --create -z --listed-incremental=$listFile $folders \ + | ib_storage_run $storageName $taskName "${fileBaseName}-${DATE}.tar.gz" +} diff --git a/lib/tasks/tarball.sh b/lib/tasks/tarball.sh new file mode 100644 index 0000000..59e0413 --- /dev/null +++ b/lib/tasks/tarball.sh @@ -0,0 +1,47 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + + +## +# This method execute a task of type "tarball" +# @param taskName The name of the task to execute +## +function ib_task_tarball_run() { + local taskName="$1" + local folders=$(ib_get_conf_value "IB_TASK_${taskName}_FOLDERS") + local fileBaseName=$(ib_get_conf_value "IB_TASK_${taskName}_FILE_BASENAME") + local storageName=$(ib_get_conf_value "IB_TASK_${taskName}_STORAGE") + + if [ -z "$fileBaseName" ] + then + fileBaseName="backup" + fi + + if [ -z "$folders" ]; then echo "No valid IB_TASK_${taskName}_FOLDERS found"; return -1; fi + if [ -z "$storageName" ]; then echo "No valid IB_TASK_${taskName}_STORAGE found"; return -1; fi + + tar --create --absolute-names -z $folders | ib_storage_run $storageName $taskName "${fileBaseName}-${DATE}.tar.gz" +} diff --git a/lib/utils.sh b/lib/utils.sh new file mode 100644 index 0000000..62f29e6 --- /dev/null +++ b/lib/utils.sh @@ -0,0 +1,52 @@ +################################################################################ +# +# The MIT License (MIT) +# +# Copyright (c) 2015 Indigen-Solution +# See the AUTHORS file for details. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +function ib_get_conf_value() { + local varname="$1" + if [[ -z "${!varname}" ]] + then + return -1 + fi + echo "${!varname}" + return 0 +} + +function ib_print_usage() { + echo "USAGE" + echo " indi-backup [options] tasks ..." + echo "" + echo "OPTIONS" + echo " -c path, --config path" + echo " Overide the default config file location. (/etc/indi-backup.conf)" + echo " -h, --help" + echo " Show summary of options" + echo " --version" + echo " Print the version and exit" + echo " --list-tasks" + echo " List all available tasks" + echo " --list-storages" + echo " List all available storages" +} diff --git a/manpages/indi-backup.1 b/manpages/indi-backup.1 new file mode 100644 index 0000000..9a6b206 --- /dev/null +++ b/manpages/indi-backup.1 @@ -0,0 +1,31 @@ +.TH INDI-BACKUP 1 "August 12, 2015" +.SH NAME +indi-backup \- A modular backup system write in bash with minimal dependency. +.SH SYNOPSIS +.B indi-backup +.RI [ options ] " tasks ..." +.PP +.SH CONFIGURATION +The default configuration file is /etc/indi-backup.conf +.TP +The complete configuration documentation is availabale at +.BR https://github.com/indigen-solutions/indi-backup/wiki +.SH OPTIONS +.TP +.B \-h, \-\-help +Show summary of options. +.TP +.B \-\-version +Print the version and exit +.TP +.B \-c, \-\-config +Overide the default config file location. +.TP +.B \-\-list\-tasks +List all available tasks +.TP +.B \-\-list\-storages +List all available storages +.SH AUTHOR +.br +Jérôme Quéré