scripts/duplicati/monitor.backup.sh

609 lines
21 KiB
Bash
Executable File
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/sh
#
# Purpose {{{
# This script will …
# 1. List all users directories with a minimum size.
# 2. For each, get the list of backup directories.
# 3. For each backup directories.
# a. Ignore those that contain a .ignore.me hidden file.
# b. Get current backup directory size.
# c. Compare current size with a previous value (if available) from db.
# d. Increase informations iterations for this backup (N-1 become N-2,…).
# e. Store current backup dir size to N-1.
# f. Prepare email's body if difference is not significant.
# 4. Send email to user if a content exists.
# 5. Remove all backup informations with iterations ≤ -10.
# …
#
# 2022-12-07
# }}}
# Sqlite3 database info {{{
# Table name : SQLITE_TABLE_NAME (default : "backups_info").
# 6 fields :
# name TEXT (eg. username/duplicati_mr013370)
# user TEXT (eg. username)
# size INT (eg. 16000)
# iteration INT (eg. -2)
# date TEXT (eg. 2022-12-05)
# emailed TEXT (eg True)
#
# A primary key is built from those 6 fields to avoid duplicate entries.
#
# The script will try to get the last iteration of a backup (iteration = -1) for
# a comparison with the current backup directory size.
# The script will decrease all iterations number of a backup (iteration - 1).
# The script will finally remove all entries with an iteration below -10.
# }}}
# Vars {{{
PROGNAME=$(basename "${0}"); readonly PROGNAME
PROGDIR=$(readlink -m $(dirname "${0}")); readonly PROGDIR
ARGS="${*}"; readonly ARGS
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && DEBUG=1
## Export DEBUG for sub-script
export DEBUG
## Default values for some vars
readonly DUP_DIR_DEFAULT="/home"
readonly SQLITE_TABLE_NAME="backups_info"
## Temp files
DUP_TMP_DIR=$(mktemp -d -t duplicati-monitor-backup-XXXXXX.tmp) ; readonly DUP_TMP_DIR
readonly DUP_USERS_DIR_LIST="${DUP_TMP_DIR}/users.dir.list"
## Minimum size for a user|backup directory (for du comparison)
readonly MIN_SIZE_INT_DEFAULT="10"
readonly MIN_SIZE_UNIT_DEFAULT="MB"
readonly MIN_SIZE_DEFAULT="${MIN_SIZE_INT_DEFAULT}${MIN_SIZE_UNIT_DEFAULT}"
## Default domain for email addresses
readonly USER_EMAIL_DOMAIN_DEFAULT=$(hostname -d)
## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}
usage() { # {{{
cat <<- HELP
usage: $PROGNAME [-d|-h|-m|-s]
Monitor users backups for a Duplicati backend.
Store some informations in a sqlite3 db file.
And send an email if the size between two backups is not significant.
EXAMPLES:
- Apply to default directory (${DUP_DIR_DEFAULT})
${PROGNAME}
- Apply to a specific directory (/mnt/remote.duplicati)
${PROGNAME} --dir /mnt/remote.duplicati
OPTIONS:
-d,--dir,--directory
Path where users home directory are stored
(default: ${DUP_DIR_DEFAULT}).
--db,--sqlite3
Path to sqlite3 database file to store backups informations
(default relative to --directory: ${DUP_DIR_DEFAULT}/.duplicati/monitor.db).
--domain,--mail-domain
Domain used to build user's email address.
(default: ${USER_EMAIL_DOMAIN_DEFAULT}).
--debug
Enable debug messages.
-h,--help
Print this help message.
-m,--mail,--mail-template,--template
Template used to build email content for users.
(default relative to --directory: ${DUP_DIR_DEFAULT}/.duplicati/mail.template).
-s,--size,--min-size
Minimal size for a user directory.
This value is also used to compare backup directory size between two runs.
Expect to have both an integer AND a unit. If one is missing, the default
one will be used.
(default: ${MIN_SIZE_DEFAULT}).
HELP
}
# }}}
debug_message() { # {{{
local_debug_message="${1}"
## Print message if DEBUG is enable (=0)
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG ${PROGNAME}: ${local_debug_message}"
unset local_debug_message
return 0
}
# }}}
error_message() { # {{{
local_error_message="${1}"
local_error_code="${2}"
## Print message
printf '%b\n' "ERROR ${PROGNAME}: ${RED}${local_error_message}${RESET}"
unset local_error_message
exit "${local_error_code:=66}"
}
# }}}
define_vars() { # {{{
## If dup_dir wasn't defined (argument) {{{
if [ -z "${dup_dir}" ]; then
## Use default value
readonly dup_dir="${DUP_DIR_DEFAULT}"
fi
## }}}
## If dup_info wasn't defined {{{
if [ -z "${dup_info}" ]; then
## Store it in previous define dup_dir
readonly dup_info="${dup_dir}/.duplicati/monitor_backup"
fi
## }}}
## If mail_template wasn't defined {{{
if [ -z "${mail_template}" ]; then
## Store it in previous define dup_dir
readonly mail_template="${dup_info}/mail.template"
fi
## }}}
## If mail_template_bottom wasn't defined {{{
if [ -z "${mail_template_bottom}" ]; then
## Store it in previous define dup_dir
readonly mail_template_bottom="${mail_template}.bottom"
fi
## }}}
## If db_path wasn't defined {{{
if [ -z "${db_path}" ]; then
## Store it in previous dup_info directory
readonly db_path="${dup_info}/monitor.db"
fi
## }}}
## If min_size wasn't defined (argument) {{{
if [ -z "${min_size}" ]; then
## Use default value
readonly min_size="${MIN_SIZE_DEFAULT}"
fi
## }}}
## Define min_size_int {{{
min_size_int=$(printf -- '%s' "${min_size}" \
| tr --delete --complement -- '0-9')
if [ -z "${min_size_int}" ]; then
min_size_int="${MIN_SIZE_INT_DEFAULT}"
readonly min_size_int
fi
## }}}
## Define min_size_unit {{{
min_size_unit=$(printf -- '%s' "${min_size}" \
| tr --delete --complement -- 'a-zA-Z')
if [ -z "${min_size_unit}" ]; then
min_size_unit="${MIN_SIZE_UNIT_DEFAULT}"
readonly min_size_unit
fi
## }}}
## If user_email_domain wasn't defined (argument) {{{
if [ -z "${user_email_domain}" ]; then
## Use default value
readonly user_email_domain="${USER_EMAIL_DOMAIN_DEFAULT}"
fi
## }}}
}
# }}}
is_directory_absent() { # {{{
local_directory_absent="${1}"
debug_prefix="${2:-}"
## Directory doesn't exists by default
return_is_directory_absent="0"
### Check if the directory exists
# shellcheck disable=SC2086
if test -d "${local_directory_absent}"; then
return_is_directory_absent="1"
debug_message "${debug_prefix}is_directory_absent \
The directory ${RED}${local_directory_absent}${COLOR_DEBUG} exists."
else
return_is_directory_absent="0"
debug_message "${debug_prefix}is_directory_absent \
The directory ${RED}${local_directory_absent}${COLOR_DEBUG} doesn't exist."
fi
unset local_directory_absent
unset debug_prefix
return "${return_is_directory_absent}"
}
# }}}
is_file_absent() { # {{{
local_file_absent="${1}"
debug_prefix="${2:-}"
## File exists by default
return_is_file_absent="1"
### Check if the file exists
# shellcheck disable=SC2086
if find ${local_file_absent} > /dev/null 2>&1; then
return_is_file_absent="1"
debug_message "${debug_prefix}is_file_absent \
The file ${RED}${local_file_absent}${COLOR_DEBUG} exists."
else
return_is_file_absent="0"
debug_message "${debug_prefix}is_file_absent \
The file ${RED}${local_file_absent}${COLOR_DEBUG} doesn't exist."
fi
unset local_file_absent
unset debug_prefix
return "${return_is_file_absent}"
}
# }}}
is_file_present() { # {{{
local_file_present="${1}"
debug_prefix="${2:-}"
## File doesn't exist by default
return_is_file_present="1"
### Check if the file exists
# shellcheck disable=SC2086
if find ${local_file_present} > /dev/null 2>&1; then
return_is_file_present="0"
debug_message "${debug_prefix}is_file_present \
The file ${RED}${local_file_present}${COLOR_DEBUG} exists."
else
return_is_file_present="1"
debug_message "${debug_prefix}is_file_present \
The file ${RED}${local_file_present}${COLOR_DEBUG} doesn't exist."
fi
unset local_file_present
unset debug_prefix
return "${return_is_file_present}"
}
# }}}
clean_temp_files() { # {{{
## Remove temp files if DEBUG is not set
if [ ! "${DEBUG}" -eq "0" ]; then
rm --recursive --force -- "${DUP_TMP_DIR}"
else
debug_message "| Temp files are available in ${RED}${DUP_TMP_DIR}${COLOR_DEBUG} for any debug."
fi
return "0"
}
# }}}
main() { # {{{
# Define all vars
define_vars
debug_message "--- MAIN BEGIN"
debug_message "| Temp files are availabe here : ${RED}${DUP_TMP_DIR}${COLOR_DEBUG} ."
# Requirements {{{
if [ ! $(command -v sqlite3) ]; then
debug_message "| Need to install script dependency (${RED}sqlite3${COLOR_DEBUG})."
sudo aptitude install sqlite3
fi
# }}}
# If dup_dir doesn't exists {{{
# AND exit with error
is_directory_absent "${dup_dir}" "| " \
&& error_message "The directory (${dup_dir}) doesn't exists. Check your configuration or use -d|--dir option to give an existing directory. Use --help for more informations." 11
# }}}
# If dup_info doesn't exists {{{
# AND exit with error
is_directory_absent "${dup_info}" "| " \
&& debug_message "| Create the ${RED}${dup_info}${COLOR_DEBUG} directory." \
&& mkdir --parent -- "${dup_info}"
# }}}
# If mail_template doesn't exists {{{
# AND exit with error
is_file_absent "${mail_template}" "| " \
&& error_message "Email template (${mail_template}) doesn't exists. Please provide a path to a valid file (--template option) or create a template in appropriate directory (${dup_info}). See --help for more informations." 12
# }}}
# If db_path doesn't exists {{{
# Init a table in a new sqlite3 database file
if is_file_absent "${db_path}" "| "; then
debug_message "| Init table (${RED}${SQLITE_TABLE_NAME}${COLOR_DEBUG}) in a new SQLite database file (${RED}${db_path}${COLOR_DEBUG})."
sqlite3 "${db_path}" "CREATE table ${SQLITE_TABLE_NAME}( name TEXT, user TEXT, size INT, iteration INT, date TEXT, emailed TEXT, PRIMARY KEY (name, user, size, iteration, date, emailed) );" \
|| error_message "Can't init a new TABLE (${SQLITE_TABLE_NAME}) in SQLite database (${db_path})." 13
fi
# }}}
# Move to dup_dir
debug_message "| Move to users backup base path (${RED}${dup_dir}${COLOR_DEBUG})."
cd -- "${dup_dir}" >/dev/null \
|| error_message "Can't move to directory (${dup_dir}). Check the permissions or try with 'sudo'." 14
# Get the sorted list of users directories. {{{
find . -mindepth 1 -maxdepth 1 -type d -not -name '.*' -printf "%f\n" | sort > "${DUP_USERS_DIR_LIST}"
# Test if the users directories list exists and is not empty
# OR exit with error
is_file_present "${DUP_USERS_DIR_LIST}" "| " \
|| error_message "The users list (${DUP_USERS_DIR_LIST}) seems empty or doesn't exists." 15
# }}}
# While loop for each user directory
while IFS= read -r user_dir; do
## Define path for an email content
user_email_template="${DUP_TMP_DIR}/${user_dir}.mail"
## Check the size of user directory {{{
if ! du --summarize --block-size="${min_size}" -- "${user_dir}" >/dev/null 2>&1; then
error_message "Can't get size of ${user_dir} user directory. Check the permissions or try with 'sudo'" 21
fi
user_dir_size=$(du --summarize --block-size="${min_size}" -- "${user_dir}" \
| cut --fields=1 -- )
### Multiply by min_size_int to have the correct size order
: $(( user_dir_size=user_dir_size*min_size_int ))
if [ "${user_dir_size}" -gt "${min_size_int}" ]; then
debug_message "|-- Manage ${RED}${user_dir}${COLOR_DEBUG} directory BEGIN"
else
debug_message "| ${RED}Skip ${user_dir}${COLOR_DEBUG} directory because it's below the minimum size (~${user_dir_size} ${min_size_unit}${min_size})."
unset user_dir_size
## Continue the while loop with next argument
continue
fi
unset user_dir_size
## }}}
## Get the list of user backups {{{
DUP_USER_BACKUP_LIST="${DUP_TMP_DIR}/${user_dir}.backup.list"
find "${user_dir}" -mindepth 1 -maxdepth 1 -type d -not -name '.*' -printf "%f\n" | sort > "${DUP_USER_BACKUP_LIST}"
## Test if the user backups list exists and is not empty
## OR exit with error
is_file_present "${DUP_USER_BACKUP_LIST}" "|| " \
|| error_message "Backups list (${DUP_USER_BACKUP_LIST}) for user (${user_dir}) seems empty or doesn't exists." 22
## }}}
## While loop for each backup directory
while IFS= read -r backup_dir; do
### Ignore directory if a .ignore.me file exists {{{
find "${user_dir}/${backup_dir}/.ignore.me" >/dev/null 2>&1 \
&& debug_message "|| ${RED}Skip ${user_dir}/${backup_dir}${COLOR_DEBUG} directory because a .ignore.me file exists." \
&& continue
### }}}
debug_message "||- Manage ${RED}${user_dir}/${backup_dir}${COLOR_DEBUG} directory BEGIN"
### Get current size {{{
if ! du --summarize --block-size="${min_size}" -- "${user_dir}/${backup_dir}" >/dev/null 2>&1; then
error_message "Can't get size of ${user_dir}/${backup_dir} backup directory. Check the permissions or try with 'sudo'" 31
fi
backup_dir_current_size=$(du --summarize --block-size="${min_size}" -- "${user_dir}/${backup_dir}" \
| cut --fields=1 -- )
### Multiply by min_size_int to have the correct size order
: $(( backup_dir_current_size=backup_dir_current_size*min_size_int ))
debug_message "||| Get backup current size (~${RED}${backup_dir_current_size}${COLOR_DEBUG} ${min_size_unit})."
### }}}
### Get previous size {{{
if [ $(sqlite3 "${db_path}" "SELECT size FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' AND iteration='-1'" | wc --lines --) -eq "1" ] ; then
backup_dir_previous_size=$(sqlite3 "${db_path}" "SELECT size FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' AND iteration='-1'")
debug_message "||| Use backup previous size (${RED}${backup_dir_previous_size}${COLOR_DEBUG} ${min_size_unit}) from SQLite database."
else
backup_dir_previous_size="0"
debug_message "||| Set backup previous size to zero (${RED}${backup_dir_previous_size}${COLOR_DEBUG} ${min_size_unit})."
fi
### }}}
### Compare current and previous sizes {{{
: $(( backup_dir_compare_size=backup_dir_current_size-backup_dir_previous_size ))
if [ "${backup_dir_compare_size}" -lt "${min_size_int}" ] ; then
debug_message "||| The differences between current and previous sizes is not enough (~${RED}${backup_dir_compare_size}${COLOR_DEBUG} ${min_size_unit})."
backup_dir_email="True"
else
debug_message "||| The differences between current and previous sizes is correct (~${RED}${backup_dir_compare_size}${COLOR_DEBUG} ${min_size_unit})."
backup_dir_email="False"
fi
### }}}
### Increase iterations for registred informations {{{
sqlite3 "${db_path}" "UPDATE ${SQLITE_TABLE_NAME} SET iteration = iteration - 1 WHERE name='${user_dir}/${backup_dir}'" \
|| error_message "Can't increase iterations on SQLite db file (${db_path}) for backup dir (${user_dir}/${backup_dir})." 32
### }}}
### Add a new record {{{
sqlite3 "${db_path}" "INSERT INTO ${SQLITE_TABLE_NAME} values( \
'${user_dir}/${backup_dir}', \
'${user_dir}', \
${backup_dir_current_size}, \
-1, \
'$(date +%Y/%m/%d-%H:%M)', \
'${backup_dir_email}' \
)" || error_message "Can't add a new on SQLite db file (${db_path}) for backup dir (${user_dir}/${backup_dir})." 33
### }}}
### Build email if required {{{
if [ "${backup_dir_email}" = "True" ]; then
### Use default email template to build email body for this user
is_file_absent "${user_email_template}" "| " \
&& cp -- "${mail_template}" "${user_email_template}"
### Add informations specific to this backup {{{
printf '%b' "## ${backup_dir}
Taille précédente pour cette sauvegarde/Previous size for this backup : ~${backup_dir_previous_size} ${min_size_unit} (compress).
Taille actuelle pour cette sauvegarde/Current size for this backup : ~${backup_dir_current_size} ${min_size_unit} (compress).
La différence de taille (~${backup_dir_compare_size} ${min_size_unit}) est inférieure au minimum attendu (${min_size}).
---
The difference (~${backup_dir_compare_size} ${min_size_unit}) is bellow the minimum expected size (${min_size}).
" >> "${user_email_template}"
### }}}
### Add all recorded informations from SQLite db {{{
if [ $(sqlite3 "${db_path}" "SELECT * FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}'" | wc --lines --) -ge "1" ] ; then
printf '%b' "\nInformations issues des tests précédents :
Recorded informations from previous tests :
" >> "${user_email_template}"
printf ".mode box\nSELECT * FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' ORDER BY name, iteration\n" | sqlite3 "${db_path}" >> "${user_email_template}"
#sqlite3 "${db_path}" "SELECT * FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' ORDER BY name, iteration"
fi
printf '%b\n' "" >> "${user_email_template}"
### }}}
fi
### }}}
## Unset vars {{{
unset backup_dir_current_size
unset backup_dir_previous_size
unset backup_dir_email
#unset
## }}}
debug_message "||- Manage ${RED}${user_dir}/${backup_dir}${COLOR_DEBUG} directory END"
done < "${DUP_USER_BACKUP_LIST}"
## Done while loop for each backup directory
## Send email if a content exists {{{
if is_file_present "${user_email_template}" "| "; then
user_email_address="${user_dir}@${user_email_domain}"
### Add final informations if available {{{
is_file_present "${mail_template_bottom}" "| " \
&& debug_message "| Add extra content (${RED}${mail_template_bottom}${COLOR_DEBUG}) at the end of email for user." \
&& cat "${mail_template_bottom}" >> "${user_email_template}"
### }}}
### Send email {{{
debug_message "| Send email content (${user_email_template}) to user email address (${RED}${user_email_address}${COLOR_DEBUG})."
mail -r "duplicati@duplicati.$(hostname --domain)" -s "Duplicati backup warning" "${user_email_address}" < "${user_email_template}" \
|| error_message "Can't send content (${user_email_template}) by mail to user (${user_email_address}). See --debug option for more options." 23
### }}}
fi
## }}}
## Remove temp list if DEBUG is not set
[ ! "${DEBUG}" -eq "0" ] && rm --force -- "${DUP_USER_BACKUP_LIST}"
## Unset vars {{{
unset DUP_USER_BACKUP_LIST
unset backup_dir
unset user_email_template
## }}}
debug_message "|-- Manage ${RED}${user_dir}${COLOR_DEBUG} directory END"
done < "${DUP_USERS_DIR_LIST}"
# Done while loop for each user directory
# Remove SQLite records with an iterations ≤ -10
sqlite3 "${db_path}" "DELETE FROM ${SQLITE_TABLE_NAME} WHERE iteration<=-10" \
|| error_message "Can't remove from SQLite database (${db_path}) records with iterations ≤ -10." 15
debug_message "| Move back to previous directory (${RED}before ${dup_dir}${COLOR_DEBUG})."
cd -- - >/dev/null \
|| error_message "Can't move back to previous directory (before ${dup_dir}). Check the permissions or try with 'sudo'." 16
## Remove temp files if DEBUG is not set {{{
if [ ! "${DEBUG}" -eq "0" ]; then
debug_message "| ${RED}Clean temp files.${COLOR_DEBUG}"
clean_temp_files
else
mv -- "${DUP_TMP_DIR}" "${dup_info}/$(date +%Y%m%d).logs"
debug_message "| Temp files are available in ${RED}${dup_info}/$(date +%Y%m%d).logs${COLOR_DEBUG} for any debug."
fi
## }}}
debug_message "--- MAIN END"
}
# }}}
# Manage arguments # {{{
# This code can't be in a function due to argument management
if [ ! "${NBARGS}" -eq "0" ]; then
manage_arg="0"
## If the first argument is not an option
if ! printf -- '%s' "${1}" | grep -q -E -- "^-+";
then
## Print help message and exit
printf '%b\n' "${RED}Invalid option: ${1}${RESET}"
printf '%b\n' "---"
usage
exit 1
fi
# Parse all options (start with a "-") one by one
while printf -- '%s' "${1}" | grep -q -E -- "^-+"; do
case "${1}" in
-d|--dir|--directory ) ## Define dup_dir
## Move to the next argument
shift
## Define var
readonly dup_dir="${1}"
;;
--db|--sqlite3 ) ## Define db_path
## Move to the next argument
shift
## Define var
readonly db_path="${1}"
;;
--domain|--mail-domain ) ## Define user_email_domain
## Move to the next argument
shift
## Define var
readonly user_email_domain="${1}"
;;
--debug ) ## debug
DEBUG=0
debug_message "--- Manage argument BEGIN"
;;
-h|--help ) ## help
usage
## Exit after help informations
exit 0
;;
-m,--mail,--mail-template,--template ) ## Define mail_template
## Move to the next argument
shift
## Define var
readonly mail_template="${1}"
;;
-s,--size,--min-size ) ## Define min_size
## Move to the next argument
shift
## Define var
readonly min_size="${1}"
;;
* ) ## unknow option
printf '%b\n' "${RED}Invalid option: ${1}${RESET}"
printf '%b\n' "---"
usage
exit 1
;;
esac
debug_message "| ${RED}${1}${COLOR_DEBUG} option managed."
## Move to the next argument
shift
manage_arg=$((manage_arg+1))
done
debug_message "| ${RED}${manage_arg}${COLOR_DEBUG} argument(s) successfully managed."
else
debug_message "| No arguments/options to manage."
fi
debug_message "--- Manage argument END"
# }}}
main
exit 0