From c97ede2df53ada11ebb02c29f59c35ddbb2aa6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gardais=20J=C3=A9r=C3=A9my?= Date: Fri, 16 Dec 2022 11:29:56 +0100 Subject: [PATCH] Add new script to monitor Duplicati backend --- duplicati/monitor.backup.sh | 608 ++++++++++++++++++++++++++++++++++++ 1 file changed, 608 insertions(+) create mode 100755 duplicati/monitor.backup.sh diff --git a/duplicati/monitor.backup.sh b/duplicati/monitor.backup.sh new file mode 100755 index 0000000..6823fbc --- /dev/null +++ b/duplicati/monitor.backup.sh @@ -0,0 +1,608 @@ +#!/bin/sh +# +# Purpose {{{ +# This script will … +# 1. List all users directories with a minimum size. +# 2. For each, get the list of backup directories. +# 3. For each backup directories. +# a. Ignore those that contain a .ignore.me hidden file. +# b. Get current backup directory size. +# c. Compare current size with a previous value (if available) from db. +# d. Increase informations iterations for this backup (N-1 become N-2,…). +# e. Store current backup dir size to N-1. +# f. Prepare email's body if difference is not significant. +# 4. Send email to user if a content exists. +# 5. Remove all backup informations with iterations ≤ -10. +# … +# +# 2022-12-07 +# }}} +# Sqlite3 database info {{{ +# Table name : SQLITE_TABLE_NAME (default : "backups_info"). +# 6 fields : +# name TEXT (eg. username/duplicati_mr013370) +# user TEXT (eg. username) +# size INT (eg. 16000) +# iteration INT (eg. -2) +# date TEXT (eg. 2022-12-05) +# emailed TEXT (eg True) +# +# A primary key is built from those 6 fields to avoid duplicate entries. +# +# The script will try to get the last iteration of a backup (iteration = -1) for +# a comparison with the current backup directory size. +# The script will decrease all iterations number of a backup (iteration - 1). +# The script will finally remove all entries with an iteration below -10. +# }}} +# Vars {{{ +PROGNAME=$(basename "${0}"); readonly PROGNAME +PROGDIR=$(readlink -m $(dirname "${0}")); readonly PROGDIR +ARGS="${*}"; readonly ARGS +readonly NBARGS="${#}" +[ -z "${DEBUG}" ] && DEBUG=1 +## Export DEBUG for sub-script +export DEBUG + +## Default values for some vars +readonly DUP_DIR_DEFAULT="/home" +readonly SQLITE_TABLE_NAME="backups_info" +## Temp files +DUP_TMP_DIR=$(mktemp -d -t duplicati-monitor-backup-XXXXXX.tmp) ; readonly DUP_TMP_DIR +readonly DUP_USERS_DIR_LIST="${DUP_TMP_DIR}/users.dir.list" +## Minimum size for a user|backup directory (for du comparison) +readonly MIN_SIZE_INT_DEFAULT="10" +readonly MIN_SIZE_UNIT_DEFAULT="MB" +readonly MIN_SIZE_DEFAULT="${MIN_SIZE_INT_DEFAULT}${MIN_SIZE_UNIT_DEFAULT}" +## Default domain for email addresses +readonly USER_EMAIL_DOMAIN_DEFAULT=$(hostname -d) + +## Colors +readonly PURPLE='\033[1;35m' +readonly RED='\033[0;31m' +readonly RESET='\033[0m' +readonly COLOR_DEBUG="${PURPLE}" +# }}} +usage() { # {{{ + + cat <<- HELP +usage: $PROGNAME [-d|-h|-m|-s] + +Monitor users backups for a Duplicati backend. +Store some informations in a sqlite3 db file. +And send an email if the size between two backups is not significant. + +EXAMPLES : + - Apply to default directory (${DUP_DIR_DEFAULT}) + ${PROGNAME} + + - Apply to a specific directory (/mnt/remote.duplicati) + ${PROGNAME} --dir /mnt/remote.duplicati + +OPTIONS : + -d,--dir,--directory + Path where users home directory are stored + (default: ${DUP_DIR_DEFAULT}). + + --db,--sqlite3 + Path to sqlite3 database file to store backups informations + (default relative to --directory: ${DUP_DIR_DEFAULT}/.duplicati/monitor.db). + + --domain,--mail-domain + Domain used to build user's email address. + (default: ${USER_EMAIL_DOMAIN_DEFAULT}). + + --debug + Enable debug messages. + + -h,--help + Print this help message. + + -m,--mail,--mail-template,--template + Template used to build email content for users. + (default relative to --directory: ${DUP_DIR_DEFAULT}/.duplicati/mail.template). + + -s,--size,--min-size + Minimal size for a user directory. + This value is also used to compare backup directory size between two runs. + Expect to have both an integer AND a unit. If one is missing, the default + one will be used. + (default: ${MIN_SIZE_DEFAULT}). +HELP +} +# }}} +debug_message() { # {{{ + + local_debug_message="${1}" + + ## Print message if DEBUG is enable (=0) + [ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG − ${PROGNAME} : ${local_debug_message}" + + unset local_debug_message + + return 0 +} +# }}} +error_message() { # {{{ + + local_error_message="${1}" + local_error_code="${2}" + + ## Print message + printf '%b\n' "ERROR − ${PROGNAME} : ${RED}${local_error_message}${RESET}" + + unset local_error_message + + exit "${local_error_code:=66}" +} +# }}} +define_vars() { # {{{ + + ## If dup_dir wasn't defined (argument) {{{ + if [ -z "${dup_dir}" ]; then + ## Use default value + readonly dup_dir="${DUP_DIR_DEFAULT}" + fi + ## }}} + ## If dup_info wasn't defined {{{ + if [ -z "${dup_info}" ]; then + ## Store it in previous define dup_dir + readonly dup_info="${dup_dir}/.duplicati/monitor_backup" + fi + ## }}} + ## If mail_template wasn't defined {{{ + if [ -z "${mail_template}" ]; then + ## Store it in previous define dup_dir + readonly mail_template="${dup_info}/mail.template" + fi + ## }}} + ## If mail_template_bottom wasn't defined {{{ + if [ -z "${mail_template_bottom}" ]; then + ## Store it in previous define dup_dir + readonly mail_template_bottom="${mail_template}.bottom" + fi + ## }}} + ## If db_path wasn't defined {{{ + if [ -z "${db_path}" ]; then + ## Store it in previous dup_info directory + readonly db_path="${dup_info}/monitor.db" + fi + ## }}} + ## If min_size wasn't defined (argument) {{{ + if [ -z "${min_size}" ]; then + ## Use default value + readonly min_size="${MIN_SIZE_DEFAULT}" + fi + ## }}} + ## Define min_size_int {{{ + min_size_int=$(printf -- '%s' "${min_size}" \ + | tr --delete --complement -- '0-9') + if [ -z "${min_size_int}" ]; then + min_size_int="${MIN_SIZE_INT_DEFAULT}" + readonly min_size_int + fi + ## }}} + ## Define min_size_unit {{{ + min_size_unit=$(printf -- '%s' "${min_size}" \ + | tr --delete --complement -- 'a-zA-Z') + if [ -z "${min_size_unit}" ]; then + min_size_unit="${MIN_SIZE_UNIT_DEFAULT}" + readonly min_size_unit + fi + ## }}} + ## If user_email_domain wasn't defined (argument) {{{ + if [ -z "${user_email_domain}" ]; then + ## Use default value + readonly user_email_domain="${USER_EMAIL_DOMAIN_DEFAULT}" + fi + ## }}} + +} +# }}} + +is_directory_absent() { # {{{ + + local_directory_absent="${1}" + debug_prefix="${2:-}" + + ## Directory doesn't exists by default + return_is_directory_absent="0" + + ### Check if the directory exists + # shellcheck disable=SC2086 + if test -d "${local_directory_absent}"; then + return_is_directory_absent="1" + debug_message "${debug_prefix}is_directory_absent − \ +The directory ${RED}${local_directory_absent}${COLOR_DEBUG} exists." + else + return_is_directory_absent="0" + debug_message "${debug_prefix}is_directory_absent − \ +The directory ${RED}${local_directory_absent}${COLOR_DEBUG} doesn't exist." + fi + + unset local_directory_absent + unset debug_prefix + + return "${return_is_directory_absent}" +} +# }}} +is_file_absent() { # {{{ + + local_file_absent="${1}" + debug_prefix="${2:-}" + + ## File exists by default + return_is_file_absent="1" + + ### Check if the file exists + # shellcheck disable=SC2086 + if find ${local_file_absent} > /dev/null 2>&1; then + return_is_file_absent="1" + debug_message "${debug_prefix}is_file_absent − \ +The file ${RED}${local_file_absent}${COLOR_DEBUG} exists." + else + return_is_file_absent="0" + debug_message "${debug_prefix}is_file_absent − \ +The file ${RED}${local_file_absent}${COLOR_DEBUG} doesn't exist." + fi + + unset local_file_absent + unset debug_prefix + + return "${return_is_file_absent}" +} +# }}} +is_file_present() { # {{{ + + local_file_present="${1}" + debug_prefix="${2:-}" + + ## File doesn't exist by default + return_is_file_present="1" + + ### Check if the file exists + # shellcheck disable=SC2086 + if find ${local_file_present} > /dev/null 2>&1; then + return_is_file_present="0" + debug_message "${debug_prefix}is_file_present − \ +The file ${RED}${local_file_present}${COLOR_DEBUG} exists." + else + return_is_file_present="1" + debug_message "${debug_prefix}is_file_present − \ +The file ${RED}${local_file_present}${COLOR_DEBUG} doesn't exist." + fi + + unset local_file_present + unset debug_prefix + + return "${return_is_file_present}" +} +# }}} +clean_temp_files() { # {{{ + + ## Remove temp files if DEBUG is not set + if [ ! "${DEBUG}" -eq "0" ]; then + rm --recursive --force -- "${DUP_TMP_DIR}" + else + debug_message "| Temp files are available in ${RED}${DUP_TMP_DIR}${COLOR_DEBUG} for any debug." + fi + + return "0" +} +# }}} + +main() { # {{{ + + # Define all vars + define_vars + + debug_message "--- MAIN BEGIN" + + debug_message "| Temp files are availabe here : ${RED}${DUP_TMP_DIR}${COLOR_DEBUG} ." + + # Requirements {{{ + if [ ! $(command -v sqlite3) ]; then + debug_message "| Need to install script dependency (${RED}sqlite3${COLOR_DEBUG})." + sudo aptitude install sqlite3 + fi + # }}} + + # If dup_dir doesn't exists {{{ + # AND exit with error + is_directory_absent "${dup_dir}" "| " \ + && error_message "The directory (${dup_dir}) doesn't exists. Check your configuration or use -d|--dir option to give an existing directory. Use --help for more informations." 11 + # }}} + # If dup_info doesn't exists {{{ + # AND exit with error + is_directory_absent "${dup_info}" "| " \ + && debug_message "| Create the ${RED}${dup_info}${COLOR_DEBUG} directory." \ + && mkdir --parent -- "${dup_info}" + # }}} + # If mail_template doesn't exists {{{ + # AND exit with error + is_file_absent "${mail_template}" "| " \ + && error_message "Email template (${mail_template}) doesn't exists. Please provide a path to a valid file (--template option) or create a template in appropriate directory (${dup_info}). See --help for more informations." 12 + # }}} + # If db_path doesn't exists {{{ + # Init a table in a new sqlite3 database file + if is_file_absent "${db_path}" "| "; then + debug_message "| Init table (${RED}${SQLITE_TABLE_NAME}${COLOR_DEBUG}) in a new SQLite database file (${RED}${db_path}${COLOR_DEBUG})." + sqlite3 "${db_path}" "CREATE table ${SQLITE_TABLE_NAME}( name TEXT, user TEXT, size INT, iteration INT, date TEXT, emailed TEXT, PRIMARY KEY (name, user, size, iteration, date, emailed) );" \ + || error_message "Can't init a new TABLE (${SQLITE_TABLE_NAME}) in SQLite database (${db_path})." 13 + fi + # }}} + + # Move to dup_dir + debug_message "| Move to users backup base path (${RED}${dup_dir}${COLOR_DEBUG})." + cd -- "${dup_dir}" >/dev/null \ + || error_message "Can't move to directory (${dup_dir}). Check the permissions or try with 'sudo'." 14 + + # Get the sorted list of users directories. {{{ + find . -mindepth 1 -maxdepth 1 -type d -not -name '.*' -printf "%f\n" | sort > "${DUP_USERS_DIR_LIST}" + + # Test if the users directories list exists and is not empty + # OR exit with error + is_file_present "${DUP_USERS_DIR_LIST}" "| " \ + || error_message "The users list (${DUP_USERS_DIR_LIST}) seems empty or doesn't exists." 15 + # }}} + + # While loop for each user directory + while IFS= read -r user_dir; do + ## Define path for an email content + user_email_template="${DUP_TMP_DIR}/${user_dir}.mail" + + ## Check the size of user directory {{{ + if ! du --summarize --block-size="${min_size}" -- "${user_dir}" >/dev/null 2>&1; then + error_message "Can't get size of ${user_dir} user directory. Check the permissions or try with 'sudo'" 21 + fi + user_dir_size=$(du --summarize --block-size="${min_size}" -- "${user_dir}" \ + | cut --fields=1 -- ) + ### Multiply by min_size_int to have the correct size order + : $(( user_dir_size=user_dir_size*min_size_int )) + + if [ "${user_dir_size}" -gt "${min_size_int}" ]; then + debug_message "|-- Manage ${RED}${user_dir}${COLOR_DEBUG} directory BEGIN" + else + debug_message "| ${RED}Skip ${user_dir}${COLOR_DEBUG} directory because it's below the minimum size (~${user_dir_size} ${min_size_unit} ≤ ${min_size})." + unset user_dir_size + ## Continue the while loop with next argument + continue + fi + unset user_dir_size + ## }}} + ## Get the list of user backups {{{ + DUP_USER_BACKUP_LIST="${DUP_TMP_DIR}/${user_dir}.backup.list" + find "${user_dir}" -mindepth 1 -maxdepth 1 -type d -not -name '.*' -printf "%f\n" | sort > "${DUP_USER_BACKUP_LIST}" + + ## Test if the user backups list exists and is not empty + ## OR exit with error + is_file_present "${DUP_USER_BACKUP_LIST}" "|| " \ + || error_message "Backups list (${DUP_USER_BACKUP_LIST}) for user (${user_dir}) seems empty or doesn't exists." 22 + ## }}} + + ## While loop for each backup directory + while IFS= read -r backup_dir; do + ### Ignore directory if a .ignore.me file exists {{{ + find "${user_dir}/${backup_dir}/.ignore.me" >/dev/null 2>&1 \ + && debug_message "|| ${RED}Skip ${user_dir}/${backup_dir}${COLOR_DEBUG} directory because a .ignore.me file exists." \ + && continue + ### }}} + debug_message "||- Manage ${RED}${user_dir}/${backup_dir}${COLOR_DEBUG} directory BEGIN" + ### Get current size {{{ + if ! du --summarize --block-size="${min_size}" -- "${user_dir}/${backup_dir}" >/dev/null 2>&1; then + error_message "Can't get size of ${user_dir}/${backup_dir} backup directory. Check the permissions or try with 'sudo'" 31 + fi + backup_dir_current_size=$(du --summarize --block-size="${min_size}" -- "${user_dir}/${backup_dir}" \ + | cut --fields=1 -- ) + ### Multiply by min_size_int to have the correct size order + : $(( backup_dir_current_size=backup_dir_current_size*min_size_int )) + debug_message "||| Get backup current size (~${RED}${backup_dir_current_size}${COLOR_DEBUG} ${min_size_unit})." + ### }}} + ### Get previous size {{{ + if [ $(sqlite3 "${db_path}" "SELECT size FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' AND iteration='-1'" | wc --lines --) -eq "1" ] ; then + backup_dir_previous_size=$(sqlite3 "${db_path}" "SELECT size FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' AND iteration='-1'") + debug_message "||| Use backup previous size (${RED}${backup_dir_previous_size}${COLOR_DEBUG} ${min_size_unit}) from SQLite database." + else + backup_dir_previous_size="0" + debug_message "||| Set backup previous size to zero (${RED}${backup_dir_previous_size}${COLOR_DEBUG} ${min_size_unit})." + fi + ### }}} + ### Compare current and previous sizes {{{ + : $(( backup_dir_compare_size=backup_dir_current_size-backup_dir_previous_size )) + if [ "${backup_dir_compare_size}" -lt "${min_size_int}" ] ; then + debug_message "||| The differences between current and previous sizes is not enough (~${RED}${backup_dir_compare_size}${COLOR_DEBUG} ${min_size_unit})." + backup_dir_email="True" + else + debug_message "||| The differences between current and previous sizes is correct (~${RED}${backup_dir_compare_size}${COLOR_DEBUG} ${min_size_unit})." + backup_dir_email="False" + fi + ### }}} + ### Increase iterations for registred informations {{{ + sqlite3 "${db_path}" "UPDATE ${SQLITE_TABLE_NAME} SET iteration = iteration - 1 WHERE name='${user_dir}/${backup_dir}'" \ + || error_message "Can't increase iterations on SQLite db file (${db_path}) for backup dir (${user_dir}/${backup_dir})." 32 + ### }}} + ### Add a new record {{{ + sqlite3 "${db_path}" "INSERT INTO ${SQLITE_TABLE_NAME} values( \ + '${user_dir}/${backup_dir}', \ + '${user_dir}', \ + ${backup_dir_current_size}, \ + -1, \ + '$(date +%Y/%m/%d-%H:%M)', \ + '${backup_dir_email}' \ + )" || error_message "Can't add a new on SQLite db file (${db_path}) for backup dir (${user_dir}/${backup_dir})." 33 + ### }}} + ### Build email if required {{{ + if [ "${backup_dir_email}" = "True" ]; then + ### Use default email template to build email body for this user + is_file_absent "${user_email_template}" "| " \ + && cp -- "${mail_template}" "${user_email_template}" + + ### Add informations specific to this backup {{{ + printf '%b' "## ${backup_dir} +Taille précédente pour cette sauvegarde/Previous size for this backup : ~${backup_dir_previous_size} ${min_size_unit} (compress). +Taille actuelle pour cette sauvegarde/Current size for this backup : ~${backup_dir_current_size} ${min_size_unit} (compress). +La différence de taille (~${backup_dir_compare_size} ${min_size_unit}) est inférieure au minimum attendu (${min_size}). +--- +The difference (~${backup_dir_compare_size} ${min_size_unit}) is bellow the minimum expected size (${min_size}). +" >> "${user_email_template}" + ### }}} + ### Add all recorded informations from SQLite db {{{ + if [ $(sqlite3 "${db_path}" "SELECT * FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}'" | wc --lines --) -ge "1" ] ; then + printf '%b' "\nInformations issues des tests précédents : +Recorded informations from previous tests : +" >> "${user_email_template}" + + printf ".mode box\nSELECT * FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' ORDER BY name, iteration\n" | sqlite3 "${db_path}" >> "${user_email_template}" + #sqlite3 "${db_path}" "SELECT * FROM ${SQLITE_TABLE_NAME} WHERE name='${user_dir}/${backup_dir}' ORDER BY name, iteration" + fi + printf '%b\n' "" >> "${user_email_template}" + ### }}} + + fi + ### }}} + ## Unset vars {{{ + unset backup_dir_current_size + unset backup_dir_previous_size + unset backup_dir_email + #unset + ## }}} + debug_message "||- Manage ${RED}${user_dir}/${backup_dir}${COLOR_DEBUG} directory END" + done < "${DUP_USER_BACKUP_LIST}" + ## Done while loop for each backup directory + + ## Send email if a content exists {{{ + if is_file_present "${user_email_template}" "| "; then + user_email_address="${user_dir}@${user_email_domain}" + ### Add final informations if available {{{ + is_file_present "${mail_template_bottom}" "| " \ + && debug_message "| Add extra content (${RED}${mail_template_bottom}${COLOR_DEBUG}) at the end of email for user." \ + && cat "${mail_template_bottom}" >> "${user_email_template}" + ### }}} + ### Send email {{{ + debug_message "| Send email content (${user_email_template}) to user email address (${RED}${user_email_address}${COLOR_DEBUG})." + mail -r "duplicati@duplicati.$(hostname --domain)" -s "Duplicati backup warning" "${user_email_address}" < "${user_email_template}" \ + || error_message "Can't send content (${user_email_template}) by mail to user (${user_email_address}). See --debug option for more options." 23 + ### }}} + fi + ## }}} + + ## Remove temp list if DEBUG is not set + [ ! "${DEBUG}" -eq "0" ] && rm --force -- "${DUP_USER_BACKUP_LIST}" + ## Unset vars {{{ + unset DUP_USER_BACKUP_LIST + unset backup_dir + unset user_email_template + ## }}} + debug_message "|-- Manage ${RED}${user_dir}${COLOR_DEBUG} directory END" + done < "${DUP_USERS_DIR_LIST}" + # Done while loop for each user directory + + # Remove SQLite records with an iterations ≤ -10 + sqlite3 "${db_path}" "DELETE FROM ${SQLITE_TABLE_NAME} WHERE iteration<=-10" \ + || error_message "Can't remove from SQLite database (${db_path}) records with iterations ≤ -10." 15 + + debug_message "| Move back to previous directory (${RED}before ${dup_dir}${COLOR_DEBUG})." + cd -- - >/dev/null \ + || error_message "Can't move back to previous directory (before ${dup_dir}). Check the permissions or try with 'sudo'." 16 + + ## Remove temp files if DEBUG is not set {{{ + if [ ! "${DEBUG}" -eq "0" ]; then + debug_message "| ${RED}Clean temp files.${COLOR_DEBUG}" + clean_temp_files + else + mv -- "${DUP_TMP_DIR}" "${dup_info}/$(date +%Y%m%d).logs" + debug_message "| Temp files are available in ${RED}${dup_info}/$(date +%Y%m%d).logs${COLOR_DEBUG} for any debug." + fi + ## }}} + + debug_message "--- MAIN END" +} +# }}} + +# Manage arguments # {{{ +# This code can't be in a function due to argument management + +if [ ! "${NBARGS}" -eq "0" ]; then + + manage_arg="0" + + ## If the first argument is not an option + if ! printf -- '%s' "${1}" | grep -q -E -- "^-+"; + then + ## Print help message and exit + printf '%b\n' "${RED}Invalid option: ${1}${RESET}" + printf '%b\n' "---" + usage + + exit 1 + fi + + # Parse all options (start with a "-") one by one + while printf -- '%s' "${1}" | grep -q -E -- "^-+"; do + + case "${1}" in + -d|--dir|--directory ) ## Define dup_dir + ## Move to the next argument + shift + ## Define var + readonly dup_dir="${1}" + ;; + --db|--sqlite3 ) ## Define db_path + ## Move to the next argument + shift + ## Define var + readonly db_path="${1}" + ;; + --domain|--mail-domain ) ## Define user_email_domain + ## Move to the next argument + shift + ## Define var + readonly user_email_domain="${1}" + ;; + --debug ) ## debug + DEBUG=0 + debug_message "--- Manage argument BEGIN" + ;; + -h|--help ) ## help + usage + ## Exit after help informations + exit 0 + ;; + -m,--mail,--mail-template,--template ) ## Define mail_template + ## Move to the next argument + shift + ## Define var + readonly mail_template="${1}" + ;; + -s,--size,--min-size ) ## Define min_size + ## Move to the next argument + shift + ## Define var + readonly min_size="${1}" + ;; + * ) ## unknow option + printf '%b\n' "${RED}Invalid option: ${1}${RESET}" + printf '%b\n' "---" + usage + exit 1 + ;; + esac + + debug_message "| ${RED}${1}${COLOR_DEBUG} option managed." + + ## Move to the next argument + shift + manage_arg=$((manage_arg+1)) + + done + + debug_message "| ${RED}${manage_arg}${COLOR_DEBUG} argument(s) successfully managed." +else + debug_message "| No arguments/options to manage." +fi + + debug_message "--- Manage argument END" +# }}} + +main + +exit 0