scripts/cluster/apt.apply.update.sh

297 lines
7.3 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/sh
# This script will try to apply APT upgrades if all conditions are satisfied
# 1. All SGE queues are disable
# 2. No SGE jobs are running
# 3. No process related to Maco are running
# This script can be called by a cronjob (eg. hourly)
# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly PROGDIR=$(readlink -m $(dirname "${0}"))
readonly ARGS="${*}"
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && readonly DEBUG=1
## Export DEBUG for sub-script
export DEBUG
# APT temp file to monitor
readonly APT_TMP_FILE="/tmp/.apt.upgrade"
## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}
manage_args() { # {{{
case "${NBARGS}" in
0 ) ## Nothing to do
;;
* )
printf '%b\n' "${RED}Don't expect any arguments.${RESET}"
printf '%b\n' "---"
usage
exit 1
;;
esac
}
# }}}
usage() { # {{{
cat <<- EOF
usage: $PROGNAME
Apply any APT package upgrade if the host is free:
* All SGE queues are disable
* No SGE jobs are running
* No other upgrades are running
EXAMPLES:
- Apply upgrade on the current host
${PROGNAME}
EOF
}
# }}}
debug_message() { # {{{
local_message="${1}"
## Print message if DEBUG is enable (=0)
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG ${PROGNAME}: ${local_message}"
return 0
}
# }}}
is_apt_upgrade_absent() { # {{{
## Check if temp APT upgrade file is absent
if [ ! -f "${APT_TMP_FILE}" ]; then
return_apt_upgrade_absent="0"
debug_message "is_apt_upgrade_absent \
NO APT upgrade available for this system."
else
return_apt_upgrade_absent="1"
debug_message "is_apt_upgrade_absent \
APT upgrade seems available for this system."
fi
return "${return_apt_upgrade_absent}"
}
# }}}
is_apt_upgrade_present() { # {{{
## Check if temp APT upgrade file exists
if [ -f "${APT_TMP_FILE}" ]; then
return_apt_upgrade_present="0"
debug_message "is_apt_upgrade_absent \
APT upgrade seems available for this system."
else
return_apt_upgrade_present="1"
debug_message "is_apt_upgrade_absent \
NO APT upgrade available for this system."
fi
return "${return_apt_upgrade_present}"
}
# }}}
is_queue_enable() { # {{{
local_sge_hostname="${1}"
local_sge_queue_name="${2}"
## List all queues with 'disable' state and filter to the expected queue name
## with a fake_user to avoid pending jobs for this queue
### And count returned lines
local_sge_queue_test=$(qstat -f -qs d -q "${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null}" -u fake_user \
| wc -l)
case "${local_sge_queue_test}" in
0 ) ## No result so the queue is enable
local_sge_queue_state="enable"
return_queue_enable="0"
;;
3 ) ## Results (header + queue name) so the queue is disable
local_sge_queue_state="disable"
return_queue_enable="1"
;;
* ) ## Unexpected result
printf '%b\n' "${RED}Not able to determine the state of ${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null} queue (command return ${local_sge_queue_test} lines).${RESET}"
exit 2
;;
esac
## Simple debug message to valid current variables
debug_message "is_queue_enable \
SGE queue: ${RED}${local_sge_queue_name:=/dev/null}${COLOR_DEBUG} \
state is: ${RED}${local_sge_queue_state:=/dev/null}${COLOR_DEBUG}."
return "${return_queue_enable}"
}
# }}}
is_any_queue_enable() { # {{{
local_any_queue_enable_hostname="${1}"
local_any_queue_enable_name="${2}"
## By default, all queues are disable
return_any_queue_enable="1"
## Test all queues one by one
for loop_enable_queue in ${local_any_queue_enable_name}; do
### If a queue is enable
#### Change the return value
is_queue_enable "${local_any_queue_enable_hostname}" "${loop_enable_queue}" \
&& return_any_queue_enable="0"
done
return "${return_any_queue_enable}"
}
# }}}
is_job_running() { # {{{
local_sge_hostname="${1}"
## List SGE informations about the host
### And get the number of used slots from all queues
### Sort the results
### Only get the last result (greater number of used slots)
local_sge_slots_used=$(qhost -h "${local_sge_hostname:=/dev/null}" -q -xml \
| grep slots_used \
| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;' \
| sort --numeric-sort \
| tail --lines=1)
case "${local_sge_slots_used}" in
0 ) ## No jobs are running
return_job_running="1"
;;
* ) ## Some jobs are running
return_job_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_job_running \
jobs running on ${local_sge_hostname} host: ${RED}${local_sge_slots_used}${COLOR_DEBUG}."
return "${return_job_running}"
}
# }}}
is_proc_running() { # {{{
local_proc_pattern="${1}"
local_count_proc_pattern="$(pgrep -f -- "${local_proc_pattern}" | wc -l)"
case "${local_count_proc_pattern}" in
0 ) ## No procs related to this pattern are running
return_proc_running="1"
;;
* ) ## At least one proc seems running
return_proc_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_proc_running \
procs running (with the pattern: ${RED}${local_proc_pattern}${COLOR_DEBUG}) on the current host: ${RED}${local_count_proc_pattern}${COLOR_DEBUG}."
return "${return_proc_running}"
}
# }}}
prepare_host() { # {{{
debug_message "prepare_host \
Forbid SSH logins,…".
touch /etc/nologin
}
# }}}
upgrade_system() { # {{{
debug_message "upgrade_system \
Try to apply APT upgrades".
## First update repositories to get all available upgrades
aptitude update >> "${APT_TMP_FILE}" 2>&1
## And apply full-upgrade
DEBIAN_FRONTEND=noninteractive aptitude -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade >> "${APT_TMP_FILE}" 2>&1
}
# }}}
clean_host() { # {{{
debug_message "clean_host \
Try to clean temp files, downloaded packages,…".
aptitude clean >> "${APT_TMP_FILE}" 2>&1 \
&& rm -f -- "${APT_TMP_FILE}" /etc/nologin
}
# }}}
main() { # {{{
manage_args "${ARGS}"
sge_hostname="$(hostname -f)"
sge_queues_name="$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
| grep "queue name" \
| cut -d"'" -f2 )"
maco_proc_pattern="(/opt/maco/bin/maco.autoupdate.sh)"
## If NO APT package upgrade is available
### Exit
is_apt_upgrade_absent \
&& exit 0
## If any SGE queue is enable
### Exit
is_any_queue_enable "${sge_hostname}" "${sge_queues_name}" \
&& exit 0
## If any SGE job runs
### Exit
is_job_running "${sge_hostname}" \
&& exit 0
## If anything related to maco is currently running
### Exit
is_proc_running "${maco_proc_pattern}" \
&& exit 0
## Prepare the host for upgrade
prepare_host
## Try to upgrade the system
### If error: Exit 50
upgrade_system \
|| exit 50 \
## Finish by cleaning temp files
### and reboot the system
clean_host \
&& systemctl reboot
}
# }}}
main
exit 255