scripts/cluster/maco.apply.update.sh

305 lines
7.7 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/sh
# This script will try to apply Maco upgrade if all conditions are satisfied
# 1. All SGE queues are disable
# 2. No SGE jobs are running
# 3. No process related to Maco are already running
# 4. No process related to APT are currently running
# This script can be called by a cronjob (eg. hourly)
# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly PROGDIR=$(readlink -m $(dirname "${0}"))
readonly ARGS="${*}"
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && readonly DEBUG=1
## Export DEBUG for sub-script
export DEBUG
# Maco temp file
readonly MACO_TMP_FILE="/tmp/.maco.upgrade"
readonly MACO_TMP_URGENT_FILE="/tmp/.maco.urgent.upgrade"
# APT temp file to monitor
readonly APT_TMP_FILE="/tmp/.apt.upgrade"
## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}
manage_args() { # {{{
case "${NBARGS}" in
0 ) ## Nothing to do
;;
* )
printf '%b\n' "${RED}Don't expect any arguments.${RESET}"
printf '%b\n' "---"
usage
exit 1
;;
esac
}
# }}}
usage() { # {{{
cat <<- EOF
usage: $PROGNAME
Apply any Maco upgrade if the host is free:
* All SGE queues are disable
* No SGE jobs are running
* No other upgrades are running
EXAMPLES:
- Apply Maco upgrade on the current host
${PROGNAME}
EOF
}
# }}}
debug_message() { # {{{
local_message="${1}"
## Print message if DEBUG is enable (=0)
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG ${PROGNAME}: ${local_message}"
return 0
}
# }}}
is_apt_upgrade_present() { # {{{
## Check if temp APT upgrade file exists
if [ -f "${APT_TMP_FILE}" ]; then
return_apt_upgrade_present="0"
debug_message "is_apt_upgrade_absent \
APT upgrade seems available for this system."
else
return_apt_upgrade_present="1"
debug_message "is_apt_upgrade_absent \
NO APT upgrade available for this system."
fi
return "${return_apt_upgrade_present}"
}
# }}}
is_maco_upgrade_absent() { # {{{
return_maco_upgrade_absent="0"
## Check if temp Maco upgrade file is present
if [ -f "${MACO_TMP_FILE}" ]; then
return_maco_upgrade_absent="1"
debug_message "is_maco_upgrade_absent \
Maco upgrade seems available."
## Check if temp Maco urgent upgrade file is present
elif [ -f "${MACO_TMP_URGENT_FILE}" ]; then
return_maco_upgrade_absent="1"
debug_message "is_maco_upgrade_absent \
Maco urgent upgrade seems available."
else
debug_message "is_maco_upgrade_absent \
No Maco upgrade require."
fi
return "${return_maco_upgrade_absent}"
}
# }}}
is_queue_enable() { # {{{
local_sge_hostname="${1}"
local_sge_queue_name="${2}"
## List all queues with 'disable' state and filter to the expected queue name
## with a fake_user to avoid pending jobs for this queue
### And count returned lines
local_sge_queue_test=$(qstat -f -qs d -q "${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null}" -u fake_user \
| wc -l)
case "${local_sge_queue_test}" in
0 ) ## No result so the queue is enable
local_sge_queue_state="enable"
return_queue_enable="0"
;;
3 ) ## Results (header + queue name) so the queue is disable
local_sge_queue_state="disable"
return_queue_enable="1"
;;
* ) ## Unexpected result
printf '%b\n' "${RED}Not able to determine the state of ${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null} queue (command return ${local_sge_queue_test} lines).${RESET}"
exit 2
;;
esac
## Simple debug message to valid current variables
debug_message "is_queue_enable \
SGE queue: ${RED}${local_sge_queue_name:=/dev/null}${COLOR_DEBUG} \
state is: ${RED}${local_sge_queue_state:=/dev/null}${COLOR_DEBUG}."
return "${return_queue_enable}"
}
# }}}
is_any_queue_enable() { # {{{
local_any_queue_enable_hostname="${1}"
local_any_queue_enable_name="${2}"
## By default, all queues are disable
return_any_queue_enable="1"
## Test all queues one by one
for loop_enable_queue in ${local_any_queue_enable_name}; do
### If a queue is enable
#### Change the return value
is_queue_enable "${local_any_queue_enable_hostname}" "${loop_enable_queue}" \
&& return_any_queue_enable="0"
done
return "${return_any_queue_enable}"
}
# }}}
is_job_running() { # {{{
local_sge_hostname="${1}"
## List SGE informations about the host
### And get the number of used slots from all queues
### Sort the results
### Only get the last result (greater number of used slots)
local_sge_slots_used=$(qhost -h "${local_sge_hostname:=/dev/null}" -q -xml \
| grep slots_used \
| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;' \
| sort --numeric-sort \
| tail --lines=1)
case "${local_sge_slots_used}" in
0 ) ## No jobs are running
return_job_running="1"
;;
* ) ## Some jobs are running
return_job_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_job_running \
jobs running on ${local_sge_hostname} host: ${RED}${local_sge_slots_used}${COLOR_DEBUG}."
return "${return_job_running}"
}
# }}}
is_proc_running() { # {{{
local_proc_pattern="${1}"
local_count_proc_pattern="$(pgrep -f -- "${local_proc_pattern}" | wc -l)"
case "${local_count_proc_pattern}" in
0 ) ## No procs related to this pattern are running
return_proc_running="1"
;;
* ) ## At least one proc seems running
return_proc_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_proc_running \
procs running (with the pattern: ${RED}${local_proc_pattern}${COLOR_DEBUG}) on the current host: ${RED}${local_count_proc_pattern}${COLOR_DEBUG}."
return "${return_proc_running}"
}
# }}}
upgrade_maco() { # {{{
debug_message "upgrade_maco \
Try to apply Maco upgrade".
/opt/maco/bin/maco.autoupdate.sh
}
# }}}
clean_host() { # {{{
debug_message "clean_host \
Try to clean temp files,…".
rm -f -- "${MACO_TMP_FILE}" "${MACO_TMP_URGENT_FILE}"
}
# }}}
main() { # {{{
manage_args "${ARGS}"
sge_hostname="$(hostname -f)"
sge_queues_name="$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
| grep "queue name" \
| cut -d"'" -f2 )"
maco_proc_pattern="(/opt/maco/bin/maco.autoupdate.sh)"
apt_proc_pattern="(aptitude.*full-upgrade|/usr/bin/dpkg.*--configure|dpkg-deb|/bin/sh /usr/lib/needrestart/dpkg-status)"
## If APT package upgrade is available
### Exit (wait for APT upgrade to be applied first)
is_apt_upgrade_present \
&& exit 0
## If Maco upgrade is absent
### Exit
is_maco_upgrade_absent \
&& exit 0
## If any SGE queue is enable
### Exit
is_any_queue_enable "${sge_hostname}" "${sge_queues_name}" \
&& exit 0
## If any SGE job runs
### Exit
is_job_running "${sge_hostname}" \
&& exit 0
## If anything related to maco is currently running
### Exit
is_proc_running "${maco_proc_pattern}" \
&& exit 0
## If anything related to APT is currently running
### Exit
is_proc_running "${apt_proc_pattern}" \
&& exit 0
## Maco is ready to be upgraded
### First, ensure maco.service is enable for next reboot
systemctl --quiet enable maco.service > /dev/null 2>&1
## Then try to upgrade Maco
### If error: Exit 50
### If success: Clean host (temp files,…)
### then reboot the system
upgrade_maco \
|| exit 50 \
&& clean_host \
&& systemctl reboot
}
# }}}
main
exit 255