scripts/cluster/maco.apply.update.sh

303 lines
7.7 KiB
Bash
Raw Normal View History

2020-06-19 10:19:03 +02:00
#!/bin/sh
# This script will try to apply Maco upgrade if all conditions are satisfied
# 1. All SGE queues are disable
2020-06-19 10:19:03 +02:00
# 2. No SGE jobs are running
# 3. No process related to Maco are already running
2020-06-22 11:00:24 +02:00
# 4. No process related to APT are currently running
2020-06-19 10:19:03 +02:00
2020-06-22 11:00:24 +02:00
# This script can be called by a cronjob (eg. hourly)
2020-06-19 10:19:03 +02:00
# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly PROGDIR=$(readlink -m $(dirname "${0}"))
readonly ARGS="${*}"
readonly NBARGS="${#}"
2020-06-24 16:03:33 +02:00
[ -z "${DEBUG}" ] && readonly DEBUG=1
2020-06-19 10:19:03 +02:00
## Export DEBUG for sub-script
export DEBUG
# Maco temp file
readonly MACO_TMP_FILE="/tmp/.maco.upgrade"
readonly MACO_TMP_URGENT_FILE="/tmp/.maco.urgent.upgrade"
# APT temp file to monitor
readonly APT_TMP_FILE="/tmp/.apt.upgrade"
## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}
manage_args() { # {{{
case "${NBARGS}" in
0 ) ## Nothing to do
;;
* )
printf '%b\n' "${RED}Don't expect any arguments.${RESET}"
printf '%b\n' "---"
usage
exit 1
;;
esac
}
# }}}
usage() { # {{{
cat <<- EOF
usage: $PROGNAME
Apply any Maco upgrade if the host is free:
* All SGE queues are disable
2020-06-19 10:19:03 +02:00
* No SGE jobs are running
* No other upgrades are running
2020-06-19 10:19:03 +02:00
EXAMPLES:
- Apply Maco upgrade on the current host
${PROGNAME}
EOF
}
# }}}
debug_message() { # {{{
local_message="${1}"
## Print message if DEBUG is enable (=0)
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG ${PROGNAME}: ${local_message}"
return 0
2020-06-19 10:19:03 +02:00
}
# }}}
is_apt_upgrade_present() { # {{{
## Check if temp APT upgrade file exists
if [ -f "${APT_TMP_FILE}" ]; then
return_apt_upgrade_present="0"
debug_message "is_apt_upgrade_absent \
APT upgrade seems available for this system."
else
return_apt_upgrade_present="1"
debug_message "is_apt_upgrade_absent \
NO APT upgrade available for this system."
fi
return "${return_apt_upgrade_present}"
}
# }}}
is_maco_upgrade_absent() { # {{{
2020-06-19 10:19:03 +02:00
return_maco_upgrade_absent="0"
## Check if temp Maco upgrade file is present
if [ -f "${MACO_TMP_FILE}" ]; then
return_maco_upgrade_absent="1"
debug_message "is_maco_upgrade_absent \
Maco upgrade seems available."
## Check if temp Maco urgent upgrade file is present
elif [ -f "${MACO_TMP_URGENT_FILE}" ]; then
return_maco_upgrade_absent="1"
debug_message "is_maco_upgrade_absent \
Maco urgent upgrade seems available."
else
debug_message "is_maco_upgrade_absent \
No Maco upgrade require."
fi
return "${return_maco_upgrade_absent}"
}
# }}}
is_queue_enable() { # {{{
local_sge_hostname="${1}"
local_sge_queue_name="${2}"
## List all queues with 'disable' state and filter to the expected queue name
## with a fake_user to avoid pending jobs for this queue
### And count returned lines
local_sge_queue_test=$(qstat -f -qs d -q "${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null}" -u fake_user \
| wc -l)
case "${local_sge_queue_test}" in
0 ) ## No result so the queue is enable
local_sge_queue_state="enable"
return_queue_enable="0"
;;
3 ) ## Results (header + queue name) so the queue is disable
local_sge_queue_state="disable"
return_queue_enable="1"
;;
* ) ## Unexpected result
printf '%b\n' "${RED}Not able to determine the state of ${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null} queue (command return ${local_sge_queue_test} lines).${RESET}"
exit 2
;;
esac
## Simple debug message to valid current variables
debug_message "is_queue_enable \
SGE queue: ${RED}${local_sge_queue_name:=/dev/null}${COLOR_DEBUG} \
state is: ${RED}${local_sge_queue_state:=/dev/null}${COLOR_DEBUG}."
return "${return_queue_enable}"
}
# }}}
is_any_queue_enable() { # {{{
local_any_queue_enable_hostname="${1}"
local_any_queue_enable_name="${2}"
## By default, all queues are disable
return_any_queue_enable="1"
## Test all queues one by one
for loop_enable_queue in ${local_any_queue_enable_name}; do
### If a queue is enable
#### Change the return value
is_queue_enable "${local_any_queue_enable_hostname}" "${loop_enable_queue}" \
&& return_any_queue_enable="0"
done
return "${return_any_queue_enable}"
2020-06-19 10:19:03 +02:00
}
# }}}
is_job_running() { # {{{
local_sge_hostname="${1}"
## List SGE informations about the host
### And get the number of used slots from all queues
### Sort the results
### Only get the last result (greater number of used slots)
local_sge_slots_used=$(qhost -h "${local_sge_hostname:=/dev/null}" -q -xml \
| grep slots_used \
| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;' \
| sort --numeric-sort \
| tail --lines=1)
case "${local_sge_slots_used}" in
0 ) ## No jobs are running
return_job_running="1"
;;
* ) ## Some jobs are running
return_job_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_job_running \
jobs running on ${local_sge_hostname} host: ${RED}${local_sge_slots_used}${COLOR_DEBUG}."
return "${return_job_running}"
}
# }}}
is_proc_running() { # {{{
local_proc_pattern="${1}"
local_count_proc_pattern="$(pgrep -f -- "${local_proc_pattern}" | wc -l)"
case "${local_count_proc_pattern}" in
0 ) ## No procs related to this pattern are running
return_proc_running="1"
;;
* ) ## At least one proc seems running
return_proc_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_proc_running \
procs running (with the pattern: ${RED}${local_proc_pattern}${COLOR_DEBUG}) on the current host: ${RED}${local_count_proc_pattern}${COLOR_DEBUG}."
return "${return_proc_running}"
2020-06-22 11:00:24 +02:00
}
# }}}
upgrade_maco() { # {{{
debug_message "upgrade_maco \
Try to apply Maco upgrade".
/opt/maco/bin/maco.autoupdate.sh
}
# }}}
clean_host() { # {{{
debug_message "clean_host \
Try to clean temp files,…".
rm -f -- "${MACO_TMP_FILE}" "${MACO_TMP_URGENT_FILE}"
2020-06-19 10:19:03 +02:00
}
# }}}
main() { # {{{
manage_args "${ARGS}"
sge_hostname="$(hostname -f)"
sge_queues_name="$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
2020-06-19 10:19:03 +02:00
| grep "queue name" \
| cut -d"'" -f2 )"
maco_proc_pattern="(/opt/maco/bin/maco.autoupdate.sh)"
2020-06-19 10:30:13 +02:00
apt_proc_pattern="(aptitude.*full-upgrade|/usr/bin/dpkg.*--configure|dpkg-deb|/bin/sh /usr/lib/needrestart/dpkg-status)"
2020-06-19 10:19:03 +02:00
## If APT package upgrade is available
2020-06-22 11:00:24 +02:00
### Exit (wait for APT upgrade to be applied first)
2020-06-19 10:19:03 +02:00
is_apt_upgrade_present \
&& exit 0
## If Maco upgrade is absent
### Exit
is_maco_upgrade_absent \
&& exit 0
## If any SGE queue is enable
2020-06-19 10:19:03 +02:00
### Exit
is_any_queue_enable "${sge_hostname}" "${sge_queues_name}" \
2020-06-19 10:19:03 +02:00
&& exit 0
## If any SGE job runs
### Exit
is_job_running "${sge_hostname}" \
&& exit 0
## If anything related to maco is currently running
### Exit
is_proc_running "${maco_proc_pattern}" \
&& exit 0
2020-06-19 10:30:13 +02:00
## If anything related to APT is currently running
### Exit
is_proc_running "${apt_proc_pattern}" \
&& exit 0
## Try to upgrade Maco
2020-06-22 11:00:24 +02:00
### If error: Exit 50
### If success: Clean host (temp files,…)
### Ensure maco.service is enable for next reboot
2020-06-22 11:23:20 +02:00
### then reboot the system
2020-06-22 11:00:24 +02:00
upgrade_maco \
|| exit 50 \
2020-06-22 11:23:20 +02:00
&& clean_host \
&& systemctl --quiet enable maco.service > /dev/null 2>&1 \
2020-06-22 11:23:20 +02:00
&& systemctl reboot
2020-06-22 11:00:24 +02:00
2020-06-19 10:19:03 +02:00
}
# }}}
main
exit 255