scripts/cluster/maco.apply.update.sh

362 lines
9.4 KiB
Bash
Executable File
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/sh
# This script will try to apply Maco upgrade if all conditions are satisfied
# 1. All SGE queues are disable
# 2. No SGE jobs are running
# 3. No process related to Maco are already running
# 4. No process related to APT are currently running
# This script can be called by a cronjob (eg. hourly)
# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly ARGS="${*}"
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && readonly DEBUG=1
## Export DEBUG for sub-script
export DEBUG
# Maco temp file
readonly MACO_LOCAL_DIR="/opt/maco"
readonly MACO_TMP_FILE="${MACO_LOCAL_DIR}/.maco.upgrade"
readonly MACO_TMP_URGENT_FILE="${MACO_LOCAL_DIR}/.maco.urgent.upgrade"
# APT temp file to monitor
readonly APT_TMP_FILE="/tmp/.apt.upgrade"
## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}
manage_args() { # {{{
case "${NBARGS}" in
0 ) ## Nothing to do
;;
* )
printf '%b\n' "${RED}Don't expect any arguments.${RESET}"
printf '%b\n' "---"
usage
exit 1
;;
esac
}
# }}}
usage() { # {{{
cat <<- EOF
usage: $PROGNAME
Apply any Maco upgrade if the host is free:
* All SGE queues are disable
* No SGE jobs are running
* No other upgrades are running
EXAMPLES:
- Apply Maco upgrade on the current host
${PROGNAME}
EOF
}
# }}}
debug_message() { # {{{
local_message="${1}"
## Print message if DEBUG is enable (=0)
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG ${PROGNAME}: ${local_message}"
return 0
}
# }}}
is_sge_host() { # {{{
## Check if SGE commands (qhost) are available
if [ "$(command -v qhost)" ]; then
return_is_sge_host="0"
debug_message "is_sge_host \
SGE seems present on this host."
else
return_is_sge_host="1"
debug_message "is_sge_host \
SGE is not present on this host."
fi
return "${return_is_sge_host}"
}
# }}}
is_sge_master_available() { # {{{
## Check with Netcat if SGE master (sge_qmaster) is reachable from this host.
### -z: Only scan for listening daemons, without sending any data to them.
### -w 10: Timeout the test after 10 seconds.
if nc -z -w 10 "${sge_master_uri}" "${sge_master_port}"; then
return_is_sge_master_available="0"
debug_message "is_sge_master_available \
SGE Master (${sge_master_uri}:${sge_master_port}) is reachable from this host."
else
return_is_sge_master_available="1"
debug_message "is_sge_master_available \
SGE Master (${sge_master_uri}:${sge_master_port}) is not reachable from this host."
fi
return "${return_is_sge_master_available}"
}
# }}}
is_apt_upgrade_present() { # {{{
## Check if temp APT upgrade file exists
if [ -f "${APT_TMP_FILE}" ]; then
return_apt_upgrade_present="0"
debug_message "is_apt_upgrade_absent \
APT upgrade seems available for this system."
else
return_apt_upgrade_present="1"
debug_message "is_apt_upgrade_absent \
NO APT upgrade available for this system."
fi
return "${return_apt_upgrade_present}"
}
# }}}
is_maco_upgrade_absent() { # {{{
return_maco_upgrade_absent="0"
## Check if temp Maco upgrade file is present
if [ -f "${MACO_TMP_FILE}" ]; then
return_maco_upgrade_absent="1"
debug_message "is_maco_upgrade_absent \
Maco upgrade seems available."
## Check if temp Maco urgent upgrade file is present
elif [ -f "${MACO_TMP_URGENT_FILE}" ]; then
return_maco_upgrade_absent="1"
debug_message "is_maco_upgrade_absent \
Maco urgent upgrade seems available."
else
debug_message "is_maco_upgrade_absent \
No Maco upgrade require."
fi
return "${return_maco_upgrade_absent}"
}
# }}}
is_queue_enable() { # {{{
local_sge_hostname="${1}"
local_sge_queue_name="${2}"
## List all queues with 'disable' state and filter to the expected queue name
## with a fake_user to avoid pending jobs for this queue
### And count returned lines
local_sge_queue_test=$(qstat -f -qs d -q "${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null}" -u fake_user \
| wc -l)
case "${local_sge_queue_test}" in
0 ) ## No result so the queue is enable
local_sge_queue_state="enable"
return_queue_enable="0"
;;
3 ) ## Results (header + queue name) so the queue is disable
local_sge_queue_state="disable"
return_queue_enable="1"
;;
* ) ## Unexpected result
printf '%b\n' "${RED}Not able to determine the state of ${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null} queue (command return ${local_sge_queue_test} lines).${RESET}"
exit 2
;;
esac
## Simple debug message to valid current variables
debug_message "is_queue_enable \
SGE queue: ${RED}${local_sge_queue_name:=/dev/null}${COLOR_DEBUG} \
state is: ${RED}${local_sge_queue_state:=/dev/null}${COLOR_DEBUG}."
return "${return_queue_enable}"
}
# }}}
is_any_queue_enable() { # {{{
local_any_queue_enable_hostname="${1}"
local_any_queue_enable_name="${2}"
## By default, all queues are disable
return_any_queue_enable="1"
## Test all queues one by one
for loop_enable_queue in ${local_any_queue_enable_name}; do
### If a queue is enable
#### Change the return value
is_queue_enable "${local_any_queue_enable_hostname}" "${loop_enable_queue}" \
&& return_any_queue_enable="0"
done
return "${return_any_queue_enable}"
}
# }}}
is_job_running() { # {{{
local_sge_hostname="${1}"
## List SGE informations about the host
### And get the number of used slots from all queues
### Sort the results
### Only get the last result (greater number of used slots)
local_sge_slots_used=$(qhost -h "${local_sge_hostname:=/dev/null}" -q -xml \
| grep slots_used \
| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;' \
| sort --numeric-sort \
| tail --lines=1)
case "${local_sge_slots_used}" in
0 ) ## No jobs are running
return_job_running="1"
;;
* ) ## Some jobs are running
return_job_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_job_running \
jobs running on ${local_sge_hostname} host: ${RED}${local_sge_slots_used}${COLOR_DEBUG}."
return "${return_job_running}"
}
# }}}
is_proc_running() { # {{{
local_proc_pattern="${1}"
local_count_proc_pattern="$(pgrep -f -- "${local_proc_pattern}" | wc -l)"
case "${local_count_proc_pattern}" in
0 ) ## No procs related to this pattern are running
return_proc_running="1"
;;
* ) ## At least one proc seems running
return_proc_running="0"
;;
esac
## Simple debug message to valid current variables
debug_message "is_proc_running \
procs running (with the pattern: ${RED}${local_proc_pattern}${COLOR_DEBUG}) on the current host: ${RED}${local_count_proc_pattern}${COLOR_DEBUG}."
return "${return_proc_running}"
}
# }}}
upgrade_maco() { # {{{
debug_message "upgrade_maco \
Try to apply Maco upgrade".
/opt/maco/bin/maco.autoupdate.sh
}
# }}}
clean_host() { # {{{
debug_message "clean_host \
Try to clean temp files,…"
rm -f -- "${MACO_TMP_FILE}" "${MACO_TMP_URGENT_FILE}"
}
# }}}
main() { # {{{
## If SGE is not yet available on this host
### Exit
is_sge_host \
|| exit 0
manage_args "${ARGS}"
sge_hostname="$(hostname -f)"
sge_queues_name="$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
| grep "queue name" \
| cut -d"'" -f2 )"
maco_proc_pattern="(/opt/maco/bin/maco.autoupdate.sh)"
apt_proc_pattern="(aptitude.*full-upgrade|/usr/bin/dpkg.*--configure|dpkg-deb|/bin/sh /usr/lib/needrestart/dpkg-status)"
## If sge_master_uri wasn't defined (environment variable,…)
if [ -z "${sge_master_uri}" ]; then
## Use local host for sge_master_uri
sge_master_uri="physix-master.ipr.univ-rennes1.fr"
fi
## If sge_master_port wasn't defined (environment variable,…)
if [ -z "${sge_master_port}" ]; then
## Use local host for sge_master_port
sge_master_port="6444"
fi
## If SGE Master is not reachable from this host {{{
### Exit
is_sge_master_available \
|| exit 0
## }}}
## If APT package upgrade is available
### Exit (wait for APT upgrade to be applied first)
is_apt_upgrade_present \
&& exit 0
## If Maco upgrade is absent
### Exit
is_maco_upgrade_absent \
&& exit 0
## If any SGE queue is enable
### Exit
is_any_queue_enable "${sge_hostname}" "${sge_queues_name}" \
&& exit 0
## If any SGE job runs
### Exit
is_job_running "${sge_hostname}" \
&& exit 0
## If anything related to maco is currently running
### Exit
is_proc_running "${maco_proc_pattern}" \
&& exit 0
## If anything related to APT is currently running
### Exit
is_proc_running "${apt_proc_pattern}" \
&& exit 0
## Maco is ready to be upgraded
### First, ensure maco.service is enable for next reboot
systemctl --quiet enable maco.service > /dev/null 2>&1
## Then, Maco is ready for upgrade
### First clean temp files on the host
### And try to upgrade Maco
### If success: exit 0 (leave reboot to maco/atd)
clean_host \
&& upgrade_maco \
&& exit 0
}
# }}}
main
exit 255