scripts/cluster/maco.check.update.sh

497 lines
14 KiB
Bash
Executable File
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/sh
# This script will check if Maco require an upgrade and
# will prepare the host in order to apply upgrade:
# 1. Disable SGE queue
# 2. Create a temp file (${MACO_LOCAL_DIR}/.maco.upgrade or ${MACO_LOCAL_DIR}/.maco.urgent.upgrade)
# This script can be call by a cronjob (eg. weekly or daily with --fail option).
# Another script should try to apply upgrades also with cron (eg. hourly).
# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly PROGDIR=$(readlink -m $(dirname "${0}"))
readonly ARGS="${*}"
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && DEBUG=1
## Export DEBUG for sub-script
export DEBUG
# If output message should be displayed
[ -z "${OUTPUT_MESSAGE}" ] && OUTPUT_MESSAGE=0
## Export OUTPUT_MESSAGE for sub-script
export OUTPUT_MESSAGE
# If modifications should be applied
[ -z "${SIMULATE_MODE}" ] && SIMULATE_MODE=1
# If only urgent upgrade should be checked
[ -z "${URGENT_ONLY_MODE}" ] && URGENT_ONLY_MODE=1
## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
## Maco
readonly MACO_LOCAL_DIR="/opt/maco"
readonly MACO_INSTALL_DIR="/mnt/store.ipr/InstallProgs/ipr/maco"
readonly MACO_TMP_FILE="${MACO_LOCAL_DIR}/.maco.upgrade"
readonly MACO_TMP_URGENT_FILE="${MACO_LOCAL_DIR}/.maco.urgent.upgrade"
# }}}
usage() { # {{{
cat <<- EOF
usage: $PROGNAME [-d|-h|-q|-s|-u|-w]
Compare current version of Maco script with the latest and
the urgent versions then try to prepare the host by:
* Disabling SGE queue
EXAMPLES:
- Verify Maco's upgrade and prepare the current host
${PROGNAME}
- Verify only urgent upgrade for Maco and prepare the current host
${PROGNAME} --urgent
- Get Maco's new versions from a webserver instead of the filesystem
${PROGNAME} --web
OPTIONS:
-d,--debug
Enable debug messages.
-f,--fail
Check update only if the last Maco update failed.
-h,--help
Print this help message.
-q,--quiet
Disable messages on standard output (except for error).
-s,--simulate
Only display messages and don't manage temp files.
-u,--urgent
Check only for urgent upgrade.
-w,--web
Get Maco's versions from a webserver.
EOF
}
# }}}
debug_message() { # {{{
local_debug_message="${1}"
## Print message if DEBUG is enable (=0)
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG ${PROGNAME}: ${local_debug_message}"
return 0
}
# }}}
message() { # {{{
local_message="${1}"
## Print message if OUTPUT_MESSAGE is enable (=0)
[ "${OUTPUT_MESSAGE}" -eq "0" ] && printf '%b\n' "${local_message}"
return 0
}
# }}}
is_var_empty() { # {{{
## Return False by default
return_var_empty="1"
## Total number of variables to test
local_total_var="${#}"
loop_count_var_empty="0"
## While it remains a variable to test
while [ "${local_total_var}" -gt "${loop_count_var_empty}" ]; do
debug_message "is_var_empty \
Test var: ${1}."
### Test if this is empty and set return value to True
[ -z "${1}" ] && return_var_empty="0"
### Increase the number of tested variables
loop_count_var_empty=$((loop_count_var_empty+1))
### Shift to the next variable
shift
done
return "${return_var_empty}"
}
# }}}
is_sge_host() { # {{{
## Check if SGE commands (qconf) are available
if [ "$(command -v qconf)" ]; then
debug_message "is_sge_host \
SGE seems present on this host."
### And verify if the host is fully configured as a submit host
if qconf -ss 2>/dev/null | grep --word-regexp --quiet $(hostname -f); then
debug_message "is_sge_host \
The host seems configured as a SGE submit host."
return_is_sge_host="0"
else
return_is_sge_host="1"
debug_message "is_sge_host \
This host is not yet configured as a SGE submit host."
fi
else
return_is_sge_host="1"
debug_message "is_sge_host \
SGE is not present on this host."
fi
return "${return_is_sge_host}"
}
# }}}
is_sge_master_available() { # {{{
## Check with Netcat if SGE master (sge_qmaster) is reachable from this host.
### -z: Only scan for listening daemons, without sending any data to them.
### -w 10: Timeout the test after 10 seconds.
if nc -z -w 10 "${sge_master_uri}" "${sge_master_port}"; then
return_is_sge_master_available="0"
debug_message "is_sge_master_available \
SGE Master (${sge_master_uri}:${sge_master_port}) is reachable from this host."
else
return_is_sge_master_available="1"
debug_message "is_sge_master_available \
SGE Master (${sge_master_uri}:${sge_master_port}) is not reachable from this host."
fi
return "${return_is_sge_master_available}"
}
# }}}
is_maco_urgent_upgrade_require() { # {{{
return_urgent_upgrade=""
if [ "${URGENT_TIMESTAMP}" -gt "${CURRENT_TIMESTAMP}" ]; then
debug_message "is_maco_urgent_upgrade_require \
Urgent Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${URGENT_MACO_VERSION})."
message "Urgent Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${URGENT_MACO_VERSION})."
return_urgent_upgrade="0"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
### Also create an urgent temp file that can be monitored
### touch and append to keep existing content
touch "${MACO_TMP_URGENT_FILE}" && echo "Urgent Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${URGENT_MACO_VERSION})." >> "${MACO_TMP_URGENT_FILE}"
fi
else
debug_message "is_maco_urgent_upgrade_require \
Local Maco (${CURRENT_MACO_VERSION}) is newer than urgent upgrade (${URGENT_MACO_VERSION})."
return_urgent_upgrade="1"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
## Ensure urgent temp file is not present
rm -f -- "${MACO_TMP_URGENT_FILE}"
fi
fi
return "${return_urgent_upgrade}"
}
# }}}
is_maco_upgrade_require() { # {{{
return_upgrade=""
if [ "${LATEST_TIMESTAMP}" -gt "${CURRENT_TIMESTAMP}" ]; then
debug_message "is_maco_upgrade_require \
Latest Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${LATEST_MACO_VERSION})."
message "Latest Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${LATEST_MACO_VERSION})."
return_upgrade="0"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
### Also create a temp file that can be monitored
### touch and append to keep existing content
touch "${MACO_TMP_FILE}" && echo "Latest Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${LATEST_MACO_VERSION})." >> "${MACO_TMP_FILE}"
fi
else
debug_message "is_maco_upgrade_require \
Local Maco (${CURRENT_MACO_VERSION}) seems up to date (latest: ${LATEST_MACO_VERSION})."
return_upgrade="1"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
## Ensure temp file is not present
rm -f -- "${MACO_TMP_FILE}"
fi
fi
return "${return_upgrade}"
}
# }}}
is_maco_uptodate() { # {{{
return_uptodate=""
if [ "${CURRENT_TIMESTAMP}" = "${LATEST_TIMESTAMP}" ]; then
debug_message "is_maco_uptodate \
Local Maco (${CURRENT_MACO_VERSION}) is up to date (latest: ${LATEST_MACO_VERSION})."
message "Local Maco (${CURRENT_MACO_VERSION}) is up to date (latest: ${LATEST_MACO_VERSION})."
return_uptodate="0"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
## Ensure to remove temp files
rm -f -- "${MACO_TMP_FILE}" "${MACO_TMP_URGENT_FILE}"
fi
else
debug_message "is_maco_uptodate \
Local Maco version (${CURRENT_MACO_VERSION}) is different from latest version (${LATEST_MACO_VERSION})."
return_uptodate="1"
fi
return "${return_uptodate}"
}
# }}}
is_maco_status_ok() { # {{{
## Maco status not ok by default
return_maco_status_ok="1"
## Check if Maco status file is present
if [ -f "${MACO_STATUS_FILE}" ]; then
debug_message "is_maco_status_ok \
Maco status file (${MACO_STATUS_FILE}) exists."
local_maco_status=$(grep --max-count=1 -- MacoStatus "${MACO_STATUS_FILE}" | cut --delimiter="=" --fields=2)
## Check current Maco status
if [ "${local_maco_status}" = "last-update-succeeded" ]; then
debug_message "is_maco_status_ok \
Last Maco upgrade succeed (${local_maco_status})."
return_maco_status_ok="0"
else
debug_message "is_maco_status_ok \
Maco require upgrade/maintenance (current state: ${local_maco_status})."
fi
else
debug_message "is_maco_status_ok \
Maco status file (${MACO_STATUS_FILE}) doesn't exists."
fi
return "${return_maco_status_ok}"
}
# }}}
prepare_host_for_upgrade() { # {{{
sge_disable_host_queue_script="${PROGDIR}/sge.disable.host.queue.sh"
## If SIMULATE_MODE is enable
if [ "${SIMULATE_MODE}" -eq 0 ]; then
return 0
else
debug_message "prepare_host_for_upgrade \
Disable SGE queue."
sh "${sge_disable_host_queue_script}"
return "${?}"
fi
}
# }}}
main() { # {{{
## Test if SGE Master is reachable {{{
### If sge_master_uri wasn't defined (environment variable,…) {{{
if [ -z "${sge_master_uri}" ]; then
## Get SGE master from current configuration
sge_master_uri=$(grep --max-count=1 -- "" /var/lib/gridengine/default/common/act_qmaster 2>/dev/null || echo "localhost")
fi
### }}}
### If sge_master_port wasn't defined (environment variable,…) {{{
if [ -z "${sge_master_port}" ]; then
## Use local host for sge_master_port
sge_master_port="6444"
fi
### }}}
### If SGE Master is not reachable from this host {{{
#### Exit
is_sge_master_available \
|| exit 0
### }}}
## }}}
## If SGE is not yet available on this host {{{
### Exit
is_sge_host \
|| exit 0
## }}}
## If MACO_FAILED_MODE wasn't defined (argument, environment variable,…) {{{
if [ -z "${MACO_FAILED_MODE}" ]; then
### Set False by default
MACO_FAILED_MODE="1"
fi
## }}}
## Get all Maco's versions (date)
readonly CURRENT_MACO_VERSION=$(< "${MACO_LOCAL_DIR}/maco-version.txt" awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }' )
## If WEB_MODE wasn't defined (argument, environment variable,…) {{{
if [ -z "${WEB_MODE}" ]; then
### Get Maco's versions from filesystem
debug_message "define_vars \
Get Maco's versions from filesystem."
readonly LATEST_MACO_VERSION=$(< "${MACO_INSTALL_DIR}/maco-version.txt" awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }' )
readonly URGENT_MACO_VERSION=$(< "${MACO_INSTALL_DIR}/urgent-maco-version.txt" awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }' )
else
### Get Maco's versions from webserver
debug_message "define_vars \
Get Maco's versions from webserver."
### TODO: Replace --silent option with --no-progress-meter once curl > v7.74.0-*
### will be available on the cluster (at least buster-backports).
readonly LATEST_MACO_VERSION=$(curl --silent --insecure https://store.ipr.univ-rennes.fr/InstallProgs/ipr/maco/maco-version.txt | awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }')
readonly URGENT_MACO_VERSION=$(curl --silent --insecure https://store.ipr.univ-rennes.fr/InstallProgs/ipr/maco/urgent-maco-version.txt | awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }')
fi
## }}}
## Convert version to timestamp
readonly CURRENT_TIMESTAMP=$(date -d "${CURRENT_MACO_VERSION}" "+%s")
readonly LATEST_TIMESTAMP=$(date -d "${LATEST_MACO_VERSION}" "+%s")
readonly URGENT_TIMESTAMP=$(date -d "${URGENT_MACO_VERSION}" "+%s")
# Maco status file
readonly MACO_STATUS_FILE="/var/fr.univ-rennes1.ipr.maco.machinestate.txt"
## If a variable is empty
## Exit with error code
is_var_empty "${CURRENT_MACO_VERSION}" "${LATEST_MACO_VERSION}" "${URGENT_MACO_VERSION}" "${CURRENT_TIMESTAMP}" "${LATEST_TIMESTAMP}" "${URGENT_TIMESTAMP}" \
&& exit 1
## If MACO_FAILED_MODE is set {{{
### Exit if Maco last-update-succeeded
if [ "${MACO_FAILED_MODE}" -eq "0" ]; then
### If Macostatus=last-update-succeeded
### Exit
is_maco_status_ok \
&& exit 0
fi
## }}}
is_maco_uptodate \
&& exit 0
## If URGENT_ONLY_MODE is set
if [ "${URGENT_ONLY_MODE}" -eq "0" ]; then
### Check only for urgent upgrade {{{
is_maco_urgent_upgrade_require \
&& prepare_host_for_upgrade \
&& exit 0
### }}}
else
### Check for latest upgrade {{{
is_maco_upgrade_require \
&& prepare_host_for_upgrade \
&& exit 0
### }}}
### Check for urgent upgrade {{{
is_maco_urgent_upgrade_require \
&& prepare_host_for_upgrade \
&& exit 0
### }}}
fi
}
# }}}
# Manage arguments # {{{
# This code can't be in a function due to arguments
if [ ! "${NBARGS}" -eq "0" ]; then
manage_arg="0"
# Parse all options (start with a "-") one by one
while printf -- '%s' "${1}" | grep -q -E -- "^-+"; do
case "${1}" in
-f|--fail ) ## Failed mode
MACO_FAILED_MODE="0"
;;
-h|--help ) ## help
usage
## Exit after help informations
exit 0
;;
-d|--debug ) ## debug
DEBUG=0
## Re-export new DEBUG value
export DEBUG
;;
-q|--quiet ) ## Silent mode
## Avoid to display any message on standard output
OUTPUT_MESSAGE=1
;;
-s|--simulate ) ## Simulate mode
## Only display messages
SIMULATE_MODE=0
;;
-u|--urgent ) ## Urgent upgrade only
## Check only for urgent upgrade
URGENT_ONLY_MODE=0
;;
-w|--web ) ## Web mode
## Get versions from webserver
WEB_MODE=0
;;
-- ) ## End of options list
## End the while loop
break
;;
* )
printf '%b\n' "${RED}Invalid option: ${1}${RESET}"
printf '%b\n' "---"
usage
exit 1
;;
esac
debug_message "Arguments management \
${1} option managed."
## Next arg
shift
manage_arg=$((manage_arg+1))
done
debug_message "Arguments management \
${manage_arg} argument(s) successfully managed."
else
debug_message "Arguments management \
No arguments/options to manage."
fi
# }}}
main
exit 0