scripts/cluster/maco.check.update.sh

497 lines
14 KiB
Bash
Raw Normal View History

2020-06-12 12:34:23 +02:00
#!/bin/sh
# This script will check if Maco require an upgrade and
# will prepare the host in order to apply upgrade:
# 1. Disable SGE queue
# 2. Create a temp file (${MACO_LOCAL_DIR}/.maco.upgrade or ${MACO_LOCAL_DIR}/.maco.urgent.upgrade)
2020-06-12 12:34:23 +02:00
# This script can be call by a cronjob (eg. weekly or daily with --fail option).
# Another script should try to apply upgrades also with cron (eg. hourly).
2020-06-12 12:34:23 +02:00
# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly PROGDIR=$(readlink -m $(dirname "${0}"))
readonly ARGS="${*}"
readonly NBARGS="${#}"
2020-06-12 15:13:53 +02:00
[ -z "${DEBUG}" ] && DEBUG=1
2020-06-12 12:34:23 +02:00
## Export DEBUG for sub-script
export DEBUG
# If output message should be displayed
[ -z "${OUTPUT_MESSAGE}" ] && OUTPUT_MESSAGE=0
## Export OUTPUT_MESSAGE for sub-script
export OUTPUT_MESSAGE
# If modifications should be applied
[ -z "${SIMULATE_MODE}" ] && SIMULATE_MODE=1
# If only urgent upgrade should be checked
[ -z "${URGENT_ONLY_MODE}" ] && URGENT_ONLY_MODE=1
2020-06-12 12:34:23 +02:00
## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
## Maco
readonly MACO_LOCAL_DIR="/opt/maco"
readonly MACO_INSTALL_DIR="/mnt/store.ipr/InstallProgs/ipr/maco"
readonly MACO_TMP_FILE="${MACO_LOCAL_DIR}/.maco.upgrade"
readonly MACO_TMP_URGENT_FILE="${MACO_LOCAL_DIR}/.maco.urgent.upgrade"
2020-06-12 12:34:23 +02:00
# }}}
usage() { # {{{
cat <<- EOF
usage: $PROGNAME [-d|-h|-q|-s|-u|-w]
2020-06-12 12:34:23 +02:00
Compare current version of Maco script with the latest and
2020-06-16 10:04:01 +02:00
the urgent versions then try to prepare the host by:
2020-06-12 12:34:23 +02:00
* Disabling SGE queue
2020-06-16 10:04:01 +02:00
EXAMPLES:
2020-06-12 12:34:23 +02:00
- Verify Maco's upgrade and prepare the current host
${PROGNAME}
- Verify only urgent upgrade for Maco and prepare the current host
${PROGNAME} --urgent
- Get Maco's new versions from a webserver instead of the filesystem
${PROGNAME} --web
OPTIONS:
-d,--debug
Enable debug messages.
-f,--fail
Check update only if the last Maco update failed.
-h,--help
Print this help message.
-q,--quiet
Disable messages on standard output (except for error).
-s,--simulate
Only display messages and don't manage temp files.
2020-06-29 09:19:03 +02:00
-u,--urgent
Check only for urgent upgrade.
-w,--web
Get Maco's versions from a webserver.
EOF
2020-06-12 12:34:23 +02:00
}
# }}}
debug_message() { # {{{
local_debug_message="${1}"
2020-06-12 12:34:23 +02:00
## Print message if DEBUG is enable (=0)
2020-06-16 10:04:01 +02:00
[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG ${PROGNAME}: ${local_debug_message}"
return 0
}
# }}}
2020-06-12 17:24:34 +02:00
message() { # {{{
local_message="${1}"
## Print message if OUTPUT_MESSAGE is enable (=0)
2020-06-12 17:24:34 +02:00
[ "${OUTPUT_MESSAGE}" -eq "0" ] && printf '%b\n' "${local_message}"
return 0
2020-06-12 12:34:23 +02:00
2020-06-15 09:23:50 +02:00
}
# }}}
is_var_empty() { # {{{
## Return False by default
return_var_empty="1"
## Total number of variables to test
local_total_var="${#}"
loop_count_var_empty="0"
## While it remains a variable to test
while [ "${local_total_var}" -gt "${loop_count_var_empty}" ]; do
debug_message "is_var_empty \
Test var: ${1}."
### Test if this is empty and set return value to True
[ -z "${1}" ] && return_var_empty="0"
### Increase the number of tested variables
loop_count_var_empty=$((loop_count_var_empty+1))
### Shift to the next variable
shift
done
return "${return_var_empty}"
2021-12-22 11:03:26 +01:00
}
# }}}
is_sge_host() { # {{{
## Check if SGE commands (qconf) are available
if [ "$(command -v qconf)" ]; then
debug_message "is_sge_host \
SGE seems present on this host."
### And verify if the host is fully configured as a submit host
if qconf -ss 2>/dev/null | grep --word-regexp --quiet $(hostname -f); then
debug_message "is_sge_host \
The host seems configured as a SGE submit host."
return_is_sge_host="0"
else
return_is_sge_host="1"
debug_message "is_sge_host \
This host is not yet configured as a SGE submit host."
fi
else
return_is_sge_host="1"
debug_message "is_sge_host \
SGE is not present on this host."
fi
return "${return_is_sge_host}"
}
# }}}
is_sge_master_available() { # {{{
## Check with Netcat if SGE master (sge_qmaster) is reachable from this host.
### -z: Only scan for listening daemons, without sending any data to them.
### -w 10: Timeout the test after 10 seconds.
if nc -z -w 10 "${sge_master_uri}" "${sge_master_port}"; then
return_is_sge_master_available="0"
debug_message "is_sge_master_available \
SGE Master (${sge_master_uri}:${sge_master_port}) is reachable from this host."
else
return_is_sge_master_available="1"
debug_message "is_sge_master_available \
SGE Master (${sge_master_uri}:${sge_master_port}) is not reachable from this host."
fi
return "${return_is_sge_master_available}"
2020-06-12 12:34:23 +02:00
}
# }}}
2020-06-12 17:24:34 +02:00
is_maco_urgent_upgrade_require() { # {{{
2020-06-12 12:34:23 +02:00
return_urgent_upgrade=""
if [ "${URGENT_TIMESTAMP}" -gt "${CURRENT_TIMESTAMP}" ]; then
debug_message "is_maco_urgent_upgrade_require \
Urgent Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${URGENT_MACO_VERSION})."
message "Urgent Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${URGENT_MACO_VERSION})."
2020-06-12 12:34:23 +02:00
return_urgent_upgrade="0"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
### Also create an urgent temp file that can be monitored
### touch and append to keep existing content
touch "${MACO_TMP_URGENT_FILE}" && echo "Urgent Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${URGENT_MACO_VERSION})." >> "${MACO_TMP_URGENT_FILE}"
fi
2020-06-12 12:34:23 +02:00
else
debug_message "is_maco_urgent_upgrade_require \
Local Maco (${CURRENT_MACO_VERSION}) is newer than urgent upgrade (${URGENT_MACO_VERSION})."
return_urgent_upgrade="1"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
## Ensure urgent temp file is not present
rm -f -- "${MACO_TMP_URGENT_FILE}"
fi
2020-06-12 12:34:23 +02:00
fi
return "${return_urgent_upgrade}"
}
# }}}
2020-06-12 17:24:34 +02:00
is_maco_upgrade_require() { # {{{
2020-06-12 12:34:23 +02:00
return_upgrade=""
if [ "${LATEST_TIMESTAMP}" -gt "${CURRENT_TIMESTAMP}" ]; then
debug_message "is_maco_upgrade_require \
Latest Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${LATEST_MACO_VERSION})."
message "Latest Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${LATEST_MACO_VERSION})."
2020-06-12 12:34:23 +02:00
return_upgrade="0"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
### Also create a temp file that can be monitored
### touch and append to keep existing content
touch "${MACO_TMP_FILE}" && echo "Latest Maco upgrade is available (from version ${CURRENT_MACO_VERSION} to ${LATEST_MACO_VERSION})." >> "${MACO_TMP_FILE}"
fi
2020-06-12 12:34:23 +02:00
else
debug_message "is_maco_upgrade_require \
2020-06-16 10:04:01 +02:00
Local Maco (${CURRENT_MACO_VERSION}) seems up to date (latest: ${LATEST_MACO_VERSION})."
2020-06-12 12:34:23 +02:00
return_upgrade="1"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
## Ensure temp file is not present
rm -f -- "${MACO_TMP_FILE}"
fi
2020-06-12 12:34:23 +02:00
fi
return "${return_upgrade}"
}
# }}}
2020-06-12 17:24:34 +02:00
is_maco_uptodate() { # {{{
2020-06-12 12:34:23 +02:00
return_uptodate=""
if [ "${CURRENT_TIMESTAMP}" = "${LATEST_TIMESTAMP}" ]; then
debug_message "is_maco_uptodate \
2020-06-16 10:04:01 +02:00
Local Maco (${CURRENT_MACO_VERSION}) is up to date (latest: ${LATEST_MACO_VERSION})."
message "Local Maco (${CURRENT_MACO_VERSION}) is up to date (latest: ${LATEST_MACO_VERSION})."
2020-06-12 12:34:23 +02:00
return_uptodate="0"
## If SIMULATE_MODE is not enable
if [ ! "${SIMULATE_MODE}" -eq 0 ]; then
## Ensure to remove temp files
rm -f -- "${MACO_TMP_FILE}" "${MACO_TMP_URGENT_FILE}"
fi
2020-06-12 12:34:23 +02:00
else
debug_message "is_maco_uptodate \
Local Maco version (${CURRENT_MACO_VERSION}) is different from latest version (${LATEST_MACO_VERSION})."
return_uptodate="1"
fi
return "${return_uptodate}"
}
# }}}
is_maco_status_ok() { # {{{
## Maco status not ok by default
return_maco_status_ok="1"
## Check if Maco status file is present
if [ -f "${MACO_STATUS_FILE}" ]; then
debug_message "is_maco_status_ok \
Maco status file (${MACO_STATUS_FILE}) exists."
local_maco_status=$(grep --max-count=1 -- MacoStatus "${MACO_STATUS_FILE}" | cut --delimiter="=" --fields=2)
## Check current Maco status
if [ "${local_maco_status}" = "last-update-succeeded" ]; then
debug_message "is_maco_status_ok \
Last Maco upgrade succeed (${local_maco_status})."
return_maco_status_ok="0"
else
debug_message "is_maco_status_ok \
Maco require upgrade/maintenance (current state: ${local_maco_status})."
fi
else
debug_message "is_maco_status_ok \
Maco status file (${MACO_STATUS_FILE}) doesn't exists."
fi
return "${return_maco_status_ok}"
2020-06-12 17:24:34 +02:00
}
# }}}
prepare_host_for_upgrade() { # {{{
2020-06-22 11:25:11 +02:00
sge_disable_host_queue_script="${PROGDIR}/sge.disable.host.queue.sh"
2020-06-12 17:24:34 +02:00
## If SIMULATE_MODE is enable
if [ "${SIMULATE_MODE}" -eq 0 ]; then
return 0
else
debug_message "prepare_host_for_upgrade \
2020-06-12 17:24:34 +02:00
Disable SGE queue."
2020-06-22 11:25:11 +02:00
sh "${sge_disable_host_queue_script}"
2020-06-12 17:24:34 +02:00
return "${?}"
fi
2020-06-12 17:24:34 +02:00
2020-06-12 12:34:23 +02:00
}
# }}}
main() { # {{{
## Test if SGE Master is reachable {{{
### If sge_master_uri wasn't defined (environment variable,…) {{{
if [ -z "${sge_master_uri}" ]; then
2021-12-09 15:33:27 +01:00
## Get SGE master from current configuration
sge_master_uri=$(grep --max-count=1 -- "" /var/lib/gridengine/default/common/act_qmaster 2>/dev/null || echo "localhost")
fi
### }}}
### If sge_master_port wasn't defined (environment variable,…) {{{
if [ -z "${sge_master_port}" ]; then
## Use local host for sge_master_port
sge_master_port="6444"
fi
### }}}
### If SGE Master is not reachable from this host {{{
#### Exit
is_sge_master_available \
|| exit 0
### }}}
## }}}
2021-12-22 11:03:26 +01:00
## If SGE is not yet available on this host {{{
### Exit
is_sge_host \
|| exit 0
## }}}
## If MACO_FAILED_MODE wasn't defined (argument, environment variable,…) {{{
if [ -z "${MACO_FAILED_MODE}" ]; then
### Set False by default
MACO_FAILED_MODE="1"
fi
## }}}
2020-06-12 12:34:23 +02:00
## Get all Maco's versions (date)
2020-06-15 09:23:50 +02:00
readonly CURRENT_MACO_VERSION=$(< "${MACO_LOCAL_DIR}/maco-version.txt" awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }' )
## If WEB_MODE wasn't defined (argument, environment variable,…) {{{
if [ -z "${WEB_MODE}" ]; then
### Get Maco's versions from filesystem
debug_message "define_vars \
Get Maco's versions from filesystem."
readonly LATEST_MACO_VERSION=$(< "${MACO_INSTALL_DIR}/maco-version.txt" awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }' )
readonly URGENT_MACO_VERSION=$(< "${MACO_INSTALL_DIR}/urgent-maco-version.txt" awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }' )
else
### Get Maco's versions from webserver
debug_message "define_vars \
Get Maco's versions from webserver."
2021-06-09 08:08:27 +02:00
### TODO: Replace --silent option with --no-progress-meter once curl > v7.74.0-*
### will be available on the cluster (at least buster-backports).
2023-10-20 10:14:08 +02:00
readonly LATEST_MACO_VERSION=$(curl --silent --insecure https://store.ipr.univ-rennes.fr/InstallProgs/ipr/maco/maco-version.txt | awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }')
readonly URGENT_MACO_VERSION=$(curl --silent --insecure https://store.ipr.univ-rennes.fr/InstallProgs/ipr/maco/urgent-maco-version.txt | awk -v FS=. '{ print $1 "-" $2 "-" $3 "T" $4 ":" $5 ":" $6 }')
fi
## }}}
2020-06-12 12:34:23 +02:00
## Convert version to timestamp
readonly CURRENT_TIMESTAMP=$(date -d "${CURRENT_MACO_VERSION}" "+%s")
readonly LATEST_TIMESTAMP=$(date -d "${LATEST_MACO_VERSION}" "+%s")
readonly URGENT_TIMESTAMP=$(date -d "${URGENT_MACO_VERSION}" "+%s")
# Maco status file
readonly MACO_STATUS_FILE="/var/fr.univ-rennes1.ipr.maco.machinestate.txt"
2020-06-12 12:34:23 +02:00
2020-06-15 09:23:50 +02:00
## If a variable is empty
## Exit with error code
is_var_empty "${CURRENT_MACO_VERSION}" "${LATEST_MACO_VERSION}" "${URGENT_MACO_VERSION}" "${CURRENT_TIMESTAMP}" "${LATEST_TIMESTAMP}" "${URGENT_TIMESTAMP}" \
&& exit 1
## If MACO_FAILED_MODE is set {{{
### Exit if Maco last-update-succeeded
if [ "${MACO_FAILED_MODE}" -eq "0" ]; then
### If Macostatus=last-update-succeeded
### Exit
is_maco_status_ok \
&& exit 0
fi
## }}}
is_maco_uptodate \
2020-06-12 12:34:23 +02:00
&& exit 0
## If URGENT_ONLY_MODE is set
if [ "${URGENT_ONLY_MODE}" -eq "0" ]; then
### Check only for urgent upgrade {{{
is_maco_urgent_upgrade_require \
&& prepare_host_for_upgrade \
&& exit 0
### }}}
2020-06-12 12:34:23 +02:00
else
### Check for latest upgrade {{{
is_maco_upgrade_require \
&& prepare_host_for_upgrade \
&& exit 0
### }}}
### Check for urgent upgrade {{{
is_maco_urgent_upgrade_require \
&& prepare_host_for_upgrade \
&& exit 0
### }}}
fi
2020-06-12 12:34:23 +02:00
}
# }}}
2020-06-12 17:24:34 +02:00
# Manage arguments # {{{
2020-06-12 15:13:53 +02:00
# This code can't be in a function due to arguments
if [ ! "${NBARGS}" -eq "0" ]; then
manage_arg="0"
# Parse all options (start with a "-") one by one
2020-06-22 18:23:22 +02:00
while printf -- '%s' "${1}" | grep -q -E -- "^-+"; do
2020-06-12 15:13:53 +02:00
case "${1}" in
-f|--fail ) ## Failed mode
MACO_FAILED_MODE="0"
;;
-h|--help ) ## help
2020-06-12 15:13:53 +02:00
usage
## Exit after help informations
exit 0
;;
-d|--debug ) ## debug
2020-06-12 15:13:53 +02:00
DEBUG=0
## Re-export new DEBUG value
export DEBUG
;;
-q|--quiet ) ## Silent mode
## Avoid to display any message on standard output
OUTPUT_MESSAGE=1
;;
-s|--simulate ) ## Simulate mode
## Only display messages
SIMULATE_MODE=0
;;
-u|--urgent ) ## Urgent upgrade only
## Check only for urgent upgrade
URGENT_ONLY_MODE=0
;;
-w|--web ) ## Web mode
## Get versions from webserver
WEB_MODE=0
;;
-- ) ## End of options list
2020-06-12 15:13:53 +02:00
## End the while loop
break
;;
* )
printf '%b\n' "${RED}Invalid option: ${1}${RESET}"
printf '%b\n' "---"
usage
exit 1
;;
esac
debug_message "Arguments management \
${1} option managed."
## Next arg
shift
manage_arg=$((manage_arg+1))
done
debug_message "Arguments management \
${manage_arg} argument(s) successfully managed."
else
debug_message "Arguments management \
No arguments/options to manage."
fi
# }}}
2020-06-12 12:34:23 +02:00
main
exit 0