scripts/cluster/maco.apply.update.sh

#!/bin/sh

# This script will try to apply Maco upgrade if all conditions are satisfied
#   1. All SGE queues are disable
#   2. No SGE jobs are running
#   3. No process related to Maco are already running
#   4. No process related to APT are currently running

# This script can be called by a cronjob (eg. hourly)

# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly ARGS="${*}"
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && readonly DEBUG=1
## Export DEBUG for sub-script
export DEBUG

# Maco temp file
readonly MACO_LOCAL_DIR="/opt/maco"
readonly MACO_TMP_FILE="${MACO_LOCAL_DIR}/.maco.upgrade"
readonly MACO_TMP_URGENT_FILE="${MACO_LOCAL_DIR}/.maco.urgent.upgrade"

# APT temp file to monitor
readonly APT_TMP_FILE="/tmp/.apt.upgrade"

## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}

manage_args() {                                                 # {{{

	case "${NBARGS}" in
		0 ) ## Nothing to do
			;;
		* )
			printf '%b\n' "${RED}Don't expect any arguments.${RESET}"
			printf '%b\n' "---"
			usage
			exit 1
			;;
	esac

}
# }}}
usage() {                                                       # {{{

	cat <<- EOF
usage: $PROGNAME

Apply any Maco upgrade if the host is free:
	* All SGE queues are disable
	* No SGE jobs are running
	* No other upgrades are running

EXAMPLES:
    - Apply Maco upgrade on the current host
        ${PROGNAME}
	EOF

}
# }}}
debug_message() {                                               # {{{

	local_message="${1}"

	## Print message if DEBUG is enable (=0)
	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG − ${PROGNAME}: ${local_message}"

	return 0
}
# }}}
is_sge_host() {                                                 # {{{

	## Check if SGE commands (qconf) are available
	if [ "$(command -v qconf)" ]; then
		debug_message "is_sge_host − \
SGE seems present on this host."

		### And verify if the host is fully configured as a submit host
		if qconf -ss 2>/dev/null | grep --word-regexp --quiet $(hostname -f); then
			debug_message "is_sge_host − \
The host seems configured as a SGE submit host."
			return_is_sge_host="0"
		else
			return_is_sge_host="1"
			debug_message "is_sge_host − \
This host is not yet configured as a SGE submit host."
		fi
	else
		return_is_sge_host="1"
		debug_message "is_sge_host − \
SGE is not present on this host."
	fi

	return "${return_is_sge_host}"

}
# }}}
is_sge_master_available() {                                     # {{{

	## Check with Netcat if SGE master (sge_qmaster) is reachable from this host.
	### -z: Only scan for listening daemons, without sending any data to them.
	### -w 10: Timeout the test after 10 seconds.
	if nc -z -w 10 "${sge_master_uri}" "${sge_master_port}"; then
		return_is_sge_master_available="0"
		debug_message "is_sge_master_available − \
SGE Master (${sge_master_uri}:${sge_master_port}) is reachable from this host."
	else
		return_is_sge_master_available="1"
		debug_message "is_sge_master_available − \
SGE Master (${sge_master_uri}:${sge_master_port}) is not reachable from this host."
	fi

	return "${return_is_sge_master_available}"

}
# }}}
is_apt_upgrade_present() {                                      # {{{

	## Check if temp APT upgrade file exists
	if [ -f "${APT_TMP_FILE}" ]; then
		return_apt_upgrade_present="0"
		debug_message "is_apt_upgrade_absent − \
APT upgrade seems available for this system."
	else
		return_apt_upgrade_present="1"
		debug_message "is_apt_upgrade_absent − \
NO APT upgrade available for this system."
	fi

	return "${return_apt_upgrade_present}"

}
# }}}
is_maco_upgrade_absent() {                                      # {{{

	return_maco_upgrade_absent="0"

	## Check if temp Maco upgrade file is present
	if [ -f "${MACO_TMP_FILE}" ]; then
		return_maco_upgrade_absent="1"
		debug_message "is_maco_upgrade_absent − \
Maco upgrade seems available."

	## Check if temp Maco urgent upgrade file is present
	elif [ -f "${MACO_TMP_URGENT_FILE}" ]; then
		return_maco_upgrade_absent="1"
		debug_message "is_maco_upgrade_absent − \
Maco urgent upgrade seems available."
	else
		debug_message "is_maco_upgrade_absent − \
No Maco upgrade require."
	fi

	return "${return_maco_upgrade_absent}"

}
# }}}
is_queue_enable() {                                             # {{{

	local_sge_hostname="${1}"
	local_sge_queue_name="${2}"
	## List all queues with 'disable' state and filter to the expected queue name
	## with a fake_user to avoid pending jobs for this queue
	### And count returned lines
	local_sge_queue_test=$(qstat -f -qs d -q "${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null}" -u fake_user \
		| wc -l)

	case "${local_sge_queue_test}" in
		0 ) ## No result so the queue is enable
			local_sge_queue_state="enable"
			return_queue_enable="0"
			;;
		3 ) ## Results (header + queue name) so the queue is disable
			local_sge_queue_state="disable"
			return_queue_enable="1"
			;;
		* ) ## Unexpected result
			printf '%b\n' "${RED}Not able to determine the state of ${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null} queue (command return ${local_sge_queue_test} lines).${RESET}"
			exit 2
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_queue_enable − \
SGE queue: ${RED}${local_sge_queue_name:=/dev/null}${COLOR_DEBUG} \
state is: ${RED}${local_sge_queue_state:=/dev/null}${COLOR_DEBUG}."

	return "${return_queue_enable}"

}
# }}}
is_any_queue_enable() {                                         # {{{

	local_any_queue_enable_hostname="${1}"
	local_any_queue_enable_name="${2}"

	## By default, all queues are disable
	return_any_queue_enable="1"

	## Test all queues one by one
	for loop_enable_queue in ${local_any_queue_enable_name}; do

		### If a queue is enable
		#### Change the return value
		is_queue_enable "${local_any_queue_enable_hostname}" "${loop_enable_queue}" \
			&& return_any_queue_enable="0"

	done

	return "${return_any_queue_enable}"

}
# }}}
is_job_running() {                                              # {{{

	local_sge_hostname="${1}"
	## List SGE informations about the host
	### And get the number of used slots from all queues
	### Sort the results
	### Only get the last result (greater number of used slots)
	local_sge_slots_used=$(qhost -h "${local_sge_hostname:=/dev/null}" -q -xml \
		| grep slots_used \
		| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;' \
		| sort --numeric-sort \
		| tail --lines=1)

	case "${local_sge_slots_used}" in
		0 ) ## No jobs are running
			return_job_running="1"
			;;
		* ) ## Some jobs are running
			return_job_running="0"
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_job_running − \
jobs running on ${local_sge_hostname} host: ${RED}${local_sge_slots_used}${COLOR_DEBUG}."

	return "${return_job_running}"

}
# }}}
is_proc_running() {                                             # {{{

	local_proc_pattern="${1}"

	local_count_proc_pattern="$(pgrep -f -- "${local_proc_pattern}" | wc -l)"

	case "${local_count_proc_pattern}" in
		0 ) ## No procs related to this pattern are running
			return_proc_running="1"
			;;
		* ) ## At least one proc seems running
			return_proc_running="0"
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_proc_running − \
procs running (with the pattern: ${RED}${local_proc_pattern}${COLOR_DEBUG}) on the current host: ${RED}${local_count_proc_pattern}${COLOR_DEBUG}."

	return "${return_proc_running}"

}
# }}}
upgrade_maco() {                                                # {{{

	debug_message "upgrade_maco − \
Try to apply Maco upgrade".

	/opt/maco/bin/maco.autoupdate.sh

}
# }}}
clean_host() {                                                  # {{{

	debug_message "clean_host − \
Try to clean temp files,…"

	rm -f -- "${MACO_TMP_FILE}" "${MACO_TMP_URGENT_FILE}"
}
# }}}
main() {                                                        # {{{

	## Test if SGE Master is reachable {{{
	### If sge_master_uri wasn't defined (environment variable,…) {{{
	if [ -z "${sge_master_uri}" ]; then
		## Get SGE master from current configuration
		sge_master_uri=$(grep --max-count=1 -- "" /var/lib/gridengine/default/common/act_qmaster 2>/dev/null || echo "localhost")
	fi
	### }}}
	### If sge_master_port wasn't defined (environment variable,…) {{{
	if [ -z "${sge_master_port}" ]; then
		## Use local host for sge_master_port
		sge_master_port="6444"
	fi
	### }}}

	### If SGE Master is not reachable from this host {{{
	#### Exit
	is_sge_master_available \
		|| exit 0
	### }}}
	## }}}

	## If SGE is not yet available on this host {{{
	### Exit
	is_sge_host \
		|| exit 0
	## }}}

	manage_args "${ARGS}"

	sge_hostname="$(hostname -f)"
	sge_queues_name="$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
		| grep "queue name" \
		| cut -d"'" -f2 )"

	maco_proc_pattern="(/opt/maco/bin/maco.autoupdate.sh)"
	apt_proc_pattern="(aptitude.*full-upgrade|/usr/bin/dpkg.*--configure|dpkg-deb|/bin/sh /usr/lib/needrestart/dpkg-status)"

	## If APT package upgrade is available
	### Exit (wait for APT upgrade to be applied first)
	is_apt_upgrade_present \
		&& exit 0

	## If Maco upgrade is absent
	### Exit
	is_maco_upgrade_absent \
		&& exit 0

	## If any SGE queue is enable
	### Exit
	is_any_queue_enable "${sge_hostname}" "${sge_queues_name}" \
		&& exit 0

	## If any SGE job runs
	### Exit
	is_job_running "${sge_hostname}" \
		&& exit 0

	## If anything related to maco is currently running
	### Exit
	is_proc_running "${maco_proc_pattern}" \
		&& exit 0

	## If anything related to APT is currently running
	### Exit
	is_proc_running "${apt_proc_pattern}" \
		&& exit 0

	## Maco is ready to be upgraded
	### First, ensure maco.service is enable for next reboot
	systemctl --quiet enable maco.service > /dev/null 2>&1

	## Then, Maco is ready for upgrade
	### First clean temp files on the host
	### And try to upgrade Maco
	### If success: exit 0 (leave reboot to maco/atd)
	clean_host \
		&& upgrade_maco \
		&& exit 0

}
# }}}

main

exit 255