scripts/cluster/maco.apply.update.sh

#!/bin/sh

# This script will try to apply Maco upgrade if all conditions are satisfied
#   1. All SGE queues are disable
#   2. No SGE jobs are running
#   3. No process related to Maco are already running
#   4. No process related to APT are currently running

# This script can be called by a cronjob (eg. hourly)

# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly PROGDIR=$(readlink -m $(dirname "${0}"))
readonly ARGS="${*}"
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && readonly DEBUG=1
## Export DEBUG for sub-script
export DEBUG

# Maco temp file
readonly MACO_TMP_FILE="/tmp/.maco.upgrade"
readonly MACO_TMP_URGENT_FILE="/tmp/.maco.urgent.upgrade"

# APT temp file to monitor
readonly APT_TMP_FILE="/tmp/.apt.upgrade"

## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}

manage_args() {                                                 # {{{

	case "${NBARGS}" in
		0 ) ## Nothing to do
			;;
		* )
			printf '%b\n' "${RED}Don't expect any arguments.${RESET}"
			printf '%b\n' "---"
			usage
			exit 1
			;;
	esac

}
# }}}
usage() {                                                       # {{{

	cat <<- EOF
usage: $PROGNAME

Apply any Maco upgrade if the host is free:
	* All SGE queues are disable
	* No SGE jobs are running
	* No other upgrades are running

EXAMPLES:
    - Apply Maco upgrade on the current host
        ${PROGNAME}
	EOF

}
# }}}
debug_message() {                                               # {{{

	local_message="${1}"

	## Print message if DEBUG is enable (=0)
	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG − ${PROGNAME}: ${local_message}"

	return 0
}
# }}}
is_apt_upgrade_present() {                                      # {{{

	## Check if temp APT upgrade file exists
	if [ -f "${APT_TMP_FILE}" ]; then
		return_apt_upgrade_present="0"
		debug_message "is_apt_upgrade_absent − \
APT upgrade seems available for this system."
	else
		return_apt_upgrade_present="1"
		debug_message "is_apt_upgrade_absent − \
NO APT upgrade available for this system."
	fi

	return "${return_apt_upgrade_present}"

}
# }}}
is_maco_upgrade_absent() {                                      # {{{

	return_maco_upgrade_absent="0"

	## Check if temp Maco upgrade file is present
	if [ -f "${MACO_TMP_FILE}" ]; then
		return_maco_upgrade_absent="1"
		debug_message "is_maco_upgrade_absent − \
Maco upgrade seems available."

	## Check if temp Maco urgent upgrade file is present
	elif [ -f "${MACO_TMP_URGENT_FILE}" ]; then
		return_maco_upgrade_absent="1"
		debug_message "is_maco_upgrade_absent − \
Maco urgent upgrade seems available."
	else
		debug_message "is_maco_upgrade_absent − \
No Maco upgrade require."
	fi

	return "${return_maco_upgrade_absent}"

}
# }}}
is_queue_enable() {                                             # {{{

	local_sge_hostname="${1}"
	local_sge_queue_name="${2}"
	## List all queues with 'disable' state and filter to the expected queue name
	## with a fake_user to avoid pending jobs for this queue
	### And count returned lines
	local_sge_queue_test=$(qstat -f -qs d -q "${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null}" -u fake_user \
		| wc -l)

	case "${local_sge_queue_test}" in
		0 ) ## No result so the queue is enable
			local_sge_queue_state="enable"
			return_queue_enable="0"
			;;
		3 ) ## Results (header + queue name) so the queue is disable
			local_sge_queue_state="disable"
			return_queue_enable="1"
			;;
		* ) ## Unexpected result
			printf '%b\n' "${RED}Not able to determine the state of ${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null} queue (command return ${local_sge_queue_test} lines).${RESET}"
			exit 2
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_queue_enable − \
SGE queue: ${RED}${local_sge_queue_name:=/dev/null}${COLOR_DEBUG} \
state is: ${RED}${local_sge_queue_state:=/dev/null}${COLOR_DEBUG}."

	return "${return_queue_enable}"

}
# }}}
is_any_queue_enable() {                                         # {{{

	local_any_queue_enable_hostname="${1}"
	local_any_queue_enable_name="${2}"

	## By default, all queues are disable
	return_any_queue_enable="1"

	## Test all queues one by one
	for loop_enable_queue in ${local_any_queue_enable_name}; do

		### If a queue is enable
		#### Change the return value
		is_queue_enable "${local_any_queue_enable_hostname}" "${loop_enable_queue}" \
			&& return_any_queue_enable="0"

	done

	return "${return_any_queue_enable}"

}
# }}}
is_job_running() {                                              # {{{

	local_sge_hostname="${1}"
	## List SGE informations about the host
	### And get the number of used slots from all queues
	### Sort the results
	### Only get the last result (greater number of used slots)
	local_sge_slots_used=$(qhost -h "${local_sge_hostname:=/dev/null}" -q -xml \
		| grep slots_used \
		| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;' \
		| sort --numeric-sort \
		| tail --lines=1)

	case "${local_sge_slots_used}" in
		0 ) ## No jobs are running
			return_job_running="1"
			;;
		* ) ## Some jobs are running
			return_job_running="0"
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_job_running − \
jobs running on ${local_sge_hostname} host: ${RED}${local_sge_slots_used}${COLOR_DEBUG}."

	return "${return_job_running}"

}
# }}}
is_proc_running() {                                             # {{{

	local_proc_pattern="${1}"

	local_count_proc_pattern="$(pgrep -f -- "${local_proc_pattern}" | wc -l)"

	case "${local_count_proc_pattern}" in
		0 ) ## No procs related to this pattern are running
			return_proc_running="1"
			;;
		* ) ## At least one proc seems running
			return_proc_running="0"
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_proc_running − \
procs running (with the pattern: ${RED}${local_proc_pattern}${COLOR_DEBUG}) on the current host: ${RED}${local_count_proc_pattern}${COLOR_DEBUG}."

	return "${return_proc_running}"

}
# }}}
upgrade_maco() {                                                # {{{

	debug_message "upgrade_maco − \
Try to apply Maco upgrade".

	/opt/maco/bin/maco.autoupdate.sh

}
# }}}
clean_host() {                                                  # {{{

	debug_message "clean_host − \
Try to clean temp files,…".

	rm -f -- "${MACO_TMP_FILE}" "${MACO_TMP_URGENT_FILE}"

}
# }}}
main() {                                                        # {{{

	manage_args "${ARGS}"

	sge_hostname="$(hostname -f)"
	sge_queues_name="$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
		| grep "queue name" \
		| cut -d"'" -f2 )"

	maco_proc_pattern="(/opt/maco/bin/maco.autoupdate.sh)"
	apt_proc_pattern="(aptitude.*full-upgrade|/usr/bin/dpkg.*--configure|dpkg-deb|/bin/sh /usr/lib/needrestart/dpkg-status)"

	## If APT package upgrade is available
	### Exit (wait for APT upgrade to be applied first)
	is_apt_upgrade_present \
		&& exit 0

	## If Maco upgrade is absent
	### Exit
	is_maco_upgrade_absent \
		&& exit 0

	## If any SGE queue is enable
	### Exit
	is_any_queue_enable "${sge_hostname}" "${sge_queues_name}" \
		&& exit 0

	## If any SGE job runs
	### Exit
	is_job_running "${sge_hostname}" \
		&& exit 0

	## If anything related to maco is currently running
	### Exit
	is_proc_running "${maco_proc_pattern}" \
		&& exit 0

	## If anything related to APT is currently running
	### Exit
	is_proc_running "${apt_proc_pattern}" \
		&& exit 0

	## Maco is ready to be upgraded
	### First, ensure maco.service is enable for next reboot
	systemctl --quiet enable maco.service > /dev/null 2>&1

	## Then try to upgrade Maco
	### If error: Exit 50
	### If success: Clean host (temp files,…)
	### then reboot the system
	upgrade_maco \
		|| exit 50 \
		&& clean_host \
		&& systemctl reboot

}
# }}}

main

exit 255