scripts/cluster/apt.apply.update.sh

#!/bin/sh

# This script will try to apply APT upgrades if all conditions are satisfied
#   1. All SGE queues are disable
#   2. No SGE jobs are running
#   3. No process related to Maco are running

# This script can be called by a cronjob (eg. hourly)

# Vars {{{
readonly PROGNAME=$(basename "${0}")
readonly PROGDIR=$(readlink -m $(dirname "${0}"))
readonly ARGS="${*}"
readonly NBARGS="${#}"
[ -z "${DEBUG}" ] && readonly DEBUG=1
## Export DEBUG for sub-script
export DEBUG

# APT temp file to monitor
readonly APT_TMP_FILE="/tmp/.apt.upgrade"

## Colors
readonly PURPLE='\033[1;35m'
readonly RED='\033[0;31m'
readonly RESET='\033[0m'
readonly COLOR_DEBUG="${PURPLE}"
# }}}

manage_args() {                                                 # {{{

	case "${NBARGS}" in
		0 ) ## Nothing to do
			;;
		* )
			printf '%b\n' "${RED}Don't expect any arguments.${RESET}"
			printf '%b\n' "---"
			usage
			exit 1
			;;
	esac

}
# }}}
usage() {                                                       # {{{

	cat <<- EOF
usage: $PROGNAME

Apply any APT package upgrade if the host is free:
	* All SGE queues are disable
	* No SGE jobs are running
	* No other upgrades are running

EXAMPLES:
    - Apply upgrade on the current host
        ${PROGNAME}
	EOF

}
# }}}
debug_message() {                                               # {{{

	local_message="${1}"

	## Print message if DEBUG is enable (=0)
	[ "${DEBUG}" -eq "0" ] && printf '\e[1;35m%-6b\e[m\n' "DEBUG − ${PROGNAME}: ${local_message}"

	return 0
}
# }}}
is_sge_host() {                                                 # {{{

	## Check if SGE commands (qconf) are available
	if [ "$(command -v qconf)" ]; then
		debug_message "is_sge_host − \
SGE seems present on this host."

		### And verify if the host is fully configured as a submit host
		if qconf -ss | grep --word-regexp --quiet $(hostname -f); then
			debug_message "is_sge_host − \
The host seems configured as a SGE submit host."
			return_is_sge_host="0"
		else
			return_is_sge_host="1"
			debug_message "is_sge_host − \
This host is not yet configured as a SGE submit host."
		fi
	else
		return_is_sge_host="1"
		debug_message "is_sge_host − \
SGE is not present on this host."
	fi

	return "${return_is_sge_host}"

}
# }}}
is_apt_upgrade_absent() {                                       # {{{

	## Check if temp APT upgrade file is absent
	if [ ! -f "${APT_TMP_FILE}" ]; then
		return_apt_upgrade_absent="0"
		debug_message "is_apt_upgrade_absent − \
NO APT upgrade available for this system."
	else
		return_apt_upgrade_absent="1"
		debug_message "is_apt_upgrade_absent − \
APT upgrade seems available for this system."
	fi

	return "${return_apt_upgrade_absent}"

}
# }}}
is_apt_upgrade_present() {                                      # {{{

	## Check if temp APT upgrade file exists
	if [ -f "${APT_TMP_FILE}" ]; then
		return_apt_upgrade_present="0"
		debug_message "is_apt_upgrade_absent − \
APT upgrade seems available for this system."
	else
		return_apt_upgrade_present="1"
		debug_message "is_apt_upgrade_absent − \
NO APT upgrade available for this system."
	fi

	return "${return_apt_upgrade_present}"

}
# }}}
is_queue_enable() {                                             # {{{

	local_sge_hostname="${1}"
	local_sge_queue_name="${2}"
	## List all queues with 'disable' state and filter to the expected queue name
	## with a fake_user to avoid pending jobs for this queue
	### And count returned lines
	local_sge_queue_test=$(qstat -f -qs d -q "${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null}" -u fake_user \
		| wc -l)

	case "${local_sge_queue_test}" in
		0 ) ## No result so the queue is enable
			local_sge_queue_state="enable"
			return_queue_enable="0"
			;;
		3 ) ## Results (header + queue name) so the queue is disable
			local_sge_queue_state="disable"
			return_queue_enable="1"
			;;
		* ) ## Unexpected result
			printf '%b\n' "${RED}Not able to determine the state of ${local_sge_queue_name:=/dev/null}@${local_sge_hostname:=/dev/null} queue (command return ${local_sge_queue_test} lines).${RESET}"
			exit 2
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_queue_enable − \
SGE queue: ${RED}${local_sge_queue_name:=/dev/null}${COLOR_DEBUG} \
state is: ${RED}${local_sge_queue_state:=/dev/null}${COLOR_DEBUG}."

	return "${return_queue_enable}"

}
# }}}
is_sge_master_available() {                                     # {{{

	## Check with Netcat if SGE master (sge_qmaster) is reachable from this host.
	### -z: Only scan for listening daemons, without sending any data to them.
	### -w 10: Timeout the test after 10 seconds.
	if nc -z -w 10 "${sge_master_uri}" "${sge_master_port}"; then
		return_is_sge_master_available="0"
		debug_message "is_sge_master_available − \
SGE Master (${sge_master_uri}:${sge_master_port}) is reachable from this host."
	else
		return_is_sge_master_available="1"
		debug_message "is_sge_master_available − \
SGE Master (${sge_master_uri}:${sge_master_port}) is not reachable from this host."
	fi

	return "${return_is_sge_master_available}"

}
# }}}
is_any_queue_enable() {                                         # {{{

	local_any_queue_enable_hostname="${1}"
	local_any_queue_enable_name="${2}"

	## By default, all queues are disable
	return_any_queue_enable="1"

	## Test all queues one by one
	for loop_enable_queue in ${local_any_queue_enable_name}; do

		### If a queue is enable
		#### Change the return value
		is_queue_enable "${local_any_queue_enable_hostname}" "${loop_enable_queue}" \
			&& return_any_queue_enable="0"

	done

	return "${return_any_queue_enable}"

}
# }}}
is_job_running() {                                              # {{{

	local_sge_hostname="${1}"
	## List SGE informations about the host
	### And get the number of used slots from all queues
	### Sort the results
	### Only get the last result (greater number of used slots)
	local_sge_slots_used=$(qhost -h "${local_sge_hostname:=/dev/null}" -q -xml \
		| grep slots_used \
		| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;' \
		| sort --numeric-sort \
		| tail --lines=1)

	case "${local_sge_slots_used}" in
		0 ) ## No jobs are running
			return_job_running="1"
			;;
		* ) ## Some jobs are running
			return_job_running="0"
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_job_running − \
jobs running on ${local_sge_hostname} host: ${RED}${local_sge_slots_used}${COLOR_DEBUG}."

	return "${return_job_running}"

}
# }}}
is_proc_running() {                                             # {{{

	local_proc_pattern="${1}"

	local_count_proc_pattern="$(pgrep -f -- "${local_proc_pattern}" | wc -l)"

	case "${local_count_proc_pattern}" in
		0 ) ## No procs related to this pattern are running
			return_proc_running="1"
			;;
		* ) ## At least one proc seems running
			return_proc_running="0"
			;;
esac

	## Simple debug message to valid current variables
	debug_message "is_proc_running − \
procs running (with the pattern: ${RED}${local_proc_pattern}${COLOR_DEBUG}) on the current host: ${RED}${local_count_proc_pattern}${COLOR_DEBUG}."

	return "${return_proc_running}"

}
# }}}
prepare_host() {                                                # {{{

	debug_message "prepare_host − \
Forbid SSH logins,…".

	touch /etc/nologin

}
# }}}
upgrade_system() {                                              # {{{

	debug_message "upgrade_system − \
Try to apply APT upgrades".

	## First update repositories to get all available upgrades
	aptitude update >> "${APT_TMP_FILE}" 2>&1

	## And apply full-upgrade
	DEBIAN_FRONTEND=noninteractive aptitude -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade >> "${APT_TMP_FILE}" 2>&1

}
# }}}
clean_host() {                                                  # {{{

	debug_message "clean_host − \
Try to clean temp files, downloaded packages,…".

	aptitude clean >> "${APT_TMP_FILE}" 2>&1 \
	&& rm -f -- "${APT_TMP_FILE}" /etc/nologin

}
# }}}
main() {                                                        # {{{

	## Test if SGE Master is reachable {{{
	### If sge_master_uri wasn't defined (environment variable,…) {{{
	if [ -z "${sge_master_uri}" ]; then
		## Get SGE master from current configuration
		sge_master_uri=$(grep --max-count=1 -- "" /var/lib/gridengine/default/common/act_qmaster 2>/dev/null || echo "localhost")
	fi
	### }}}
	### If sge_master_port wasn't defined (environment variable,…) {{{
	if [ -z "${sge_master_port}" ]; then
		## Use local host for sge_master_port
		sge_master_port="6444"
	fi
	### }}}

	### If SGE Master is not reachable from this host {{{
	#### Exit
	is_sge_master_available \
		|| exit 0
	### }}}
	## }}}

	## If SGE is not yet available on this host {{{
	### Exit
	is_sge_host \
		|| exit 0
	## }}}

	manage_args "${ARGS}"

	## Define vars
	sge_hostname="$(hostname -f)"
	sge_queues_name="$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
		| grep "queue name" \
		| cut -d"'" -f2 )"

	maco_proc_pattern="(/opt/maco/bin/maco.autoupdate.sh)"

	## If NO APT package upgrade is available
	### Exit
	is_apt_upgrade_absent \
		&& exit 0

	## If any SGE queue is enable
	### Exit
	is_any_queue_enable "${sge_hostname}" "${sge_queues_name}" \
		&& exit 0

	## If any SGE job runs
	### Exit
	is_job_running "${sge_hostname}" \
		&& exit 0

	## If anything related to maco is currently running
	### Exit
	is_proc_running "${maco_proc_pattern}" \
		&& exit 0

	## Prepare the host for upgrade
	prepare_host

	## Try to upgrade the system
	### If error: Exit 50
	upgrade_system \
		|| exit 50 \


	## Finish by cleaning temp files
	### and reboot the system
	clean_host \
		&& systemctl reboot

}
# }}}

main

exit 255