From d9beee6e273ac3faf7a6cd840c21ae2d69cf1fc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gardais=20J=C3=A9r=C3=A9my?=
 <jeremy.gardais@univ-rennes1.fr>
Date: Thu, 24 Sep 2020 15:44:01 +0200
Subject: [PATCH] Disable queue if used slots reach a percentage

---
 cluster/apt.check.update.sh | 59 +++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/cluster/apt.check.update.sh b/cluster/apt.check.update.sh
index b594f7d..8f6b853 100755
--- a/cluster/apt.check.update.sh
+++ b/cluster/apt.check.update.sh
@@ -59,6 +59,16 @@ debug_message() {                                               # {{{
 	return 0
 }
 # }}}
+define_vars() {                                                 # {{{
+
+	## If sge_hostname wasn't defined
+	if [ -z "${sge_hostname}" ]; then
+		## Use local host for sge_hostname
+		sge_hostname="$(hostname -f)"
+	fi
+
+}
+# }}}
 is_apt_upgrade_absent() {                                       # {{{
 
 	## Count the number of upgradable packages and substract 1 for the header
@@ -105,6 +115,41 @@ APT upgrade available for this system: ${RED}${local_apt_upgrade_number:=/dev/nu
 
 	return "${return_apt_upgrade_present}"
 
+}
+# }}}
+is_sge_slots_more_than_percentage() {                            # {{{
+
+	local_percentage="${1}"
+
+	## Get the number of total SGE slots
+	local_sge_slots=$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
+		| grep --max-count=1 -- "'slots'" \
+		| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;')
+
+	## Get the expected percentage of total SGE slots
+	local_sge_slots_percentage=$(echo "${local_sge_slots}" \
+		| awk -v percentage="0.${local_percentage}" '{ print int($1 * percentage) }')
+
+	## Get the number of SGE used slots
+	local_sge_slots_used=$(qhost -h "${sge_hostname:=/dev/null}" -q -xml \
+		| grep --max-count=1 -- "'slots_used'" \
+		| sed 's;.*<queuevalue.*>\(.*\)</queuevalue>;\1;')
+
+	if [ "${local_sge_slots_used}" -ge "${local_sge_slots_percentage}"  ]; then
+		## Used slots is greater or equal than expected percentage
+		return_sge_slots_percentage="0"
+		## Simple debug message to valid current variable
+		debug_message "is_sge_slots_more_than_percentage (${local_percentage}%) − \
+Used slots has reached ${local_percentage}% of total slots: ${RED}${local_sge_slots_used:=/dev/null}${COLOR_DEBUG}/${local_sge_slots}."
+	else
+		return_sge_slots_percentage="1"
+		## Simple debug message to valid current variable
+		debug_message "is_sge_slots_more_than_percentage (${local_percentage}%) − \
+Used slots did not reach ${local_percentage}% of total slots: ${RED}${local_sge_slots_used:=/dev/null}${COLOR_DEBUG}/${local_sge_slots}."
+	fi
+
+	return "${return_sge_slots_percentage}"
+
 }
 # }}}
 is_pending_job_empty() {                                        # {{{
@@ -133,6 +178,9 @@ Pending jobs for the compute cluster: ${RED}${local_pending_jobs:=/dev/null}${CO
 # }}}
 main() {                                                        # {{{
 
+	## Define all vars
+	define_vars
+
 	sge_disable_host_queue_script="${PROGDIR}/sge.disable.host.queue.sh"
 
 	## If NO APT package upgrade is available {{{
@@ -143,6 +191,17 @@ main() {                                                        # {{{
 		&& exit 0
 	## }}}
 
+	## If SGE used slots is more than 75% of total slots AND {{{
+	## APT package upgrade is available
+	### Create a temp file
+	### Disable SGE queue
+	is_sge_slots_more_than_percentage "75" \
+	&& is_apt_upgrade_present \
+		&& touch "${APT_TMP_FILE}" && echo "APT upgrade is available." >> "${APT_TMP_FILE}" \
+		&& sh "${sge_disable_host_queue_script}" \
+		&& exit 0
+	## }}}
+
 	## If pending job list is empty AND {{{
 	## APT package upgrade is available
 	### Create a temp file