#!/bin/sh
#
# SGE: Sun Grid Engine check - BB external script test
#
#####  Purpose is to report back to a central server, all Sun
#####     Grid Engine software faults.
#####
#
# version 0.3
#
# BIG BROTHER / XXXXXXXXXXXXXXXX status
#
# Written by Butch Deal <butchdeal@yahoo.com>
#	     Daniel Gomez <dgomez@tigr.org,daniel@ixplosive.com>
#
# v0.3e 10/14/08 cut down on the number of qhost runs
# v0.3d 03/31/06 added alarm/suspend state identification
# v0.3c 03/01/06 propogated yellow state upon UNAVAILABLE queue intances
# v0.3b 01/31/06 fixed yellow warning queue status for ambigious config test
# v0.3a 01/31/06 added unknown queue status and ambigious config test
# v0.3 01/26/06 fixed status reporting and optimized job status
# v0.2 08/03/05 flag disabled queues as clear
# v0.1 07/28/05 authored

########################################
# NOTE
# This has been tested with BB 1.9e and Xymon 4.2.x
#
# The color status with respects to queue status is arbitrary and should be
# reviewed for your particular environment.
#
# Tested on :
#   Solaris & Linux
########################################

########################################
# INSTALLATION
#  step 1  - update bb-bbexttab to include this script
#
#  step 4 - restart Big Brother
#
# NOTE - the TEST variable in the configuration section, this is the name used
#        as the column header.
########################################

##################################
# CONFIGURE IT HERE
##################################
readonly PLUGIN_NAME=$(basename "${0}")

readonly TEST="sge"
readonly PLUGIN_RESULT="${XYMONTMP}/${MACHINEDOTS}.${TEST}.plugin_result"

readonly QSTAT=$(command -v qstat)
readonly QHOST=$(command -v qhost)
readonly QSELECT=$(command -v qselect)
export QSTAT QHOST QSELECT

# define colours for graphics
# Comment these out if using older BB versions
CLEAR_PIC="&clear"
RED_PIC="&red"
YELLOW_PIC="&yellow"
GREEN_PIC="&green"
UNKNOWN_PIC="&purple"

##################################
# Start of script
##################################

get_header()
{
	echo ""
	#echo "<FONT SIZE=+2><b>$1</b></FONT> ($2)<BR>"
	echo "<FONT SIZE=+2><b>$1</b></FONT> <BR>"
	# If you do not want the header in a bigger font use line below instead
	#echo "<b>$1</b> ($2)"
	# If you want the "Paul Luzzi" look uncomment this section and comment
	# out the above sections:
	#echo "<P><DIV ALIGN=\"CENTER\"><HR>"
	#echo "<B>============== $1 ==============</B>"
	#echo "<B>--- ($2) ---</B>"
	#echo "<HR></DIV>"
	#echo "<BLOCKQUOTE>"
}
get_header_small()
{
	echo ""
	#echo "<FONT SIZE=+2><b>$1</b></FONT> ($2)<BR>"
	echo "<FONT SIZE=+1><b>$1</b></FONT> <BR>"
	# If you do not want the header in a bigger font use line below instead
	# echo "<b>$1</b> ($2)"
	# If you want the "Paul Luzzi" look uncomment this section and comment
	# out the above sections:
	#echo "<P><DIV ALIGN=\"CENTER\"><HR>"
	#echo "<B>============== $1 ==============</B>"
	#echo "<B>--- ($2) ---</B>"
	#echo "<HR></DIV>"
	#echo "<BLOCKQUOTE>"
}


get_footer()
{
	echo ""
	# If you want the "Paul Luzzi" look uncomment this section and comment
	# out the above sections:
	#echo "</BLOCKQUOTE>"
}

#####
#####  Get Status proc - used to get all responses
#####
get_status()
{
	#####
	#####  Setup some variables for use later
	#####
	COLOR="green"

	# Check defaults have been set
	if [ "${QSTAT}" = "" ]; then
		readonly QSTAT=$(command -v qstat)
		echo ""
		echo "$YELLOW_PIC QSTAT command is not defined in etc/bbsys.local - using default: ${QSTAT}"
	fi

	if [ "${QHOST}" = "" ]; then
		readonly QHOST=$(command -v qhost)
		echo ""
		echo "$YELLOW_PIC QHOST command is not defined in etc/bbsys.local - using default: ${QHOST}"
	fi

	if [ "${QSELECT}" = "" ]; then
		readonly QSELECT=$(command -v qselect)
		echo ""
		echo "$YELLOW_PIC QSELECT command is not defined in etc/bbsys.local - using default: ${QSELECT}"
	fi

	###
	### Check the jobs
	###
	get_header "Jobs" "$QSTAT -l hostname=$MACHINEDOTS"
	jobs=$(${QSTAT} -l hostname=${MACHINEDOTS} -s r)
	if [ -z "$jobs" ]; then
		echo "No Running Jobs"
	else
		${QSTAT} -l hostname=${MACHINEDOTS} -s r
	fi
	get_footer

	###
	### Check the host
	###
	get_header "Host" "$METAHS -i"
	${QHOST} -h ${MACHINEDOTS} | ${GREP} -v "global"
	get_footer

	###
	### Identify queue memberships
	###
	#get_header "Queue Membership" "$QHOST -q"
	#${QHOST} -h ${MACHINEDOTS} -q | ${TAIL} +5
	#get_footer

	###
	### Check queue instance states
	###
	queueTriggered=false;
	#  Queueset=$(${QHOST} -h ${MACHINEDOTS} -q | ${TAIL} +5 | ${AWK} '{ print $1 }')
	${QHOST} -h ${MACHINEDOTS} -q | ${TAIL} +5 > "${PLUGIN_RESULT}.QSTATE"
	Queueset=$(cat "${PLUGIN_RESULT}.QSTATE" | ${AWK} '{ print $1 }')
	for Qset in $Queueset; do
		#    qstate=$(${QHOST} -q -h ${MACHINEDOTS} | ${TAIL} +5 | $GREP " $Qset" | $AWK '{print $4}')
		qstate=$(cat "${PLUGIN_RESULT}.QSTATE" | $GREP " $Qset" | $AWK '{print $4}')

		# Order determines more significant color status
		if [ "$(echo "${qstate}" | $GREP -c d)" != "0" ]; then
			COLOR="clear"
			queueMsg=$(echo "$queueMsg<BR>$CLEAR_PIC $Qset@$HOST is DISABLED")
			queueTriggered=true;
		elif [ "$(echo "${qstate}" | $GREP -c E)" != "0" ]; then
			COLOR="red"
			queueMsg=$(echo "$queueMsg<BR>$RED_PIC $Qset@$HOST is in ERROR!")
			queueTriggered=true;
		elif [ "$(echo "${qstate}" | $GREP -c c)" != "0" ]; then
			COLOR="yellow"
			queueMsg=$(echo "$queueMsg<BR>$YELLOW_PIC $Qset@$HOST has an ambigious configuration!")
			queueTriggered=true;
		elif [ "$(echo "${qstate}" | $GREP -c a)" != "0" ] || \
			[ "$(echo "${qstate}" | $GREP -c A)" != "0" ]; then
				queueMsg=$(echo "$queueMsg<BR>$YELLOW_PIC $Qset@$HOST is in ALARM")
			elif [ "$(echo "${qstate}" | $GREP -c s)" != "0" ] || \
				[ "$(echo "${qstate}" | $GREP -c S)" != "0" ]; then
					queueMsg=$(echo "$queueMsg<BR>$YELLOW_PIC $Qset@$HOST is SUSPENDED")
				elif [ "$(echo "${qstate}" | $GREP -c u)" != "0" ]; then
					COLOR="yellow"
					queueMsg=$(echo "$queueMsg<BR>$YELLOW_PIC $Qset@$HOST is UNAVAILABLE!")
					queueTriggered=true;
				elif [ "$qstate" = "" ]; then
					queueMsg=$(echo "$queueMsg<BR>$GREEN_PIC $Qset@$HOST is OK")
				else
					queueMsg=$(echo "$queueMsg<BR>$UNKNOWN_PIC $Qset@$HOST is UNKNOWN")
					queueTriggered=true;
		fi
	done

	if [ -f "${PLUGIN_RESULT}.QSTATE" ]; then
		rm -f -- "${PLUGIN_RESULT}.QSTATE"
	fi

	get_header "Queue Instance Status Report"
	echo "$queueMsg"
	get_footer

	#####
	#####  Make sure to export COLOR so that it gets back to "central"
	#####
	export COLOR

#####
#####  End of get_status proc
#####
}

#####
#####  Main body
#####
get_status > "${PLUGIN_RESULT}"

	# NOW USE THE XYMON COMMAND TO SEND THE DATA ACROSS
	$XYMON "${XYMONSRV}" "status ${MACHINE}.${TEST} ${COLOR} $($DATE) $(cat ${PLUGIN_RESULT})"
	#For testing only
	#       echo $XYMON "${XYMONSRV}" "status ${MACHINE}.${TEST} ${COLOR} $($DATE) $(cat ${PLUGIN_RESULT})" > /tmp/sgetmp

# Clean up our mess
# Checking for existence of each file since the whole test may be optional
#   and may not actually run on every client
#
if [ -f "${PLUGIN_RESULT}" ]; then
	rm -f -- "${PLUGIN_RESULT}"
fi
##############################################
# end of script
##############################################