cellinfo
/
cocluto
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fixes made a few weeks ago to make maco's release machanism work from a linux machine (graffy-ws2) 

- Bug 2664 - mettre graffy-ws2 en production
This commit is contained in:
Guillaume Raffy 2019-11-14 12:08:01 +00:00
parent e39a65c288
commit 788798c74c
1 changed files with 47 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
ClusterController/QstatParser.py
View File

@ -50,6 +50,41 @@ class QstatParser:
assert False, 'unhandled queue machine state flag :"' + c + '"'
return queueMachineState
def parseQstatOutput( self, qstatOutput ):
"""
parses result of command 'qstat -f -u \* -pri'
"""
def parse_pending_tasks(task_ranges_sequence):
"""
parses a job's task ids encoded in the form of a string containing a sequence of ranges
:param str task_ranges_sequence: a job's task ids encoded in the form of a string containing a sequence of non overlapping ranges separated with a comma. Each range is expected to be in the form "<min_index>-<max_index>:<step>"
:return list(int): the list of task ids
for example, this function would return [1, 2, 3, 4, 6, 7, 8] for the input string "1-4:1,6-8:1"
"""
task_ids = []
astrRanges = re.split(',', task_ranges_sequence)
for strRange in astrRanges:
singleIndexMatch = re.match('^(?P<elementIndex>[0-9]+)$', strRange)
if singleIndexMatch:
iElementIndex = int(singleIndexMatch.group('elementIndex'))
task_ids.extend(range(iElementIndex, iElementIndex+1))
else:
# we expect strRange to be of the form "1-4:1", where :
# the 1st number is the min element index (sge imposes it to be greater than 0)
# the 2nd number is the max element index
# the 3rd number is the step between consecutive element indices
rangeMatch = re.match( '^(?P<minElementIndex>[0-9]+)-(?P<maxElementIndex>[0-9]+):(?P<stepBetweenIndices>[0-9]+)$', strRange)
if rangeMatch == None:
logError('unexpected format for job array details : "%s" (line="%s"' % (strRange, line) )
assert(False)
iMinElementIndex=int(rangeMatch.group('minElementIndex'))
iMaxElementIndex=int(rangeMatch.group('maxElementIndex'))
iStepBetweenIndices=int(rangeMatch.group('stepBetweenIndices'))
task_ids.extend(range(iMinElementIndex, iMaxElementIndex+1, iStepBetweenIndices))
return task_ids
# ugly hack to work around the fact that qstat truncates the fqdn of cluster nodes
# graffy@physix-master:~$ qstat -f -u \*
@ -72,7 +107,13 @@ class QstatParser:
# 43619 0.56000 SimpleJobA raffy r 08/20/2009 18:13:03 1 3
# a typical job array line in the pending jobs section looks like this
# 43646 0.00000 SimpleJobA raffy qw 08/21/2009 09:56:40 1 1-4:1
jobRegularExp = re.compile( '^[ ]*(?P<jobId>[^ ]+)[ ]+[0-9.]+[ ]+(?P<jobScriptName>[^ ]+)[ ]+(?P<jobOwner>[^ ]+)[ ]+(?P<jobStatus>[^ ]+)[ ]+(?P<jobStartOrSubmitTime>[0-9][0-9]/[0-9][0-9]/[0-9][0-9][0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9])[ ]+(?P<numSlots>[0-9]+)[ ]+(?P<jobArrayDetails>[^\n]*)[\s]*$' )
# nurg The job's total urgency value in normalized fashion.
# npprior The job's -p priority in normalized fashion.
# ntckts The job's ticket amount in normalized fashion.
# ppri The job's -p priority as specified by the user.
jobRegularExp = re.compile( '^[ ]*(?P<jobId>[^ ]+)[ ]+(?P<JobPriority>[0-9.]+)[ ]+(?P<nurg>[0-9.]+)[ ]+(?P<npprior>[0-9.]+)[ ]+(?P<ntckts>[0-9.]+)[ ]+(?P<ppri>[0-9]+)[ ]+(?P<jobScriptName>[^ ]+)[ ]+(?P<jobOwner>[^ ]+)[ ]+(?P<jobStatus>[^ ]+)[ ]+(?P<jobStartOrSubmitTime>[0-9][0-9]/[0-9][0-9]/[0-9][0-9][0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9])[ ]+(?P<numSlots>[0-9]+)[ ]+(?P<jobArrayDetails>[^\n]*)[\s]*$' )
# example of machine line :
# allintel.q@simpatix34.univ-ren BIP 0/6/8 6.00 darwin-x86
machineRegularExp = re.compile( '^(?P<queueName>[^@]+)@(?P<machineName>[^ ]+)[ ]+(?P<queueTypeString>[^ ]+)[ ]+(?P<numReservedSlots>[^/]+)/(?P<numUsedSlots>[^/]+)/(?P<numTotalSlots>[^ ]+)[ ]+(?P<cpuLoad>[^ ]+)[\s]+(?P<archName>[^ ]+)[\s]+(?P<queueMachineStatus>[^\s]*)' )
@ -93,38 +134,19 @@ class QstatParser:
#logDebug('strJobArrayDetails = "%s", bIsJobArray=%d' % (strJobArrayDetails, int(bIsJobArray)))
# each element of a job array is treated as a separate job for the sake of simplicity.
# For these elements, the job id in sge sense is the same, but they are different in this program's sense
jobElementsIndexRange = range(0,1) # just one element, unless it's a job array
task_ids = range(0,1) # just one element, unless it's a job array
if bIsJobArray:
if bInPendingJobsSection:
jobElementsIndexRange = []
astrRanges = re.split(',', strJobArrayDetails)
for strRange in astrRanges:
singleIndexMatch = re.match('^(?P<elementIndex>[0-9]+)$', strRange)
if singleIndexMatch:
iElementIndex = int(singleIndexMatch.group('elementIndex'))
jobElementsIndexRange.extend(range(iElementIndex, iElementIndex+1))
else:
# we expect strRange to be of the form "1-4:1", where :
# the 1st number is the min element index (sge imposes it to be greater than 0)
# the 2nd number is the max element index
# the 3rd number is the step between consecutive element indices
rangeMatch = re.match( '^(?P<minElementIndex>[0-9]+)-(?P<maxElementIndex>[0-9]+):(?P<stepBetweenIndices>[0-9]+)$', strRange)
if rangeMatch == None:
logError('unexpected format for job array details : "%s" (line="%s"' % (strRange, line) )
assert(False)
iMinElementIndex=int(rangeMatch.group('minElementIndex'))
iMaxElementIndex=int(rangeMatch.group('maxElementIndex'))
iStepBetweenIndices=int(rangeMatch.group('stepBetweenIndices'))
jobElementsIndexRange.extend(range(iMinElementIndex, iMaxElementIndex+1, iStepBetweenIndices))
task_ids = parse_pending_tasks(strJobArrayDetails)
else:
# we are in the running jobs section, and here we expect the strJobArrayDetails to just contain the index of the job array element
iJobArrayElementIndex = int(strJobArrayDetails)
assert(iJobArrayElementIndex != 0) # sge does not allow element indices to be 0
jobElementsIndexRange = range(iJobArrayElementIndex,iJobArrayElementIndex+1)
for iElementIndex in jobElementsIndexRange:
task_ids = range(iJobArrayElementIndex,iJobArrayElementIndex+1)
for task_id in task_ids:
jobId = None
if bIsJobArray:
jobId = JobId(iJobId, iElementIndex)
jobId = JobId(iJobId, task_id)
else:
jobId = JobId(iJobId)
job = jobsState.getJob(jobId)