From 788798c74c9374d978c4d05b8e318d9a7a1d0ab5 Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Thu, 14 Nov 2019 12:08:01 +0000 Subject: [PATCH] fixes made a few weeks ago to make maco's release machanism work from a linux machine (graffy-ws2) - Bug 2664 - mettre graffy-ws2 en production --- ClusterController/QstatParser.py | 72 +++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/ClusterController/QstatParser.py b/ClusterController/QstatParser.py index c334697..e87a633 100644 --- a/ClusterController/QstatParser.py +++ b/ClusterController/QstatParser.py @@ -50,6 +50,41 @@ class QstatParser: assert False, 'unhandled queue machine state flag :"' + c + '"' return queueMachineState def parseQstatOutput( self, qstatOutput ): + """ + parses result of command 'qstat -f -u \* -pri' + """ + + def parse_pending_tasks(task_ranges_sequence): + """ + parses a job's task ids encoded in the form of a string containing a sequence of ranges + + :param str task_ranges_sequence: a job's task ids encoded in the form of a string containing a sequence of non overlapping ranges separated with a comma. Each range is expected to be in the form "-:" + :return list(int): the list of task ids + + for example, this function would return [1, 2, 3, 4, 6, 7, 8] for the input string "1-4:1,6-8:1" + """ + task_ids = [] + astrRanges = re.split(',', task_ranges_sequence) + for strRange in astrRanges: + singleIndexMatch = re.match('^(?P[0-9]+)$', strRange) + if singleIndexMatch: + iElementIndex = int(singleIndexMatch.group('elementIndex')) + task_ids.extend(range(iElementIndex, iElementIndex+1)) + else: + # we expect strRange to be of the form "1-4:1", where : + # the 1st number is the min element index (sge imposes it to be greater than 0) + # the 2nd number is the max element index + # the 3rd number is the step between consecutive element indices + rangeMatch = re.match( '^(?P[0-9]+)-(?P[0-9]+):(?P[0-9]+)$', strRange) + if rangeMatch == None: + logError('unexpected format for job array details : "%s" (line="%s"' % (strRange, line) ) + assert(False) + iMinElementIndex=int(rangeMatch.group('minElementIndex')) + iMaxElementIndex=int(rangeMatch.group('maxElementIndex')) + iStepBetweenIndices=int(rangeMatch.group('stepBetweenIndices')) + task_ids.extend(range(iMinElementIndex, iMaxElementIndex+1, iStepBetweenIndices)) + return task_ids + # ugly hack to work around the fact that qstat truncates the fqdn of cluster nodes # graffy@physix-master:~$ qstat -f -u \* @@ -72,7 +107,13 @@ class QstatParser: # 43619 0.56000 SimpleJobA raffy r 08/20/2009 18:13:03 1 3 # a typical job array line in the pending jobs section looks like this # 43646 0.00000 SimpleJobA raffy qw 08/21/2009 09:56:40 1 1-4:1 - jobRegularExp = re.compile( '^[ ]*(?P[^ ]+)[ ]+[0-9.]+[ ]+(?P[^ ]+)[ ]+(?P[^ ]+)[ ]+(?P[^ ]+)[ ]+(?P[0-9][0-9]/[0-9][0-9]/[0-9][0-9][0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9])[ ]+(?P[0-9]+)[ ]+(?P[^\n]*)[\s]*$' ) + + # nurg The job's total urgency value in normalized fashion. + # npprior The job's -p priority in normalized fashion. + # ntckts The job's ticket amount in normalized fashion. + # ppri The job's -p priority as specified by the user. + + jobRegularExp = re.compile( '^[ ]*(?P[^ ]+)[ ]+(?P[0-9.]+)[ ]+(?P[0-9.]+)[ ]+(?P[0-9.]+)[ ]+(?P[0-9.]+)[ ]+(?P[0-9]+)[ ]+(?P[^ ]+)[ ]+(?P[^ ]+)[ ]+(?P[^ ]+)[ ]+(?P[0-9][0-9]/[0-9][0-9]/[0-9][0-9][0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9])[ ]+(?P[0-9]+)[ ]+(?P[^\n]*)[\s]*$' ) # example of machine line : # allintel.q@simpatix34.univ-ren BIP 0/6/8 6.00 darwin-x86 machineRegularExp = re.compile( '^(?P[^@]+)@(?P[^ ]+)[ ]+(?P[^ ]+)[ ]+(?P[^/]+)/(?P[^/]+)/(?P[^ ]+)[ ]+(?P[^ ]+)[\s]+(?P[^ ]+)[\s]+(?P[^\s]*)' ) @@ -93,38 +134,19 @@ class QstatParser: #logDebug('strJobArrayDetails = "%s", bIsJobArray=%d' % (strJobArrayDetails, int(bIsJobArray))) # each element of a job array is treated as a separate job for the sake of simplicity. # For these elements, the job id in sge sense is the same, but they are different in this program's sense - jobElementsIndexRange = range(0,1) # just one element, unless it's a job array + task_ids = range(0,1) # just one element, unless it's a job array if bIsJobArray: if bInPendingJobsSection: - jobElementsIndexRange = [] - astrRanges = re.split(',', strJobArrayDetails) - for strRange in astrRanges: - singleIndexMatch = re.match('^(?P[0-9]+)$', strRange) - if singleIndexMatch: - iElementIndex = int(singleIndexMatch.group('elementIndex')) - jobElementsIndexRange.extend(range(iElementIndex, iElementIndex+1)) - else: - # we expect strRange to be of the form "1-4:1", where : - # the 1st number is the min element index (sge imposes it to be greater than 0) - # the 2nd number is the max element index - # the 3rd number is the step between consecutive element indices - rangeMatch = re.match( '^(?P[0-9]+)-(?P[0-9]+):(?P[0-9]+)$', strRange) - if rangeMatch == None: - logError('unexpected format for job array details : "%s" (line="%s"' % (strRange, line) ) - assert(False) - iMinElementIndex=int(rangeMatch.group('minElementIndex')) - iMaxElementIndex=int(rangeMatch.group('maxElementIndex')) - iStepBetweenIndices=int(rangeMatch.group('stepBetweenIndices')) - jobElementsIndexRange.extend(range(iMinElementIndex, iMaxElementIndex+1, iStepBetweenIndices)) + task_ids = parse_pending_tasks(strJobArrayDetails) else: # we are in the running jobs section, and here we expect the strJobArrayDetails to just contain the index of the job array element iJobArrayElementIndex = int(strJobArrayDetails) assert(iJobArrayElementIndex != 0) # sge does not allow element indices to be 0 - jobElementsIndexRange = range(iJobArrayElementIndex,iJobArrayElementIndex+1) - for iElementIndex in jobElementsIndexRange: + task_ids = range(iJobArrayElementIndex,iJobArrayElementIndex+1) + for task_id in task_ids: jobId = None if bIsJobArray: - jobId = JobId(iJobId, iElementIndex) + jobId = JobId(iJobId, task_id) else: jobId = JobId(iJobId) job = jobsState.getJob(jobId)