cellinfo
/
cocluto
4
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

migrated code using dict that was no longer compatible with python3 

iteritems() was removed in python3, see https://wiki.python.org/moin/Python3.0#Built-In_Changes

work related to Bug 3315 - make simpaweb django app a packageable application
This commit is contained in:
Guillaume Raffy 2023-05-23 11:29:59 +02:00
parent 66fb58148d
commit b5c2c886f1
9 changed files with 42 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cocluto/ClusterController/ClusterController.py
View File

@ -127,7 +127,7 @@ class ClusterController:
idleMachines = self.m_clusterStatus.getIdleMachines()
# logInfo('idleMachines :')
self.m_machinesThatNeedToSleep = []
for machineName, idleMachine in idleMachines.iteritems():
for machineName, idleMachine in idleMachines.items():
if idleMachine.getPowerState() == PowerState.ON:
# logInfo('\t%s' % machineName)
if idleMachine.getName() != 'simpatix10': # never put simpatix10 to sleep because it's the sge master and is also server for other things
@ -152,7 +152,7 @@ class ClusterController:
pendingJobs = self.m_clusterStatus.getPendingJobs()
"""
logInfo('pending jobs :')
for job in pendingJobs.itervalues():
for job in pendingJobs.values():
logInfo('\t%d' % job.getId().asStr())
"""
if len(pendingJobs) != 0:
@ -252,7 +252,7 @@ class ClusterController:
iNumMachines = len(self.m_clusterStatus.m_clusterNodes)
iNumMachinesOn = 0
iNumSleepingMachines = 0
for machine in self.m_clusterStatus.m_clusterNodes.itervalues():
for machine in self.m_clusterStatus.m_clusterNodes.values():
ePowerState = machine.getPowerState()
if ePowerState == PowerState.ON:
iNumMachinesOn+=1
@ -284,7 +284,7 @@ def storeClusterNodeStatus( clusterNodeStatus ):
r=conn.store_result()
print r.fetch_row()[0]
'''
for key, sensor in clusterNodeStatus.m_sensors.iteritems():
for key, sensor in clusterNodeStatus.m_sensors.items():
sensorId = clusterNodeStatus.m_clusterNodeName + '_' + sensor.m_name
if sensor.typeName() == 'Fan':
sqlCommand = """INSERT INTO `fan_rpm_logs` (`fan_id`, `rpm`, `date`) VALUES ('"""+sensorId+"""', """+str(sensor.m_rpms)+""", NOW());"""

26
cocluto/ClusterController/ClusterStatus.py
View File

@ -40,7 +40,7 @@ class ClusterStatus:
def setControlOnMachine(self, machineName, bControl):
if bControl:
# add machineName under control of ClusterController
for k, v in self.m_clusterNodes.iteritems():
for k, v in self.m_clusterNodes.items():
if v.getName() == machineName :
return # nothing to do : machineName is already under the control of ClusterController
@ -64,12 +64,12 @@ class ClusterStatus:
return self.m_clusterNodes
def startReadingThreads( self ):
for k, v in self.m_clusterNodes.iteritems():
for k, v in self.m_clusterNodes.items():
v.m_machineStatusUpdater.start()
self.m_jobsStateUpdater.start()
def stopReadingThreads( self ):
for k, v in self.m_clusterNodes.iteritems():
for k, v in self.m_clusterNodes.items():
v.m_machineStatusUpdater.m_bStop = True
v.m_machineStatusUpdater.join()
self.m_jobsStateUpdater.m_bStop = True
@ -86,7 +86,7 @@ class ClusterStatus:
return self.m_jobsState.getJobsOnMachine( machineName )
def isReady( self ):
for k, v in self.m_clusterNodes.iteritems():
for k, v in self.m_clusterNodes.items():
if not v.isReady():
logInfo( 'ClusterStatus::isReady : not ready because of ' + v.getName() )
return False
@ -108,7 +108,7 @@ class ClusterStatus:
logError('ClusterStatus::getIdleMachines : age of jobs state is too old (%f s). This is bug 00000009.' % (fJobsStateAge))
assert( False )
idleMachines = {}
for machineName, machine in self.m_clusterNodes.iteritems():
for machineName, machine in self.m_clusterNodes.items():
if machine.getPowerState() == PowerState.ON:
jobsOnThisMachine = self.getJobsOnMachine( machineName )
if len(jobsOnThisMachine) == 0:
@ -137,7 +137,7 @@ class ClusterStatus:
returns an estimate of the energy consumption since the start of the cluster controller (in joules)
"""
fEnergyConsumption = 0.0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
if machine.isReady(): # there are cases where the machine is not ready yet (for example, it's just been added to clustercontroller's control)
fEnergyConsumption += machine.getEnergyConsumption()
return fEnergyConsumption
@ -147,21 +147,21 @@ class ClusterStatus:
returns an estimate of the energy saving since the start of the cluster controller (in joules)
"""
fEnergySavings = 0.0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
if machine.isReady():
fEnergySavings += machine.getEnergySavings()
return fEnergySavings
def getCurrentPowerConsumption( self ):
fPowerConsumption = 0.0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
if machine.isReady():
fPowerConsumption += machine.getPowerConsumption()
return fPowerConsumption
def getCurrentPowerSavings( self ):
fPowerSavings = 0.0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
if machine.isReady():
fPowerSavings += machine.getPowerConsumptionForPowerState( PowerState.ON ) - machine.getPowerConsumption()
return fPowerSavings
@ -169,7 +169,7 @@ class ClusterStatus:
def getNumControlledSlots( self ):
self.m_lock.acquire()
iNumControlledSlots = 0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
iNumControlledSlots += queueMachine.getNumSlots()
self.m_lock.release()
@ -178,7 +178,7 @@ class ClusterStatus:
def getNumUsedSlots( self ):
self.m_lock.acquire()
iNumUsedSlots = 0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
iNumUsedSlotsOnThisMachine = queueMachine.getNumSlots() - self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine)
assert(iNumUsedSlotsOnThisMachine >= 0)
@ -189,7 +189,7 @@ class ClusterStatus:
def getNumWastedSlots( self ):
self.m_lock.acquire()
iNumWastedSlots = 0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
if machine.getPowerState() == PowerState.ON:
queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
iNumWastedSlots += self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine)
@ -199,7 +199,7 @@ class ClusterStatus:
def getNumSleepingSlots( self ):
self.m_lock.acquire()
iNumSleepingSlots = 0
for machine in self.m_clusterNodes.itervalues():
for machine in self.m_clusterNodes.values():
if machine.getPowerState() == PowerState.SLEEP:
queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
iNumSleepingSlots += self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine)

10
cocluto/ClusterController/JobsState.py
View File

@ -38,8 +38,8 @@ class JobsState:
def getJobsOnMachine( self, machineName ):
jobsOnMachine = {}
for jobId, job in self.m_jobs.iteritems():
for queueMachineName, numSlots in job.getSlots().iteritems():
for jobId, job in self.m_jobs.items():
for queueMachineName, numSlots in job.getSlots().items():
jobMachineName = queueMachineName.split('@')[1]
if jobMachineName == machineName:
jobsOnMachine[ jobId ] = job
@ -48,7 +48,7 @@ class JobsState:
def getNumFreeSlotsOnQueueMachine( self, queueMachine ):
#logInfo('getNumFreeSlotsOnQueueMachine : looking for free slots on queuemachine %s' % queueMachine.getName() )
numUsedSlots = 0
for job in self.m_jobs.itervalues():
for job in self.m_jobs.values():
numUsedSlotsByThisJob = job.getSlots().get( queueMachine.getName() )
if numUsedSlotsByThisJob != None:
#logInfo('getNumFreeSlotsOnQueueMachine : job %d uses %d slots' % (job.getId().asStr(), numUsedSlotsByThisJob) )
@ -68,7 +68,7 @@ class JobsState:
finds the queue machine associated with a machine
"""
queueMachine = None
for qmName, qm in self.m_queueMachines.iteritems():
for qmName, qm in self.m_queueMachines.items():
if qm.m_machineName == machineName:
assert( queueMachine == None ) # to be sure that no more than one queue machine is on a given machine
queueMachine = qm
@ -79,7 +79,7 @@ class JobsState:
def getPendingJobs( self ):
pendingJobs = {}
for jobId, job in self.m_jobs.iteritems():
for jobId, job in self.m_jobs.items():
if job.isPending():
pendingJobs[ job.getId() ] = job
return pendingJobs

12
cocluto/ClusterController/SlotAllocator.py
View File

@ -25,13 +25,13 @@ class SimpleSlotAllocator( SlotAllocator ):
highestPriorityPendingJob = pendingJobs.values()[0]
logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : looking for free slots for job ' + highestPriorityPendingJob.getId().asStr() )
numFreeSlots = {} # contains the number of free slots for each queueMachine
for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
for queueMachine in clusterState.getJobsState().getQueueMachines().values():
numFreeSlots[ queueMachine ] = clusterState.getJobsState().getNumFreeSlotsOnQueueMachine( queueMachine )
logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : init numFreeSlots[ %s ] with %d ' % (queueMachine.getName(), numFreeSlots[ queueMachine ]) )
remainingNumSlotsToAllocate = highestPriorityPendingJob.m_jobRequirements.m_numSlots
logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : still %d slots to find' % remainingNumSlotsToAllocate )
# first look in running machines if there are available slots
for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
for queueMachine in clusterState.getJobsState().getQueueMachines().values():
logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : examining queueMachine %s ' % queueMachine.getName() )
machine = clusterState.getMachines()[ queueMachine.getMachineName() ]
if machine.getPowerState() == PowerState.ON:
@ -47,7 +47,7 @@ class SimpleSlotAllocator( SlotAllocator ):
break
if remainingNumSlotsToAllocate > 0:
# now look into machines that are asleep
for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
for queueMachine in clusterState.getJobsState().getQueueMachines().values():
logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : examining queueMachine %s ' % queueMachine.getName() )
machine = clusterState.getMachines()[ queueMachine.getMachineName() ]
if machine.getPowerState() == PowerState.SLEEP:
@ -100,7 +100,7 @@ class DecoupledSlotAllocator( SlotAllocator ):
print 'id(oldJobs) : ', id(oldJobs)
print 'id(newJobs) : ', id(newJobs)
"""
for newJob in newJobs.itervalues():
for newJob in newJobs.values():
#logDebug('DecoupledSlotAllocator::jobsStateHasChanged newJob id=%s' % newJob.getId().asStr())
if newJob.getId() in oldJobs:
#logDebug('DecoupledSlotAllocator::jobsStateHasChanged job id=%d is in old jobs' % newJob.getId())
@ -110,7 +110,7 @@ class DecoupledSlotAllocator( SlotAllocator ):
logInfo('A new job (jobId =%s) has been detected ' % newJob.getId().asStr() )
bJobsHaveChanged = True
if len(oldJobsOnly) != 0:
for oldJob in oldJobsOnly.itervalues():
for oldJob in oldJobsOnly.values():
logInfo('Job (jobId =%s) has finished' % oldJob.getId().asStr() )
# at least one old job has finished, freeing some slots
bJobsHaveChanged = True
@ -129,7 +129,7 @@ class DecoupledSlotAllocator( SlotAllocator ):
logInfo('DecoupledSlotAllocator::getMachinesThatNeedWakeUp : waking up machines that are asleep because jobs state has changed')
else:
logInfo('DecoupledSlotAllocator::getMachinesThatNeedWakeUp : waking up machines that are asleep for periodic check (to be sure pending jobs get a chance to start)')
for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
for queueMachine in clusterState.getJobsState().getQueueMachines().values():
if queueMachine.getMachineName() in clusterState.getMachines():
# this means that the machine is under the cluster controller's control
machine = clusterState.getMachines()[ queueMachine.getMachineName() ]

2
cocluto/ClusterController/SunGridEngine.py
View File

@ -24,7 +24,7 @@ class SunGridEngine:
# read the requirements for pending jobs (which parallel environment, which queue, which architecture) from sge
if False: # no need for job details at the moment and since it's very slow, it's been disabled
for unused_jobId, job in jobsState.getPendingJobs().iteritems():
for unused_jobId, job in jobsState.getPendingJobs().items():
(returnCode, stdout, stderr) = executeProgram( ['qstat', '-j', job.getId().asStr()] )
assert returnCode != 0, 'prout'
QstatParser().parseJobDetails( stdout, job )

2
cocluto/ClusterController/WebServer.py
View File

@ -47,7 +47,7 @@ class MyHandler(BaseHTTPRequestHandler):
controlledMachinesElement = doc.createElement("ControlledMachines")
doc.appendChild(controlledMachinesElement)
for machine in self.server.m_clusterController.m_clusterStatus.m_clusterNodes.itervalues():
for machine in self.server.m_clusterController.m_clusterStatus.m_clusterNodes.values():
# Create the main <card> element
controlledMachineElement = doc.createElement("Machine")
controlledMachineElement.setAttribute("name", machine.getName())

4
cocluto/Ipmi/ClusterNodeSensorsReadings.py
View File

@ -18,7 +18,7 @@ class ClusterNodeSensorsReadings:
def addSensor(self, sensor):
self.m_sensors[sensor.m_name] = sensor
def dump(self):
for key,sensor in self.m_sensors.iteritems():
for key,sensor in self.m_sensors.items():
sensor.dump()
return
#def getPowerState(self):
@ -27,7 +27,7 @@ class ClusterNodeSensorsReadings:
#log('ClusterNodeSensorsReadings::getLowestTemperature : start')
lowestTemperature = 0.0
lowestTemperatureIsDefined = False
for key,sensor in self.m_sensors.iteritems():
for key,sensor in self.m_sensors.items():
#log('ClusterNodeSensorsReadings::getLowestTemperature : start')
if sensor.typeName() == 'Temperature':
sensor.m_temperature

14
cocluto/PowerDiagram.py
View File

@ -68,7 +68,7 @@ class Machine(object):
powered_machines[self.name] = self
for conn in self.get_outgoing_connections():
conn_powered_machines = conn.to_plug.machine.get_powered_machines()
for machine in conn_powered_machines.itervalues():
for machine in conn_powered_machines.values():
if machine not in powered_machines.keys():
powered_machines[machine.name] = machine
return powered_machines
@ -82,7 +82,7 @@ class Machine(object):
if worst_case_scenario:
# this machine has to provide the full power to all machine it powers, assuming all the power backup providers are dead
power_consumption = 0.0
for machine in self.get_powered_machines().itervalues():
for machine in self.get_powered_machines().values():
power_consumption += machine.power_consumption
else:
power_consumption = self.power_consumption
@ -171,7 +171,7 @@ class Connection(object):
my_power_provider = self.get_power_provider()
# find the first sibling cable that has the same provider as self
first_cable_with_same_provider = None
for input_plug in to_machine.input_plugs.itervalues():
for input_plug in to_machine.input_plugs.values():
sibling_cable = input_plug.get_incoming_connection()
if sibling_cable.get_power_provider() == my_power_provider:
first_cable_with_same_provider = sibling_cable
@ -209,7 +209,7 @@ class Connection(object):
power_consumption = to_machine.get_power_consumption(worst_case_scenario)
else:
num_input_cables = 0
for input_plug in to_machine.input_plugs.itervalues():
for input_plug in to_machine.input_plugs.values():
input_cable = input_plug.get_incoming_connection()
assert input_cable
num_input_cables += 1
@ -447,7 +447,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True):
if machine.name not in rack['machines']:
rack['machines'].append(machine)
for machine in power_config.machines.itervalues():
for machine in power_config.machines.values():
graph.add_node(machine.name)
node = graph.get_node(machine.name)
machine_total_power_consumption = int(machine.get_power_consumption(worst_case_scenario=worst_case_scenario))
@ -465,7 +465,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True):
if False:
x = 0.0
for rack in racks.itervalues():
for rack in racks.values():
y = 0.0
for machine in rack['machines']:
node = graph.get_node(machine.name)
@ -498,7 +498,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True):
# graph.add_edge(con.from_plug.machine.name, con.to_plug.machine.name, color="%s:%s" % (color, wsc_color), label=label, penwidth="%s:%s" % (penwidth, penwidth))
graph.add_edge(con.from_plug.machine.name, con.to_plug.machine.name, color=color, label=label, penwidth=penwidth)
for rack_id, rack in racks.iteritems():
for rack_id, rack in racks.items():
# sub = graph.add_subgraph(rack, name='cluster_%s' % rack_id, rank='same')
machine_names = list(machine.name for machine in rack['machines'])
sub = graph.add_subgraph(machine_names, name='cluster_%s' % rack_id, style='rounded')

6
cocluto/cluster_stats.py
View File

@ -110,12 +110,12 @@ def get_computer_value_over_time(inventory, computer_id, time_value, flops_price
# """
# # matplot 1.1.1 doesn't have the stackplot method in Axes
# if 'toto_stackplot' in dir(ax):
# ax.stackplot(x_signal, list(y_signals.itervalues()) )
# ax.stackplot(x_signal, list(y_signals.values()) )
# plt.legend(list(y_signals.keys()))
# else:
# colors = ['blue', 'orange', 'green', 'purple', 'yellow']
# # emulating missing Axes.stackplot method
# y = np.row_stack(list(y_signals.itervalues()))
# y = np.row_stack(list(y_signals.values()))
# # this call to 'cumsum' (cumulative sum), passing in your y data,
# # is necessary to avoid having to manually order the datasets
# y_stack = np.cumsum(y, axis=0) # a 3x10 array
@ -236,7 +236,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type
fig, ax = plt.subplots()
ax.set_title(graph_type)
# for dept, cluster_value_for_dept in cluster_value.iteritems():
# for dept, cluster_value_for_dept in cluster_value.items():
# ax.plot(time_value, cluster_value_for_dept)
stackplot(ax, time_value, cluster_value, legend_location='upper left')