From b5c2c886f14f32388f74787602ad5b1bd36d71ba Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Tue, 23 May 2023 11:29:59 +0200 Subject: [PATCH] migrated code using dict that was no longer compatible with python3 iteritems() was removed in python3, see https://wiki.python.org/moin/Python3.0#Built-In_Changes work related to Bug 3315 - make simpaweb django app a packageable application --- .../ClusterController/ClusterController.py | 8 +++--- cocluto/ClusterController/ClusterStatus.py | 26 +++++++++---------- cocluto/ClusterController/JobsState.py | 10 +++---- cocluto/ClusterController/SlotAllocator.py | 12 ++++----- cocluto/ClusterController/SunGridEngine.py | 2 +- cocluto/ClusterController/WebServer.py | 2 +- cocluto/Ipmi/ClusterNodeSensorsReadings.py | 4 +-- cocluto/PowerDiagram.py | 14 +++++----- cocluto/cluster_stats.py | 6 ++--- 9 files changed, 42 insertions(+), 42 deletions(-) diff --git a/cocluto/ClusterController/ClusterController.py b/cocluto/ClusterController/ClusterController.py index 80c974c..4e23156 100644 --- a/cocluto/ClusterController/ClusterController.py +++ b/cocluto/ClusterController/ClusterController.py @@ -127,7 +127,7 @@ class ClusterController: idleMachines = self.m_clusterStatus.getIdleMachines() # logInfo('idleMachines :') self.m_machinesThatNeedToSleep = [] - for machineName, idleMachine in idleMachines.iteritems(): + for machineName, idleMachine in idleMachines.items(): if idleMachine.getPowerState() == PowerState.ON: # logInfo('\t%s' % machineName) if idleMachine.getName() != 'simpatix10': # never put simpatix10 to sleep because it's the sge master and is also server for other things @@ -152,7 +152,7 @@ class ClusterController: pendingJobs = self.m_clusterStatus.getPendingJobs() """ logInfo('pending jobs :') - for job in pendingJobs.itervalues(): + for job in pendingJobs.values(): logInfo('\t%d' % job.getId().asStr()) """ if len(pendingJobs) != 0: @@ -252,7 +252,7 @@ class ClusterController: iNumMachines = len(self.m_clusterStatus.m_clusterNodes) iNumMachinesOn = 0 iNumSleepingMachines = 0 - for machine in self.m_clusterStatus.m_clusterNodes.itervalues(): + for machine in self.m_clusterStatus.m_clusterNodes.values(): ePowerState = machine.getPowerState() if ePowerState == PowerState.ON: iNumMachinesOn+=1 @@ -284,7 +284,7 @@ def storeClusterNodeStatus( clusterNodeStatus ): r=conn.store_result() print r.fetch_row()[0] ''' - for key, sensor in clusterNodeStatus.m_sensors.iteritems(): + for key, sensor in clusterNodeStatus.m_sensors.items(): sensorId = clusterNodeStatus.m_clusterNodeName + '_' + sensor.m_name if sensor.typeName() == 'Fan': sqlCommand = """INSERT INTO `fan_rpm_logs` (`fan_id`, `rpm`, `date`) VALUES ('"""+sensorId+"""', """+str(sensor.m_rpms)+""", NOW());""" diff --git a/cocluto/ClusterController/ClusterStatus.py b/cocluto/ClusterController/ClusterStatus.py index b300eac..031e016 100644 --- a/cocluto/ClusterController/ClusterStatus.py +++ b/cocluto/ClusterController/ClusterStatus.py @@ -40,7 +40,7 @@ class ClusterStatus: def setControlOnMachine(self, machineName, bControl): if bControl: # add machineName under control of ClusterController - for k, v in self.m_clusterNodes.iteritems(): + for k, v in self.m_clusterNodes.items(): if v.getName() == machineName : return # nothing to do : machineName is already under the control of ClusterController @@ -64,12 +64,12 @@ class ClusterStatus: return self.m_clusterNodes def startReadingThreads( self ): - for k, v in self.m_clusterNodes.iteritems(): + for k, v in self.m_clusterNodes.items(): v.m_machineStatusUpdater.start() self.m_jobsStateUpdater.start() def stopReadingThreads( self ): - for k, v in self.m_clusterNodes.iteritems(): + for k, v in self.m_clusterNodes.items(): v.m_machineStatusUpdater.m_bStop = True v.m_machineStatusUpdater.join() self.m_jobsStateUpdater.m_bStop = True @@ -86,7 +86,7 @@ class ClusterStatus: return self.m_jobsState.getJobsOnMachine( machineName ) def isReady( self ): - for k, v in self.m_clusterNodes.iteritems(): + for k, v in self.m_clusterNodes.items(): if not v.isReady(): logInfo( 'ClusterStatus::isReady : not ready because of ' + v.getName() ) return False @@ -108,7 +108,7 @@ class ClusterStatus: logError('ClusterStatus::getIdleMachines : age of jobs state is too old (%f s). This is bug 00000009.' % (fJobsStateAge)) assert( False ) idleMachines = {} - for machineName, machine in self.m_clusterNodes.iteritems(): + for machineName, machine in self.m_clusterNodes.items(): if machine.getPowerState() == PowerState.ON: jobsOnThisMachine = self.getJobsOnMachine( machineName ) if len(jobsOnThisMachine) == 0: @@ -137,7 +137,7 @@ class ClusterStatus: returns an estimate of the energy consumption since the start of the cluster controller (in joules) """ fEnergyConsumption = 0.0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): if machine.isReady(): # there are cases where the machine is not ready yet (for example, it's just been added to clustercontroller's control) fEnergyConsumption += machine.getEnergyConsumption() return fEnergyConsumption @@ -147,21 +147,21 @@ class ClusterStatus: returns an estimate of the energy saving since the start of the cluster controller (in joules) """ fEnergySavings = 0.0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): if machine.isReady(): fEnergySavings += machine.getEnergySavings() return fEnergySavings def getCurrentPowerConsumption( self ): fPowerConsumption = 0.0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): if machine.isReady(): fPowerConsumption += machine.getPowerConsumption() return fPowerConsumption def getCurrentPowerSavings( self ): fPowerSavings = 0.0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): if machine.isReady(): fPowerSavings += machine.getPowerConsumptionForPowerState( PowerState.ON ) - machine.getPowerConsumption() return fPowerSavings @@ -169,7 +169,7 @@ class ClusterStatus: def getNumControlledSlots( self ): self.m_lock.acquire() iNumControlledSlots = 0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): queueMachine = self.m_jobsState.getQueueMachine( machine.getName() ) iNumControlledSlots += queueMachine.getNumSlots() self.m_lock.release() @@ -178,7 +178,7 @@ class ClusterStatus: def getNumUsedSlots( self ): self.m_lock.acquire() iNumUsedSlots = 0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): queueMachine = self.m_jobsState.getQueueMachine( machine.getName() ) iNumUsedSlotsOnThisMachine = queueMachine.getNumSlots() - self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine) assert(iNumUsedSlotsOnThisMachine >= 0) @@ -189,7 +189,7 @@ class ClusterStatus: def getNumWastedSlots( self ): self.m_lock.acquire() iNumWastedSlots = 0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): if machine.getPowerState() == PowerState.ON: queueMachine = self.m_jobsState.getQueueMachine( machine.getName() ) iNumWastedSlots += self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine) @@ -199,7 +199,7 @@ class ClusterStatus: def getNumSleepingSlots( self ): self.m_lock.acquire() iNumSleepingSlots = 0 - for machine in self.m_clusterNodes.itervalues(): + for machine in self.m_clusterNodes.values(): if machine.getPowerState() == PowerState.SLEEP: queueMachine = self.m_jobsState.getQueueMachine( machine.getName() ) iNumSleepingSlots += self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine) diff --git a/cocluto/ClusterController/JobsState.py b/cocluto/ClusterController/JobsState.py index 437bcc8..2ec24f3 100644 --- a/cocluto/ClusterController/JobsState.py +++ b/cocluto/ClusterController/JobsState.py @@ -38,8 +38,8 @@ class JobsState: def getJobsOnMachine( self, machineName ): jobsOnMachine = {} - for jobId, job in self.m_jobs.iteritems(): - for queueMachineName, numSlots in job.getSlots().iteritems(): + for jobId, job in self.m_jobs.items(): + for queueMachineName, numSlots in job.getSlots().items(): jobMachineName = queueMachineName.split('@')[1] if jobMachineName == machineName: jobsOnMachine[ jobId ] = job @@ -48,7 +48,7 @@ class JobsState: def getNumFreeSlotsOnQueueMachine( self, queueMachine ): #logInfo('getNumFreeSlotsOnQueueMachine : looking for free slots on queuemachine %s' % queueMachine.getName() ) numUsedSlots = 0 - for job in self.m_jobs.itervalues(): + for job in self.m_jobs.values(): numUsedSlotsByThisJob = job.getSlots().get( queueMachine.getName() ) if numUsedSlotsByThisJob != None: #logInfo('getNumFreeSlotsOnQueueMachine : job %d uses %d slots' % (job.getId().asStr(), numUsedSlotsByThisJob) ) @@ -68,7 +68,7 @@ class JobsState: finds the queue machine associated with a machine """ queueMachine = None - for qmName, qm in self.m_queueMachines.iteritems(): + for qmName, qm in self.m_queueMachines.items(): if qm.m_machineName == machineName: assert( queueMachine == None ) # to be sure that no more than one queue machine is on a given machine queueMachine = qm @@ -79,7 +79,7 @@ class JobsState: def getPendingJobs( self ): pendingJobs = {} - for jobId, job in self.m_jobs.iteritems(): + for jobId, job in self.m_jobs.items(): if job.isPending(): pendingJobs[ job.getId() ] = job return pendingJobs diff --git a/cocluto/ClusterController/SlotAllocator.py b/cocluto/ClusterController/SlotAllocator.py index ee3af29..b76d9a3 100644 --- a/cocluto/ClusterController/SlotAllocator.py +++ b/cocluto/ClusterController/SlotAllocator.py @@ -25,13 +25,13 @@ class SimpleSlotAllocator( SlotAllocator ): highestPriorityPendingJob = pendingJobs.values()[0] logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : looking for free slots for job ' + highestPriorityPendingJob.getId().asStr() ) numFreeSlots = {} # contains the number of free slots for each queueMachine - for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues(): + for queueMachine in clusterState.getJobsState().getQueueMachines().values(): numFreeSlots[ queueMachine ] = clusterState.getJobsState().getNumFreeSlotsOnQueueMachine( queueMachine ) logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : init numFreeSlots[ %s ] with %d ' % (queueMachine.getName(), numFreeSlots[ queueMachine ]) ) remainingNumSlotsToAllocate = highestPriorityPendingJob.m_jobRequirements.m_numSlots logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : still %d slots to find' % remainingNumSlotsToAllocate ) # first look in running machines if there are available slots - for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues(): + for queueMachine in clusterState.getJobsState().getQueueMachines().values(): logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : examining queueMachine %s ' % queueMachine.getName() ) machine = clusterState.getMachines()[ queueMachine.getMachineName() ] if machine.getPowerState() == PowerState.ON: @@ -47,7 +47,7 @@ class SimpleSlotAllocator( SlotAllocator ): break if remainingNumSlotsToAllocate > 0: # now look into machines that are asleep - for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues(): + for queueMachine in clusterState.getJobsState().getQueueMachines().values(): logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : examining queueMachine %s ' % queueMachine.getName() ) machine = clusterState.getMachines()[ queueMachine.getMachineName() ] if machine.getPowerState() == PowerState.SLEEP: @@ -100,7 +100,7 @@ class DecoupledSlotAllocator( SlotAllocator ): print 'id(oldJobs) : ', id(oldJobs) print 'id(newJobs) : ', id(newJobs) """ - for newJob in newJobs.itervalues(): + for newJob in newJobs.values(): #logDebug('DecoupledSlotAllocator::jobsStateHasChanged newJob id=%s' % newJob.getId().asStr()) if newJob.getId() in oldJobs: #logDebug('DecoupledSlotAllocator::jobsStateHasChanged job id=%d is in old jobs' % newJob.getId()) @@ -110,7 +110,7 @@ class DecoupledSlotAllocator( SlotAllocator ): logInfo('A new job (jobId =%s) has been detected ' % newJob.getId().asStr() ) bJobsHaveChanged = True if len(oldJobsOnly) != 0: - for oldJob in oldJobsOnly.itervalues(): + for oldJob in oldJobsOnly.values(): logInfo('Job (jobId =%s) has finished' % oldJob.getId().asStr() ) # at least one old job has finished, freeing some slots bJobsHaveChanged = True @@ -129,7 +129,7 @@ class DecoupledSlotAllocator( SlotAllocator ): logInfo('DecoupledSlotAllocator::getMachinesThatNeedWakeUp : waking up machines that are asleep because jobs state has changed') else: logInfo('DecoupledSlotAllocator::getMachinesThatNeedWakeUp : waking up machines that are asleep for periodic check (to be sure pending jobs get a chance to start)') - for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues(): + for queueMachine in clusterState.getJobsState().getQueueMachines().values(): if queueMachine.getMachineName() in clusterState.getMachines(): # this means that the machine is under the cluster controller's control machine = clusterState.getMachines()[ queueMachine.getMachineName() ] diff --git a/cocluto/ClusterController/SunGridEngine.py b/cocluto/ClusterController/SunGridEngine.py index 2467a73..a6e6794 100644 --- a/cocluto/ClusterController/SunGridEngine.py +++ b/cocluto/ClusterController/SunGridEngine.py @@ -24,7 +24,7 @@ class SunGridEngine: # read the requirements for pending jobs (which parallel environment, which queue, which architecture) from sge if False: # no need for job details at the moment and since it's very slow, it's been disabled - for unused_jobId, job in jobsState.getPendingJobs().iteritems(): + for unused_jobId, job in jobsState.getPendingJobs().items(): (returnCode, stdout, stderr) = executeProgram( ['qstat', '-j', job.getId().asStr()] ) assert returnCode != 0, 'prout' QstatParser().parseJobDetails( stdout, job ) diff --git a/cocluto/ClusterController/WebServer.py b/cocluto/ClusterController/WebServer.py index f17fd38..7862850 100644 --- a/cocluto/ClusterController/WebServer.py +++ b/cocluto/ClusterController/WebServer.py @@ -47,7 +47,7 @@ class MyHandler(BaseHTTPRequestHandler): controlledMachinesElement = doc.createElement("ControlledMachines") doc.appendChild(controlledMachinesElement) - for machine in self.server.m_clusterController.m_clusterStatus.m_clusterNodes.itervalues(): + for machine in self.server.m_clusterController.m_clusterStatus.m_clusterNodes.values(): # Create the main element controlledMachineElement = doc.createElement("Machine") controlledMachineElement.setAttribute("name", machine.getName()) diff --git a/cocluto/Ipmi/ClusterNodeSensorsReadings.py b/cocluto/Ipmi/ClusterNodeSensorsReadings.py index 1d7d35b..3c29af8 100644 --- a/cocluto/Ipmi/ClusterNodeSensorsReadings.py +++ b/cocluto/Ipmi/ClusterNodeSensorsReadings.py @@ -18,7 +18,7 @@ class ClusterNodeSensorsReadings: def addSensor(self, sensor): self.m_sensors[sensor.m_name] = sensor def dump(self): - for key,sensor in self.m_sensors.iteritems(): + for key,sensor in self.m_sensors.items(): sensor.dump() return #def getPowerState(self): @@ -27,7 +27,7 @@ class ClusterNodeSensorsReadings: #log('ClusterNodeSensorsReadings::getLowestTemperature : start') lowestTemperature = 0.0 lowestTemperatureIsDefined = False - for key,sensor in self.m_sensors.iteritems(): + for key,sensor in self.m_sensors.items(): #log('ClusterNodeSensorsReadings::getLowestTemperature : start') if sensor.typeName() == 'Temperature': sensor.m_temperature diff --git a/cocluto/PowerDiagram.py b/cocluto/PowerDiagram.py index a529f46..6999e40 100644 --- a/cocluto/PowerDiagram.py +++ b/cocluto/PowerDiagram.py @@ -68,7 +68,7 @@ class Machine(object): powered_machines[self.name] = self for conn in self.get_outgoing_connections(): conn_powered_machines = conn.to_plug.machine.get_powered_machines() - for machine in conn_powered_machines.itervalues(): + for machine in conn_powered_machines.values(): if machine not in powered_machines.keys(): powered_machines[machine.name] = machine return powered_machines @@ -82,7 +82,7 @@ class Machine(object): if worst_case_scenario: # this machine has to provide the full power to all machine it powers, assuming all the power backup providers are dead power_consumption = 0.0 - for machine in self.get_powered_machines().itervalues(): + for machine in self.get_powered_machines().values(): power_consumption += machine.power_consumption else: power_consumption = self.power_consumption @@ -171,7 +171,7 @@ class Connection(object): my_power_provider = self.get_power_provider() # find the first sibling cable that has the same provider as self first_cable_with_same_provider = None - for input_plug in to_machine.input_plugs.itervalues(): + for input_plug in to_machine.input_plugs.values(): sibling_cable = input_plug.get_incoming_connection() if sibling_cable.get_power_provider() == my_power_provider: first_cable_with_same_provider = sibling_cable @@ -209,7 +209,7 @@ class Connection(object): power_consumption = to_machine.get_power_consumption(worst_case_scenario) else: num_input_cables = 0 - for input_plug in to_machine.input_plugs.itervalues(): + for input_plug in to_machine.input_plugs.values(): input_cable = input_plug.get_incoming_connection() assert input_cable num_input_cables += 1 @@ -447,7 +447,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True): if machine.name not in rack['machines']: rack['machines'].append(machine) - for machine in power_config.machines.itervalues(): + for machine in power_config.machines.values(): graph.add_node(machine.name) node = graph.get_node(machine.name) machine_total_power_consumption = int(machine.get_power_consumption(worst_case_scenario=worst_case_scenario)) @@ -465,7 +465,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True): if False: x = 0.0 - for rack in racks.itervalues(): + for rack in racks.values(): y = 0.0 for machine in rack['machines']: node = graph.get_node(machine.name) @@ -498,7 +498,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True): # graph.add_edge(con.from_plug.machine.name, con.to_plug.machine.name, color="%s:%s" % (color, wsc_color), label=label, penwidth="%s:%s" % (penwidth, penwidth)) graph.add_edge(con.from_plug.machine.name, con.to_plug.machine.name, color=color, label=label, penwidth=penwidth) - for rack_id, rack in racks.iteritems(): + for rack_id, rack in racks.items(): # sub = graph.add_subgraph(rack, name='cluster_%s' % rack_id, rank='same') machine_names = list(machine.name for machine in rack['machines']) sub = graph.add_subgraph(machine_names, name='cluster_%s' % rack_id, style='rounded') diff --git a/cocluto/cluster_stats.py b/cocluto/cluster_stats.py index bf131c8..20cf52e 100644 --- a/cocluto/cluster_stats.py +++ b/cocluto/cluster_stats.py @@ -110,12 +110,12 @@ def get_computer_value_over_time(inventory, computer_id, time_value, flops_price # """ # # matplot 1.1.1 doesn't have the stackplot method in Axes # if 'toto_stackplot' in dir(ax): -# ax.stackplot(x_signal, list(y_signals.itervalues()) ) +# ax.stackplot(x_signal, list(y_signals.values()) ) # plt.legend(list(y_signals.keys())) # else: # colors = ['blue', 'orange', 'green', 'purple', 'yellow'] # # emulating missing Axes.stackplot method -# y = np.row_stack(list(y_signals.itervalues())) +# y = np.row_stack(list(y_signals.values())) # # this call to 'cumsum' (cumulative sum), passing in your y data, # # is necessary to avoid having to manually order the datasets # y_stack = np.cumsum(y, axis=0) # a 3x10 array @@ -236,7 +236,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type fig, ax = plt.subplots() ax.set_title(graph_type) - # for dept, cluster_value_for_dept in cluster_value.iteritems(): + # for dept, cluster_value_for_dept in cluster_value.items(): # ax.plot(time_value, cluster_value_for_dept) stackplot(ax, time_value, cluster_value, legend_location='upper left')