migrated code using dict that was no longer compatible with python3

iteritems() was removed in python3, see https://wiki.python.org/moin/Python3.0#Built-In_Changes

work related to Bug 3315 - make simpaweb django app a packageable application

cocluto/ClusterController/ClusterController.py

@@ -127,7 +127,7 @@ class ClusterController:
 		idleMachines = self.m_clusterStatus.getIdleMachines()
 		# logInfo('idleMachines :')
 		self.m_machinesThatNeedToSleep = []
-		for machineName, idleMachine in idleMachines.iteritems():
+		for machineName, idleMachine in idleMachines.items():
 			if idleMachine.getPowerState() == PowerState.ON:
 				# logInfo('\t%s' % machineName)
 				if idleMachine.getName() != 'simpatix10': # never put simpatix10 to sleep because it's the sge master and is also server for other things
@@ -152,7 +152,7 @@ class ClusterController:
 		pendingJobs = self.m_clusterStatus.getPendingJobs()
 		"""
 		logInfo('pending jobs :')
-		for job in 	pendingJobs.itervalues():
+		for job in 	pendingJobs.values():
 			logInfo('\t%d' % job.getId().asStr())
 		"""
 		if len(pendingJobs) != 0:
@@ -252,7 +252,7 @@ class ClusterController:
 				iNumMachines = len(self.m_clusterStatus.m_clusterNodes)
 				iNumMachinesOn = 0
 				iNumSleepingMachines = 0
-				for machine in self.m_clusterStatus.m_clusterNodes.itervalues():
+				for machine in self.m_clusterStatus.m_clusterNodes.values():
 					ePowerState = machine.getPowerState()
 					if ePowerState == PowerState.ON:
 						iNumMachinesOn+=1
@@ -284,7 +284,7 @@ def storeClusterNodeStatus( clusterNodeStatus ):
 	r=conn.store_result()
 	print r.fetch_row()[0]
 	'''
-	for key, sensor in clusterNodeStatus.m_sensors.iteritems():
+	for key, sensor in clusterNodeStatus.m_sensors.items():
 		sensorId = clusterNodeStatus.m_clusterNodeName + '_' + sensor.m_name
 		if sensor.typeName() == 'Fan':
 			sqlCommand = """INSERT INTO `fan_rpm_logs` (`fan_id`, `rpm`, `date`) VALUES ('"""+sensorId+"""', """+str(sensor.m_rpms)+""", NOW());"""

cocluto/ClusterController/ClusterStatus.py

@@ -40,7 +40,7 @@ class ClusterStatus:
 	def setControlOnMachine(self, machineName, bControl):
 		if bControl:
 			# add machineName under control of ClusterController
-			for k, v in self.m_clusterNodes.iteritems():
+			for k, v in self.m_clusterNodes.items():
 				if v.getName() == machineName :
 					return # nothing to do : machineName is already under the control of ClusterController
 
@@ -64,12 +64,12 @@ class ClusterStatus:
 		return self.m_clusterNodes
 	
 	def startReadingThreads( self ):
-		for k, v in self.m_clusterNodes.iteritems():
+		for k, v in self.m_clusterNodes.items():
 			v.m_machineStatusUpdater.start()
 		self.m_jobsStateUpdater.start()
 		
 	def stopReadingThreads( self ):
-		for k, v in self.m_clusterNodes.iteritems():
+		for k, v in self.m_clusterNodes.items():
 			v.m_machineStatusUpdater.m_bStop = True
 			v.m_machineStatusUpdater.join()
 		self.m_jobsStateUpdater.m_bStop = True
@@ -86,7 +86,7 @@ class ClusterStatus:
 		return self.m_jobsState.getJobsOnMachine( machineName )
 		
 	def isReady( self ):
-		for k, v in self.m_clusterNodes.iteritems():
+		for k, v in self.m_clusterNodes.items():
 			if not v.isReady():
 				logInfo( 'ClusterStatus::isReady : not ready because of ' + v.getName() )
 				return False
@@ -108,7 +108,7 @@ class ClusterStatus:
 				logError('ClusterStatus::getIdleMachines : age of jobs state is too old (%f s). This is bug 00000009.' % (fJobsStateAge))
 				assert( False )
 		idleMachines = {}
-		for machineName, machine in self.m_clusterNodes.iteritems():
+		for machineName, machine in self.m_clusterNodes.items():
 			if machine.getPowerState() == PowerState.ON:
 				jobsOnThisMachine = self.getJobsOnMachine( machineName )
 				if len(jobsOnThisMachine) == 0:
@@ -137,7 +137,7 @@ class ClusterStatus:
 			returns an estimate of the energy consumption since the start of the cluster controller (in joules)
 		"""
 		fEnergyConsumption = 0.0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			if machine.isReady(): # there are cases where the machine is not ready yet (for example, it's just been added to clustercontroller's control)
 				fEnergyConsumption += machine.getEnergyConsumption()
 		return fEnergyConsumption
@@ -147,21 +147,21 @@ class ClusterStatus:
 			returns an estimate of the energy saving since the start of the cluster controller (in joules)
 		"""
 		fEnergySavings = 0.0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			if machine.isReady():
 				fEnergySavings += machine.getEnergySavings()
 		return fEnergySavings
 		
 	def getCurrentPowerConsumption( self ):
 		fPowerConsumption = 0.0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			if machine.isReady():
 				fPowerConsumption += machine.getPowerConsumption()
 		return fPowerConsumption
 		
 	def getCurrentPowerSavings( self ):
 		fPowerSavings = 0.0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			if machine.isReady():
 				fPowerSavings += machine.getPowerConsumptionForPowerState( PowerState.ON ) - machine.getPowerConsumption()
 		return fPowerSavings
@@ -169,7 +169,7 @@ class ClusterStatus:
 	def getNumControlledSlots( self ):
 		self.m_lock.acquire()
 		iNumControlledSlots = 0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
 			iNumControlledSlots	+= queueMachine.getNumSlots()
 		self.m_lock.release()
@@ -178,7 +178,7 @@ class ClusterStatus:
 	def getNumUsedSlots( self ):
 		self.m_lock.acquire()
 		iNumUsedSlots = 0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
 			iNumUsedSlotsOnThisMachine = queueMachine.getNumSlots() - self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine)
 			assert(iNumUsedSlotsOnThisMachine >= 0)
@@ -189,7 +189,7 @@ class ClusterStatus:
 	def getNumWastedSlots( self ):
 		self.m_lock.acquire()
 		iNumWastedSlots = 0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			if machine.getPowerState() == PowerState.ON:
 				queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
 				iNumWastedSlots += self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine)
@@ -199,7 +199,7 @@ class ClusterStatus:
 	def getNumSleepingSlots( self ):
 		self.m_lock.acquire()
 		iNumSleepingSlots = 0
-		for machine in self.m_clusterNodes.itervalues():
+		for machine in self.m_clusterNodes.values():
 			if machine.getPowerState() == PowerState.SLEEP:
 				queueMachine = self.m_jobsState.getQueueMachine( machine.getName() )
 				iNumSleepingSlots += self.m_jobsState.getNumFreeSlotsOnQueueMachine(queueMachine)

cocluto/ClusterController/JobsState.py

@@ -38,8 +38,8 @@ class JobsState:
 	
 	def getJobsOnMachine( self, machineName ):
 		jobsOnMachine = {}
-		for jobId, job in self.m_jobs.iteritems():
+		for jobId, job in self.m_jobs.items():
-			for queueMachineName, numSlots in job.getSlots().iteritems():
+			for queueMachineName, numSlots in job.getSlots().items():
 				jobMachineName = queueMachineName.split('@')[1]
 				if jobMachineName == machineName:
 					jobsOnMachine[ jobId ] = job
@@ -48,7 +48,7 @@ class JobsState:
 	def getNumFreeSlotsOnQueueMachine( self, queueMachine ):
 		#logInfo('getNumFreeSlotsOnQueueMachine : looking for free slots on queuemachine %s' % queueMachine.getName() )
 		numUsedSlots = 0
-		for job in self.m_jobs.itervalues():
+		for job in self.m_jobs.values():
 			numUsedSlotsByThisJob = job.getSlots().get( queueMachine.getName() )
 			if numUsedSlotsByThisJob != None:
 				#logInfo('getNumFreeSlotsOnQueueMachine : job %d uses %d slots' % (job.getId().asStr(), numUsedSlotsByThisJob) )
@@ -68,7 +68,7 @@ class JobsState:
 			finds the queue machine associated with a machine
 		"""
 		queueMachine = None
-		for qmName, qm in self.m_queueMachines.iteritems():
+		for qmName, qm in self.m_queueMachines.items():
 			if qm.m_machineName == machineName:
 				assert( queueMachine == None ) # to be sure that no more than one queue machine is on a given machine
 				queueMachine = qm 
@@ -79,7 +79,7 @@ class JobsState:
 	
 	def getPendingJobs( self ):
 		pendingJobs = {}
-		for jobId, job in self.m_jobs.iteritems():
+		for jobId, job in self.m_jobs.items():
 			if job.isPending():
 				pendingJobs[ job.getId() ] = job
 		return pendingJobs

cocluto/ClusterController/SlotAllocator.py

@@ -25,13 +25,13 @@ class SimpleSlotAllocator( SlotAllocator ):
 		highestPriorityPendingJob = pendingJobs.values()[0]
 		logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : looking for free slots for job ' + highestPriorityPendingJob.getId().asStr() )
 		numFreeSlots = {} # contains the number of free slots for each queueMachine
-		for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
+		for queueMachine in clusterState.getJobsState().getQueueMachines().values():
 			numFreeSlots[ queueMachine ] = clusterState.getJobsState().getNumFreeSlotsOnQueueMachine( queueMachine )
 			logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : init numFreeSlots[ %s ] with %d ' % (queueMachine.getName(), numFreeSlots[ queueMachine ]) )
 		remainingNumSlotsToAllocate = highestPriorityPendingJob.m_jobRequirements.m_numSlots
 		logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : still %d slots to find' % remainingNumSlotsToAllocate )
 		# first look in running machines if there are available slots
-		for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
+		for queueMachine in clusterState.getJobsState().getQueueMachines().values():
 			logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : examining queueMachine %s ' % queueMachine.getName() )
 			machine = clusterState.getMachines()[ queueMachine.getMachineName() ]
 			if machine.getPowerState() == PowerState.ON:
@@ -47,7 +47,7 @@ class SimpleSlotAllocator( SlotAllocator ):
 						break
 		if remainingNumSlotsToAllocate > 0:
 			# now look into machines that are asleep
-			for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
+			for queueMachine in clusterState.getJobsState().getQueueMachines().values():
 				logInfo('SimpleSlotAllocator::getMachinesThatNeedWakeUp : examining queueMachine %s ' % queueMachine.getName() )
 				machine = clusterState.getMachines()[ queueMachine.getMachineName() ]
 				if machine.getPowerState() == PowerState.SLEEP:
@@ -100,7 +100,7 @@ class DecoupledSlotAllocator( SlotAllocator ):
 		print 'id(oldJobs) : ', id(oldJobs)
 		print 'id(newJobs) : ', id(newJobs)
 		"""
-		for newJob in newJobs.itervalues():
+		for newJob in newJobs.values():
 			#logDebug('DecoupledSlotAllocator::jobsStateHasChanged newJob id=%s' % newJob.getId().asStr())
 			if newJob.getId() in oldJobs:
 				#logDebug('DecoupledSlotAllocator::jobsStateHasChanged job id=%d is in old jobs' % newJob.getId())
@@ -110,7 +110,7 @@ class DecoupledSlotAllocator( SlotAllocator ):
 				logInfo('A new job (jobId =%s) has been detected ' % newJob.getId().asStr() )
 				bJobsHaveChanged = True
 		if len(oldJobsOnly) != 0:
-			for oldJob in oldJobsOnly.itervalues():
+			for oldJob in oldJobsOnly.values():
 				logInfo('Job (jobId =%s) has finished' % oldJob.getId().asStr() )
 			# at least one old job has finished, freeing some slots
 			bJobsHaveChanged = True
@@ -129,7 +129,7 @@ class DecoupledSlotAllocator( SlotAllocator ):
 				logInfo('DecoupledSlotAllocator::getMachinesThatNeedWakeUp : waking up machines that are asleep because jobs state has changed')
 			else:
 				logInfo('DecoupledSlotAllocator::getMachinesThatNeedWakeUp : waking up machines that are asleep for periodic check (to be sure pending jobs get a chance to start)')
-			for queueMachine in clusterState.getJobsState().getQueueMachines().itervalues():
+			for queueMachine in clusterState.getJobsState().getQueueMachines().values():
 				if queueMachine.getMachineName() in clusterState.getMachines():
 					# this means that the machine is under the cluster controller's control
 					machine = clusterState.getMachines()[ queueMachine.getMachineName() ]

cocluto/ClusterController/SunGridEngine.py

@@ -24,7 +24,7 @@ class SunGridEngine:
 		
 		# read the requirements for pending jobs (which parallel environment, which queue, which architecture) from sge
 		if False: # no need for job details at the moment and since it's very slow, it's been disabled
-			for unused_jobId, job in jobsState.getPendingJobs().iteritems():
+			for unused_jobId, job in jobsState.getPendingJobs().items():
 				(returnCode, stdout, stderr) = executeProgram( ['qstat', '-j', job.getId().asStr()] )
 				assert returnCode != 0, 'prout'
 				QstatParser().parseJobDetails( stdout, job )

cocluto/ClusterController/WebServer.py

@@ -47,7 +47,7 @@ class MyHandler(BaseHTTPRequestHandler):
                 controlledMachinesElement = doc.createElement("ControlledMachines")
                 doc.appendChild(controlledMachinesElement)
 
-                for machine in self.server.m_clusterController.m_clusterStatus.m_clusterNodes.itervalues():
+                for machine in self.server.m_clusterController.m_clusterStatus.m_clusterNodes.values():
                     # Create the main <card> element
                     controlledMachineElement = doc.createElement("Machine")
                     controlledMachineElement.setAttribute("name", machine.getName())

cocluto/Ipmi/ClusterNodeSensorsReadings.py

@@ -18,7 +18,7 @@ class ClusterNodeSensorsReadings:
 	def addSensor(self, sensor):
 		self.m_sensors[sensor.m_name] = sensor
 	def dump(self):
-		for key,sensor in self.m_sensors.iteritems():
+		for key,sensor in self.m_sensors.items():
 			sensor.dump()
 		return
 	#def getPowerState(self):
@@ -27,7 +27,7 @@ class ClusterNodeSensorsReadings:
 		#log('ClusterNodeSensorsReadings::getLowestTemperature : start')
 		lowestTemperature = 0.0
 		lowestTemperatureIsDefined = False
-		for key,sensor in self.m_sensors.iteritems():
+		for key,sensor in self.m_sensors.items():
 			#log('ClusterNodeSensorsReadings::getLowestTemperature : start')
 			if sensor.typeName() == 'Temperature':
 				sensor.m_temperature

cocluto/PowerDiagram.py

@@ -68,7 +68,7 @@ class Machine(object):
         powered_machines[self.name] = self
         for conn in self.get_outgoing_connections():
             conn_powered_machines = conn.to_plug.machine.get_powered_machines()
-            for machine in conn_powered_machines.itervalues():
+            for machine in conn_powered_machines.values():
                 if machine not in powered_machines.keys():
                     powered_machines[machine.name] = machine
         return powered_machines
@@ -82,7 +82,7 @@ class Machine(object):
         if worst_case_scenario:
             # this machine has to provide the full power to all machine it powers, assuming all the power backup providers are dead
             power_consumption = 0.0
-            for machine in self.get_powered_machines().itervalues():
+            for machine in self.get_powered_machines().values():
                 power_consumption += machine.power_consumption
         else:
             power_consumption = self.power_consumption
@@ -171,7 +171,7 @@ class Connection(object):
         my_power_provider = self.get_power_provider()
         # find the first sibling cable that has the same provider as self
         first_cable_with_same_provider = None
-        for input_plug in to_machine.input_plugs.itervalues():
+        for input_plug in to_machine.input_plugs.values():
             sibling_cable = input_plug.get_incoming_connection()
             if sibling_cable.get_power_provider() == my_power_provider:
                 first_cable_with_same_provider = sibling_cable
@@ -209,7 +209,7 @@ class Connection(object):
                 power_consumption = to_machine.get_power_consumption(worst_case_scenario)
         else:
             num_input_cables = 0
-            for input_plug in to_machine.input_plugs.itervalues():
+            for input_plug in to_machine.input_plugs.values():
                 input_cable = input_plug.get_incoming_connection()
                 assert input_cable
                 num_input_cables += 1
@@ -447,7 +447,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True):
                 if machine.name not in rack['machines']:
                     rack['machines'].append(machine)
 
-    for machine in power_config.machines.itervalues():
+    for machine in power_config.machines.values():
         graph.add_node(machine.name)
         node = graph.get_node(machine.name)
         machine_total_power_consumption = int(machine.get_power_consumption(worst_case_scenario=worst_case_scenario))
@@ -465,7 +465,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True):
 
     if False:
         x = 0.0
-        for rack in racks.itervalues():
+        for rack in racks.values():
             y = 0.0
             for machine in rack['machines']:
                 node = graph.get_node(machine.name)
@@ -498,7 +498,7 @@ def power_config_to_svg(power_config, svg_file_path, worst_case_scenario=True):
             # graph.add_edge(con.from_plug.machine.name, con.to_plug.machine.name, color="%s:%s" % (color, wsc_color), label=label, penwidth="%s:%s" % (penwidth, penwidth))
             graph.add_edge(con.from_plug.machine.name, con.to_plug.machine.name, color=color, label=label, penwidth=penwidth)
 
-    for rack_id, rack in racks.iteritems():
+    for rack_id, rack in racks.items():
         # sub = graph.add_subgraph(rack, name='cluster_%s' % rack_id, rank='same')
         machine_names = list(machine.name for machine in rack['machines'])
         sub = graph.add_subgraph(machine_names, name='cluster_%s' % rack_id, style='rounded')

cocluto/cluster_stats.py

@@ -110,12 +110,12 @@ def get_computer_value_over_time(inventory, computer_id, time_value, flops_price
 #     """
 #     # matplot 1.1.1 doesn't have the stackplot method in Axes
 #     if 'toto_stackplot' in dir(ax):
-#         ax.stackplot(x_signal, list(y_signals.itervalues()) )
+#         ax.stackplot(x_signal, list(y_signals.values()) )
 #         plt.legend(list(y_signals.keys()))
 #     else:
 #         colors = ['blue', 'orange', 'green', 'purple', 'yellow']
 #         # emulating missing Axes.stackplot method
-#         y = np.row_stack(list(y_signals.itervalues()))
+#         y = np.row_stack(list(y_signals.values()))
 #         # this call to 'cumsum' (cumulative sum), passing in your y data,
 #         # is necessary to avoid having to manually order the datasets
 #         y_stack = np.cumsum(y, axis=0)   # a 3x10 array
@@ -236,7 +236,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type
 
     fig, ax = plt.subplots()
     ax.set_title(graph_type)
-    # for dept, cluster_value_for_dept in cluster_value.iteritems():
+    # for dept, cluster_value_for_dept in cluster_value.items():
     #    ax.plot(time_value, cluster_value_for_dept)
 
     stackplot(ax, time_value, cluster_value, legend_location='upper left')