various changes

alignment.py

@@ -6,7 +6,6 @@ import joblib
 import collections
 import numpy as np
 import matplotlib.pyplot as plt
-import utils
 import time
 
 # /--------\
@@ -79,7 +78,7 @@ kw_2dplot = dict(
   )
 
 fit_ret = collections.namedtuple("fit_ret",["fit_result","init_pars","final_pars",\
-  "final_transform1","final_transform2","im1","im2","E","p1","p2","fom","ratio","tneeded"] )
+  "final_transform1","final_transform2","im1","im2","E","p1","p1_sum","p2","p2_sum","fom","ratio","tneeded"] )
 
 def calcFOM(p1,p2,ratio):
   idx = ( p1>p1.max()/10 ) & (p2>p2.max()/10)
@@ -266,33 +265,39 @@ def saveAlignment(fname,transform,roi1,roi2):
 def loadAlignment(fname):
   return np.load(fname).item()
 
-def unravel_results(res):
+def unravel_results(res,nSaveImg='all'):
   final_pars = dict()
+  # res[0].fit_result is a list if we are trying to unravel retult from getShots
+  if isinstance(res[0].fit_result,list):
+    parnames = res[0].fit_result[0].parameters
+  else:
     parnames = res[0].fit_result.parameters
   final_pars = dict()
   init_pars  = dict()
   for n in parnames:
-    final_pars[n] = np.asarray( [r.final_pars[n] for r in res])
+    final_pars[n] = np.hstack( [r.final_pars[n] for r in res])
-    init_pars[n] = np.asarray( [r.init_pars[n] for r in res])
+    init_pars[n] = np.hstack( [r.init_pars[n] for r in res])
-  #out["ratio"] = np.asarray( [r.ratio for r in res])
+  im1  = np.asarray( [r.im1 for r in res])
-  #out["p1"] =    np.asarray( [r.p1 for r in res]  )
+  im2  = np.asarray( [r.im2 for r in res])
-  #out["p2"] =    np.asarray( [r.p2 for r in res]  )
+  if nSaveImg != 'all':
-  #out["fom"] =   np.asarray( [r.fom for r in res] )
+    im1 = im1[:nSaveImg]
-  #out["E"]  =    res[0].E
+    im2 = im2[:nSaveImg]
   return fit_ret(
     fit_result   = [r.fit_result for r in res],
     init_pars    = init_pars,
     final_pars   = final_pars,
     final_transform1 = [r.final_transform1 for r in res],
     final_transform2 = [r.final_transform2 for r in res],
-    im1          = np.asarray( [r.im1 for r in res]),
+    im1          = im1,
-    im2          = np.asarray( [r.im2 for r in res]),
+    im2          = im2,
     E            = defaultE,
-    p1           = np.hstack( [r.p1 for r in res]),
+    p1           = np.vstack( [r.p1 for r in res]),
-    p2           = np.asarray( [r.p2 for r in res]),
+    p1_sum       = np.hstack( [r.p1_sum for r in res]),
-    ratio        = np.asarray( [r.ratio for r in res]),
+    p2           = np.vstack( [r.p2 for r in res]),
-    fom          = np.asarray( [r.fom for r in res] ),
+    p2_sum       = np.hstack( [r.p2_sum for r in res]),
-    tneeded      = np.asarray( [r.tneeded for r in res])
+    ratio        = np.vstack( [r.ratio for r in res]),
+    fom          = np.hstack( [r.fom for r in res] ),
+    tneeded      = np.hstack( [r.tneeded for r in res])
   )
 
 
@@ -443,7 +448,9 @@ def transformIminuit(im1,im2,init_transform=dict(),show=False,verbose=True,zeroT
     im2          = i2,
     E            = defaultE,
     p1           = p1,
+    p1_sum       = p1.sum(),
     p2           = p2,
+    p2_sum       = p2.sum(),
     ratio        = r,
     fom          = fom,
     tneeded      = time.time()-t0
@@ -567,17 +574,18 @@ def doShot(i1,i2,init_pars,doFit=True,show=False):
   return r
 
 
-def doShots(imgs1,imgs2,initpars,nJobs=16,doFit=False,returnBestTransform=False):
+def doShots(imgs1,imgs2,initpars,nJobs=16,doFit=False,returnBestTransform=False,nSaveImg='all'):
   clearCache()
   N = imgs1.shape[0]
   pool = joblib.Parallel(backend="threading",n_jobs=nJobs) \
     (joblib.delayed(doShot)(imgs1[i],imgs2[i],initpars,doFit=doFit) for i in range(N))
+  res = unravel_results(pool,nSaveImg=nSaveImg)
   if returnBestTransform:
-    idx = np.argmin( np.abs([p.fom for p in pool]) ); # abs because sometime fit screws up the give negative spectra...
+    idx = np.argmin( res.fom );
-    print("FOM for best alignment %.2f"%pool[idx].fom)
+    print("FOM for best alignment %.2f"%res.fom[fom])
-    return pool,pool[idx].final_transform1
+    return res,res.final_transform1[idx]
   else:
-    return pool
+    return res
 #  out = collections.OrderedDict( enumerate(pool) )
 #  return out
     

x3py_config

@@ -1 +1,2 @@
 #cachepath = "/reg/d/psdm/xpp/xppl3716/scratch/mc/cache"
+readlimit_GB = 6

xanes_analyzeRun.py

@@ -174,26 +174,25 @@ class AnalyzeRun(object):
     gui.save(fname)
 
 
-  def analyzeScan(self,initpars=None,nShotsPerCalib=20,nC=None,doFit=False,fitEveryCalib=False):
+  def analyzeScan(self,initpars=None,nShotsPerCalib=20,nC=None,doFit=False,fitEveryCalib=False,nSaveImg=5):
     """ this is a comment """
     if initpars is None: initpars= self.initAlign
-    d = self.data
+    if nC is None: nC = self.nCalib
-    if nC is None: nC = d.opal1.nCalib
-    shots = slice(nShotsPerCalib)
     out = []
     for i in range(nC):
+      if nShotsPerCalib == 'all':
+        shots = slice(self.nShotsPerCalib[i])
+      else:
+        shots = slice(nShotsPerCalib)
       s1,s2 = self.getShot(shots,calib=i)
       if fitEveryCalib is not False:
-        res = alignment.doShots(s1[:fitEveryCalib],s2[:fitEveryCalib],doFit=True,\
+        ret,bestTransf = alignment.doShots(s1[:fitEveryCalib],s2[:fitEveryCalib],doFit=True,\
-                initpars=initpars); #
+                initpars=initpars,nSaveImg=nSaveImg,returnBestTransform=True);
-        idx = np.argmin( [p.fom for p in res] )
+        initpars = bestTransf; self.initAlign=bestTransf
-        res = res[idx]
+      ret = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit,nSaveImg=nSaveImg)
-        initpars = res.final_pars; self.initAlign=res.final_pars
+      self.results[i] = ret
-        print("Calib cycle %d -> %.3f aligned (best FOM: %.2f)" % (i,self.data.scan.scanmotor0_values[i],res.fom))
+      print("Calib cycle %d -> %.3f (best FOM: %.2f)" % (i,self.scanpos[i],np.min(ret.fom)))
-      ret = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit)
+      out.append(ret)
-      res = alignment.unravel_results(ret)
-      self.results[i] = res
-      out.append(res)
     return out
 
   def doShot(self,shot=0,calib=None,initpars=None,im1=None,im2=None,doFit=True,show=False,showInit=False,save=False,savePlot="auto"):
@@ -212,17 +211,20 @@ class AnalyzeRun(object):
     if save: self.saveTransform()
     return r
 
-  def doShots(self,shots=slice(0,50),calib=None,initpars=None,doFit=False,returnBestTransform=False,unravel=True):
+  def doShots(self,shots=slice(0,50),calib=None,initpars=None,doFit=False,returnBestTransform=False,nSaveImg='all'):
+    """
+        shots   : slice to define shots to read, use 'all' for all shots in calibcycle
+        nSaveImg : save saveImg images in memory (self.results), use 'all' for all
+                                 useful for decreasing memory footprint
+    """
     if initpars is None: initpars= self.initAlign
-    d = self.data
+    if shots == "all": shots = slice(self.nShotsPerCalib[calib])
     s1,s2 = self.getShot(shots,calib=calib)
     ret,transformForBestFit = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit,\
-            returnBestTransform=True)
+            returnBestTransform=True,nSaveImg=nSaveImg)
     if doFit: self.initAlign = transformForBestFit
-    ret_unravel = alignment.unravel_results(ret)
     # keep it for later !
-    self.results[calib] = ret_unravel
+    self.results[calib] = ret
-    if unravel: ret = ret_unravel
     if returnBestTransform:
       return ret,transformForBestFit
     else: