From b0914f15f83e0f26b70dee28dc9522e8aae2b9f6 Mon Sep 17 00:00:00 2001 From: Marco Cammarata Date: Wed, 23 Nov 2016 17:16:39 +0100 Subject: [PATCH] various changes --- alignment.py | 54 ++++++++++++++++++++++++++------------------- x3py_config | 1 + xanes_analyzeRun.py | 42 ++++++++++++++++++----------------- 3 files changed, 54 insertions(+), 43 deletions(-) diff --git a/alignment.py b/alignment.py index 99aaccd..227ac67 100644 --- a/alignment.py +++ b/alignment.py @@ -6,7 +6,6 @@ import joblib import collections import numpy as np import matplotlib.pyplot as plt -import utils import time # /--------\ @@ -79,7 +78,7 @@ kw_2dplot = dict( ) fit_ret = collections.namedtuple("fit_ret",["fit_result","init_pars","final_pars",\ - "final_transform1","final_transform2","im1","im2","E","p1","p2","fom","ratio","tneeded"] ) + "final_transform1","final_transform2","im1","im2","E","p1","p1_sum","p2","p2_sum","fom","ratio","tneeded"] ) def calcFOM(p1,p2,ratio): idx = ( p1>p1.max()/10 ) & (p2>p2.max()/10) @@ -266,33 +265,39 @@ def saveAlignment(fname,transform,roi1,roi2): def loadAlignment(fname): return np.load(fname).item() -def unravel_results(res): +def unravel_results(res,nSaveImg='all'): final_pars = dict() - parnames = res[0].fit_result.parameters + # res[0].fit_result is a list if we are trying to unravel retult from getShots + if isinstance(res[0].fit_result,list): + parnames = res[0].fit_result[0].parameters + else: + parnames = res[0].fit_result.parameters final_pars = dict() init_pars = dict() for n in parnames: - final_pars[n] = np.asarray( [r.final_pars[n] for r in res]) - init_pars[n] = np.asarray( [r.init_pars[n] for r in res]) - #out["ratio"] = np.asarray( [r.ratio for r in res]) - #out["p1"] = np.asarray( [r.p1 for r in res] ) - #out["p2"] = np.asarray( [r.p2 for r in res] ) - #out["fom"] = np.asarray( [r.fom for r in res] ) - #out["E"] = res[0].E + final_pars[n] = np.hstack( [r.final_pars[n] for r in res]) + init_pars[n] = np.hstack( [r.init_pars[n] for r in res]) + im1 = np.asarray( [r.im1 for r in res]) + im2 = np.asarray( [r.im2 for r in res]) + if nSaveImg != 'all': + im1 = im1[:nSaveImg] + im2 = im2[:nSaveImg] return fit_ret( fit_result = [r.fit_result for r in res], init_pars = init_pars, final_pars = final_pars, final_transform1 = [r.final_transform1 for r in res], final_transform2 = [r.final_transform2 for r in res], - im1 = np.asarray( [r.im1 for r in res]), - im2 = np.asarray( [r.im2 for r in res]), + im1 = im1, + im2 = im2, E = defaultE, - p1 = np.hstack( [r.p1 for r in res]), - p2 = np.asarray( [r.p2 for r in res]), - ratio = np.asarray( [r.ratio for r in res]), - fom = np.asarray( [r.fom for r in res] ), - tneeded = np.asarray( [r.tneeded for r in res]) + p1 = np.vstack( [r.p1 for r in res]), + p1_sum = np.hstack( [r.p1_sum for r in res]), + p2 = np.vstack( [r.p2 for r in res]), + p2_sum = np.hstack( [r.p2_sum for r in res]), + ratio = np.vstack( [r.ratio for r in res]), + fom = np.hstack( [r.fom for r in res] ), + tneeded = np.hstack( [r.tneeded for r in res]) ) @@ -443,7 +448,9 @@ def transformIminuit(im1,im2,init_transform=dict(),show=False,verbose=True,zeroT im2 = i2, E = defaultE, p1 = p1, + p1_sum = p1.sum(), p2 = p2, + p2_sum = p2.sum(), ratio = r, fom = fom, tneeded = time.time()-t0 @@ -567,17 +574,18 @@ def doShot(i1,i2,init_pars,doFit=True,show=False): return r -def doShots(imgs1,imgs2,initpars,nJobs=16,doFit=False,returnBestTransform=False): +def doShots(imgs1,imgs2,initpars,nJobs=16,doFit=False,returnBestTransform=False,nSaveImg='all'): clearCache() N = imgs1.shape[0] pool = joblib.Parallel(backend="threading",n_jobs=nJobs) \ (joblib.delayed(doShot)(imgs1[i],imgs2[i],initpars,doFit=doFit) for i in range(N)) + res = unravel_results(pool,nSaveImg=nSaveImg) if returnBestTransform: - idx = np.argmin( np.abs([p.fom for p in pool]) ); # abs because sometime fit screws up the give negative spectra... - print("FOM for best alignment %.2f"%pool[idx].fom) - return pool,pool[idx].final_transform1 + idx = np.argmin( res.fom ); + print("FOM for best alignment %.2f"%res.fom[fom]) + return res,res.final_transform1[idx] else: - return pool + return res # out = collections.OrderedDict( enumerate(pool) ) # return out diff --git a/x3py_config b/x3py_config index 71e46a5..5a47d43 100644 --- a/x3py_config +++ b/x3py_config @@ -1 +1,2 @@ #cachepath = "/reg/d/psdm/xpp/xppl3716/scratch/mc/cache" +readlimit_GB = 6 diff --git a/xanes_analyzeRun.py b/xanes_analyzeRun.py index c55b069..2b2b608 100644 --- a/xanes_analyzeRun.py +++ b/xanes_analyzeRun.py @@ -174,26 +174,25 @@ class AnalyzeRun(object): gui.save(fname) - def analyzeScan(self,initpars=None,nShotsPerCalib=20,nC=None,doFit=False,fitEveryCalib=False): + def analyzeScan(self,initpars=None,nShotsPerCalib=20,nC=None,doFit=False,fitEveryCalib=False,nSaveImg=5): """ this is a comment """ if initpars is None: initpars= self.initAlign - d = self.data - if nC is None: nC = d.opal1.nCalib - shots = slice(nShotsPerCalib) + if nC is None: nC = self.nCalib out = [] for i in range(nC): + if nShotsPerCalib == 'all': + shots = slice(self.nShotsPerCalib[i]) + else: + shots = slice(nShotsPerCalib) s1,s2 = self.getShot(shots,calib=i) if fitEveryCalib is not False: - res = alignment.doShots(s1[:fitEveryCalib],s2[:fitEveryCalib],doFit=True,\ - initpars=initpars); # - idx = np.argmin( [p.fom for p in res] ) - res = res[idx] - initpars = res.final_pars; self.initAlign=res.final_pars - print("Calib cycle %d -> %.3f aligned (best FOM: %.2f)" % (i,self.data.scan.scanmotor0_values[i],res.fom)) - ret = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit) - res = alignment.unravel_results(ret) - self.results[i] = res - out.append(res) + ret,bestTransf = alignment.doShots(s1[:fitEveryCalib],s2[:fitEveryCalib],doFit=True,\ + initpars=initpars,nSaveImg=nSaveImg,returnBestTransform=True); + initpars = bestTransf; self.initAlign=bestTransf + ret = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit,nSaveImg=nSaveImg) + self.results[i] = ret + print("Calib cycle %d -> %.3f (best FOM: %.2f)" % (i,self.scanpos[i],np.min(ret.fom))) + out.append(ret) return out def doShot(self,shot=0,calib=None,initpars=None,im1=None,im2=None,doFit=True,show=False,showInit=False,save=False,savePlot="auto"): @@ -212,17 +211,20 @@ class AnalyzeRun(object): if save: self.saveTransform() return r - def doShots(self,shots=slice(0,50),calib=None,initpars=None,doFit=False,returnBestTransform=False,unravel=True): + def doShots(self,shots=slice(0,50),calib=None,initpars=None,doFit=False,returnBestTransform=False,nSaveImg='all'): + """ + shots : slice to define shots to read, use 'all' for all shots in calibcycle + nSaveImg : save saveImg images in memory (self.results), use 'all' for all + useful for decreasing memory footprint + """ if initpars is None: initpars= self.initAlign - d = self.data + if shots == "all": shots = slice(self.nShotsPerCalib[calib]) s1,s2 = self.getShot(shots,calib=calib) ret,transformForBestFit = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit,\ - returnBestTransform=True) + returnBestTransform=True,nSaveImg=nSaveImg) if doFit: self.initAlign = transformForBestFit - ret_unravel = alignment.unravel_results(ret) # keep it for later ! - self.results[calib] = ret_unravel - if unravel: ret = ret_unravel + self.results[calib] = ret if returnBestTransform: return ret,transformForBestFit else: