mcammara
/
dispersiveXanes
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

first attempt to work with chunks to bypass memory limit

This commit is contained in:
Marco Cammarata 2016-11-27 23:52:54 +01:00
parent 9d22137e98
commit 3d10ebb852
1 changed files with 30 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
xanes_analyzeRun.py
View File

@ -141,7 +141,8 @@ def medianRatio(p1,p2,threshold=0.03):
ratio = p2/p1 ratio = p2/p1
return np.ma.average(ratio,axis=0,weights=p1) return np.ma.average(ratio,axis=0,weights=p1)
def sliceToIndices(shot_slice,nShots):
return list(range(*shot_slice.indices(nShots)))
class AnalyzeRun(object): class AnalyzeRun(object):
def __init__(self,run,initAlign="auto",swapx=g_swapx,swapy=g_swapy): def __init__(self,run,initAlign="auto",swapx=g_swapx,swapy=g_swapy):
@ -221,27 +222,31 @@ class AnalyzeRun(object):
gui.save(fname) gui.save(fname)
def analyzeScan(self,initpars=None,shots=slice(0,30),calibs="all",nImagesToFit=0,nSaveImg=5): def analyzeScan(self,initpars=None,nShotsPerCalib="all",calibs="all",calibsToFit="all",nImagesToFit=0,nSaveImg=5):
""" nImagesToFit: number of images to Fit per calibcycle, (int or "all") """ """ nImagesToFit: number of images to Fit per calibcycle, (int or "all")
calibs to fit could be 'all','even','odd'
"""
if initpars is None: initpars= self.initAlign if initpars is None: initpars= self.initAlign
if calibs == "all": calibs=list(range(self.nCalib)) if calibs == "all": calibs=list(range(self.nCalib))
if isinstance(calibs,slice): calibs=list(range(self.nCalib))[calibs] if isinstance(calibs,slice): calibs=list(range(self.nCalib))[calibs]
nC = len(calibs) nC = len(calibs)
for ic,calib in enumerate(calibs): for ic,calib in enumerate(calibs):
shots = list(range(self.nShotsPerCalib[calib]))
if nShotsPerCalib != "all": shots = shots[:nShotsPerCalib]
if nImagesToFit == "all": if nImagesToFit == "all":
nToFit = self.nShotsPerCalib[calib] nToFit = self.nShotsPerCalib[calib]
else: else:
nToFit = nImagesToFit nToFit = nImagesToFit
#print("Memory available 1",x3py.toolsOS.memAvailable()) if calibsToFit == 'even' and (calib%2==1): nToFit=0
s1,s2 = self.getShots(shots,calib=calib) if calibsToFit == 'odd' and (calib%2==0): nToFit=0
#print("Memory available 2",x3py.toolsOS.memAvailable()) print("Calib %d, tofit %d"%(calib,nToFit))
ret = None ret = None
if nImagesToFit > 0: if nToFit > 0:
ret,bestTransf = alignment.doShots(s1[:nToFit],s2[:nToFit],doFit=True,\ ret,bestTransf = self.doShots(shots=shots[:nToFit],calib=calib,doFit=True,\
initpars=initpars,nSaveImg=nSaveImg,returnBestTransform=True); initpars=initpars,nSaveImg=nSaveImg,returnBestTransform=True);
initpars = bestTransf; self.initAlign=bestTransf initpars = bestTransf; self.initAlign=bestTransf
if nToFit < s1.shape[0]: if nToFit < len(shots):
ret2 = alignment.doShots(s1[nToFit:],s2[nToFit:],initpars=initpars,doFit=False,nSaveImg=0) ret2 = self.doShots(shots[nToFit:],initpars=initpars,doFit=False,nSaveImg=0)
if ret is None: if ret is None:
ret = ret2 ret = ret2
else: else:
@ -268,17 +273,27 @@ class AnalyzeRun(object):
if save: self.saveTransform() if save: self.saveTransform()
return r return r
def doShots(self,shots=slice(0,50),calib=None,initpars=None,doFit=False,returnBestTransform=False,nSaveImg='all'): def doShots(self,shots=slice(0,50),calib=None,initpars=None,doFit=False,returnBestTransform=False,nSaveImg='all',nInChunks=250):
""" """
shots : slice to define shots to read, use 'all' for all shots in calibcycle shots : slice to define shots to read, use 'all' for all shots in calibcycle
nSaveImg : save saveImg images in memory (self.results), use 'all' for all nSaveImg : save saveImg images in memory (self.results), use 'all' for all
useful for decreasing memory footprint useful for decreasing memory footprint
""" """
if initpars is None: initpars= self.initAlign if initpars is None: initpars= self.initAlign
if shots == "all": shots = slice(self.nShotsPerCalib[calib]) if shots == "all": shots = list(range(slice(self.nShotsPerCalib[calib])))
s1,s2 = self.getShots(shots,calib=calib) if isinstance(shots,slice):
ret,transformForBestFit = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit,\ nmax = self.nShotsPerCalib[calib] if calib is not None else self.data.spec1.nShots
returnBestTransform=True,nSaveImg=nSaveImg) shots = sliceToIndices(shots,nmax)
chunks = x3py.toolsVarious.chunk(shots,nInChunks)
ret = []
for chunk in chunks:
s1,s2 = self.getShots(chunk,calib=calib)
temp = alignment.doShots(s1,s2,initpars=initpars,doFit=doFit,\
returnBestTransform=False,nSaveImg=nSaveImg)
ret.append(temp)
ret = alignment.unravel_results(ret)
idxBest = np.nanargmin(ret.fom)
transformForBestFit = dict( [ (p,ret.final_pars[p][idxBest]) for p in ret.final_pars.keys() ] )
if doFit: self.initAlign = transformForBestFit if doFit: self.initAlign = transformForBestFit
# keep it for later ! # keep it for later !
self.results[calib] = ret self.results[calib] = ret