mcammara
/
mcutils
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

implemented dezinger with pyfai separate function, changes related to the id9 logfile and the way the information is stored inside azav data storage

This commit is contained in:
Marco Cammarata 2017-03-15 18:23:20 +01:00
parent 35757665ad
commit 45e6e44549
2 changed files with 57 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
xray/azav.py
View File

@ -63,9 +63,10 @@ def ai_as_str(ai):
"# rot[1,2,3] [rad]: %.3f,%.3f,%.3f" % (ai.rot1,ai.rot2,ai.rot3) ] "# rot[1,2,3] [rad]: %.3f,%.3f,%.3f" % (ai.rot1,ai.rot2,ai.rot3) ]
return "\n".join(s) return "\n".join(s)
def do1d(ai, imgs, mask = None, npt_radial = 600, method = 'csr',safe=True,dark=10., polCorr = 1): def do1d(ai, imgs, mask = None, npt_radial = 600, method = 'csr',safe=True,dark=10., polCorr = 1,dezinger=None):
""" ai is a pyFAI azimuthal intagrator """ ai is a pyFAI azimuthal intagrator
it can be defined with pyFAI.load(ponifile) it can be defined with pyFAI.load(ponifile)
dezinger: None or float (used as percentile of ai.separate)
mask: True are points to be masked out """ mask: True are points to be masked out """
# force float to be sure of type casting for img # force float to be sure of type casting for img
if isinstance(dark,int): dark = float(dark); if isinstance(dark,int): dark = float(dark);
@ -73,6 +74,9 @@ def do1d(ai, imgs, mask = None, npt_radial = 600, method = 'csr',safe=True,dark=
out_i = np.empty( ( len(imgs), npt_radial) ) out_i = np.empty( ( len(imgs), npt_radial) )
out_s = np.empty( ( len(imgs), npt_radial) ) out_s = np.empty( ( len(imgs), npt_radial) )
for _i,img in enumerate(imgs): for _i,img in enumerate(imgs):
if dezinger is not None and dezinger > 0:
_,img=ai.separate(img,npt_rad=npt_radial,npt_azim=512,unit='q_A^-1',
method=method,mask=mask,percentile=dezinger)
q,i, sig = ai.integrate1d(img-dark, npt_radial, mask= mask, safe = safe,\ q,i, sig = ai.integrate1d(img-dark, npt_radial, mask= mask, safe = safe,\
unit="q_A^-1", method = method, error_model = "poisson", unit="q_A^-1", method = method, error_model = "poisson",
polarization_factor = polCorr) polarization_factor = polCorr)
@ -202,14 +206,14 @@ def removeBackground(data,qlims=(0,10),max_iter=30,background_regions=[],force=F
storageFile=None,save=True,**removeBkg): storageFile=None,save=True,**removeBkg):
""" similar function to the zray.utils one, this works on dataset created by """ similar function to the zray.utils one, this works on dataset created by
doFolder """ doFolder """
idx = utils.findSlice(data.q_orig,qlims) idx = utils.findSlice(data.orig.q,qlims)
# see if there are some to do ... # see if there are some to do ...
if force: if force:
idx_start = 0 idx_start = 0
else: else:
idx_start = len(data.data) idx_start = len(data.data)
if idx_start < len(data.data_orig): if idx_start < len(data.orig.data):
_q,_data = utils.removeBackground(data.q_orig[idx],data.data_orig[:,idx], _q,_data = utils.removeBackground(data.orig.q[idx],data.orig.data[:,idx],
max_iter=max_iter,background_regions=background_regions,**removeBkg) max_iter=max_iter,background_regions=background_regions,**removeBkg)
data.q = _q data.q = _q
data.data = np.concatenate( (data.data,_data ) ) data.data = np.concatenate( (data.data,_data ) )
@ -218,8 +222,9 @@ def removeBackground(data,qlims=(0,10),max_iter=30,background_regions=[],force=F
return data return data
def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm='auto',save_pyfai=False, def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,
saveChi=True,poni='pyfai.poni',storageFile='auto',save=True,diagnostic=None,skip_first=0,last=None): norm='auto',save_pyfai=False,saveChi=True,poni='pyfai.poni',
storageFile='auto',save=True,logDict=None,dezinger=None,skip_first=0,last=None):
""" calc 1D curves from files in folder, returning a dictionary of stuff """ calc 1D curves from files in folder, returning a dictionary of stuff
nQ : number of Q-points (equispaced) nQ : number of Q-points (equispaced)
force : if True, redo from beginning even if previous data are found force : if True, redo from beginning even if previous data are found
@ -227,7 +232,9 @@ def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm=
mask : can be a list of [filenames|array of booleans|mask string] mask : can be a list of [filenames|array of booleans|mask string]
pixels that are True are dis-regarded pixels that are True are dis-regarded
saveChi: self-explanatory saveChi: self-explanatory
save_pyfai: save all pyfai's internal arrays (~110 MB) dezinger: None or 0 to disable; good value is ~50. Needs good center and mask
logDict: dictionary(-like) structure. has to have 'file' key
save_pyfai: store all pyfai's internal arrays (~110 MB)
poni : could be: poni : could be:
→ an AzimuthalIntegrator instance → an AzimuthalIntegrator instance
→ a filename that will be look for in → a filename that will be look for in
@ -248,16 +255,19 @@ def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm=
else: else:
saved = None saved = None
files = utils.getFiles(folder,files)[skip_first:last] files = utils.getFiles(folder,files)
if saved is not None: if logDict is not None:
files = [f for f in files if utils.getBasename(f) not in saved["files"]] files = [f for f in files if utils.getBasename(f) in logDict['file'] ]
files = files[skip_first:last]
if diagnostic is not None:
key = list( diagnostic.keys() )[0] if saved is not None:
files_diagnostic = list(diagnostic[key].keys()) files = [f for f in files if f not in saved["files"]]
files = [ f for f in files if utils.getBasename(f) in files_diagnostic ]
log.info("Will do azimuthal integration for %d files"%(len(files))) log.info("Will do azimuthal integration for %d files"%(len(files)))
files = np.asarray(files)
basenames = np.asarray( [ utils.getBasename(file) for file in files] )
if len(files) > 0: if len(files) > 0:
# which poni file to use: # which poni file to use:
ai = getAI(poni,folder) ai = getAI(poni,folder)
@ -277,22 +287,20 @@ def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm=
chi_fname = utils.removeExt(fname) + ".chi" chi_fname = utils.removeExt(fname) + ".chi"
utils.saveTxt(chi_fname,q,np.vstack((i,e)),info=ai_as_str(ai),overwrite=True) utils.saveTxt(chi_fname,q,np.vstack((i,e)),info=ai_as_str(ai),overwrite=True)
files = [ utils.getBasename(f) for f in files ]
files = np.asarray(files)
if saved is not None: if saved is not None:
files = np.concatenate( (saved["files"] ,files ) ) files = np.concatenate( (saved["files"] ,basenames ) )
data = np.concatenate( (saved["data"] ,data ) ) data = np.concatenate( (saved["data"] ,data ) )
err = np.concatenate( (saved["err"] ,err ) ) err = np.concatenate( (saved["err"] ,err ) )
ret = dict(q=q,folder=folder,files=files,data=data,err=err, ret = dict(q=q,folder=folder,files=files,data=data,err=err,
data_orig=data.copy(),err_orig=err.copy(),q_orig=q.copy(), orig = dict(data=data.copy(),err=err.copy(),q=q.copy()),
pyfai=ai_as_dict(ai),pyfai_info=ai_as_str(ai),mask=mask) pyfai=ai_as_dict(ai),pyfai_info=ai_as_str(ai),mask=mask)
if not save_pyfai: del ret['pyfai'] if not save_pyfai: del ret['pyfai']
# add info from diagnostic if provided
if diagnostic is not None:
for k in diagnostic:
ret[k] = np.asarray( [diagnostic[k][f] for f in ret['files']] )
ret = storage.DataStorage(ret) ret = storage.DataStorage(ret)
# add info from logDict if provided
if logDict is not None:
ret['log']=logDict
# sometime saving is not necessary (if one has to do it after subtracting background # sometime saving is not necessary (if one has to do it after subtracting background
if storageFile is not None and save: ret.save(storageFile) if storageFile is not None and save: ret.save(storageFile)
else: else:

44
xray/id9.py
View File

@ -33,7 +33,7 @@ def _readDiagnostic(fname,retry=3):
# it should not arrive here # it should not arrive here
raise ValueError("Could not read diagnostic file after %d attempts"%retry) raise ValueError("Could not read diagnostic file after %d attempts"%retry)
def readDelayFromDiagnostic(fname): def readDiagnostic(fname):
""" return an ordered dict dictionary of filename; for each key a rounded """ return an ordered dict dictionary of filename; for each key a rounded
value of delay is associated """ value of delay is associated """
if os.path.isdir(fname): fname += "/diagnostics.log" if os.path.isdir(fname): fname += "/diagnostics.log"
@ -44,9 +44,10 @@ def readDelayFromDiagnostic(fname):
# skip lines that cannot be interpreted as float (like done, etc) # skip lines that cannot be interpreted as float (like done, etc)
idx_ok = np.isfinite( delays ) idx_ok = np.isfinite( delays )
files = files[idx_ok] files = files[idx_ok]
files = np.asarray( [utils.getBasename(f) for f in files])
delays = delays[idx_ok] delays = delays[idx_ok]
delays = np.round(delays.astype(float),12) delays = np.round(delays.astype(float),12)
return collections.OrderedDict( zip(files,delays) ) return dict( file = files, scan = delays)
def _findDark(line): def _findDark(line):
_,value = line.split(":") _,value = line.split(":")
@ -98,27 +99,27 @@ def readLogFile(fnameOrFolder,subtractDark=False,skip_first=0,asDataStorage=True
def doFolder_azav(folder,nQ=1500,files='*.edf*',force=False,mask=None, def doFolder_azav(folder,nQ=1500,files='*.edf*',force=False,mask=None,
saveChi=True,poni='pyfai.poni',storageFile='auto',dark=9.9,zingerFilter=30,qlims=(0,10), saveChi=True,poni='pyfai.poni',storageFile='auto',dark=9.9,dezinger=None,
removeBack=False,removeBack_kw=dict()): qlims=(0,10),removeBack=False,removeBack_kw=dict(),skip_first=0,
last=None):
""" very small wrapper around azav.doFolder, essentially just reading """ very small wrapper around azav.doFolder, essentially just reading
the diagnostics.log """ the id9 logfile or diagnostics.log """
diag = dict( delays = readDelayFromDiagnostic(folder) ) try:
loginfo = readLogFile(folder,skip_first=skip_first,last=last)
except Exception as e:
log.warn("Could not read log file, trying to read diagnostics.log")
loginfo = readDiagnostic(folder)
if storageFile == 'auto' : storageFile = folder + "/" + "pyfai_1d" + default_extension if storageFile == 'auto' : storageFile = folder + "/" + "pyfai_1d" + default_extension
data = azav.doFolder(folder,files=files,nQ=nQ,force=force,mask=mask, data = azav.doFolder(folder,files=files,nQ=nQ,force=force,mask=mask,
saveChi=saveChi,poni=poni,storageFile=storageFile,diagnostic=diag,dark=dark,save=False) saveChi=saveChi,poni=poni,storageFile=storageFile,logDict=loginfo,dark=dark,save=False,dezinger=dezinger)
#try: #try:
# if removeBack is not None: # if removeBack is not None:
# _,data.data = azav.removeBackground(data,qlims=qlims,**removeBack_kw) # _,data.data = azav.removeBackground(data,qlims=qlims,**removeBack_kw)
#except Exception as e: #except Exception as e:
# log.error("Could not remove background, error was %s"%(str(e))) # log.error("Could not remove background, error was %s"%(str(e)))
if zingerFilter > 0:
data.data = filters.removeZingers(data.data,threshold=zingerFilter)
#data.save(storageFile); it does not save err ?
# idx = utils.findSlice(data.q,qlims) # idx = utils.findSlice(data.q,qlims)
# n = np.nanmean(data.data[:,idx],axis=1) # n = np.nanmean(data.data[:,idx],axis=1)
# data.norm_range = qlims # data.norm_range = qlims
@ -135,7 +136,15 @@ def doFolder_azav(folder,nQ=1500,files='*.edf*',force=False,mask=None,
def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean, def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean,
qlims=None,outStorageFile='auto',reference='min'): qlims=None,outStorageFile='auto',reference='min'):
""" azavStorage if a DataStorage instance or the filename to read """ """ azavStorage if a DataStorage instance or the filename to read
monitor : normalization vector that can be given as
1. numpy array
2. a list (interpreted as q-range normalization)
3. a string to look for as key in the log, e.g.
monitor="pd2ic" would reult in using
azavStorage.log.pd2ic
"""
if isinstance(azavStorage,storage.DataStorage): if isinstance(azavStorage,storage.DataStorage):
data = azavStorage data = azavStorage
@ -156,11 +165,11 @@ def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean,
data.err = data.err[:,idx] data.err = data.err[:,idx]
data.q = data.q[idx] data.q = data.q[idx]
if isinstance(monitor,str): monitor = data['log'][monitor]
# calculate differences # calculate differences
diffs = dataReduction.calcTimeResolvedSignal(data.delays,data.data,err=data.err, diffs = dataReduction.calcTimeResolvedSignal(data.log.delay,data.data,
q=data.q,reference=reference,monitor=monitor, err=data.err,q=data.q,reference=reference,monitor=monitor,
funcForAveraging=funcForAveraging) funcForAveraging=funcForAveraging)
# save txt and npz file # save txt and npz file
@ -171,7 +180,7 @@ def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean,
return data,diffs return data,diffs
def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax=20): def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax=20,force=False):
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
if folder == "./": folder = os.path.abspath(folder) if folder == "./": folder = os.path.abspath(folder)
fig = plt.figure() fig = plt.figure()
@ -196,6 +205,7 @@ def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax=
else: else:
retryNum += 1 retryNum += 1
plt.pause(30) plt.pause(30)
if force: force=False; # does not make sense to have always True ...
except KeyboardInterrupt: except KeyboardInterrupt:
keepGoing = False keepGoing = False
if not online: keepGoing = False if not online: keepGoing = False