From 45e6e44549504581424496afa91b5790ddccd90e Mon Sep 17 00:00:00 2001 From: Marco Cammarata Date: Wed, 15 Mar 2017 18:23:20 +0100 Subject: [PATCH] implemented dezinger with pyfai separate function, changes related to the id9 logfile and the way the information is stored inside azav data storage --- xray/azav.py | 52 ++++++++++++++++++++++++++++++---------------------- xray/id9.py | 44 +++++++++++++++++++++++++++----------------- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/xray/azav.py b/xray/azav.py index 9090d27..ba14cd4 100644 --- a/xray/azav.py +++ b/xray/azav.py @@ -63,9 +63,10 @@ def ai_as_str(ai): "# rot[1,2,3] [rad]: %.3f,%.3f,%.3f" % (ai.rot1,ai.rot2,ai.rot3) ] return "\n".join(s) -def do1d(ai, imgs, mask = None, npt_radial = 600, method = 'csr',safe=True,dark=10., polCorr = 1): +def do1d(ai, imgs, mask = None, npt_radial = 600, method = 'csr',safe=True,dark=10., polCorr = 1,dezinger=None): """ ai is a pyFAI azimuthal intagrator it can be defined with pyFAI.load(ponifile) + dezinger: None or float (used as percentile of ai.separate) mask: True are points to be masked out """ # force float to be sure of type casting for img if isinstance(dark,int): dark = float(dark); @@ -73,6 +74,9 @@ def do1d(ai, imgs, mask = None, npt_radial = 600, method = 'csr',safe=True,dark= out_i = np.empty( ( len(imgs), npt_radial) ) out_s = np.empty( ( len(imgs), npt_radial) ) for _i,img in enumerate(imgs): + if dezinger is not None and dezinger > 0: + _,img=ai.separate(img,npt_rad=npt_radial,npt_azim=512,unit='q_A^-1', + method=method,mask=mask,percentile=dezinger) q,i, sig = ai.integrate1d(img-dark, npt_radial, mask= mask, safe = safe,\ unit="q_A^-1", method = method, error_model = "poisson", polarization_factor = polCorr) @@ -202,14 +206,14 @@ def removeBackground(data,qlims=(0,10),max_iter=30,background_regions=[],force=F storageFile=None,save=True,**removeBkg): """ similar function to the zray.utils one, this works on dataset created by doFolder """ - idx = utils.findSlice(data.q_orig,qlims) + idx = utils.findSlice(data.orig.q,qlims) # see if there are some to do ... if force: idx_start = 0 else: idx_start = len(data.data) - if idx_start < len(data.data_orig): - _q,_data = utils.removeBackground(data.q_orig[idx],data.data_orig[:,idx], + if idx_start < len(data.orig.data): + _q,_data = utils.removeBackground(data.orig.q[idx],data.orig.data[:,idx], max_iter=max_iter,background_regions=background_regions,**removeBkg) data.q = _q data.data = np.concatenate( (data.data,_data ) ) @@ -218,8 +222,9 @@ def removeBackground(data,qlims=(0,10),max_iter=30,background_regions=[],force=F return data -def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm='auto',save_pyfai=False, - saveChi=True,poni='pyfai.poni',storageFile='auto',save=True,diagnostic=None,skip_first=0,last=None): +def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10, + norm='auto',save_pyfai=False,saveChi=True,poni='pyfai.poni', + storageFile='auto',save=True,logDict=None,dezinger=None,skip_first=0,last=None): """ calc 1D curves from files in folder, returning a dictionary of stuff nQ : number of Q-points (equispaced) force : if True, redo from beginning even if previous data are found @@ -227,7 +232,9 @@ def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm= mask : can be a list of [filenames|array of booleans|mask string] pixels that are True are dis-regarded saveChi: self-explanatory - save_pyfai: save all pyfai's internal arrays (~110 MB) + dezinger: None or 0 to disable; good value is ~50. Needs good center and mask + logDict: dictionary(-like) structure. has to have 'file' key + save_pyfai: store all pyfai's internal arrays (~110 MB) poni : could be: → an AzimuthalIntegrator instance → a filename that will be look for in @@ -248,16 +255,19 @@ def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm= else: saved = None - files = utils.getFiles(folder,files)[skip_first:last] - if saved is not None: - files = [f for f in files if utils.getBasename(f) not in saved["files"]] + files = utils.getFiles(folder,files) + if logDict is not None: + files = [f for f in files if utils.getBasename(f) in logDict['file'] ] + files = files[skip_first:last] - if diagnostic is not None: - key = list( diagnostic.keys() )[0] - files_diagnostic = list(diagnostic[key].keys()) - files = [ f for f in files if utils.getBasename(f) in files_diagnostic ] + + if saved is not None: + files = [f for f in files if f not in saved["files"]] log.info("Will do azimuthal integration for %d files"%(len(files))) + files = np.asarray(files) + basenames = np.asarray( [ utils.getBasename(file) for file in files] ) + if len(files) > 0: # which poni file to use: ai = getAI(poni,folder) @@ -277,22 +287,20 @@ def doFolder(folder,files='*.edf*',nQ = 1500,force=False,mask=None,dark=10,norm= chi_fname = utils.removeExt(fname) + ".chi" utils.saveTxt(chi_fname,q,np.vstack((i,e)),info=ai_as_str(ai),overwrite=True) - files = [ utils.getBasename(f) for f in files ] - files = np.asarray(files) if saved is not None: - files = np.concatenate( (saved["files"] ,files ) ) + files = np.concatenate( (saved["files"] ,basenames ) ) data = np.concatenate( (saved["data"] ,data ) ) err = np.concatenate( (saved["err"] ,err ) ) ret = dict(q=q,folder=folder,files=files,data=data,err=err, - data_orig=data.copy(),err_orig=err.copy(),q_orig=q.copy(), + orig = dict(data=data.copy(),err=err.copy(),q=q.copy()), pyfai=ai_as_dict(ai),pyfai_info=ai_as_str(ai),mask=mask) if not save_pyfai: del ret['pyfai'] - # add info from diagnostic if provided - if diagnostic is not None: - for k in diagnostic: - ret[k] = np.asarray( [diagnostic[k][f] for f in ret['files']] ) ret = storage.DataStorage(ret) + + # add info from logDict if provided + if logDict is not None: + ret['log']=logDict # sometime saving is not necessary (if one has to do it after subtracting background if storageFile is not None and save: ret.save(storageFile) else: diff --git a/xray/id9.py b/xray/id9.py index 6c94cb7..a71886b 100644 --- a/xray/id9.py +++ b/xray/id9.py @@ -33,7 +33,7 @@ def _readDiagnostic(fname,retry=3): # it should not arrive here raise ValueError("Could not read diagnostic file after %d attempts"%retry) -def readDelayFromDiagnostic(fname): +def readDiagnostic(fname): """ return an ordered dict dictionary of filename; for each key a rounded value of delay is associated """ if os.path.isdir(fname): fname += "/diagnostics.log" @@ -44,9 +44,10 @@ def readDelayFromDiagnostic(fname): # skip lines that cannot be interpreted as float (like done, etc) idx_ok = np.isfinite( delays ) files = files[idx_ok] + files = np.asarray( [utils.getBasename(f) for f in files]) delays = delays[idx_ok] delays = np.round(delays.astype(float),12) - return collections.OrderedDict( zip(files,delays) ) + return dict( file = files, scan = delays) def _findDark(line): _,value = line.split(":") @@ -98,27 +99,27 @@ def readLogFile(fnameOrFolder,subtractDark=False,skip_first=0,asDataStorage=True def doFolder_azav(folder,nQ=1500,files='*.edf*',force=False,mask=None, - saveChi=True,poni='pyfai.poni',storageFile='auto',dark=9.9,zingerFilter=30,qlims=(0,10), - removeBack=False,removeBack_kw=dict()): + saveChi=True,poni='pyfai.poni',storageFile='auto',dark=9.9,dezinger=None, + qlims=(0,10),removeBack=False,removeBack_kw=dict(),skip_first=0, + last=None): """ very small wrapper around azav.doFolder, essentially just reading - the diagnostics.log """ + the id9 logfile or diagnostics.log """ - diag = dict( delays = readDelayFromDiagnostic(folder) ) + try: + loginfo = readLogFile(folder,skip_first=skip_first,last=last) + except Exception as e: + log.warn("Could not read log file, trying to read diagnostics.log") + loginfo = readDiagnostic(folder) if storageFile == 'auto' : storageFile = folder + "/" + "pyfai_1d" + default_extension data = azav.doFolder(folder,files=files,nQ=nQ,force=force,mask=mask, - saveChi=saveChi,poni=poni,storageFile=storageFile,diagnostic=diag,dark=dark,save=False) + saveChi=saveChi,poni=poni,storageFile=storageFile,logDict=loginfo,dark=dark,save=False,dezinger=dezinger) #try: # if removeBack is not None: # _,data.data = azav.removeBackground(data,qlims=qlims,**removeBack_kw) #except Exception as e: # log.error("Could not remove background, error was %s"%(str(e))) - if zingerFilter > 0: - data.data = filters.removeZingers(data.data,threshold=zingerFilter) - #data.save(storageFile); it does not save err ? - - # idx = utils.findSlice(data.q,qlims) # n = np.nanmean(data.data[:,idx],axis=1) # data.norm_range = qlims @@ -135,7 +136,15 @@ def doFolder_azav(folder,nQ=1500,files='*.edf*',force=False,mask=None, def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean, qlims=None,outStorageFile='auto',reference='min'): - """ azavStorage if a DataStorage instance or the filename to read """ + """ azavStorage if a DataStorage instance or the filename to read + monitor : normalization vector that can be given as + 1. numpy array + 2. a list (interpreted as q-range normalization) + 3. a string to look for as key in the log, e.g. + monitor="pd2ic" would reult in using + azavStorage.log.pd2ic + + """ if isinstance(azavStorage,storage.DataStorage): data = azavStorage @@ -156,11 +165,11 @@ def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean, data.err = data.err[:,idx] data.q = data.q[idx] - + if isinstance(monitor,str): monitor = data['log'][monitor] # calculate differences - diffs = dataReduction.calcTimeResolvedSignal(data.delays,data.data,err=data.err, - q=data.q,reference=reference,monitor=monitor, + diffs = dataReduction.calcTimeResolvedSignal(data.log.delay,data.data, + err=data.err,q=data.q,reference=reference,monitor=monitor, funcForAveraging=funcForAveraging) # save txt and npz file @@ -171,7 +180,7 @@ def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean, return data,diffs -def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax=20): +def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax=20,force=False): import matplotlib.pyplot as plt if folder == "./": folder = os.path.abspath(folder) fig = plt.figure() @@ -196,6 +205,7 @@ def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax= else: retryNum += 1 plt.pause(30) + if force: force=False; # does not make sense to have always True ... except KeyboardInterrupt: keepGoing = False if not online: keepGoing = False