mcutils/xray/id9.py

import logging
log = logging.getLogger(__name__)

import os
import collections
import numpy as np
from . import azav
from . import dataReduction
from . import utils
from . import storage
from . import filters

default_extension = ".npz"

def _conv(x):
    try:
        x = float(x)
    except:
        x = np.nan
    return x

def _readDiagnostic(fname,retry=3):
  ntry = 0
  while ntry<retry:
    try:
      data = np.genfromtxt(fname,usecols=(2,3),\
             dtype=None,converters={3: lambda x: _conv(x)},
             names = ['fname','delay'])
      return data
    except Exception as e:
      log.warn("Could not read diagnostic file, retrying soon,error was %s"%e)
      ntry += 1
  # it should not arrive here
  raise ValueError("Could not read diagnostic file after %d attempts"%retry)

def readDelayFromDiagnostic(fname):
  """ return an ordered dict dictionary of filename; for each key a rounded
      value of delay is associated """
  if os.path.isdir(fname): fname += "/diagnostics.log"
  # try to read diagnostic couple of times
  data = _readDiagnostic(fname,retry=4)
  files = data['fname'].astype(str)
  delays = data['delay']
  # skip lines that cannot be interpreted as float (like done, etc)
  idx_ok = np.isfinite( delays )
  files = files[idx_ok]
  delays = delays[idx_ok]
  delays = np.round(delays.astype(float),12)
  return collections.OrderedDict( zip(files,delays) )

def _findDark(line):
  _,value = line.split(":")
  return float(value)

def _delayToNum(delay):
  if delay.decode('ascii') == 'off':
    delay = -10
  else:
    delay=utils.strToTime(delay)
  return delay

def findLogFile(folder):
  files = utils.getFiles(folder,basename='*.log')
  files.remove(os.path.join(folder,"diagnostics.log"))
  logfile = files[0]
  if len(files)>1: log.warn("Found more than one *.log file that is not diagnostics.log: %s"%files)
  return logfile

def readLogFile(fnameOrFolder,subtractDark=False,skip_first=0,asDataStorage=True,last=None):
  """ read id9 style logfile """
  if os.path.isdir(fnameOrFolder):
    fname = findLogFile(fnameOrFolder)
  else:
    fname = fnameOrFolder
  f = open(fname,"r")
  lines = f.readlines()
  f.close()
  lines = [line.strip() for line in lines]
  darks = {}
  for line in lines:
    if line.find("pd1 dark/sec")>=0: darks['pd1ic'] = _findDark(line)
    if line.find("pd2 dark/sec")>=0: darks['pd2ic'] = _findDark(line)
    if line.find("pd3 dark/sec")>=0: darks['pd3ic'] = _findDark(line)
    if line.find("pd4 dark/sec")>=0: darks['pd4ic'] = _findDark(line)
  for iline,line in enumerate(lines):
    if line.lstrip()[0] != "#": break
  data=np.genfromtxt(fname,skip_header=iline-1,names=True,comments="%",dtype=None,converters = {'delay': lambda s: _delayToNum(s)})
  if subtractDark:
    for diode in ['pd1ic','pd2ic','pd3ic','pd4ic']:
      if diode in darks: data[diode]=data[diode]-darks[diode]*data['timeic']
  data = data[skip_first:last]
  if asDataStorage:
    # rstrip("_") is used to clean up last _ that appera for some reason in file_
    data = storage.DataStorage( dict((name.rstrip("_"),data[name]) for name in data.dtype.names ) )
  data.file = data.file.astype(str)
  return data



def doFolder_azav(folder,nQ=1500,files='*.edf*',force=False,mask=None,
  saveChi=True,poni='pyfai.poni',storageFile='auto',dark=9.9,zingerFilter=30,qlims=(0,10),
  removeBack=False,removeBack_kw=dict()):
  """ very small wrapper around azav.doFolder, essentially just reading
      the diagnostics.log """

  diag = dict( delays = readDelayFromDiagnostic(folder) )
  if storageFile == 'auto' : storageFile = folder +  "/" + "pyfai_1d" + default_extension

  data = azav.doFolder(folder,files=files,nQ=nQ,force=force,mask=mask,
    saveChi=saveChi,poni=poni,storageFile=storageFile,diagnostic=diag,dark=dark,save=False)
  #try:
  #  if removeBack is not None:
  #    _,data.data = azav.removeBackground(data,qlims=qlims,**removeBack_kw)
  #except Exception as e:
  #  log.error("Could not remove background, error was %s"%(str(e)))

  if zingerFilter > 0:
    data.data = filters.removeZingers(data.data,threshold=zingerFilter)
      #data.save(storageFile); it does not save err ?


#  idx = utils.findSlice(data.q,qlims)
#  n   = np.nanmean(data.data[:,idx],axis=1)
#  data.norm_range = qlims
#  data.norm = n
#  n   = utils.reshapeToBroadcast(n,data.data)
#  data.data_norm = data.data/n

  data.save(storageFile)


  return data



def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean,
                     qlims=None,outStorageFile='auto',reference='min'):
  """ azavStorage if a DataStorage instance or the filename to read """

  if isinstance(azavStorage,storage.DataStorage):
    data = azavStorage
    folder = azavStorage.folder
  elif os.path.isfile(azavStorage):
    folder = os.path.dirname(azavStorage)
    data = storage.DataStorage(azavStorage)
  else:
    # assume is just a folder name
    folder = azavStorage
    azavStorage  = folder +  "/pyfai_1d" + default_extension
    data = storage.DataStorage(azavStorage)

  #assert data.q.shape[0] == data.data.shape[1] == data.err.shape[1]
  if qlims is not None:
    idx = (data.q>qlims[0]) & (data.q<qlims[1])
    data.data = data.data[:,idx]
    data.err  = data.err[:,idx]
    data.q    = data.q[idx]



  # calculate differences
  diffs = dataReduction.calcTimeResolvedSignal(data.delays,data.data,err=data.err,
          q=data.q,reference=reference,monitor=monitor,
          funcForAveraging=funcForAveraging)

  # save txt and npz file
  dataReduction.saveTxt(folder,diffs,info=data.pyfai_info)
  if outStorageFile == 'auto':
    outStorageFile = folder + "/diffs" + default_extension
  diffs.save(outStorageFile)

  return data,diffs

def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax=20):
  import matplotlib.pyplot as plt
  if folder == "./": folder = os.path.abspath(folder)
  fig = plt.figure()
  lastNum = None
  keepGoing = True
  lines   = None
  retryNum = 0
  if online: print("Press Ctrl+C to stop")
  while keepGoing and retryNum < retryMax:
    try:
      data = doFolder_azav(folder,**azav_kw)
      # check if there are new data
      if lastNum is None or lastNum<data.data.shape[0]:
        data,diffs = doFolder_dataRed(data,**datared_kw)
        if lines is None or len(lines) != diffs.data.shape[0]:
          lines,_ = utils.plotdiffs(diffs,fig=fig,title=folder)
        else:
          utils.updateLines(lines,diffs.data)
        plt.draw()
        lastNum = data.data.shape[0]
        retryNum = 0
      else:
        retryNum += 1
      plt.pause(30)
    except KeyboardInterrupt:
      keepGoing = False
    if not online: keepGoing = False
  return data,diffs