diff --git a/xray/id9.py b/xray/id9.py
index 4bbe6c5..1561045 100644
--- a/xray/id9.py
+++ b/xray/id9.py
@@ -8,6 +8,7 @@ from . import azav
 from . import dataReduction
 from . import utils
 from . import storage
+from . import filters
 
 default_extension = ".npz"
 
@@ -18,13 +19,26 @@ def _conv(x):
         x = np.nan
     return x
 
+def _readDiagnostic(fname,retry=3):
+  ntry = 0
+  while ntry<retry:
+    try:
+      data = np.genfromtxt(fname,usecols=(2,3),\
+             dtype=None,converters={3: lambda x: _conv(x)},
+             names = ['fname','delay'])
+      return data
+    except Exception as e:
+      log.warn("Could not read diagnostic file, retrying soon,error was %s"%e)
+      ntry += 1
+  # it should not arrive here
+  raise ValueError("Could not read diagnostic file after %d attempts"%retry)
+
 def readDelayFromDiagnostic(fname):
   """ return an ordered dict dictionary of filename; for each key a rounded
       value of delay is associated """
   if os.path.isdir(fname): fname += "/diagnostics.log"
-  data = np.genfromtxt(fname,usecols=(2,3),\
-          dtype=None,converters={3: lambda x: _conv(x)},
-          names = ['fname','delay'])
+  # try to read diagnostic couple of times
+  data = _readDiagnostic(fname,retry=4)
   files = data['fname'].astype(str)
   delays = data['delay']
   # skip lines that cannot be interpreted as float (like done, etc)
@@ -35,18 +49,44 @@ def readDelayFromDiagnostic(fname):
   return collections.OrderedDict( zip(files,delays) )
 
 
-def doFolder_azav(folder,nQ=1500,force=False,mask=None,saveChi=True,
-  poni='pyfai.poni',storageFile='auto'):
+def doFolder_azav(folder,nQ=1500,files='*.edf*',force=False,mask=None,
+  saveChi=True,poni='pyfai.poni',storageFile='auto',dark=9.9,zingerFilter=30,qlims=(0,10),
+  removeBack=False,removeBack_kw=dict()):
   """ very small wrapper around azav.doFolder, essentially just reading
       the diagnostics.log """
 
   diag = dict( delays = readDelayFromDiagnostic(folder) )
   if storageFile == 'auto' : storageFile = folder +  "/" + "pyfai_1d" + default_extension
-  return azav.doFolder(folder,files="*.edf*",nQ=nQ,force=force,mask=mask,
-    saveChi=saveChi,poni=poni,storageFile=storageFile,diagnostic=diag)
+
+  data = azav.doFolder(folder,files=files,nQ=nQ,force=force,mask=mask,
+    saveChi=saveChi,poni=poni,storageFile=storageFile,diagnostic=diag,dark=dark,save=False)
+  #try:
+  #  if removeBack is not None:
+  #    _,data.data = azav.removeBackground(data,qlims=qlims,**removeBack_kw)
+  #except Exception as e:
+  #  log.error("Could not remove background, error was %s"%(str(e)))
+
+  if zingerFilter > 0:
+    data.data = filters.removeZingers(data.data,threshold=zingerFilter)
+      #data.save(storageFile); it does not save err ?
+
+
+#  idx = utils.findSlice(data.q,qlims)
+#  n   = np.nanmean(data.data[:,idx],axis=1)
+#  data.norm_range = qlims
+#  data.norm = n
+#  n   = utils.reshapeToBroadcast(n,data.data)
+#  data.data_norm = data.data/n
+
+  data.save(storageFile)
+
+
+  return data
+
+
 
 def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean,
-                     errMask=5,chi2Mask=2,qlims=None,outStorageFile='auto'):
+                     qlims=None,outStorageFile='auto',reference='min'):
   """ azavStorage if a DataStorage instance or the filename to read """
 
   if isinstance(azavStorage,storage.DataStorage):
@@ -61,22 +101,20 @@ def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean,
     azavStorage  = folder +  "/pyfai_1d" + default_extension
     data = storage.DataStorage(azavStorage)
 
+  #assert data.q.shape[0] == data.data.shape[1] == data.err.shape[1]
   if qlims is not None:
     idx = (data.q>qlims[0]) & (data.q<qlims[1])
     data.data = data.data[:,idx]
+    data.err  = data.err[:,idx]
     data.q    = data.q[idx]
-  
-  # calculate differences
-  diffs = dataReduction.calcTimeResolvedSignal(data.delays,data.data,q=data.q,\
-          reference="min",monitor=monitor,funcForAveraging=funcForAveraging)
 
-  # mask if asked so
-  if errMask>0:
-    diffs = dataReduction.errorMask(diffs,threshold=errMask)
-  if chi2Mask>0:
-    diffs = dataReduction.chi2Mask(diffs,threshold=chi2Mask)
-  diffs = dataReduction.applyMasks(diffs)
-  
+
+
+  # calculate differences
+  diffs = dataReduction.calcTimeResolvedSignal(data.delays,data.data,err=data.err,
+          q=data.q,reference=reference,monitor=monitor,
+          funcForAveraging=funcForAveraging)
+
   # save txt and npz file
   dataReduction.saveTxt(folder,diffs,info=data.pyfai_info)
   if outStorageFile == 'auto':
@@ -84,3 +122,33 @@ def doFolder_dataRed(azavStorage,monitor=None,funcForAveraging=np.nanmean,
   diffs.save(outStorageFile)
 
   return data,diffs
+
+def doFolder(folder,azav_kw = dict(), datared_kw = dict(),online=True, retryMax=20):
+  import matplotlib.pyplot as plt
+  if folder == "./": folder = os.path.abspath(folder)
+  fig = plt.figure()
+  lastNum = None
+  keepGoing = True
+  lines   = None
+  retryNum = 0
+  if online: print("Press Ctrl+C to stop")
+  while keepGoing and retryNum < retryMax:
+    try:
+      data = doFolder_azav(folder,**azav_kw)
+      # check if there are new data
+      if lastNum is None or lastNum<data.data.shape[0]:
+        data,diffs = doFolder_dataRed(data,**datared_kw)
+        if lines is None or len(lines) != diffs.data.shape[0]:
+          lines,_ = utils.plotdiffs(diffs,fig=fig,title=folder)
+        else:
+          utils.updateLines(lines,diffs.data)
+        plt.draw()
+        lastNum = data.data.shape[0]
+        retryNum = 0
+      else:
+        retryNum += 1
+      plt.pause(30)
+    except KeyboardInterrupt:
+      keepGoing = False
+    if not online: keepGoing = False
+  return data,diffs