improved storage module, now list of arrays can be saved in h5py (and read back)

storage.py

@@ -21,14 +21,18 @@ def unwrapArray(a,recursive=True,readH5pyDataset=True):
   # is h5py dataset convert to array
   if isinstance(a,h5py.Dataset) and readH5pyDataset: a = a[...]
   if isinstance(a,h5py.Dataset) and a.shape == (): a = a[...]
+  if isinstance(a,h5py.Group)   and "IS_LIST_OF_ARRAYS" in a.attrs:
+    items = list(a.keys())
+    items.sort()
+    a = np.asarray( [a[item][...] for item in items] )
   if isinstance(a,np.ndarray)   and a.ndim  == 0 : a = a.item()
   if isinstance(a,np.ndarray)   and a.dtype.char == "S": a = a.astype(str)
   if recursive:
     if "items" in dir(a): # dict, h5py groups, npz file
       a = dict(a); # convert to dict, otherwise can't asssign values
       for key,value in a.items(): a[key] = unwrapArray(value)
-    elif isinstance(a,list):
-      for index in range(len(a)): a[index] = unwrapArray(a[index])
+    elif isinstance(a,(list,tuple)):
+      a = [unwrapArray(element) for element in a]
     else:
       pass
   if isinstance(a,dict): a = DataStorage(a)
@@ -49,12 +53,20 @@ def dictToH5Group(d,group):
       # h5py can't handle numpy unicode arrays
       if isinstance(value,np.ndarray) and value.dtype.char == "U":
         value = np.asarray([vv.encode('ascii') for vv in value])
-      # h5py can't save None
-      if value is None: value="NONE_PYTHON_OBJECT"
-      try:
         group[key] = value
-      except TypeError:
-        log.error("Can't save %s"%(key))
+      # check if it is list of array
+      elif isinstance(value,np.ndarray) and value.ndim == 1 and isinstance(value[0],np.ndarray):
+        group.create_group(key)
+        group[key].attrs["IS_LIST_OF_ARRAYS"] = True
+        for index,array in enumerate(value): group["%s/index%05d"%(key,index)] = array
+      # h5py can't save None
+      elif value is None:
+        group[key] = "NONE_PYTHON_OBJECT"
+      else:
+        try:
+          group[key] = value
+        except TypeError:
+          log.error("Can't save %s"%(key))
 
 def dictToH5(h5,d):
   """ Save a dictionary into an hdf5 file