diff --git a/storage.py b/storage.py index 4e8ecd1..b3f2d06 100644 --- a/storage.py +++ b/storage.py @@ -21,14 +21,18 @@ def unwrapArray(a,recursive=True,readH5pyDataset=True): # is h5py dataset convert to array if isinstance(a,h5py.Dataset) and readH5pyDataset: a = a[...] if isinstance(a,h5py.Dataset) and a.shape == (): a = a[...] + if isinstance(a,h5py.Group) and "IS_LIST_OF_ARRAYS" in a.attrs: + items = list(a.keys()) + items.sort() + a = np.asarray( [a[item][...] for item in items] ) if isinstance(a,np.ndarray) and a.ndim == 0 : a = a.item() if isinstance(a,np.ndarray) and a.dtype.char == "S": a = a.astype(str) if recursive: if "items" in dir(a): # dict, h5py groups, npz file a = dict(a); # convert to dict, otherwise can't asssign values for key,value in a.items(): a[key] = unwrapArray(value) - elif isinstance(a,list): - for index in range(len(a)): a[index] = unwrapArray(a[index]) + elif isinstance(a,(list,tuple)): + a = [unwrapArray(element) for element in a] else: pass if isinstance(a,dict): a = DataStorage(a) @@ -49,12 +53,20 @@ def dictToH5Group(d,group): # h5py can't handle numpy unicode arrays if isinstance(value,np.ndarray) and value.dtype.char == "U": value = np.asarray([vv.encode('ascii') for vv in value]) - # h5py can't save None - if value is None: value="NONE_PYTHON_OBJECT" - try: group[key] = value - except TypeError: - log.error("Can't save %s"%(key)) + # check if it is list of array + elif isinstance(value,np.ndarray) and value.ndim == 1 and isinstance(value[0],np.ndarray): + group.create_group(key) + group[key].attrs["IS_LIST_OF_ARRAYS"] = True + for index,array in enumerate(value): group["%s/index%05d"%(key,index)] = array + # h5py can't save None + elif value is None: + group[key] = "NONE_PYTHON_OBJECT" + else: + try: + group[key] = value + except TypeError: + log.error("Can't save %s"%(key)) def dictToH5(h5,d): """ Save a dictionary into an hdf5 file