mcammara
/
mcutils
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

improved storage module, now list of arrays can be saved in h5py (and read back)

This commit is contained in:
marco cammarata 2017-03-05 22:09:08 +01:00
parent 906ec537d8
commit 2c6f7663f6
1 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
storage.py
View File

@ -21,14 +21,18 @@ def unwrapArray(a,recursive=True,readH5pyDataset=True):
# is h5py dataset convert to array # is h5py dataset convert to array
if isinstance(a,h5py.Dataset) and readH5pyDataset: a = a[...] if isinstance(a,h5py.Dataset) and readH5pyDataset: a = a[...]
if isinstance(a,h5py.Dataset) and a.shape == (): a = a[...] if isinstance(a,h5py.Dataset) and a.shape == (): a = a[...]
if isinstance(a,h5py.Group) and "IS_LIST_OF_ARRAYS" in a.attrs:
items = list(a.keys())
items.sort()
a = np.asarray( [a[item][...] for item in items] )
if isinstance(a,np.ndarray) and a.ndim == 0 : a = a.item() if isinstance(a,np.ndarray) and a.ndim == 0 : a = a.item()
if isinstance(a,np.ndarray) and a.dtype.char == "S": a = a.astype(str) if isinstance(a,np.ndarray) and a.dtype.char == "S": a = a.astype(str)
if recursive: if recursive:
if "items" in dir(a): # dict, h5py groups, npz file if "items" in dir(a): # dict, h5py groups, npz file
a = dict(a); # convert to dict, otherwise can't asssign values a = dict(a); # convert to dict, otherwise can't asssign values
for key,value in a.items(): a[key] = unwrapArray(value) for key,value in a.items(): a[key] = unwrapArray(value)
elif isinstance(a,list): elif isinstance(a,(list,tuple)):
for index in range(len(a)): a[index] = unwrapArray(a[index]) a = [unwrapArray(element) for element in a]
else: else:
pass pass
if isinstance(a,dict): a = DataStorage(a) if isinstance(a,dict): a = DataStorage(a)
@ -49,12 +53,20 @@ def dictToH5Group(d,group):
# h5py can't handle numpy unicode arrays # h5py can't handle numpy unicode arrays
if isinstance(value,np.ndarray) and value.dtype.char == "U": if isinstance(value,np.ndarray) and value.dtype.char == "U":
value = np.asarray([vv.encode('ascii') for vv in value]) value = np.asarray([vv.encode('ascii') for vv in value])
# h5py can't save None
if value is None: value="NONE_PYTHON_OBJECT"
try:
group[key] = value group[key] = value
except TypeError: # check if it is list of array
log.error("Can't save %s"%(key)) elif isinstance(value,np.ndarray) and value.ndim == 1 and isinstance(value[0],np.ndarray):
group.create_group(key)
group[key].attrs["IS_LIST_OF_ARRAYS"] = True
for index,array in enumerate(value): group["%s/index%05d"%(key,index)] = array
# h5py can't save None
elif value is None:
group[key] = "NONE_PYTHON_OBJECT"
else:
try:
group[key] = value
except TypeError:
log.error("Can't save %s"%(key))
def dictToH5(h5,d): def dictToH5(h5,d):
""" Save a dictionary into an hdf5 file """ Save a dictionary into an hdf5 file