Skip to content

GitLab

Commit 91bcdaf2 authored by Pierre Paleo's avatar Pierre Paleo
Browse files

Merge branch 'add_hdf5' of gitlab.esrf.fr:tomotools/tomoscan into add_hdf5

parents 744e6dbf f5d0938f
Pipeline #22279 failed with stages
in 1 minute and 2 seconds
Hide whitespace changes
Inline Side-by-side
tomoscan/esrf/edfscan.py
View file @ 91bcdaf2
......@@ -207,7 +207,7 @@ class EDFTomoScan(TomoScanBase):
def is_abort(self, **kwargs) -> bool:
abort_file = os.path.basename(self.path) + self.ABORT_FILE
abort_file = os.path.join(self.path, abort_file)
if 'src_pattern' in kwargs and kwargs['src_pattern' is not None]:
if 'src_pattern' in kwargs and kwargs['src_pattern'] is not None:
assert 'dest_pattern' in kwargs
abort_file = abort_file.replace(kwargs['src_pattern'],
kwargs['dest_pattern'])
......@@ -235,9 +235,10 @@ class EDFTomoScan(TomoScanBase):
@docstring(TomoScanBase.update)
def update(self):
self.projections = EDFTomoScan.get_proj_urls(self.path, n_frames=self._edf_n_frames)
self._darks = EDFTomoScan.get_darks_url(self.path)
self._flats = EDFTomoScan.get_refs_url(self.path)
if self.path is not None:
self.projections = EDFTomoScan.get_proj_urls(self.path, n_frames=self._edf_n_frames)
self._darks = EDFTomoScan.get_darks_url(self.path)
self._flats = EDFTomoScan.get_refs_url(self.path)
@docstring(TomoScanBase.load_from_dict)
def load_from_dict(self, desc: Union[dict, io.TextIOWrapper]):
......@@ -245,11 +246,11 @@ class EDFTomoScan(TomoScanBase):
data = json.load(desc)
else:
data = desc
if not (self._DICT_TYPE_KEY in data and data[self._DICT_TYPE_KEY] == self._TYPE):
if not (self.DICT_TYPE_KEY in data and data[self.DICT_TYPE_KEY] == self._TYPE):
raise ValueError('Description is not an EDFScan json description')
assert self._DICT_PATH_KEY in data
self.path = data[self._DICT_PATH_KEY]
assert self.DICT_PATH_KEY in data
self.path = data[self.DICT_PATH_KEY]
return self
@staticmethod
......
tomoscan/esrf/hdf5scan.py
View file @ 91bcdaf2
......@@ -84,6 +84,8 @@ class HDF5TomoScan(TomoScanBase):
_IMG_KEY_PATH = 'instrument/detector/image_key'
_IMG_KEY_CONTROL_PATH = 'instrument/detector/image_key_control'
_X_PIXEL_SIZE_PATH = 'instrument/detector/x_pixel_size'
_Y_PIXEL_SIZE_PATH = 'instrument/detector/y_pixel_size'
......@@ -127,6 +129,8 @@ class HDF5TomoScan(TomoScanBase):
else:
self._entry = entry or self._get_entry_at(index=index,
file_path=self.master_file)
if self._entry is None:
raise ValueError('unable to find a valid entry for %s' % self.master_file)
# for now the default entry is 1_tomo but should change with time
# data caches
......@@ -148,6 +152,7 @@ class HDF5TomoScan(TomoScanBase):
# pixel dimensions (tuple)
self._frames = None
self._image_keys = None
self._image_keys_control = None
self._rotation_angles = None
self._distance = None
self._energy = None
......@@ -182,6 +187,7 @@ class HDF5TomoScan(TomoScanBase):
self._y_pixel_size = None
self._rotation_angles = None
self._distance = None
self._image_keys_control = None
@staticmethod
def _get_entry_at(index: int, file_path: str) -> str:
......@@ -191,14 +197,14 @@ class HDF5TomoScan(TomoScanBase):
:param file_path:
:return:
"""
entries = HDF5TomoScan._get_valid_entries(file_path)
entries = HDF5TomoScan.get_valid_entries(file_path)
if len(entries) == 0:
return None
else:
return entries[index]
@staticmethod
def _get_valid_entries(file_path: str) -> tuple:
def get_valid_entries(file_path: str) -> tuple:
"""
return the list of 'Nxtomo' entries at the root level
......@@ -218,9 +224,9 @@ class HDF5TomoScan(TomoScanBase):
for root_node in h5f.keys():
node = h5f[root_node]
if HDF5TomoScan.node_is_nxtomo(node) is True:
res_buf.append(node)
res_buf.append(root_node) # cannnot be node because of sym links
[res.append(node.name) for node in res_buf]
[res.append(node) for node in res_buf]
res.sort()
return tuple(res)
......@@ -246,7 +252,7 @@ class HDF5TomoScan(TomoScanBase):
else:
master_file = HDF5TomoScan.get_master_file(scan_path=directory)
if master_file:
entries = HDF5TomoScan._get_valid_entries(file_path=master_file)
entries = HDF5TomoScan.get_valid_entries(file_path=master_file)
return len(entries) > 0
@docstring(TomoScanBase.is_abort)
......@@ -277,13 +283,13 @@ class HDF5TomoScan(TomoScanBase):
data = json.load(_dict)
else:
data = _dict
if not (self._DICT_TYPE_KEY in data and data[self._DICT_TYPE_KEY] == self._TYPE):
if not (self.DICT_TYPE_KEY in data and data[self.DICT_TYPE_KEY] == self._TYPE):
raise ValueError('Description is not an HDF5Scan json description')
if HDF5TomoScan._DICT_ENTRY_KEY not in data:
raise ValueError('No hdf5 entry specified')
assert self._DICT_PATH_KEY in data
self.path = data[self._DICT_PATH_KEY]
assert self.DICT_PATH_KEY in data
self.path = data[self.DICT_PATH_KEY]
self._entry = data[self._DICT_ENTRY_KEY]
return self
......@@ -297,7 +303,7 @@ class HDF5TomoScan(TomoScanBase):
"""projections / radio, does not include the return projections"""
if self._projections is None:
if self.frames:
proj_frames = tuple(filter(lambda x: x.image_key is ImageKey.PROJECTION and x.is_return is False, self.frames))
proj_frames = tuple(filter(lambda x: x.image_key is ImageKey.PROJECTION and x.is_control is False, self.frames))
self._projections = {}
for proj_frame in proj_frames:
self._projections[proj_frame.index] = proj_frame.url
......@@ -333,7 +339,7 @@ class HDF5TomoScan(TomoScanBase):
def update(self) -> None:
"""update list of radio and reconstruction by parsing the scan folder
"""
if not os.path.exists(self.master_file):
if self.master_file is None or not os.path.exists(self.master_file):
return
self.projections = self._get_projections_url()
# TODO: update darks and flats too
......@@ -373,7 +379,7 @@ class HDF5TomoScan(TomoScanBase):
""""""
frames = self.frames
if frames:
return_frames = list(filter(lambda x: x.is_return is True, frames))
return_frames = list(filter(lambda x: x.is_control is True, frames))
return return_frames
else:
return None
......@@ -381,6 +387,7 @@ class HDF5TomoScan(TomoScanBase):
@property
def rotation_angle(self) -> typing.Union[None, list]:
if self._rotation_angles is None:
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r') as h5_file:
_rotation_angles = h5_file[self._entry][self._ROTATION_ANGLE_PATH][()]
# cast in float
......@@ -389,12 +396,23 @@ class HDF5TomoScan(TomoScanBase):
@property
def image_key(self) -> typing.Union[list, None]:
if self._image_keys is None:
if self._entry and self._image_keys is None:
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r') as h5_file:
assert self._entry in h5_file
self._image_keys = h5_file[self._entry][self._IMG_KEY_PATH][()]
return self._image_keys
@property
def image_key_control(self) -> typing.Union[list, None]:
if self._entry and self._image_keys_control is None:
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r') as h5_file:
if self._IMG_KEY_CONTROL_PATH in h5_file[self._entry]:
self._image_keys_control = h5_file[self._entry][self._IMG_KEY_CONTROL_PATH][()]
else:
self._image_keys_control = None
return self._image_keys_control
@docstring(TomoScanBase.dark_n)
@property
def dark_n(self) -> typing.Union[None, int]:
......@@ -483,6 +501,7 @@ class HDF5TomoScan(TomoScanBase):
def distance(self) -> typing.Union[None, float]:
if (self._distance is None and self.master_file and
os.path.exists(self.master_file)):
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r') as h5_file:
distance_dataset = h5_file[self._entry][self._DISTANCE_PATH]
self._distance = self._get_value(distance_dataset, default_unit='m')
......@@ -492,6 +511,7 @@ class HDF5TomoScan(TomoScanBase):
def energy(self) -> typing.Union[None, float]:
if (self._energy is None and self.master_file and
os.path.exists(self.master_file)):
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r') as h5_file:
energy_dataset = h5_file[self._entry][self._ENERGY_PATH]
self._energy = self._get_value(energy_dataset, default_unit='keV')
......@@ -534,11 +554,15 @@ class HDF5TomoScan(TomoScanBase):
frame = Frame(index=i_frame, url=url, image_key=img_key,
rotation_angle=rot_a)
return_already_reach, delta_angle = is_return(lframe=frame,
llast_proj_frame=last_proj_frame,
ldelta_angle=delta_angle,
return_already_reach=return_already_reach)
frame.is_return = return_already_reach
if self.image_key_control is not None:
is_control_frame = self.image_key_control[frame.index]
else:
return_already_reach, delta_angle = is_return(lframe=frame,
llast_proj_frame=last_proj_frame,
ldelta_angle=delta_angle,
return_already_reach=return_already_reach)
is_control_frame = return_already_reach
frame.is_control_frame = is_control_frame
self._frames.append(frame)
last_proj_frame = frame
self._frames = tuple(self._frames)
......@@ -550,7 +574,7 @@ class HDF5TomoScan(TomoScanBase):
res = {}
for frame in self.frames:
if frame.image_key is ImageKey.PROJECTION:
if frame.is_return is False:
if frame.is_control is False:
res[frame.rotation_angle] = frame.url
else:
res[str(frame.rotation_angle) + '(1)'] = frame.url
......@@ -578,19 +602,29 @@ class HDF5TomoScan(TomoScanBase):
unit = default_unit
return value * metricsystem.MetricSystem.from_value(unit).value
def _check_hdf5scan_validity(self):
if self.master_file is None:
raise ValueError('No master file provided')
if self.entry is None:
raise ValueError('No entry provided')
with h5py.File(self.master_file, 'r') as h5_file:
if self._entry not in h5_file:
raise ValueError('Given entry %s is not in the master '
'file %s' % (self._entry, self.master_file))
class Frame:
"""class to store all metadata information of a frame"""
def __init__(self, index: int, url: typing.Union[None, DataUrl] = None,
image_key: typing.Union[None, ImageKey, int] = None,
rotation_angle: typing.Union[None, float] = None,
is_return_proj: bool = False):
is_control_proj: bool = False):
assert type(index) is int
self._index = index
self._image_key = ImageKey.from_value(image_key)
self._rotation_angle = rotation_angle
self._url = url
self._is_return_frame = is_return_proj
self._is_control_frame = is_control_proj
self._data = None
@property
......@@ -618,9 +652,9 @@ class Frame:
return self._url
@property
def is_return(self) -> bool:
return self._is_return_frame
def is_control(self) -> bool:
return self._is_control_frame
@is_return.setter
def is_return(self, is_return: bool):
self._is_return_frame = is_return
@is_control.setter
def is_control(self, is_return: bool):
self._is_control_frame = is_return
tomoscan/esrf/test/test_edfscan.py
View file @ 91bcdaf2
......@@ -307,10 +307,6 @@ class TestScanValidatorFindFiles(unittest.TestCase):
if os.path.isdir(self.path):
shutil.rmtree(self.path)
def testGetRadioPaths(self):
nFound = len(EDFTomoScan.get_proj_urls(self.path))
self.assertTrue(nFound == self.N_RADIO)
class TestRadioPath(unittest.TestCase):
"""Test static method getRadioPaths for EDFTomoScan"""
......
tomoscan/esrf/test/test_hdf5scan.py
View file @ 91bcdaf2
......@@ -97,7 +97,7 @@ class TestHDF5Scan(HDF5TestBaseClass):
self.assertTrue(isinstance(proj_2, Frame))
self.assertEqual(proj_2.index, 24)
numpy.isclose(proj_2.rotation_angle, 0.24)
self.assertFalse(proj_2.is_return)
self.assertFalse(proj_2.is_control)
self.assertEqual(proj_2.url.file_path(), self.scan.master_file)
self.assertEqual(proj_2.url.data_path(), '/entry0000/instrument/detector/data')
self.assertEqual(proj_2.url.data_slice(), 24)
......@@ -108,13 +108,13 @@ class TestHDF5Scan(HDF5TestBaseClass):
with self.subTest(frame_index=frame_index):
frame = frames[frame_index]
self.assertTrue(frame.image_key, ImageKey.PROJECTION)
self.assertFalse(frame.is_return)
self.assertFalse(frame.is_control)
# check some darks
dark_0 = frames[0]
self.assertEqual(dark_0.index, 0)
numpy.isclose(dark_0.rotation_angle, 0.0)
self.assertFalse(dark_0.is_return)
self.assertFalse(dark_0.is_control)
self.assertEqual(dark_0.url.file_path(), self.scan.master_file)
self.assertEqual(dark_0.url.data_path(), '/entry0000/instrument/detector/data')
self.assertEqual(dark_0.url.data_slice(), 0)
......@@ -125,7 +125,7 @@ class TestHDF5Scan(HDF5TestBaseClass):
ref_1 = frames[2]
self.assertEqual(ref_1.index, 2)
numpy.isclose(ref_1.rotation_angle, 0.0)
self.assertFalse(ref_1.is_return)
self.assertFalse(ref_1.is_control)
self.assertEqual(ref_1.url.file_path(), self.scan.master_file)
self.assertEqual(ref_1.url.data_path(),
'/entry0000/instrument/detector/data')
......@@ -138,7 +138,7 @@ class TestHDF5Scan(HDF5TestBaseClass):
self.assertTrue(isinstance(r_proj_0, Frame))
self.assertEqual(r_proj_0.index, 1543)
numpy.isclose(r_proj_0.rotation_angle, 180)
self.assertTrue(r_proj_0.is_return)
self.assertTrue(r_proj_0.is_control)
self.assertEqual(r_proj_0.url.file_path(), self.scan.master_file)
self.assertEqual(r_proj_0.url.data_path(),
'/entry0000/instrument/detector/data')
......
tomoscan/scanbase.py
View file @ 91bcdaf2
......@@ -46,9 +46,9 @@ class TomoScanBase:
:param scan: path to the root folder containing the scan.
:type: Union[str,None]
"""
_DICT_TYPE_KEY = 'type'
DICT_TYPE_KEY = 'type'
_DICT_PATH_KEY = 'path'
DICT_PATH_KEY = 'path'
_SCHEME = None
"""scheme to read data url for this type of acquisition"""
......@@ -212,8 +212,8 @@ class TomoScanBase:
:rtype: dict
"""
res = dict()
res[self._DICT_TYPE_KEY] = self.type
res[self._DICT_PATH_KEY] = self.path
res[self.DICT_TYPE_KEY] = self.type
res[self.DICT_PATH_KEY] = self.path
return res
def load_from_dict(self, _dict: dict):
......@@ -305,27 +305,27 @@ class TomoScanBase:
logger.warning('incoherent data information to retrieve'
'scan extra images angle')
elif len(extraImgs) == 4:
res['270(1)'] = extraImgs[0]
res['180(1)'] = extraImgs[1]
res['90(1)'] = extraImgs[2]
res['0(1)'] = extraImgs[3]
res['270(1)'] = ordered_url[extraImgs[0]]
res['180(1)'] = ordered_url[extraImgs[1]]
res['90(1)'] = ordered_url[extraImgs[2]]
res['0(1)'] = ordered_url[extraImgs[3]]
else:
res['360(1)'] = extraImgs[0]
res['270(1)'] = extraImgs[1]
res['180(1)'] = extraImgs[2]
res['90(1)'] = extraImgs[3]
res['0(1)'] = extraImgs[4]
res['360(1)'] = ordered_url[extraImgs[0]]
res['270(1)'] = ordered_url[extraImgs[1]]
res['180(1)'] = ordered_url[extraImgs[2]]
res['90(1)'] = ordered_url[extraImgs[3]]
res['0(1)'] = ordered_url[extraImgs[4]]
elif len(extraImgs) in (2, 3):
if scan_range > 180:
logger.warning('incoherent data information to retrieve'
'scan extra images angle')
elif len(extraImgs) is 3:
res['180(1)'] = extraImgs[0]
res['90(1)'] = extraImgs[1]
res['0(1)'] = extraImgs[2]
res['180(1)'] = ordered_url[extraImgs[0]]
res['90(1)'] = ordered_url[extraImgs[1]]
res['0(1)'] = ordered_url[extraImgs[2]]
else:
res['90(1)'] = extraImgs[0]
res['0(1)'] = extraImgs[1]
res['90(1)'] = ordered_url[extraImgs[0]]
res['0(1)'] = ordered_url[extraImgs[1]]
else:
raise ValueError('incoherent data information to retrieve scan'
'extra images angle')
......
tomoscan/scanfactory.py
View file @ 91bcdaf2
......@@ -72,7 +72,7 @@ class ScanFactory:
elif HDF5TomoScan.is_tomoscan_dir(scan_path):
scans = []
master_file = HDF5TomoScan.get_master_file(scan_path=scan_path)
entries = HDF5TomoScan._get_valid_entries(master_file)
entries = HDF5TomoScan.get_valid_entries(master_file)
for entry in entries:
scans.append(HDF5TomoScan(scan=scan_path, entry=entry,
index=None))
......@@ -90,13 +90,13 @@ class ScanFactory:
:return: instance of TomoScanBase
:rtype: TomoScanBase
"""
if TomoScanBase._DICT_TYPE_KEY not in _dict:
if TomoScanBase.DICT_TYPE_KEY not in _dict:
raise ValueError('given dict is not recognized. Cannot find'
'', TomoScanBase._DICT_TYPE_KEY)
elif _dict[TomoScanBase._DICT_TYPE_KEY] == EDFTomoScan._TYPE:
'', TomoScanBase.DICT_TYPE_KEY)
elif _dict[TomoScanBase.DICT_TYPE_KEY] == EDFTomoScan._TYPE:
return EDFTomoScan(scan=None).load_from_dict(_dict)
else:
raise ValueError('Scan type', _dict[TomoScanBase._DICT_TYPE_KEY],
raise ValueError('Scan type', _dict[TomoScanBase.DICT_TYPE_KEY],
'is not managed')
@staticmethod
......@@ -117,9 +117,9 @@ class ScanFactory:
"""Create a ScanBase instance from a json description"""
data = json.load(desc)
if TomoScanBase._DICT_TYPE_KEY not in data:
if TomoScanBase.DICT_TYPE_KEY not in data:
raise ValueError('json not recognize')
elif data[TomoScanBase._DICT_TYPE_KEY] == EDFTomoScan._TYPE:
elif data[TomoScanBase.DICT_TYPE_KEY] == EDFTomoScan._TYPE:
scan = EDFTomoScan(scan=None).load_from_dict(data)
return scan
else:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment