Commit ff26348e authored by payno's avatar payno
Browse files

Merge branch 'add_hdf5' into 'master'

Add hdf5

See merge request !9
parents b5d67306 39adf495
Pipeline #22665 passed with stages
in 2 minutes
TODO
\ No newline at end of file
This library is offering an abstraction to access tomography data from various file format (.edf, .hdf5)
\ No newline at end of file
tomoscan
========
tomoscan aims to provide an unified interface to read tomography data from .edf or .hdf5 (NXtomo) acquisitions.
.. toctree::
:maxdepth: 4
......
......@@ -22,6 +22,8 @@
#
#############################################################################*/
"""contains EDFTomoScan, class to be used with EDF acquisition"""
__authors__ = ["H.Payno"]
__license__ = "MIT"
......@@ -84,8 +86,25 @@ class EDFTomoScan(TomoScanBase):
self.__ref_on = None
self.__scan_range = None
self._edf_n_frames = n_frames
self.__distance = None
self.__energy = None
self.update()
@docstring(TomoScanBase.clear_caches)
def clear_caches(self):
self._darks = None
self._flats = None
self._projections = None
self.__tomo_n = None
self.__ref_n = None
self.__dark_n = None
self.__dim1 = None
self.__dim2 = None
self.__pixel_size = None
self.__ref_on = None
self.__scan_range = None
@docstring(TomoScanBase.tomo_n)
@property
def tomo_n(self) -> Union[None, int]:
......@@ -190,7 +209,7 @@ class EDFTomoScan(TomoScanBase):
def is_abort(self, **kwargs) -> bool:
abort_file = os.path.basename(self.path) + self.ABORT_FILE
abort_file = os.path.join(self.path, abort_file)
if 'src_pattern' in kwargs and kwargs['src_pattern' is not None]:
if 'src_pattern' in kwargs and kwargs['src_pattern'] is not None:
assert 'dest_pattern' in kwargs
abort_file = abort_file.replace(kwargs['src_pattern'],
kwargs['dest_pattern'])
......@@ -218,9 +237,10 @@ class EDFTomoScan(TomoScanBase):
@docstring(TomoScanBase.update)
def update(self):
self.projections = EDFTomoScan.get_proj_urls(self.path, n_frames=self._edf_n_frames)
self._darks = EDFTomoScan.get_darks_url(self.path)
self._flats = EDFTomoScan.get_refs_url(self.path)
if self.path is not None:
self.projections = EDFTomoScan.get_proj_urls(self.path, n_frames=self._edf_n_frames)
self._darks = EDFTomoScan.get_darks_url(self.path)
self._flats = EDFTomoScan.get_refs_url(self.path)
@docstring(TomoScanBase.load_from_dict)
def load_from_dict(self, desc: Union[dict, io.TextIOWrapper]):
......@@ -228,11 +248,11 @@ class EDFTomoScan(TomoScanBase):
data = json.load(desc)
else:
data = desc
if not (self._DICT_TYPE_KEY in data and data[self._DICT_TYPE_KEY] == self._TYPE):
if not (self.DICT_TYPE_KEY in data and data[self.DICT_TYPE_KEY] == self._TYPE):
raise ValueError('Description is not an EDFScan json description')
assert self._DICT_PATH_KEY in data
self.path = data[self._DICT_PATH_KEY]
assert self.DICT_PATH_KEY in data
self.path = data[self.DICT_PATH_KEY]
return self
@staticmethod
......@@ -303,8 +323,7 @@ class EDFTomoScan(TomoScanBase):
def extract_index(my_str, type_):
res = []
modified_str = copy.copy(my_str)
while modified_str[-1].isdigit():
while modified_str != "" and modified_str[-1].isdigit():
res.append(modified_str[-1])
modified_str = modified_str[:-1]
if len(res) == 0:
......@@ -315,7 +334,7 @@ class EDFTomoScan(TomoScanBase):
return int(''.join(orignalOrder)), modified_str
else:
return float('.'.join(('0', ''.join(orignalOrder)))), modified_str
_file = os.path.basename(_file)
if _file.endswith('.edf'):
name = _file.replace(basename, '', 1)
name = name.rstrip('.edf')
......@@ -407,7 +426,7 @@ class EDFTomoScan(TomoScanBase):
if self.__distance is None:
return None
else:
return self.__distance
return self.__distance * MetricSystem.MILLIMETER.value
@property
@docstring(TomoScanBase.energy)
......@@ -442,7 +461,7 @@ class EDFTomoScan(TomoScanBase):
# for now pixel size are stored in microns.
# We want to return them in meter
if value is not None:
return value * MetricSystem.MICROMETER
return value * MetricSystem.MICROMETER.value
else:
return None
......
This diff is collapsed.
......@@ -259,7 +259,9 @@ class TestProjections(unittest.TestCase):
self.assertEqual(len(scan.projections), 3)
scan.update()
self.assertEqual(len(scan.projections), 4)
self.assertTrue(isinstance(scan.projections[0], silx.io.url.DataUrl))
index_0 = list(scan.projections.keys())[0]
self.assertTrue(isinstance(scan.projections[index_0], silx.io.url.DataUrl))
def testProjectionWithExtraRadio(self):
mock = MockEDF(scan_path=self.folder, n_radio=11, n_extra_radio=2,
......@@ -305,10 +307,6 @@ class TestScanValidatorFindFiles(unittest.TestCase):
if os.path.isdir(self.path):
shutil.rmtree(self.path)
def testGetRadioPaths(self):
nFound = len(EDFTomoScan.get_proj_urls(self.path))
self.assertTrue(nFound == self.N_RADIO)
class TestRadioPath(unittest.TestCase):
"""Test static method getRadioPaths for EDFTomoScan"""
......
......@@ -33,7 +33,9 @@ import shutil
import os
import tempfile
from tomoscan.test.utils import UtilsTest
from tomoscan.esrf.hdf5scan import HDF5TomoScan
from tomoscan.esrf.hdf5scan import HDF5TomoScan, ImageKey, Frame
from ...unitsystem import metricsystem
from silx.io.utils import get_data
import numpy
......@@ -56,69 +58,148 @@ class TestHDF5Scan(HDF5TestBaseClass):
"""Basic test for the hdf5 scan"""
def setUp(self) -> None:
super(TestHDF5Scan, self).setUp()
self.dataset_file = self.get_dataset('data_test2.h5')
self.scan = HDF5TomoScan(self.dataset_file)
self.dataset_file = self.get_dataset('frm_edftomomill_twoentries.nx')
self.scan = HDF5TomoScan(scan=self.dataset_file)
def testGeneral(self):
"""some general on the HDF5Scan """
self.assertEqual(self.scan.master_file, self.dataset_file)
self.assertEqual(self.scan.path, os.path.dirname(self.dataset_file))
self.assertEqual(self.scan.type, 'hdf5')
self.assertEqual(self.scan.entry, 'entry0000')
self.assertEqual(len(self.scan.flats), 42)
self.assertEqual(len(self.scan.darks), 1)
self.assertEqual(len(self.scan.return_projs), 3)
proj_angles = self.scan.get_proj_angle_url()
self.assertEqual(len(proj_angles), 1500 + 3)
self.assertTrue(90 in proj_angles)
self.assertTrue(24.0 in proj_angles)
self.assertTrue('90.0(1)' in proj_angles)
self.assertTrue('180.0(1)' in proj_angles)
self.assertTrue(179.88 not in proj_angles)
url_1 = proj_angles[0]
self.assertTrue(url_1.is_valid())
self.assertTrue(url_1.is_absolute())
self.assertEquals(url_1.scheme(), 'silx')
# check conversion to dict
_dict = self.scan.to_dict()
scan2 = HDF5TomoScan.from_dict(_dict)
self.assertEqual(scan2.path, self.scan.path)
radios_urls = self.scan.get_proj_angle_url()
self.assertEqual(len(radios_urls), 100)
url_1 = radios_urls[0]
self.assertTrue(url_1.is_valid())
self.assertFalse(url_1.is_absolute())
self.assertEquals(url_1.scheme(), 'silx')
self.assertEqual(scan2.entry, self.scan.entry)
def testFrames(self):
"""Check the `frames` property which is massively used under the
HDF5TomoScan class"""
frames = self.scan.frames
# check some projections
proj_2 = frames[24]
self.assertTrue(isinstance(proj_2, Frame))
self.assertEqual(proj_2.index, 24)
numpy.isclose(proj_2.rotation_angle, 0.24)
self.assertFalse(proj_2.is_control)
self.assertEqual(proj_2.url.file_path(), self.scan.master_file)
self.assertEqual(proj_2.url.data_path(), 'entry0000/instrument/detector/data')
self.assertEqual(proj_2.url.data_slice(), 24)
self.assertEqual(proj_2.image_key, ImageKey.PROJECTION)
self.assertEqual(get_data(proj_2.url).shape, (20, 20))
# check last two non-return projection
for frame_index in (1520, 1542):
with self.subTest(frame_index=frame_index):
frame = frames[frame_index]
self.assertTrue(frame.image_key, ImageKey.PROJECTION)
self.assertFalse(frame.is_control)
# check some darks
dark_0 = frames[0]
self.assertEqual(dark_0.index, 0)
numpy.isclose(dark_0.rotation_angle, 0.0)
self.assertFalse(dark_0.is_control)
self.assertEqual(dark_0.url.file_path(), self.scan.master_file)
self.assertEqual(dark_0.url.data_path(), 'entry0000/instrument/detector/data')
self.assertEqual(dark_0.url.data_slice(), 0)
self.assertEqual(dark_0.image_key, ImageKey.DARK_FIELD)
self.assertEqual(get_data(dark_0.url).shape, (20, 20))
# check some refs
ref_1 = frames[2]
self.assertEqual(ref_1.index, 2)
numpy.isclose(ref_1.rotation_angle, 0.0)
self.assertFalse(ref_1.is_control)
self.assertEqual(ref_1.url.file_path(), self.scan.master_file)
self.assertEqual(ref_1.url.data_path(),
'entry0000/instrument/detector/data')
self.assertEqual(ref_1.url.data_slice(), 2)
self.assertEqual(ref_1.image_key, ImageKey.FLAT_FIELD)
self.assertEqual(get_data(ref_1.url).shape, (20, 20))
# check some return projections
r_proj_0 = frames[1543]
self.assertTrue(isinstance(r_proj_0, Frame))
self.assertEqual(r_proj_0.index, 1543)
numpy.isclose(r_proj_0.rotation_angle, 180)
self.assertTrue(r_proj_0.is_control)
self.assertEqual(r_proj_0.url.file_path(), self.scan.master_file)
self.assertEqual(r_proj_0.url.data_path(),
'entry0000/instrument/detector/data')
self.assertEqual(r_proj_0.url.data_slice(), 1543)
self.assertEqual(r_proj_0.image_key, ImageKey.PROJECTION)
self.assertEqual(get_data(r_proj_0.url).shape, (20, 20))
def testProjections(self):
"""Make sure projections are valid"""
projections = self.scan.projections
self.assertEqual(len(projections), 100)
proj_1 = projections[0]
self.assertEqual(proj_1.file_path(),
'../../../../../../users/opid19/W:/clemence/visualtomo/data_test2/tomo0001/tomo_0000.h5')
self.assertEqual(proj_1.data_slice(), ('0',))
self.assertTrue(100 not in projections)
@unittest.skip('no valid hdf5 acquisition defined yet')
self.assertEqual(len(self.scan.projections), 1500)
url_0 = projections[list(projections.keys())[0]]
self.assertEqual(url_0.file_path(), os.path.join(self.scan.master_file))
self.assertEqual(url_0.data_slice(), 22)
def testDark(self):
n_dark = 20
"""Make sure darks are valid"""
n_dark = 1
self.assertEqual(self.scan.dark_n, n_dark)
with self.assertRaises(NotImplementedError):
self.scan.darks
darks = self.scan.darks
self.assertEqual(len(darks), 1)
# TODO check accumulation time
def testFlats(self):
"""Make sure flats are valid"""
n_flats = 42
flats = self.scan.flats
self.assertEqual(len(flats), n_flats)
self.assertEqual(self.scan.ref_n, n_flats)
with self.assertRaises(NotImplementedError):
flats = self.scan.flats
n_ref = 21
self.assertEqual(self.scan.ref_n, n_ref)
self.scan.ff_interval
# for now not implemented
# not implemented at the moment
data = numpy.arange(2048*2048)
data.reshape(2048, 2048)
# not implemented at the moment
self.assertEqual(self.scan.ff_interval, 21)
def testDims(self):
self.assertEqual(self.scan.dim_1, 20)
self.assertEqual(self.scan.dim_2, 20)
def testAxisUtils(self):
self.assertEqual(self.scan.scan_range, 360)
self.assertEqual(self.scan.tomo_n, 100)
# self.assertEqual(self.scan.dim_1, 2048)
# self.assertEqual(self.scan.dim_2, 2048)
self.assertEqual(self.scan.scan_range, 180)
self.assertEqual(self.scan.tomo_n, 1500)
proj0_file_path = '../../../../../../users/opid19/W:/clemence/visualtomo/data_test2/tomo0001/tomo_0000.h5'
radios_urls_evolution = self.scan.get_proj_angle_url()
self.assertEquals(len(radios_urls_evolution), 100)
self.assertEquals(radios_urls_evolution[0].file_path(), proj0_file_path)
self.assertEquals(radios_urls_evolution[0].data_slice(), ('0',))
self.assertEquals(radios_urls_evolution[0].data_path(), '/entry_0000/measurement/pcoedge64/data')
self.assertEquals(len(radios_urls_evolution), 1503)
self.assertEquals(radios_urls_evolution[0].file_path(), self.scan.master_file)
self.assertEquals(radios_urls_evolution[0].data_slice(), 22)
self.assertEquals(radios_urls_evolution[0].data_path(), 'entry0000/instrument/detector/data')
def testDarkRefUtils(self):
self.assertEqual(self.scan.tomo_n, 100)
self.assertEqual(self.scan.pixel_size[1], 6.5)
self.assertEqual(self.scan.tomo_n, 1500)
pixel_size = self.scan.pixel_size
self.assertTrue(pixel_size is not None)
self.assertTrue(numpy.isclose(self.scan.pixel_size,
0.05 * metricsystem.MetricSystem.MICROMETER.value))
self.assertTrue(numpy.isclose(self.scan.get_pixel_size(unit='micrometer'), 0.05))
self.assertTrue(numpy.isclose(self.scan.x_pixel_size,
0.05 * metricsystem.MetricSystem.MICROMETER.value))
self.assertTrue(numpy.isclose(self.scan.y_pixel_size,
0.05 * metricsystem.MetricSystem.MICROMETER.value))
def testNabuUtil(self):
self.assertTrue(numpy.isclose(self.scan.distance, -19.9735))
self.assertTrue(numpy.isclose(self.scan.get_distance(unit='cm'), -1997.35))
def suite():
......
......@@ -46,9 +46,9 @@ class TomoScanBase:
:param scan: path to the root folder containing the scan.
:type: Union[str,None]
"""
_DICT_TYPE_KEY = 'type'
DICT_TYPE_KEY = 'type'
_DICT_PATH_KEY = 'path'
DICT_PATH_KEY = 'path'
_SCHEME = None
"""scheme to read data url for this type of acquisition"""
......@@ -57,6 +57,11 @@ class TomoScanBase:
self.path = scan
self._type = type_
def clear_caches(self):
"""clear caches. Might be call if some data changed after
first read of data or metadata"""
raise NotImplementedError('Base class')
@property
def path(self) -> Union[None, str]:
"""
......@@ -112,7 +117,7 @@ class TomoScanBase:
self._flats = flats
@property
def darks(self) -> Union[None,dict]:
def darks(self) -> Union[None, dict]:
"""list of darks files"""
return self._darks
......@@ -135,6 +140,7 @@ class TomoScanBase:
@property
def tomo_n(self) -> Union[None, int]:
"""number of projection WITHOUT the return projections"""
raise NotImplementedError('Base class')
@property
......@@ -147,7 +153,7 @@ class TomoScanBase:
def get_pixel_size(self, unit='m') -> Union[None, float]:
if self.pixel_size:
return self.pixel_size / MetricSystem.from_value(unit)
return self.pixel_size / MetricSystem.from_value(unit).value
else:
return None
......@@ -206,8 +212,8 @@ class TomoScanBase:
:rtype: dict
"""
res = dict()
res[self._DICT_TYPE_KEY] = self.type
res[self._DICT_PATH_KEY] = self.path
res[self.DICT_TYPE_KEY] = self.type
res[self.DICT_PATH_KEY] = self.path
return res
def load_from_dict(self, _dict: dict):
......@@ -237,9 +243,11 @@ class TomoScanBase:
def get_proj_angle_url(self) -> dict:
"""
Return the 'extra' radios of a scan which are used to see if the scan
moved during the acquisition. If no extra radio are found, return the
dictionary of all radios.
return a dictionary of all the projection. key is the angle of the
projection and value is the url.
Keys are int for 'standard' projections and strings for return
projections.
:return dict: angles as keys, radios as value.
"""
......@@ -260,10 +268,10 @@ class TomoScanBase:
:warning: each url should contain only one radio.
:param urls: ordered list with all the urls. First url should be
:param urls: dict with all the urls. First url should be
the first radio acquire, last url should match the last
radio acquire.
:type: list
:type: dict
:param n_projection: number of projection for the sample.
:type: int
:param scan_range: acquisition range (usually 180 or 360)
......@@ -275,16 +283,18 @@ class TomoScanBase:
are incoherent
"""
assert n_projection is not None
ordered_url = OrderedDict(sorted(urls.items(), key= lambda x: x))
ordered_url = OrderedDict(sorted(urls.items(), key=lambda x: x))
res = {}
# deal with the 'standard' acquisitions
for proj_i in range(n_projection):
angle = proj_i * scan_range / (n_projection -1)
url = list(ordered_url.values())[proj_i]
if n_projection == 1:
angle = 0.0
else:
angle = proj_i * scan_range / (n_projection - 1)
if proj_i < len(urls):
res[angle] = urls[proj_i]
# TODO: better to have them from the header ?!!
# haven't found the information in here.
res[angle] = url
if len(urls) > n_projection:
# deal with extra images (used to check if the sampled as moved for
......@@ -295,27 +305,27 @@ class TomoScanBase:
logger.warning('incoherent data information to retrieve'
'scan extra images angle')
elif len(extraImgs) == 4:
res['270(1)'] = extraImgs[0]
res['180(1)'] = extraImgs[1]
res['90(1)'] = extraImgs[2]
res['0(1)'] = extraImgs[3]
res['270(1)'] = ordered_url[extraImgs[0]]
res['180(1)'] = ordered_url[extraImgs[1]]
res['90(1)'] = ordered_url[extraImgs[2]]
res['0(1)'] = ordered_url[extraImgs[3]]
else:
res['360(1)'] = extraImgs[0]
res['270(1)'] = extraImgs[1]
res['180(1)'] = extraImgs[2]
res['90(1)'] = extraImgs[3]
res['0(1)'] = extraImgs[4]
res['360(1)'] = ordered_url[extraImgs[0]]
res['270(1)'] = ordered_url[extraImgs[1]]
res['180(1)'] = ordered_url[extraImgs[2]]
res['90(1)'] = ordered_url[extraImgs[3]]
res['0(1)'] = ordered_url[extraImgs[4]]
elif len(extraImgs) in (2, 3):
if scan_range > 180:
logger.warning('incoherent data information to retrieve'
'scan extra images angle')
elif len(extraImgs) is 3:
res['180(1)'] = extraImgs[0]
res['90(1)'] = extraImgs[1]
res['0(1)'] = extraImgs[2]
res['180(1)'] = ordered_url[extraImgs[0]]
res['90(1)'] = ordered_url[extraImgs[1]]
res['0(1)'] = ordered_url[extraImgs[2]]
else:
res['90(1)'] = extraImgs[0]
res['0(1)'] = extraImgs[1]
res['90(1)'] = ordered_url[extraImgs[0]]
res['0(1)'] = ordered_url[extraImgs[1]]
else:
raise ValueError('incoherent data information to retrieve scan'
'extra images angle')
......
......@@ -41,11 +41,10 @@ class ScanFactory:
"""
@staticmethod
def create_scan_object(scan_path):
def create_scan_object(scan_path: str) -> TomoScanBase:
"""
:param scan_path: path to the scan directory or file
:rtype: TextIOWrapper
:param str scan_path: path to the scan directory or file
:return: ScanBase instance fitting the scan folder or scan path
:rtype: TomoScanBase
"""
......@@ -55,11 +54,34 @@ class ScanFactory:
return EDFTomoScan(scan=scan_path)
elif HDF5TomoScan.is_tomoscan_dir(scan_path):
return HDF5TomoScan(scan=scan_path)
else:
raise ValueError('%s is not a valid scan path' % scan_path)
@staticmethod
def create_scan_objects(scan_path: str) -> tuple:
"""
:param str scan_path: path to the scan directory or file
:return: all possible instances of TomoScanBase contained in the given
path
:rtype: tuple
"""
scan_path = scan_path.rstrip(os.path.sep)
if EDFTomoScan.is_tomoscan_dir(scan_path):
return (EDFTomoScan(scan=scan_path), )
elif HDF5TomoScan.is_tomoscan_dir(scan_path):
scans = []
master_file = HDF5TomoScan.get_master_file(scan_path=scan_path)
entries = HDF5TomoScan.get_valid_entries(master_file)
for entry in entries:
scans.append(HDF5TomoScan(scan=scan_path, entry=entry,
index=None))
return tuple(scans)
raise ValueError('%s is not a valid scan path' % scan_path)
@staticmethod
def create_scan_object_frm_dict(_dict):
def create_scan_object_frm_dict(_dict: dict) -> TomoScanBase:
"""
Create a TomoScanBase instance from a dictionary. It should contains
the TomoScanBase._DICT_TYPE_KEY key at least.
......@@ -68,17 +90,17 @@ class ScanFactory:
:return: instance of TomoScanBase
:rtype: TomoScanBase
"""
if TomoScanBase._DICT_TYPE_KEY not in _dict:
if TomoScanBase.DICT_TYPE_KEY not in _dict:
raise ValueError('given dict is not recognized. Cannot find'
'', TomoScanBase._DICT_TYPE_KEY)
elif _dict[TomoScanBase._DICT_TYPE_KEY] == EDFTomoScan._TYPE:
'', TomoScanBase.DICT_TYPE_KEY)
elif _dict[TomoScanBase.DICT_TYPE_KEY] == EDFTomoScan._TYPE:
return EDFTomoScan(scan=None).load_from_dict(_dict)
else:
raise ValueError('Scan type', _dict[TomoScanBase._DICT_TYPE_KEY],
raise ValueError('Scan type', _dict[TomoScanBase.DICT_TYPE_KEY],
'is not managed')
@staticmethod
def is_tomoscan_dir(scan_path):
def is_tomoscan_dir(scan_path: str) -> bool:
"""
:param str scan_path: path to the scan directory or file
......@@ -91,13 +113,13 @@ class ScanFactory:
)
@staticmethod
def create_from_json(desc):
def create_from_json(desc: dict) -> TomoScanBase:
"""Create a ScanBase instance from a json description"""
data = json.load(desc)
if TomoScanBase._DICT_TYPE_KEY not in data:
if TomoScanBase.DICT_TYPE_KEY not in data:
raise ValueError('json not recognize')
elif data[TomoScanBase._DICT_TYPE_KEY] == EDFTomoScan._TYPE:
elif data[TomoScanBase.DICT_TYPE_KEY] == EDFTomoScan._TYPE:
scan = EDFTomoScan(scan=None).load_from_dict(data)
return scan
else:
......
......@@ -29,7 +29,9 @@ __date__ = "24/01/2017"
import unittest
import os
from ..esrf.edfscan import EDFTomoScan
from ..esrf.hdf5scan import HDF5TomoScan
from ..scanfactory import ScanFactory
from .utils import UtilsTest
import tempfile
......@@ -40,15 +42,52 @@ class TestScanFactory(unittest.TestCase):
valid scan type for a given file / directory