Commit 2bc2c976 authored by payno's avatar payno
Browse files

Merge branch 'master' of gitlab.esrf.fr:tomotools/tomoscan

parents ec0f7456 ff26348e
Pipeline #22669 passed with stages
in 2 minutes and 12 seconds
tomoscan tomoscan
======== ========
tomoscan aims to provide an unified interface to read tomography data from .edf or .hdf5 (NXtomo) acquisitions.
.. toctree:: .. toctree::
:maxdepth: 4 :maxdepth: 4
......
...@@ -22,6 +22,8 @@ ...@@ -22,6 +22,8 @@
# #
#############################################################################*/ #############################################################################*/
"""contains EDFTomoScan, class to be used with EDF acquisition"""
__authors__ = ["H.Payno"] __authors__ = ["H.Payno"]
__license__ = "MIT" __license__ = "MIT"
...@@ -38,7 +40,7 @@ import io ...@@ -38,7 +40,7 @@ import io
from typing import Union from typing import Union
from ..scanbase import TomoScanBase from ..scanbase import TomoScanBase
from .utils import get_parameters_frm_par_or_info, extract_urls_from_edf from .utils import get_parameters_frm_par_or_info, extract_urls_from_edf
from ..unitsystem import metricsystem from ..unitsystem.metricsystem import MetricSystem
from ..utils import docstring from ..utils import docstring
import logging import logging
...@@ -84,8 +86,25 @@ class EDFTomoScan(TomoScanBase): ...@@ -84,8 +86,25 @@ class EDFTomoScan(TomoScanBase):
self.__ref_on = None self.__ref_on = None
self.__scan_range = None self.__scan_range = None
self._edf_n_frames = n_frames self._edf_n_frames = n_frames
self.__distance = None
self.__energy = None
self.update() self.update()
@docstring(TomoScanBase.clear_caches)
def clear_caches(self):
self._darks = None
self._flats = None
self._projections = None
self.__tomo_n = None
self.__ref_n = None
self.__dark_n = None
self.__dim1 = None
self.__dim2 = None
self.__pixel_size = None
self.__ref_on = None
self.__scan_range = None
@docstring(TomoScanBase.tomo_n) @docstring(TomoScanBase.tomo_n)
@property @property
def tomo_n(self) -> Union[None, int]: def tomo_n(self) -> Union[None, int]:
...@@ -116,7 +135,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -116,7 +135,7 @@ class EDFTomoScan(TomoScanBase):
:rtype: float :rtype: float
""" """
if self.__pixel_size is None: if self.__pixel_size is None:
self.__pixel_size = EDFTomoScan.get_pixel_size(scan=self.path) self.__pixel_size = EDFTomoScan._get_pixel_size(scan=self.path)
return self.__pixel_size return self.__pixel_size
@property @property
...@@ -190,7 +209,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -190,7 +209,7 @@ class EDFTomoScan(TomoScanBase):
def is_abort(self, **kwargs) -> bool: def is_abort(self, **kwargs) -> bool:
abort_file = os.path.basename(self.path) + self.ABORT_FILE abort_file = os.path.basename(self.path) + self.ABORT_FILE
abort_file = os.path.join(self.path, abort_file) abort_file = os.path.join(self.path, abort_file)
if 'src_pattern' in kwargs and kwargs['src_pattern' is not None]: if 'src_pattern' in kwargs and kwargs['src_pattern'] is not None:
assert 'dest_pattern' in kwargs assert 'dest_pattern' in kwargs
abort_file = abort_file.replace(kwargs['src_pattern'], abort_file = abort_file.replace(kwargs['src_pattern'],
kwargs['dest_pattern']) kwargs['dest_pattern'])
...@@ -218,9 +237,10 @@ class EDFTomoScan(TomoScanBase): ...@@ -218,9 +237,10 @@ class EDFTomoScan(TomoScanBase):
@docstring(TomoScanBase.update) @docstring(TomoScanBase.update)
def update(self): def update(self):
self.projections = EDFTomoScan.get_proj_urls(self.path, n_frames=self._edf_n_frames) if self.path is not None:
self._darks = EDFTomoScan.get_darks_url(self.path) self.projections = EDFTomoScan.get_proj_urls(self.path, n_frames=self._edf_n_frames)
self._flats = EDFTomoScan.get_refs_url(self.path) self._darks = EDFTomoScan.get_darks_url(self.path)
self._flats = EDFTomoScan.get_refs_url(self.path)
@docstring(TomoScanBase.load_from_dict) @docstring(TomoScanBase.load_from_dict)
def load_from_dict(self, desc: Union[dict, io.TextIOWrapper]): def load_from_dict(self, desc: Union[dict, io.TextIOWrapper]):
...@@ -228,11 +248,11 @@ class EDFTomoScan(TomoScanBase): ...@@ -228,11 +248,11 @@ class EDFTomoScan(TomoScanBase):
data = json.load(desc) data = json.load(desc)
else: else:
data = desc data = desc
if not (self._DICT_TYPE_KEY in data and data[self._DICT_TYPE_KEY] == self._TYPE): if not (self.DICT_TYPE_KEY in data and data[self.DICT_TYPE_KEY] == self._TYPE):
raise ValueError('Description is not an EDFScan json description') raise ValueError('Description is not an EDFScan json description')
assert self._DICT_PATH_KEY in data assert self.DICT_PATH_KEY in data
self.path = data[self._DICT_PATH_KEY] self.path = data[self.DICT_PATH_KEY]
return self return self
@staticmethod @staticmethod
...@@ -303,8 +323,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -303,8 +323,7 @@ class EDFTomoScan(TomoScanBase):
def extract_index(my_str, type_): def extract_index(my_str, type_):
res = [] res = []
modified_str = copy.copy(my_str) modified_str = copy.copy(my_str)
while modified_str != "" and modified_str[-1].isdigit():
while modified_str[-1].isdigit():
res.append(modified_str[-1]) res.append(modified_str[-1])
modified_str = modified_str[:-1] modified_str = modified_str[:-1]
if len(res) == 0: if len(res) == 0:
...@@ -315,7 +334,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -315,7 +334,7 @@ class EDFTomoScan(TomoScanBase):
return int(''.join(orignalOrder)), modified_str return int(''.join(orignalOrder)), modified_str
else: else:
return float('.'.join(('0', ''.join(orignalOrder)))), modified_str return float('.'.join(('0', ''.join(orignalOrder)))), modified_str
_file = os.path.basename(_file)
if _file.endswith('.edf'): if _file.endswith('.edf'):
name = _file.replace(basename, '', 1) name = _file.replace(basename, '', 1)
name = name.rstrip('.edf') name = name.rstrip('.edf')
...@@ -330,7 +349,10 @@ class EDFTomoScan(TomoScanBase): ...@@ -330,7 +349,10 @@ class EDFTomoScan(TomoScanBase):
if part_1 is None: if part_1 is None:
return None return None
if part_2 is None: if part_2 is None:
return int(part_1) if part_1 is None:
return None
else:
return int(part_1)
else: else:
return float(part_1) + part_2 return float(part_1) + part_2
else: else:
...@@ -392,8 +414,34 @@ class EDFTomoScan(TomoScanBase): ...@@ -392,8 +414,34 @@ class EDFTomoScan(TomoScanBase):
return d1, d2 return d1, d2
@property
@docstring(TomoScanBase.distance)
def distance(self) -> Union[None, float]:
if self.__distance is None:
self.__distance = EDFTomoScan.retrieve_information(self.path,
None, "Distance",
type_=float,
key_aliases=('distance', )
)
if self.__distance is None:
return None
else:
return self.__distance * MetricSystem.MILLIMETER.value
@property
@docstring(TomoScanBase.energy)
def energy(self):
if self.__energy is None:
self.__energy = EDFTomoScan.retrieve_information(self.path,
None,
"Energy",
type_=float,
key_aliases=('energy', )
)
return self.__energy
@staticmethod @staticmethod
def get_pixel_size(scan: str) -> Union[None, int]: def _get_pixel_size(scan: str) -> Union[None, float]:
if os.path.isdir(scan) is False: if os.path.isdir(scan) is False:
return None return None
value = EDFTomoScan.retrieve_information(scan=scan, value = EDFTomoScan.retrieve_information(scan=scan,
...@@ -413,7 +461,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -413,7 +461,7 @@ class EDFTomoScan(TomoScanBase):
# for now pixel size are stored in microns. # for now pixel size are stored in microns.
# We want to return them in meter # We want to return them in meter
if value is not None: if value is not None:
return value * metricsystem.micrometer return value * MetricSystem.MICROMETER.value
else: else:
return None return None
......
This diff is collapsed.
...@@ -259,7 +259,9 @@ class TestProjections(unittest.TestCase): ...@@ -259,7 +259,9 @@ class TestProjections(unittest.TestCase):
self.assertEqual(len(scan.projections), 3) self.assertEqual(len(scan.projections), 3)
scan.update() scan.update()
self.assertEqual(len(scan.projections), 4) self.assertEqual(len(scan.projections), 4)
self.assertTrue(isinstance(scan.projections[0], silx.io.url.DataUrl))
index_0 = list(scan.projections.keys())[0]
self.assertTrue(isinstance(scan.projections[index_0], silx.io.url.DataUrl))
def testProjectionWithExtraRadio(self): def testProjectionWithExtraRadio(self):
mock = MockEDF(scan_path=self.folder, n_radio=11, n_extra_radio=2, mock = MockEDF(scan_path=self.folder, n_radio=11, n_extra_radio=2,
...@@ -305,10 +307,6 @@ class TestScanValidatorFindFiles(unittest.TestCase): ...@@ -305,10 +307,6 @@ class TestScanValidatorFindFiles(unittest.TestCase):
if os.path.isdir(self.path): if os.path.isdir(self.path):
shutil.rmtree(self.path) shutil.rmtree(self.path)
def testGetRadioPaths(self):
nFound = len(EDFTomoScan.get_proj_urls(self.path))
self.assertTrue(nFound == self.N_RADIO)
class TestRadioPath(unittest.TestCase): class TestRadioPath(unittest.TestCase):
"""Test static method getRadioPaths for EDFTomoScan""" """Test static method getRadioPaths for EDFTomoScan"""
......
...@@ -33,7 +33,9 @@ import shutil ...@@ -33,7 +33,9 @@ import shutil
import os import os
import tempfile import tempfile
from tomoscan.test.utils import UtilsTest from tomoscan.test.utils import UtilsTest
from tomoscan.esrf.hdf5scan import HDF5TomoScan from tomoscan.esrf.hdf5scan import HDF5TomoScan, ImageKey, Frame
from ...unitsystem import metricsystem
from silx.io.utils import get_data
import numpy import numpy
...@@ -56,69 +58,148 @@ class TestHDF5Scan(HDF5TestBaseClass): ...@@ -56,69 +58,148 @@ class TestHDF5Scan(HDF5TestBaseClass):
"""Basic test for the hdf5 scan""" """Basic test for the hdf5 scan"""
def setUp(self) -> None: def setUp(self) -> None:
super(TestHDF5Scan, self).setUp() super(TestHDF5Scan, self).setUp()
self.dataset_file = self.get_dataset('data_test2.h5') self.dataset_file = self.get_dataset('frm_edftomomill_twoentries.nx')
self.scan = HDF5TomoScan(self.dataset_file) self.scan = HDF5TomoScan(scan=self.dataset_file)
def testGeneral(self): def testGeneral(self):
"""some general on the HDF5Scan """ """some general on the HDF5Scan """
self.assertEqual(self.scan.master_file, self.dataset_file) self.assertEqual(self.scan.master_file, self.dataset_file)
self.assertEqual(self.scan.path, os.path.dirname(self.dataset_file)) self.assertEqual(self.scan.path, os.path.dirname(self.dataset_file))
self.assertEqual(self.scan.type, 'hdf5') self.assertEqual(self.scan.type, 'hdf5')
self.assertEqual(self.scan.entry, 'entry0000')
self.assertEqual(len(self.scan.flats), 42)
self.assertEqual(len(self.scan.darks), 1)
self.assertEqual(len(self.scan.return_projs), 3)
proj_angles = self.scan.get_proj_angle_url()
self.assertEqual(len(proj_angles), 1500 + 3)
self.assertTrue(90 in proj_angles)
self.assertTrue(24.0 in proj_angles)
self.assertTrue('90.0(1)' in proj_angles)
self.assertTrue('180.0(1)' in proj_angles)
self.assertTrue(179.88 not in proj_angles)
url_1 = proj_angles[0]
self.assertTrue(url_1.is_valid())
self.assertTrue(url_1.is_absolute())
self.assertEquals(url_1.scheme(), 'silx')
# check conversion to dict
_dict = self.scan.to_dict() _dict = self.scan.to_dict()
scan2 = HDF5TomoScan.from_dict(_dict) scan2 = HDF5TomoScan.from_dict(_dict)
self.assertEqual(scan2.path, self.scan.path) self.assertEqual(scan2.path, self.scan.path)
radios_urls = self.scan.get_proj_angle_url() self.assertEqual(scan2.entry, self.scan.entry)
self.assertEqual(len(radios_urls), 100)
url_1 = radios_urls[0] def testFrames(self):
self.assertTrue(url_1.is_valid()) """Check the `frames` property which is massively used under the
self.assertFalse(url_1.is_absolute()) HDF5TomoScan class"""
self.assertEquals(url_1.scheme(), 'silx') frames = self.scan.frames
# check some projections
proj_2 = frames[24]
self.assertTrue(isinstance(proj_2, Frame))
self.assertEqual(proj_2.index, 24)
numpy.isclose(proj_2.rotation_angle, 0.24)
self.assertFalse(proj_2.is_control)
self.assertEqual(proj_2.url.file_path(), self.scan.master_file)
self.assertEqual(proj_2.url.data_path(), 'entry0000/instrument/detector/data')
self.assertEqual(proj_2.url.data_slice(), 24)
self.assertEqual(proj_2.image_key, ImageKey.PROJECTION)
self.assertEqual(get_data(proj_2.url).shape, (20, 20))
# check last two non-return projection
for frame_index in (1520, 1542):
with self.subTest(frame_index=frame_index):
frame = frames[frame_index]
self.assertTrue(frame.image_key, ImageKey.PROJECTION)
self.assertFalse(frame.is_control)
# check some darks
dark_0 = frames[0]
self.assertEqual(dark_0.index, 0)
numpy.isclose(dark_0.rotation_angle, 0.0)
self.assertFalse(dark_0.is_control)
self.assertEqual(dark_0.url.file_path(), self.scan.master_file)
self.assertEqual(dark_0.url.data_path(), 'entry0000/instrument/detector/data')
self.assertEqual(dark_0.url.data_slice(), 0)
self.assertEqual(dark_0.image_key, ImageKey.DARK_FIELD)
self.assertEqual(get_data(dark_0.url).shape, (20, 20))
# check some refs
ref_1 = frames[2]
self.assertEqual(ref_1.index, 2)
numpy.isclose(ref_1.rotation_angle, 0.0)
self.assertFalse(ref_1.is_control)
self.assertEqual(ref_1.url.file_path(), self.scan.master_file)
self.assertEqual(ref_1.url.data_path(),
'entry0000/instrument/detector/data')
self.assertEqual(ref_1.url.data_slice(), 2)
self.assertEqual(ref_1.image_key, ImageKey.FLAT_FIELD)
self.assertEqual(get_data(ref_1.url).shape, (20, 20))
# check some return projections
r_proj_0 = frames[1543]
self.assertTrue(isinstance(r_proj_0, Frame))
self.assertEqual(r_proj_0.index, 1543)
numpy.isclose(r_proj_0.rotation_angle, 180)
self.assertTrue(r_proj_0.is_control)
self.assertEqual(r_proj_0.url.file_path(), self.scan.master_file)
self.assertEqual(r_proj_0.url.data_path(),
'entry0000/instrument/detector/data')
self.assertEqual(r_proj_0.url.data_slice(), 1543)
self.assertEqual(r_proj_0.image_key, ImageKey.PROJECTION)
self.assertEqual(get_data(r_proj_0.url).shape, (20, 20))
def testProjections(self): def testProjections(self):
"""Make sure projections are valid"""
projections = self.scan.projections projections = self.scan.projections
self.assertEqual(len(projections), 100) self.assertEqual(len(self.scan.projections), 1500)
proj_1 = projections[0] url_0 = projections[list(projections.keys())[0]]
self.assertEqual(proj_1.file_path(), self.assertEqual(url_0.file_path(), os.path.join(self.scan.master_file))
'../../../../../../users/opid19/W:/clemence/visualtomo/data_test2/tomo0001/tomo_0000.h5') self.assertEqual(url_0.data_slice(), 22)
self.assertEqual(proj_1.data_slice(), ('0',))
self.assertTrue(100 not in projections)
@unittest.skip('no valid hdf5 acquisition defined yet')
def testDark(self): def testDark(self):
n_dark = 20 """Make sure darks are valid"""
n_dark = 1
self.assertEqual(self.scan.dark_n, n_dark) self.assertEqual(self.scan.dark_n, n_dark)
with self.assertRaises(NotImplementedError): darks = self.scan.darks
self.scan.darks self.assertEqual(len(darks), 1)
# TODO check accumulation time
def testFlats(self): def testFlats(self):
"""Make sure flats are valid"""
n_flats = 42
flats = self.scan.flats
self.assertEqual(len(flats), n_flats)
self.assertEqual(self.scan.ref_n, n_flats)
with self.assertRaises(NotImplementedError): with self.assertRaises(NotImplementedError):
flats = self.scan.flats self.scan.ff_interval
n_ref = 21
self.assertEqual(self.scan.ref_n, n_ref)
# for now not implemented def testDims(self):
# not implemented at the moment self.assertEqual(self.scan.dim_1, 20)
data = numpy.arange(2048*2048) self.assertEqual(self.scan.dim_2, 20)
data.reshape(2048, 2048)
# not implemented at the moment
self.assertEqual(self.scan.ff_interval, 21)
def testAxisUtils(self): def testAxisUtils(self):
self.assertEqual(self.scan.scan_range, 360) self.assertEqual(self.scan.scan_range, 180)
self.assertEqual(self.scan.tomo_n, 100) self.assertEqual(self.scan.tomo_n, 1500)
# self.assertEqual(self.scan.dim_1, 2048)
# self.assertEqual(self.scan.dim_2, 2048)
proj0_file_path = '../../../../../../users/opid19/W:/clemence/visualtomo/data_test2/tomo0001/tomo_0000.h5'
radios_urls_evolution = self.scan.get_proj_angle_url() radios_urls_evolution = self.scan.get_proj_angle_url()
self.assertEquals(len(radios_urls_evolution), 100) self.assertEquals(len(radios_urls_evolution), 1503)
self.assertEquals(radios_urls_evolution[0].file_path(), proj0_file_path) self.assertEquals(radios_urls_evolution[0].file_path(), self.scan.master_file)
self.assertEquals(radios_urls_evolution[0].data_slice(), ('0',)) self.assertEquals(radios_urls_evolution[0].data_slice(), 22)
self.assertEquals(radios_urls_evolution[0].data_path(), '/entry_0000/measurement/pcoedge64/data') self.assertEquals(radios_urls_evolution[0].data_path(), 'entry0000/instrument/detector/data')
def testDarkRefUtils(self): def testDarkRefUtils(self):
self.assertEqual(self.scan.tomo_n, 100) self.assertEqual(self.scan.tomo_n, 1500)
self.assertEqual(self.scan.pixel_size[1], 6.5) pixel_size = self.scan.pixel_size
self.assertTrue(pixel_size is not None)
self.assertTrue(numpy.isclose(self.scan.pixel_size,
0.05 * metricsystem.MetricSystem.MICROMETER.value))
self.assertTrue(numpy.isclose(self.scan.get_pixel_size(unit='micrometer'), 0.05))
self.assertTrue(numpy.isclose(self.scan.x_pixel_size,
0.05 * metricsystem.MetricSystem.MICROMETER.value))
self.assertTrue(numpy.isclose(self.scan.y_pixel_size,
0.05 * metricsystem.MetricSystem.MICROMETER.value))
def testNabuUtil(self):
self.assertTrue(numpy.isclose(self.scan.distance, -19.9735))
self.assertTrue(numpy.isclose(self.scan.get_distance(unit='cm'), -1997.35))
def suite(): def suite():
......
...@@ -32,6 +32,7 @@ import os ...@@ -32,6 +32,7 @@ import os
import logging import logging
from typing import Union from typing import Union
from collections import OrderedDict from collections import OrderedDict
from .unitsystem.metricsystem import MetricSystem
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -45,9 +46,9 @@ class TomoScanBase: ...@@ -45,9 +46,9 @@ class TomoScanBase:
:param scan: path to the root folder containing the scan. :param scan: path to the root folder containing the scan.
:type: Union[str,None] :type: Union[str,None]
""" """
_DICT_TYPE_KEY = 'type' DICT_TYPE_KEY = 'type'
_DICT_PATH_KEY = 'path' DICT_PATH_KEY = 'path'
_SCHEME = None _SCHEME = None
"""scheme to read data url for this type of acquisition""" """scheme to read data url for this type of acquisition"""
...@@ -56,6 +57,11 @@ class TomoScanBase: ...@@ -56,6 +57,11 @@ class TomoScanBase:
self.path = scan self.path = scan
self._type = type_ self._type = type_
def clear_caches(self):
"""clear caches. Might be call if some data changed after
first read of data or metadata"""
raise NotImplementedError('Base class')
@property @property
def path(self) -> Union[None, str]: def path(self) -> Union[None, str]:
""" """
...@@ -111,7 +117,7 @@ class TomoScanBase: ...@@ -111,7 +117,7 @@ class TomoScanBase:
self._flats = flats self._flats = flats
@property @property
def darks(self) -> Union[None,dict]: def darks(self) -> Union[None, dict]:
"""list of darks files""" """list of darks files"""
return self._darks return self._darks
...@@ -134,6 +140,7 @@ class TomoScanBase: ...@@ -134,6 +140,7 @@ class TomoScanBase:
@property @property
def tomo_n(self) -> Union[None, int]: def tomo_n(self) -> Union[None, int]:
"""number of projection WITHOUT the return projections"""
raise NotImplementedError('Base class') raise NotImplementedError('Base class')
@property @property
...@@ -141,9 +148,15 @@ class TomoScanBase: ...@@ -141,9 +148,15 @@ class TomoScanBase:
raise NotImplementedError('Base class') raise NotImplementedError('Base class')
@property @property
def pixel_size(self) -> Union[None, int]: def pixel_size(self) -> Union[None, float]:
raise NotImplementedError('Base class') raise NotImplementedError('Base class')
def get_pixel_size(self, unit='m') -> Union[None, float]:
if self.pixel_size:
return self.pixel_size / MetricSystem.from_value(unit).value
else:
return None
@property @property
def dim_1(self) -> Union[None, int]: def dim_1(self) -> Union[None, int]:
raise NotImplementedError('Base class') raise NotImplementedError('Base class')
...@@ -160,6 +173,33 @@ class TomoScanBase: ...@@ -160,6 +173,33 @@ class TomoScanBase:
def scan_range(self) -> Union[None, int]: def scan_range(self) -> Union[None, int]:
raise NotImplementedError('Base class') raise NotImplementedError('Base class')
@property
def energy(self) -> Union[None, float]:
"""