Commit 3906671d authored by payno's avatar payno
Browse files

Merge branch '0.3' of https://gitlab.esrf.fr/tomotools/nxtomomill into 0.3

parents 521ced55 f75892d8
......@@ -38,6 +38,10 @@ tmp*
*.xml
*.db
*.tar.bz2
*.nx
*.hdf5
*.h5
*.edf
dataset/
datasets/
......
Change Log
==========
0.3.2: XXX
----------
* h5_to_nx:
* add set-param option to let the user define some parameters values like energy if he knows it is missing (and avoid asking him n times).
* io: add management of hdf5 files from tomoscan.io.HDF5File
0.3.1: 2020/08/19
-----------------
* add field_of_view parameter
* add plugin management (allows user to define motor position value from a python script) - PR !19
0.3.0: 2020/03/20
-----------------
......
......@@ -51,14 +51,11 @@ The result can be displayed using any hdf5 display or using silx:
silx view /data/idxx/inhouse/myname/sample1_.nx
<<<<<<< HEAD
All the .edf files in the origin directory are considered except those having '_slice_' in thir name.
=======
All the .edf files in the origin directory are considered except those having '_slice_' in thir name.
>>>>>>> 0.2
The algorithm selects raw dark fields - darkendxxxx.edf - and raw flat fields refxxxx.edf. However, if processed darks (dark.edf) and refs (refHST) exist,
All the .edf files in the origin directory are considered except those having '_slice_' in their name.
The algorithm selects raw dark fields - darkendxxxx.edf - and raw flat fields refxxxx.edf. However, if processed darks (dark.edf) and refs (refHST) exist,
they are stored in the destination file instead of the raw files.
The names of the motors are hard-coded: 'srot' for the rotation, 'sx', 'sy' and 'sz' for the positionning motors.
The names of the motors are hard-coded: 'srot' for the rotation, 'sx', 'sy' and 'sz' for the positioning motors.
.. note:: The conversion is based on a set of key value that are contained in the EDF file headers.
......
......@@ -36,7 +36,8 @@ import logging
from nxtomomill import utils
from nxtomomill.settings import (H5_Z_TRANS_KEYS, H5_Y_TRANS_KEYS,
H5_X_TRANS_KEYS, H5_VALID_CAMERA_NAMES,
H5_ROT_ANGLE_KEYS, H5_ACQ_EXPO_TIME_KEYS)
H5_ROT_ANGLE_KEYS, H5_ACQ_EXPO_TIME_KEYS,
H5_X_PIXEL_SIZE, H5_Y_PIXEL_SIZE)
from nxtomomill.settings import (H5_ALIGNMENT_TITLES, H5_PROJ_TITLES,
H5_REF_TITLES, H5_DARK_TITLES, H5_INIT_TITLES)
from nxtomomill.converter import h5_to_nx, H5FileKeys, H5ScanTitles
......@@ -48,6 +49,54 @@ _logger = logging.getLogger(__name__)
_logger.setLevel(logging.DEBUG)
_SETTABLE_PARAMETERS_UNITS = \
{
'energy': 'kev',
}
_SETTABLE_PARAMETERS = _SETTABLE_PARAMETERS_UNITS.keys()
def _getPossibleInputParams():
"""
:return: string with param1 (expected unit) ...
"""
res = []
for key, value in _SETTABLE_PARAMETERS_UNITS.items():
res.append('{} ({})'.format(key, value))
return ', '.join(res)
def _extract_param_value(key_values):
'''extract all the key / values elements from the str_list. Expected
format is `param_1_name param_1_value param_2_name param_2_value ...`
:param str str_list: raw input string as `param_1_name param_1_value
param_2_name param_2_value ...`
:return: dict of tuple (param_name, param_value)
:rtype: dict
'''
if len(key_values) % 2 != 0:
raise ValueError('Expect a pair `param_name, param_value` for each '
'parameters')
def pairwise(it):
it = iter(it)
while True:
try:
yield next(it), next(it)
except StopIteration:
# no more elements in the iterator
return
res = {}
for name, value in pairwise(key_values):
if name not in _SETTABLE_PARAMETERS:
raise ValueError('parameters {} is not managed'.format(name))
res[name] = value
return res
def main(argv):
"""
"""
......@@ -88,6 +137,10 @@ def main(argv):
help='Valid dataset name for rotation angle')
parser.add_argument('--acq_expo_time_keys', default=','.join(H5_ACQ_EXPO_TIME_KEYS),
help='Valid dataset name for acquisition exposure time')
parser.add_argument('--x_pixel_size_key', default=H5_X_PIXEL_SIZE,
help='X pixel size key to read')
parser.add_argument('--y_pixel_size_key', default=H5_Y_PIXEL_SIZE,
help='Y pixel size key to read')
# scan titles
parser.add_argument('--init_titles', default=','.join(H5_INIT_TITLES),
......@@ -100,7 +153,11 @@ def main(argv):
help='Titles corresponding to projection scans')
parser.add_argument('--align_titles', default=','.join(H5_ALIGNMENT_TITLES),
help='Titles corresponding to alignment scans')
parser.add_argument('--set-params', default='',
nargs='*',
help='Allow manual definition of some parameters. ' \
'Valid parameters (and expected input unit) '
'are: {}.'.format(_getPossibleInputParams()))
options = parser.parse_args(argv[1:])
conv = utils.get_tuple_of_keys_from_cmd
file_keys = H5FileKeys(x_trans_keys=conv(options.x_trans_keys),
......@@ -109,18 +166,22 @@ def main(argv):
valid_camera_names=conv(options.valid_camera_names),
rot_angle_keys=conv(options.rot_angle_keys),
acq_expo_time_keys=conv(options.acq_expo_time_keys),
x_pixel_size=options.x_pixel_size_key,
y_pixel_size=options.y_pixel_size_key,
)
scan_titles = H5ScanTitles(init_titles=conv(options.init_titles),
dark_titles=conv(options.dark_titles),
ref_titles=conv(options.ref_titles),
proj_titles=conv(options.proj_titles),
align_titles=conv(options.align_titles))
options.set_params = _extract_param_value(options.set_params)
h5_to_nx(input_file_path=options.input_file_path,
output_file=options.output_file, single_file=options.single_file,
file_extension=options.file_extension,
request_input=options.request_input, file_keys=file_keys,
scan_titles=scan_titles)
scan_titles=scan_titles,
param_already_defined=options.set_params)
exit(0)
......
......@@ -37,10 +37,11 @@ from tomoscan.esrf.edfscan import EDFTomoScan
from nxtomomill.utils import Progress
from nxtomomill.plugins import (get_plugins_instances_frm_env_var,
get_plugins_positioners_resources)
from nxtomomill.settings import (H5_ACQ_EXPO_TIME_KEYS, H5_ROT_ANGLE_KEYS,
from nxtomomill.settings import (H5_ROT_ANGLE_KEYS,
H5_VALID_CAMERA_NAMES, H5_X_TRANS_KEYS,
H5_Y_TRANS_KEYS, H5_Z_TRANS_KEYS,
H5_ALIGNMENT_TITLES, H5_ACQ_EXPO_TIME_KEYS,
H5_X_PIXEL_SIZE, H5_Y_PIXEL_SIZE,
H5_DARK_TITLES, H5_INIT_TITLES,
H5_PROJ_TITLES, H5_REF_TITLES)
from nxtomomill.settings import (EDF_DARK_NAMES, EDF_MOTOR_MNE, EDF_MOTOR_POS,
......@@ -48,6 +49,7 @@ from nxtomomill.settings import (EDF_DARK_NAMES, EDF_MOTOR_MNE, EDF_MOTOR_POS,
EDF_X_TRANS, EDF_Y_TRANS, EDF_Z_TRANS)
from collections import namedtuple
from tomoscan.unitsystem import metricsystem
from tomoscan.io import HDF5File
import os
import typing
import h5py
......@@ -66,7 +68,8 @@ H5ScanTitles = namedtuple('H5ScanTitles', ['init_titles',
'dark_titles',
'ref_titles',
'proj_titles',
'align_titles'])
'align_titles',
])
DEFAULT_SCAN_TITLES = H5ScanTitles(H5_INIT_TITLES, H5_DARK_TITLES,
H5_REF_TITLES, H5_PROJ_TITLES,
......@@ -77,16 +80,15 @@ H5FileKeys = namedtuple('H5FileKeys', ['acq_expo_time_keys',
'valid_camera_names',
'x_trans_keys',
'y_trans_keys',
'z_trans_keys'])
'z_trans_keys',
'x_pixel_size',
'y_pixel_size',
])
DEFAULT_H5_KEYS = H5FileKeys(H5_ACQ_EXPO_TIME_KEYS, H5_ROT_ANGLE_KEYS,
H5_VALID_CAMERA_NAMES, H5_X_TRANS_KEYS,
H5_Y_TRANS_KEYS, H5_Z_TRANS_KEYS)
EDFFileKeys = namedtuple('EDFFileKeys', ['motor_pos_key', 'motor_mne_key',
'rot_angle_key', 'x_trans_key',
'y_trans_key', 'z_trans_key',
'to_ignore', 'dark_names', 'ref_names'])
H5_Y_TRANS_KEYS, H5_Z_TRANS_KEYS, H5_X_PIXEL_SIZE,
H5_Y_PIXEL_SIZE)
EDFFileKeys = namedtuple('EDFFileKeys', ['motor_pos_key', 'motor_mne_key',
'rot_angle_key', 'x_trans_key',
......@@ -112,7 +114,7 @@ class AcquisitionStep(_Enum):
DARK = 'darks'
REFERENCE = 'references'
PROJECTION = 'projections'
ALIGNEMENT = 'alignment projections'
ALIGNMENT = 'alignment projections'
def _ask_for_file_removal(file_path):
......@@ -132,7 +134,7 @@ def edf_to_nx(scan: EDFTomoScan, output_file: str, file_extension: str,
_logger.info("Output file will be " + fileout_h5)
DARK_ACCUM_FACT = True
with h5py.File(fileout_h5, "w") as h5d:
with HDF5File(fileout_h5, "w") as h5d:
proj_urls = scan.get_proj_urls(scan=scan.path)
for dark_to_find in file_keys.dark_names:
......@@ -485,7 +487,8 @@ def h5_to_nx(input_file_path: str, output_file: str, single_file:bool,
request_input=False,
entries: typing.Union[typing.Iterable, None] = None,
input_callback=None, file_keys=DEFAULT_H5_KEYS,
scan_titles=DEFAULT_SCAN_TITLES):
scan_titles=DEFAULT_SCAN_TITLES,
param_already_defined=None):
"""
:param str input_file_path: file to be converted from .h5 to tomo .nx
......@@ -502,10 +505,14 @@ def h5_to_nx(input_file_path: str, output_file: str, single_file:bool,
parameters and return a text (that might be casted
according to the expected input type).
:param H5FileKeys file_keys: name of cameras, translation keys ...
:param Union[None, dict]: parameters for which the value has been defined
manually by the user. Like 'energy'...
:return: tuple of tuples (file_name, entry_name)
:rtype: tuple
"""
print('******set up***********')
if param_already_defined is None:
param_already_defined = {}
if not os.path.isfile(input_file_path):
raise ValueError('Given input file does not exists: %s'
......@@ -534,7 +541,7 @@ def h5_to_nx(input_file_path: str, output_file: str, single_file:bool,
plugins = []
res = []
with h5py.File(input_file_path, 'r') as h5d:
with HDF5File(input_file_path, 'r') as h5d:
groups = list(h5d.keys())
groups.sort(key=float)
# step 1: deduce acquisitions
......@@ -551,7 +558,8 @@ def h5_to_nx(input_file_path: str, output_file: str, single_file:bool,
if entry_type is AcquisitionStep.INITIALIZATION:
current_acquisition = _Acquisition(entry, file_keys,
scan_titles=scan_titles)
scan_titles=scan_titles,
param_already_defined=param_already_defined)
acquisitions.append(current_acquisition)
elif current_acquisition is not None:
current_acquisition.register_step(entry)
......@@ -572,50 +580,48 @@ def h5_to_nx(input_file_path: str, output_file: str, single_file:bool,
progress.set_max_advancement(len(acquisitions))
for i_acquisition, acquisition in enumerate(acquisitions):
if entries is not None and acquisition.initialization_entry.name not in entries:
_logger.info('skip entry ' + acquisition.initialization_entry.name)
continue
if not acquisition.is_valid():
_logger.error('unable to write nexus file for %s'
'' % acquisition.initialization_entry.name)
if single_file:
en_output_file = output_file
entry = 'entry' + str(i_acquisition).zfill(4)
else:
if single_file:
en_output_file = output_file
entry = 'entry' + str(i_acquisition).zfill(4)
else:
ext = file_extension_ or file_extension
file_name = output_file_basename + '_' + str(i_acquisition).zfill(4) + ext
en_output_file = os.path.join(os.path.dirname(output_file), file_name)
entry = 'entry'
if os.path.exists(en_output_file):
if ask_before_overwrite is False:
_logger.warning(en_output_file + ' will be removed')
_logger.info('remove ' + en_output_file)
os.remove(en_output_file)
elif _ask_for_file_removal(en_output_file) is False:
_logger.info('unable to overwrite %s, exit' % en_output_file)
exit(0)
else:
os.remove(en_output_file)
try:
acquisition.write_as_nxtomo(output_file=en_output_file,
data_path=entry,
input_file_path=input_file_path,
request_input=request_input,
input_callback=input_callback,
plugins=plugins)
# if split files create a master file with link to those entries
if single_file is False:
_logger.info('create link in %s' % output_file)
with h5py.File(output_file, 'a') as master_file:
mf_entry = 'entry' + str(i_acquisition).zfill(4)
link_file = os.path.relpath(en_output_file, os.path.dirname(output_file))
master_file[mf_entry] = h5py.ExternalLink(link_file,
entry)
res.append((output_file, mf_entry))
ext = file_extension_ or file_extension
file_name = output_file_basename + '_' + str(i_acquisition).zfill(4) + ext
en_output_file = os.path.join(os.path.dirname(output_file), file_name)
entry = 'entry'
if os.path.exists(en_output_file):
if ask_before_overwrite is False:
_logger.warning(en_output_file + ' will be removed')
_logger.info('remove ' + en_output_file)
os.remove(en_output_file)
elif _ask_for_file_removal(en_output_file) is False:
_logger.info('unable to overwrite %s, exit' % en_output_file)
exit(0)
else:
res.append((en_output_file, entry))
except Exception as e:
_logger.error('Fails to write %s. Error is %s' % (acquisition.initialization_entry.name, str(e)))
os.remove(en_output_file)
try:
acquisition.write_as_nxtomo(output_file=en_output_file,
data_path=entry,
input_file_path=input_file_path,
request_input=request_input,
input_callback=input_callback,
plugins=plugins)
# if split files create a master file with link to those entries
if single_file is False:
_logger.info('create link in %s' % output_file)
with HDF5File(output_file, 'a') as master_file:
mf_entry = 'entry' + str(i_acquisition).zfill(4)
link_file = os.path.relpath(en_output_file, os.path.dirname(output_file))
master_file[mf_entry] = h5py.ExternalLink(link_file,
entry)
res.append((output_file, mf_entry))
else:
res.append((en_output_file, entry))
except Exception as e:
_logger.error('Fails to write %s. Error is %s' % (acquisition.initialization_entry.name, str(e)))
progress.increase_advancement()
return tuple(res)
......@@ -635,7 +641,7 @@ def _get_entry_type(entry: h5py.Group, scan_titles) -> typing.Union[None, Acquis
def get_bliss_tomo_entries(input_file_path, scan_titles):
"""Util function. Used by tomwer for example"""
with h5py.File(input_file_path, 'r') as h5d:
with HDF5File(input_file_path, 'r') as h5d:
acquisitions = []
for group_name in h5d.keys():
......@@ -659,25 +665,29 @@ class _Acquisition:
_DISTANCE_PATH = 'technique/scan/sample_detector_distance'
_X_PIXEL_SIZE_PATH = 'technique/detector/pixel_size'
_Y_PIXEL_SIZE_PATH = 'technique/detector/pixel_size'
_X_MAGNIFIED_PIXEL_SIZE = ('technique/optic/sample_pixel_size',
'technique/optic/sample_pixel_size ')
# warning: we can have two cases: one with an empty space at the end or not
_Y_MAGNIFIED_PIXEL_SIZE = ('technique/optic/sample_pixel_size',
'technique/optic/sample_pixel_size ')
# warning: we can have two cases: one with an empty space at the end or not
_NAME_PATH = 'technique/scan/name'
def __init__(self, entry: h5py.Group, file_keys: H5FileKeys, scan_titles):
_FOV_PATH = 'technique/scan/field_of_view'
def __init__(self, entry: h5py.Group, file_keys: H5FileKeys, scan_titles,
param_already_defined):
self._initialization_entry = entry
self._indexes = entry[_Acquisition._SCAN_NUMBER_PATH]
self._indexes_str = tuple([str(index) for index in entry[_Acquisition._SCAN_NUMBER_PATH]])
self._registered_entries = []
self._file_keys = file_keys
self._scan_titles = scan_titles
self._param_already_defined = param_already_defined
"""user can have defined already some parameter values as energy.
The idea is to avoid asking him if """
# variables set by the `_preprocess_frames` function
self._data = None
......@@ -777,27 +787,6 @@ class _Acquisition:
else:
self._registered_entries.append(entry)
def is_valid(self) -> bool:
"""Make sure all scan number are present"""
return True
registered_entries_str = tuple([entry.name for entry in self._registered_entries])
def has_been_registered(index):
for entry_str in registered_entries_str:
if index in entry_str:
return True
return False
missing = []
for index_str in self._indexes_str:
if not has_been_registered(index_str):
missing.append(index_str)
if len(missing) > 0:
_logger.error('%s indexes are missing' % missing)
return False
else:
return True
def write_as_nxtomo(self, output_file: str, data_path: str,
input_file_path: str, request_input: bool, plugins,
input_callback=None) -> None:
......@@ -820,7 +809,7 @@ class _Acquisition:
# first retrieve the data and create some virtual dataset.
self._preprocess_frames(input_file_path)
with h5py.File(output_file, 'a') as h5_file:
with HDF5File(output_file, 'a') as h5_file:
entry = h5_file.require_group(data_path)
entry.attrs["NX_class"] = u"NXentry"
entry.attrs["definition"] = u"NXtomo"
......@@ -858,7 +847,7 @@ class _Acquisition:
if type_ is AcquisitionStep.PROJECTION:
image_key_control = ImageKey.PROJECTION
image_key = ImageKey.PROJECTION
elif type_ is AcquisitionStep.ALIGNEMENT:
elif type_ is AcquisitionStep.ALIGNMENT:
image_key_control = ImageKey.ALIGNMENT
image_key = ImageKey.PROJECTION
elif type_ is AcquisitionStep.DARK:
......@@ -981,7 +970,7 @@ class _Acquisition:
keys=self._file_keys.x_trans_keys,
info_retrieve='x translation',
expected_unit='mm')
x_tr = x_tr * metricsystem.MetricSystem.from_value(unit).value
x_tr = numpy.asarray(x_tr) * metricsystem.MetricSystem.from_value(unit).value
return x_tr, 'm'
def _get_y_translation(self, instrument_grp, n_frame) -> tuple:
......@@ -991,7 +980,7 @@ class _Acquisition:
keys=self._file_keys.y_trans_keys,
info_retrieve='y translation',
expected_unit='mm')
y_tr = y_tr * metricsystem.MetricSystem.from_value(unit).value
y_tr = numpy.asarray(y_tr) * metricsystem.MetricSystem.from_value(unit).value
return y_tr, 'm'
def _get_z_translation(self, instrument_grp, n_frame) -> tuple:
......@@ -1001,7 +990,7 @@ class _Acquisition:
keys=self._file_keys.z_trans_keys,
info_retrieve='z translation',
expected_unit='mm')
z_tr = z_tr * metricsystem.MetricSystem.from_value(unit).value
z_tr = numpy.asarray(z_tr) * metricsystem.MetricSystem.from_value(unit).value
return z_tr, 'm'
def _get_expo_time(self, detector_grp, n_frame) -> tuple:
......@@ -1061,8 +1050,12 @@ class _Acquisition:
def _write_beam(self, root_node, request_input, input_callback):
beam_node = root_node.create_group('beam')
energy, unit = self._get_energy(ask_if_0=request_input,
input_callback=input_callback)
if 'energy' in self._param_already_defined:
energy = self._param_already_defined['energy']
unit = 'kev'
else:
energy, unit = self._get_energy(ask_if_0=request_input,
input_callback=input_callback)
if energy is not None:
beam_node["incident_energy"] = energy
beam_node["incident_energy"].attrs["unit"] = unit
......@@ -1105,6 +1098,10 @@ class _Acquisition:
if y_magnified_pix_size is not None:
detector_node['y_magnified_pixel_size'] = y_magnified_pix_size
detector_node['y_magnified_pixel_size'].attrs['unit'] = unit
# write field of view
fov = self._get_field_of_fiew()
if fov is not None:
detector_node['field_of_view'] = fov
def _create_data_virtual_dataset(self, detector_node):
if (self.n_frames is None or self.dim_1 is None or self.dim_2 is None
......@@ -1180,7 +1177,7 @@ class _Acquisition:
energy = self._initialization_entry[self._ENERGY_PATH][()]
unit = self._get_unit(self._initialization_entry[self._ENERGY_PATH],
default_unit='kev')
if ask_if_0:
if energy == 0 and ask_if_0:
desc = 'Energy has not been registered. Please enter ' \
'incoming beam energy (in kev):'
if input_callback is None:
......@@ -1213,7 +1210,7 @@ class _Acquisition:
"""return tuple(pixel_size, unit)"""
assert axis in ('x', 'y')
self._check_has_metadata()
path = self._X_PIXEL_SIZE_PATH if axis == 'x' else self._Y_PIXEL_SIZE_PATH
path = self._file_keys.x_pixel_size if axis == 'x' else self._file_keys.y_pixel_size
if path in self._initialization_entry:
node = self.initialization_entry[path]
size_ = node[()][0]
......@@ -1241,6 +1238,13 @@ class _Acquisition:
_logger.warning("unable to find %s magnified pixel size. Will be skip" % axis)
return None, None
def _get_field_of_fiew(self):
if self._FOV_PATH in self._initialization_entry:
return self.initialization_entry[self._FOV_PATH][()]
else:
_logger.warning("unable to find information regarding field of view")
return None
@staticmethod
def _get_unit(node: h5py.Dataset, default_unit):
"""Simple process to retrieve unit from an attribute"""
......@@ -1252,5 +1256,3 @@ class _Acquisition:
_logger.warning('no unit found for %s, take default unit: %s'
'' % (node.name, default_unit))
return default_unit
......@@ -35,9 +35,9 @@ __date__ = "13/05/2020"
### HDF5 settings
H5_VALID_CAMERA_NAMES = ('pcolinux', 'basler1', 'frelon1')
H5_VALID_CAMERA_NAMES = ('pcolinux', 'basler1', 'frelon1', 'pco2linux', 'dimaxlinux')
H5_ROT_ANGLE_KEYS = ('hrsrot', 'srot', 'diffrz')
H5_ROT_ANGLE_KEYS = ('mhsrot', 'mrsrot', 'hrsrot', 'srot', 'diffrz')
H5_X_TRANS_KEYS = ("sx", 'd3tx')
......@@ -47,7 +47,7 @@ H5_Z_TRANS_KEYS = ("sz", 'd3tz')
H5_ACQ_EXPO_TIME_KEYS = ('acq_expo_time',)
H5_INIT_TITLES = ('tomo:basic', 'tomo:zseries', 'tomo:fullturn',
H5_INIT_TITLES = ('pcotomo:basic', 'tomo:basic', 'tomo:zseries', 'tomo:fullturn',
'sequence_of_scans', 'tomo:halfturn')
"""if a scan starts by one of those string then is considered as
initialization scan"""
......@@ -64,6 +64,10 @@ H5_ALIGNMENT_TITLES = ('static images', 'ascan diffrz 180 0 4 0.1')
"""if a scan starts by one of those string then is considered as
alignment scan"""
H5_X_PIXEL_SIZE = 'technique/detector/pixel_size'
H5_Y_PIXEL_SIZE = 'technique/detector/pixel_size'
### EDF settings
EDF_MOTOR_POS = 'motor_pos'
......
......@@ -2,4 +2,4 @@ silx
h5py
numpy
fabio
tomoscan >= 0.3
\ No newline at end of file
tomoscan >= 0.3.2
\ No newline at end of file
......@@ -189,7 +189,7 @@ def get_project_configuration(dry_run):
"numpy",
"silx",
"h5py",
"tomoscan >= 0.3",
"tomoscan >= 0.3.2",
]
setup_requires = ["setuptools",]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment