From 9c0850033a91068ad9b5ad6174ec8618f76dd367 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 19 Apr 2022 08:54:06 +0200 Subject: [PATCH 01/30] Update nxtomo.py --- nxtomomill/nexus/nxtomo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nxtomomill/nexus/nxtomo.py b/nxtomomill/nexus/nxtomo.py index 553b759..5931534 100644 --- a/nxtomomill/nexus/nxtomo.py +++ b/nxtomomill/nexus/nxtomo.py @@ -147,9 +147,9 @@ class NXtomo(NXobject): @group_size.setter def group_size(self, group_size: Optional[int]): - if not isinstance(group_size, (type(None), int)): + if not (isinstance(group_size, (type(None), int)) or numpy.isscalar(group_size)): raise TypeError( - f"group_size is expected ot be an instance of {int} or None. Not {type(group_size)}" + f"group_size is expected ot be None or a scalar. Not {type(group_size)}" ) self._group_size = group_size -- GitLab From e7ecaecb60573f3aa049adea77ef1aca488a5efd Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 19 Apr 2022 17:15:11 +0200 Subject: [PATCH 02/30] add missing import from numpy --- nxtomomill/nexus/nxtomo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nxtomomill/nexus/nxtomo.py b/nxtomomill/nexus/nxtomo.py index 5931534..9caba25 100644 --- a/nxtomomill/nexus/nxtomo.py +++ b/nxtomomill/nexus/nxtomo.py @@ -43,6 +43,7 @@ from tomoscan.nexus.paths.nxtomo import get_paths as get_nexus_paths from silx.io.url import DataUrl import logging import h5py +import numpy _logger = logging.getLogger(__name__) -- GitLab From c9ecba9dfa18d3a3344fff55cd5522f15ee81f72 Mon Sep 17 00:00:00 2001 From: Pierre Paleo Date: Wed, 20 Apr 2022 11:29:46 +0200 Subject: [PATCH 03/30] Apply black --- nxtomomill/nexus/nxtomo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nxtomomill/nexus/nxtomo.py b/nxtomomill/nexus/nxtomo.py index 9caba25..5678058 100644 --- a/nxtomomill/nexus/nxtomo.py +++ b/nxtomomill/nexus/nxtomo.py @@ -148,7 +148,9 @@ class NXtomo(NXobject): @group_size.setter def group_size(self, group_size: Optional[int]): - if not (isinstance(group_size, (type(None), int)) or numpy.isscalar(group_size)): + if not ( + isinstance(group_size, (type(None), int)) or numpy.isscalar(group_size) + ): raise TypeError( f"group_size is expected ot be None or a scalar. Not {type(group_size)}" ) -- GitLab From 8bc4cb458d0c5580c30c9b35c4656bdf1860a5a1 Mon Sep 17 00:00:00 2001 From: Pierre Paleo Date: Wed, 20 Apr 2022 11:40:34 +0200 Subject: [PATCH 04/30] str and bytes are numpy scalars --- nxtomomill/nexus/nxtomo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nxtomomill/nexus/nxtomo.py b/nxtomomill/nexus/nxtomo.py index 5678058..a17b4f3 100644 --- a/nxtomomill/nexus/nxtomo.py +++ b/nxtomomill/nexus/nxtomo.py @@ -149,7 +149,8 @@ class NXtomo(NXobject): @group_size.setter def group_size(self, group_size: Optional[int]): if not ( - isinstance(group_size, (type(None), int)) or numpy.isscalar(group_size) + isinstance(group_size, (type(None), int)) + or (numpy.isscalar(group_size) and not isinstance(group_size, (str, bytes))) ): raise TypeError( f"group_size is expected ot be None or a scalar. Not {type(group_size)}" -- GitLab From 8094c24941c01b1f1bc9f5acd4ce8d90c3a593c4 Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Thu, 21 Apr 2022 16:00:12 +0200 Subject: [PATCH 05/30] io: move io/config.py to a module --- nxtomomill/io/config/__init__.py | 39 +++++++++++++++++++ .../io/{config.py => config/hdf5config.py} | 8 ++-- 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 nxtomomill/io/config/__init__.py rename nxtomomill/io/{config.py => config/hdf5config.py} (99%) diff --git a/nxtomomill/io/config/__init__.py b/nxtomomill/io/config/__init__.py new file mode 100644 index 0000000..7a262b8 --- /dev/null +++ b/nxtomomill/io/config/__init__.py @@ -0,0 +1,39 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2015-2020 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# +# ########################################################################### + + +__authors__ = [ + "H. Payno", +] +__license__ = "MIT" +__date__ = "21/04/2022" + +from .hdf5config import ( # noqa F401,F403 + TomoHDF5Config, + XRD3DHDF5Config, + DXFileConfiguration, + generate_default_h5_config, +) diff --git a/nxtomomill/io/config.py b/nxtomomill/io/config/hdf5config.py similarity index 99% rename from nxtomomill/io/config.py rename to nxtomomill/io/config/hdf5config.py index ace01e3..af74101 100644 --- a/nxtomomill/io/config.py +++ b/nxtomomill/io/config/hdf5config.py @@ -28,7 +28,7 @@ contains the HDF5Config """ -__authors__ = ["H. Payno, J.Garriga"] +__authors__ = ["H. Payno", "J.Garriga"] __license__ = "MIT" __date__ = "08/07/2021" @@ -106,10 +106,10 @@ class TomoHDF5Config: GENERAL_SECTION_DK = "GENERAL_SECTION" - OUTPUT_FILE_DK = "output_file" - INPUT_FILE_DK = "input_file" + OUTPUT_FILE_DK = "output_file" + OVERWRITE_DK = "overwrite" FILE_EXTENSION_DK = "file_extension" @@ -128,8 +128,8 @@ class TomoHDF5Config: COMMENTS_GENERAL_SECTION = { GENERAL_SECTION_DK: "general information. \n", - OUTPUT_FILE_DK: "output file name. If not provided will use the input file basename and the file extension", INPUT_FILE_DK: "input file if not provided must be provided from the command line", + OUTPUT_FILE_DK: "output file name. If not provided will use the input file basename and the file extension", OVERWRITE_DK: "overwrite output files if exists without asking", FILE_EXTENSION_DK: "file extension. Ignored if the output file is provided and contains an extension", LOG_LEVEL_DK: 'Log level. Valid levels are "debug", "info", "warning" and "error"', -- GitLab From 9698a548d9fbd80698b73c2592e7673cc5b8d68e Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Thu, 21 Apr 2022 16:00:45 +0200 Subject: [PATCH 06/30] io/edfconfig: start to create TomoEDFConfig with necessary keys --- nxtomomill/io/config/configbase.py | 168 +++++++++ nxtomomill/io/config/edfconfig.py | 550 +++++++++++++++++++++++++++++ nxtomomill/io/config/hdf5config.py | 110 +----- nxtomomill/settings.py | 8 +- 4 files changed, 725 insertions(+), 111 deletions(-) create mode 100644 nxtomomill/io/config/configbase.py create mode 100644 nxtomomill/io/config/edfconfig.py diff --git a/nxtomomill/io/config/configbase.py b/nxtomomill/io/config/configbase.py new file mode 100644 index 0000000..cc1f65b --- /dev/null +++ b/nxtomomill/io/config/configbase.py @@ -0,0 +1,168 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2015-2020 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# +# ########################################################################### + + +__authors__ = [ + "H. Payno", +] +__license__ = "MIT" +__date__ = "21/04/2022" + + +from typing import Union, Iterable +from nxtomomill.utils import FileExtension +from nxtomomill.nexus.nxdetector import FieldOfView +import logging + + +class ConfigBase: + + __isfrozen = False + # to ease API and avoid setting wrong attributes we 'freeze' the attributes + # see https://stackoverflow.com/questions/3603502/prevent-creating-new-attributes-outside-init + + def __setattr__(self, __name, __value): + if self.__isfrozen and not hasattr(self, __name): + raise AttributeError("can't set attribute", __name) + else: + super().__setattr__(__name, __value) + + @property + def output_file(self) -> Union[None, str]: + return self._output_file + + @output_file.setter + def output_file(self, output_file: Union[None, str]): + if not isinstance(output_file, (str, type(None))): + raise TypeError("'input_file' should be None or an instance of Iterable") + elif output_file == "": + self._output_file = None + else: + self._output_file = output_file + + @property + def overwrite(self) -> bool: + return self._overwrite + + @overwrite.setter + def overwrite(self, overwrite: bool) -> None: + if not isinstance(overwrite, bool): + raise TypeError("'overwrite' should be a boolean") + else: + self._overwrite = overwrite + + @property + def file_extension(self) -> FileExtension: + return self._file_extension + + @file_extension.setter + def file_extension(self, file_extension: str): + self._file_extension = FileExtension.from_value(file_extension) + + @property + def log_level(self): + return self._log_level + + @log_level.setter + def log_level(self, level: str): + self._log_level = getattr(logging, level.upper()) + + def _set_freeze(self, freeze=True): + self.__isfrozen = freeze + + @property + def field_of_view(self) -> Union[None, FieldOfView]: + return self._field_of_view + + @field_of_view.setter + def field_of_view(self, fov: Union[None, FieldOfView, str]): + if fov is None: + self._field_of_view = fov + elif isinstance(fov, str): + self._field_of_view = FieldOfView.from_value(fov.title()) + elif isinstance(fov, FieldOfView): + self._field_of_view = fov + else: + raise TypeError( + "fov is expected to be None, a string or " + "FieldOfView. Not {}".format(type(fov)) + ) + + @property + def rotation_angle_keys(self) -> Iterable: + return self._rot_angle_keys + + @rotation_angle_keys.setter + def rotation_angle_keys(self, keys: Iterable): + if not isinstance(keys, Iterable): + raise TypeError("'keys' should be an Iterable") + else: + self._rot_angle_keys = keys + + @property + def x_trans_keys(self) -> Iterable: + return self._x_trans_keys + + @x_trans_keys.setter + def x_trans_keys(self, keys) -> None: + if not isinstance(keys, Iterable): + raise TypeError("'keys' should be an Iterable") + else: + self._x_trans_keys = keys + + @property + def y_trans_keys(self) -> Iterable: + return self._y_trans_keys + + @y_trans_keys.setter + def y_trans_keys(self, keys) -> None: + if not isinstance(keys, Iterable): + raise TypeError("'keys' should be an Iterable") + else: + self._y_trans_keys = keys + + @property + def z_trans_keys(self) -> Iterable: + return self._z_trans_keys + + @z_trans_keys.setter + def z_trans_keys(self, keys) -> None: + if not isinstance(keys, Iterable): + raise TypeError("'keys' should be an Iterable") + else: + self._z_trans_keys = keys + + def to_dict(self) -> dict: + """convert the configuration to a dictionary""" + raise NotADirectoryError("Base class") + + def load_from_dict(self, dict_: dict) -> None: + """Load the configuration from a dictionary""" + raise NotADirectoryError("Base class") + + @staticmethod + def from_dict(dict_: dict): + raise NotADirectoryError("Base class") diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py new file mode 100644 index 0000000..71de25f --- /dev/null +++ b/nxtomomill/io/config/edfconfig.py @@ -0,0 +1,550 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2015-2020 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# +# ########################################################################### + + +__authors__ = [ + "H. Payno", +] +__license__ = "MIT" +__date__ = "21/04/2022" + +from tomoscan.unitsystem.metricsystem import MetricSystem +from tomoscan.unitsystem.energysystem import EnergySI +from nxtomomill.io.config.configbase import ConfigBase +from nxtomomill.io.utils import filter_str_def +from nxtomomill.nexus.nxsource import SourceType +from nxtomomill.utils import FileExtension +from nxtomomill.settings import Tomo +from typing import Iterable, Optional, Union +import logging + + +_logger = logging.getLogger(__name__) + + +class TomoEDFConfig(ConfigBase): + """ + Configuration class to provide to the convert from h5 to nx + """ + + _valid_metric_values = MetricSystem.values() + _valid_energy_values = EnergySI.values() + + # General section keys + + GENERAL_SECTION_DK = "GENERAL_SECTION" + + INPUT_FOLDER_DK = "input_folder" + + OUTPUT_FILE_DK = "output_file" + + FILE_EXTENSION_DK = "file_extension" + + OVERWRITE_DK = "overwrite" + + DATASET_BASENAME_DK = "file_prefix" + + LOG_LEVEL_DK = "log_level" + + TITLE_DK = "title" + + IGNORE_FILE_PATTERN_DK = "patterns_to_ignores" + + COMMENTS_GENERAL_SECTION = { + GENERAL_SECTION_DK: "general information. \n", + INPUT_FOLDER_DK: "input file if not provided must be provided from the command line", + OUTPUT_FILE_DK: "output file name. If not provided will use the input file basename and the file extension", + OVERWRITE_DK: "overwrite output files if exists without asking", + FILE_EXTENSION_DK: "file extension. Ignored if the output file is provided and contains an extension", + DATASET_BASENAME_DK: "dataset file prefix. If not provided will take the folder basename", + LOG_LEVEL_DK: 'Log level. Valid levels are "debug", "info", "warning" and "error"', + TITLE_DK: "NXtomo title", + IGNORE_FILE_PATTERN_DK: "some file pattern leading to ignoring the file", + } + + # EDF KEYS SECTION + + EDF_KEYS_SECTION_DK = "EDF_KEYS_SECTION" + + MOTOR_POSITION_KEY_DK = "motor_position_key" + + MOTOR_MNE_KEY_DK = "motor_mne_key" + + X_TRANS_KEY_DK = "x_translation_key" + + Y_TRANS_KEY_DK = "y_translation_key" + + Z_TRANS_KEY_DK = "z_translation_key" + + ROT_ANGLE_KEY_DK = "rot_angle_key" + + COMMENTS_KEYS_SECTION = { + EDF_KEYS_SECTION_DK: "section to define EDF keys to pick in headers to deduce several information.\n", + MOTOR_POSITION_KEY_DK: "motor position key", + MOTOR_MNE_KEY_DK: "motor mne key", + X_TRANS_KEY_DK: "key to be used for x translation", + Y_TRANS_KEY_DK: "key to be used for y translation", + Z_TRANS_KEY_DK: "key to be used for z translation", + ROT_ANGLE_KEY_DK: "key to be used for rotation angle", + } + + # DARK AND FLAT SECTION + + FLAT_DARK_SECTION_DK = "DARK_AND_FLAT_SECTION" + + DARK_NAMES_DK = "dark_names_prefix" + + FLAT_NAMES_DK = "flat_names_prefix" + + COMMENTS_DARK_FLAT_SECTION = { + FLAT_DARK_SECTION_DK: "section to define dark and flat detection. \n", + DARK_NAMES_DK: "prefix of the dark field file(s)", + FLAT_NAMES_DK: "prefix of the flat field file(s)", + } + + # UNITS SECTION + + UNIT_SECTION_DK = "UNIT_SECTION" + + PIXEL_SIZE_EXPECTED_UNIT = "expected_unit_for_pixel_size" + + DISTANCE_EXPECTED_UNIT = "expected_unit_for_distance" + + ENERGY_EXPECTED_UNIT = "expected_unit_for_energy" + + X_TRANS_EXPECTED_UNIT = "expected_unit_for_x_translation" + + Y_TRANS_EXPECTED_UNIT = "expected_unit_for_y_translation" + + Z_TRANS_EXPECTED_UNIT = "expected_unit_for_z_translation" + + COMMENTS_UNIT_SECTION_DK = { + GENERAL_SECTION_DK: "Details units system used on SPEC side to save data. All will ne converted to NXtomo default (SI at the exception of energy-keV) \n", + PIXEL_SIZE_EXPECTED_UNIT: f"Size used to save pixel size. Must be in of {_valid_metric_values}", + DISTANCE_EXPECTED_UNIT: f"Unit used by SPEC to save sample to detector distance. Must be in of {_valid_metric_values}", + ENERGY_EXPECTED_UNIT: f"Unit used by SPEC to save energy. Must be in of {_valid_energy_values}", + X_TRANS_EXPECTED_UNIT: f"Unit used by bliss to save x translation. Must be in of {_valid_metric_values}", + Y_TRANS_EXPECTED_UNIT: f"Unit used by bliss to save y translation. Must be in of {_valid_metric_values}", + Z_TRANS_EXPECTED_UNIT: f"Unit used by bliss to save z translation. Must be in of {_valid_metric_values}", + } + + # SAMPLE SECTION + + SAMPLE_SECTION_DK = "SAMPLE_SECTION" + + SAMPLE_NAME_DK = "sample_name" + + COMMENTS_SAMPLE_SECTION_DK = { + SAMPLE_SECTION_DK: "section dedicated to sample definition.\n", + SAMPLE_NAME_DK: "name of the sample", + } + + # SOURCE SECTION + + SOURCE_SECTION_DK = "SOURCE_SECTION" + + INSTRUMENT_NAME_DK = "instrument_name" + + SOURCE_NAME_DK = "source_name" + + SOURCE_TYPE_DK = "source_type" + + COMMENTS_SOURCE_SECTION_DK = { + SOURCE_SECTION_DK: "section dedicated to source definition.\n", + INSTRUMENT_NAME_DK: "name of the instrument", + SOURCE_NAME_DK: "name of the source", + SOURCE_TYPE_DK: f"type of the source. Must be one of {SourceType.values()}", + } + + # DETECTOR SECTION + + DETECTOR_SECTION_DK = "DETECTOR_SECTION" + + FIELD_OF_VIEW_DK = "field_of_view" + + COMMENTS_DETECTOR_SECTION_DK = { + DETECTOR_SECTION_DK: "section dedicated to detector definition \n", + FIELD_OF_VIEW_DK: "Detector field of view. Must be in `Half` or `Full`", + } + + # create comments + + COMMENTS = COMMENTS_GENERAL_SECTION + COMMENTS.update(COMMENTS_KEYS_SECTION) + COMMENTS.update(COMMENTS_DARK_FLAT_SECTION) + COMMENTS.update(COMMENTS_UNIT_SECTION_DK) + COMMENTS.update(COMMENTS_SAMPLE_SECTION_DK) + COMMENTS.update(COMMENTS_SOURCE_SECTION_DK) + COMMENTS.update(COMMENTS_DETECTOR_SECTION_DK) + + def __init__(self): + # general information + self._input_folder = None + self._output_file = None + self._file_extension = FileExtension.NX + self._overwrite = False + self._dataset_basename = None + self._log_level = logging.WARNING + self._title = None + self._ignore_file_patterns = Tomo.EDF.TO_IGNORE + + # edf header keys + self._motor_position_keys = Tomo.EDF.MOTOR_POS + self._motor_mne_keys = Tomo.EDF.MOTOR_MNE + self._x_trans_keys = Tomo.EDF.X_TRANS + self._y_trans_keys = Tomo.EDF.Y_TRANS + self._z_trans_keys = Tomo.EDF.Z_TRANS + self._rot_angle_keys = Tomo.EDF.ROT_ANGLE + + # dark and flat + self._dark_names = Tomo.EDF.DARK_NAMES + self._flat_names = Tomo.EDF.REFS_NAMES + + # units + self._pixel_size_unit = MetricSystem.MICROMETER + self._distance_unit = MetricSystem.METER + self._energy_unit = EnergySI.KILOELECTRONVOLT + self._x_trans_unit = MetricSystem.METER + self._y_trans_unit = MetricSystem.METER + self._z_trans_unit = MetricSystem.METER + + # sample + self._sample_name = None + + # source + self._instrument_name = None + self._source_name = "ESRF" + self._source_type = SourceType.SYNCHROTRON_X_RAY_SOURCE + + # detector + self._field_of_view = None + + self._set_freeze(True) + + @property + def input_folder(self) -> Optional[str]: + return self._input_folder + + @property + def dataset_basename(self) -> Optional[str]: + return self._dataset_basename + + @property + def title(self) -> Optional[str]: + return self._title + + @title.setter + def title(self, title: Optional[str]) -> None: + if not isinstance(title, (type(None), str)): + raise TypeError( + f"title is expected to be None or an instance of str. Not {type(title)}" + ) + self._title = title + + @property + def ignore_file_patterns(self) -> Optional[Iterable]: + return self._ignore_file_patterns + + @ignore_file_patterns.setter + def ignore_file_patterns(self, patterns): + if not isinstance(patterns, Iterable): + raise TypeError("patterns is expected to be an Iterable") + self._patterns = patterns + + @property + def motor_position_keys(self) -> Iterable: + return self._motor_position_keys + + @motor_position_keys.setter + def motor_position_keys(self, keys: Optional[Iterable]) -> None: + if not isinstance(keys, (type(None), Iterable)): + raise TypeError("keys is expected to be None or an Iterable") + self._motor_position_keys = keys + + @property + def motor_mne_keys(self) -> Iterable: + return self._motor_mne_keys + + @motor_mne_keys.setter + def motor_mne_keys(self, keys) -> None: + if not isinstance(keys, (type(None), Iterable)): + raise TypeError("keys is expected to be None or an Iterable") + self._motor_mne_keys = keys + + @property + def dark_names(self) -> Iterable: + return self._dark_names + + @dark_names + def dark_names(self, names: Iterable) -> None: + if not isinstance(names, Iterable): + raise TypeError("names is expected to be an Iterable") + self._dark_names = names + + @property + def flat_names(self) -> Iterable: + return self._flat_names + + @flat_names.setter + def flat_names(self, names: Iterable) -> None: + if not isinstance(names, Iterable): + raise TypeError("names is expected to be an Iterable") + self._flat_names = names + + @property + def pixel_size_unit(self) -> MetricSystem: + return self._pixel_size_unit + + @pixel_size_unit.setter + def pixel_size_unit(self, unit: MetricSystem) -> None: + if not isinstance(unit, MetricSystem): + raise TypeError("unit is expected to be an instance of MetricSystem") + self._pixel_size_unit = unit + + @property + def distance_unit(self) -> MetricSystem: + return self._distance_unit + + @distance_unit.setter + def distance_unit(self, unit: MetricSystem) -> None: + if not isinstance(unit, MetricSystem): + raise TypeError("unit is expected to be an instance of MetricSystem") + self._distance_unit = unit + + @property + def energy_unit(self) -> EnergySI: + return self._energy_unit + + @energy_unit.setter + def energy_unit(self, unit: EnergySI) -> None: + if not isinstance(unit, EnergySI): + raise TypeError("unit is expected to be an instance of EnergySI") + self._energy_unit = unit + + @property + def x_trans_unit(self) -> MetricSystem: + return self._x_trans_unit + + @x_trans_unit.setter + def x_trans_unit(self, unit: MetricSystem) -> None: + if not isinstance(unit, MetricSystem): + raise TypeError("unit is expected to be an instance of MetricSystem") + self._x_trans_unit = unit + + @property + def y_trans_unit(self) -> MetricSystem: + return self._y_trans_unit + + @y_trans_unit.setter + def y_trans_unit(self, unit: MetricSystem) -> None: + if not isinstance(unit, MetricSystem): + raise TypeError("unit is expected to be an instance of MetricSystem") + self._y_trans_unit = unit + + @property + def z_trans_unit(self) -> MetricSystem: + return self._z_trans_unit + + @z_trans_unit.setter + def z_trans_unit(self, unit: MetricSystem) -> None: + if not isinstance(unit, MetricSystem): + raise TypeError("unit is expected to be an instance of MetricSystem") + self._z_trans_unit = unit + + @property + def sample_name(self) -> Optional[str]: + return self._sample_name + + @sample_name.setter + def sample_name(self, name: Optional[str]) -> None: + if not isinstance(name, (type(None), str)): + raise TypeError("name is expected to be None or an instance of str") + self._sample_name = name + + @property + def instrument_name(self) -> Optional[str]: + return self._instrument_name + + @instrument_name.setter + def instrument_name(self, name: Optional[str]): + if not isinstance(name, (type(None), str)): + raise TypeError("name is expected to be None or an instance of str") + self._instrument_name = name + + @property + def source_name(self) -> Optional[str]: + return self._source_name + + @source_name.setter + def source_name(self, name: Optional[str]) -> None: + if not isinstance(name, (type(None), str)): + raise TypeError("name is expected to be None or an instance of str") + self._source_name = name + + @property + def source_type(self) -> Optional[SourceType]: + return self._source_type + + @source_type.setter + def source_type(self, source_type: Optional[Union[SourceType, str]]): + if not isinstance(source_type, (type(None), str, SourceType)): + raise TypeError( + "source_type is expected to be None or an instance of SourceType or str" + ) + if source_type is None: + self._source_type = None + else: + self._source_name = SourceType.from_value(source_type) + + def to_dict(self) -> dict: + """convert the configuration to a dictionary""" + return { + self.GENERAL_SECTION_DK: { + self.INPUT_FOLDER_DK: self.input_folder, + self.OUTPUT_FILE_DK: self.output_file, + self.OVERWRITE_DK: self.overwrite, + self.FILE_EXTENSION_DK: self.file_extension, + self.DATASET_BASENAME_DK: self.dataset_basename, + self.LOG_LEVEL_DK: self.log_level, + self.TITLE_DK: self.title, + self.IGNORE_FILE_PATTERN_DK: self.ignore_file_patterns, + }, + self.EDF_KEYS_SECTION_DK: { + self.MOTOR_POSITION_KEY_DK: self.motor_position_keys, + self.MOTOR_MNE_KEY_DK: self.motor_mne_keys, + self.rotation_angle_keys: self.rotation_angle_keys, + self.X_TRANS_KEY_DK: self.x_trans_keys, + self.Y_TRANS_KEY_DK: self.y_trans_keys, + self.Z_TRANS_KEY_DK: self.z_trans_keys, + }, + self.FLAT_DARK_SECTION_DK: { + self.DARK_NAMES_DK: self.dark_names, + self.FLAT_NAMES_DK: self.flat_names, + }, + self.UNIT_SECTION_DK: { + self.PIXEL_SIZE_EXPECTED_UNIT: self.pixel_size_unit, + self.DISTANCE_EXPECTED_UNIT: self.distance_unit, + self.ENERGY_EXPECTED_UNIT: self.energy_unit, + self.X_TRANS_EXPECTED_UNIT: self.x_trans_unit, + self.Y_TRANS_EXPECTED_UNIT: self.y_trans_unit, + self.Z_TRANS_EXPECTED_UNIT: self.z_trans_unit, + }, + self.SAMPLE_SECTION_DK: { + self.SAMPLE_NAME_DK: self.sample_name, + }, + self.SOURCE_SECTION_DK: { + self.INSTRUMENT_NAME_DK: self.instrument_name or "", + self.SOURCE_NAME_DK: self.source_name or "", + self.SOURCE_TYPE_DK: self.source_type.value + if self.source is not None + else "", + }, + self.DETECTOR_SECTION_DK: { + self.FIELD_OF_VIEW_DK: self.field_of_view.value + if self.field_of_view is not None + else "", + }, + } + + @staticmethod + def from_dict(dict_: dict): + """ + Create a HDF5Config object and set it from values contained in the + dictionary + :param dict dict_: settings dictionary + :return: HDF5Config + """ + config = TomoEDFConfig() + config.load_from_dict(dict_) + return config + + def load_from_dict(self, dict_: dict) -> None: + """Load the configuration from a dictionary""" + sections_loaders = { + TomoEDFConfig.GENERAL_SECTION_DK: self.load_general_section, + TomoEDFConfig.EDF_KEYS_SECTION_DK: self.load_keys_section, + TomoEDFConfig.FLAT_DARK_SECTION_DK: self.load_flat_dark_section, + TomoEDFConfig.UNIT_SECTION_DK: self.load_unit_section, + TomoEDFConfig.SAMPLE_SECTION_DK: self.load_sample_section, + TomoEDFConfig.SOURCE_SECTION_DK: self.load_source_section, + TomoEDFConfig.DETECTOR_SECTION_DK: self.load_detector_section, + } + for section_key, loaded_func in sections_loaders.items(): + if section_key in dict_: + loaded_func(dict_[section_key]) + else: + _logger.error("No {} section found".format(section_key)) + + def load_general_section(self, dict_: dict) -> None: + def cast_bool(value): + if isinstance(value, bool): + return value + elif isinstance(value, str): + if value not in ("False", "True"): + raise ValueError("value should be 'True' or 'False'") + return value == "True" + else: + raise TypeError("value should be a string") + + self.input_folder = dict_.get(TomoEDFConfig.INPUT_FOLDER_DK, None) + self.output_file = dict_.get(TomoEDFConfig.OUTPUT_FILE_DK, None) + overwrite = dict_.get(TomoEDFConfig.OVERWRITE_DK, None) + if overwrite is not None: + self.overwrite = cast_bool(overwrite) + file_extension = dict_.get(TomoEDFConfig.FILE_EXTENSION_DK, None) + if file_extension is not None: + self.file_extension = filter_str_def(file_extension) + self.dataset_basename = dict_.get(TomoEDFConfig.DATASET_BASENAME_DK, None) + log_level = dict_.get(TomoEDFConfig.LOG_LEVEL_DK, None) + if log_level is not None: + self.log_level = log_level + self.title = dict_.get(TomoEDFConfig.TITLE_DK) + self.ignore_file_patterns = dict_.get(TomoEDFConfig.IGNORE_FILE_PATTERN_DK, ()) + + def load_keys_section(self, dict_: dict) -> None: + self.motor_position_keys = dict_.get(TomoEDFConfig.MOTOR_POSITION_KEY_DK) + self.motor_mne_keys = dict_.get(TomoEDFConfig.MOTOR_MNE_KEY_DK) + self.x_trans_unit = dict_.get(TomoEDFConfig.X_TRANS_KEY_DK) + + def load_flat_dark_section(self, dict_: dict) -> None: + pass + + def load_unit_section(self, dict_: dict) -> None: + pass + + def load_sample_section(self, dict_: dict) -> None: + pass + + def load_source_section(self, dict_: dict) -> None: + pass + + def load_detector_section(self, dict_: dict) -> None: + pass + + +def generate_default_edf_config() -> dict: + """generate a default configuration for converting spec-edf to NXtomo""" + return TomoEDFConfig().to_dict() diff --git a/nxtomomill/io/config/hdf5config.py b/nxtomomill/io/config/hdf5config.py index af74101..749f646 100644 --- a/nxtomomill/io/config/hdf5config.py +++ b/nxtomomill/io/config/hdf5config.py @@ -34,6 +34,7 @@ __date__ = "08/07/2021" from nxtomomill import settings +from nxtomomill.io.config.configbase import ConfigBase from nxtomomill.utils import FileExtension from nxtomomill.utils import Format from nxtomomill.nexus.nxdetector import FieldOfView @@ -97,7 +98,7 @@ def _example_fg_list(with_comment=True, with_prefix=False) -> str: ) -class TomoHDF5Config: +class TomoHDF5Config(ConfigBase): """ Configuration class to provide to the convert from h5 to nx """ @@ -367,9 +368,6 @@ class TomoHDF5Config: self._set_freeze(True) - def _set_freeze(self, freeze=True): - self.__isfrozen = freeze - @property def input_file(self) -> Union[None, str]: return self._input_file @@ -387,46 +385,6 @@ class TomoHDF5Config: else: self._input_file = input_file - @property - def output_file(self) -> Union[None, str]: - return self._output_file - - @output_file.setter - def output_file(self, output_file: Union[None, str]): - if not isinstance(output_file, (str, type(None))): - raise TypeError("'input_file' should be None or an instance of Iterable") - elif output_file == "": - self._output_file = None - else: - self._output_file = output_file - - @property - def overwrite(self) -> bool: - return self._overwrite - - @overwrite.setter - def overwrite(self, overwrite: bool) -> None: - if not isinstance(overwrite, bool): - raise TypeError("'overwrite' should be a boolean") - else: - self._overwrite = overwrite - - @property - def file_extension(self) -> FileExtension: - return self._file_extension - - @file_extension.setter - def file_extension(self, file_extension: str): - self._file_extension = FileExtension.from_value(file_extension) - - @property - def log_level(self): - return self._log_level - - @log_level.setter - def log_level(self, level: str): - self._log_level = getattr(logging, level.upper()) - @property def raises_error(self): return self._raises_error @@ -499,24 +457,6 @@ class TomoHDF5Config: else: self._bam_single_file = bam - @property - def field_of_view(self) -> Union[None, FieldOfView]: - return self._field_of_view - - @field_of_view.setter - def field_of_view(self, fov: Union[None, FieldOfView, str]): - if fov is None: - self._field_of_view = fov - elif isinstance(fov, str): - self._field_of_view = FieldOfView.from_value(fov.title()) - elif isinstance(fov, FieldOfView): - self._field_of_view = fov - else: - raise TypeError( - "fov is expected to be None, a string or " - "FieldOfView. Not {}".format(type(fov)) - ) - # Keys section @property @@ -535,50 +475,6 @@ class TomoHDF5Config: assert not isinstance(names, str), "'{}'".format(names) self._valid_camera_names = names - @property - def rotation_angle_keys(self) -> Iterable: - return self._rot_angle_keys - - @rotation_angle_keys.setter - def rotation_angle_keys(self, keys: Iterable): - if not isinstance(keys, Iterable): - raise TypeError("'keys' should be an Iterable") - else: - self._rot_angle_keys = keys - - @property - def x_trans_keys(self) -> Iterable: - return self._x_trans_keys - - @x_trans_keys.setter - def x_trans_keys(self, keys) -> None: - if not isinstance(keys, Iterable): - raise TypeError("'keys' should be an Iterable") - else: - self._x_trans_keys = keys - - @property - def y_trans_keys(self) -> Iterable: - return self._y_trans_keys - - @y_trans_keys.setter - def y_trans_keys(self, keys) -> None: - if not isinstance(keys, Iterable): - raise TypeError("'keys' should be an Iterable") - else: - self._y_trans_keys = keys - - @property - def z_trans_keys(self) -> Iterable: - return self._z_trans_keys - - @z_trans_keys.setter - def z_trans_keys(self, keys) -> None: - if not isinstance(keys, Iterable): - raise TypeError("'keys' should be an Iterable") - else: - self._z_trans_keys = keys - @property def y_rot_key(self) -> str: return self._y_rot_key @@ -896,8 +792,8 @@ class TomoHDF5Config: """convert the configuration to a dictionary""" return { self.GENERAL_SECTION_DK: { - self.OUTPUT_FILE_DK: self.output_file or "", self.INPUT_FILE_DK: self.input_file or "", + self.OUTPUT_FILE_DK: self.output_file or "", self.OVERWRITE_DK: self.overwrite, self.FILE_EXTENSION_DK: self.file_extension.value, self.LOG_LEVEL_DK: logging.getLevelName(self.log_level).lower(), diff --git a/nxtomomill/settings.py b/nxtomomill/settings.py index 3252f26..ea34f5e 100644 --- a/nxtomomill/settings.py +++ b/nxtomomill/settings.py @@ -129,13 +129,13 @@ class Tomo: MOTOR_MNE = "motor_mne" - ROT_ANGLE = "srot" + ROT_ANGLE = ("srot",) - X_TRANS = "sx" + X_TRANS = ("sx",) - Y_TRANS = "sy" + Y_TRANS = ("sy",) - Z_TRANS = "sz" + Z_TRANS = ("sz",) # EDF_TO_IGNORE = ['HST', '_slice_'] TO_IGNORE = ("_slice_",) -- GitLab From 9c9e0e6ab3a598009facd09266dbe9b9f3bb4323 Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Fri, 22 Apr 2022 14:04:58 +0200 Subject: [PATCH 07/30] io: move config test and file to the dedicated module --- nxtomomill/app/h52nx.py | 4 +- nxtomomill/app/h5_3dxrd2nx.py | 4 +- nxtomomill/io/config/confighandler.py | 358 ++++++++++++++++++ .../io/{ => config}/test/test_config.py | 0 .../{ => config}/test/test_confighandler.py | 4 +- nxtomomill/io/confighandler.py | 4 +- 6 files changed, 366 insertions(+), 8 deletions(-) create mode 100644 nxtomomill/io/config/confighandler.py rename nxtomomill/io/{ => config}/test/test_config.py (100%) rename nxtomomill/io/{ => config}/test/test_confighandler.py (98%) diff --git a/nxtomomill/app/h52nx.py b/nxtomomill/app/h52nx.py index bac06ca..92374a3 100644 --- a/nxtomomill/app/h52nx.py +++ b/nxtomomill/app/h52nx.py @@ -105,8 +105,8 @@ import logging from nxtomomill import utils from nxtomomill.utils import Progress from nxtomomill.converter import from_h5_to_nx -from nxtomomill.io.confighandler import TomoHDF5ConfigHandler -from nxtomomill.io.confighandler import SETTABLE_PARAMETERS_UNITS +from nxtomomill.io.config.confighandler import TomoHDF5ConfigHandler +from nxtomomill.io.config.confighandler import SETTABLE_PARAMETERS_UNITS from nxtomomill.utils import Format from collections.abc import Iterable import argparse diff --git a/nxtomomill/app/h5_3dxrd2nx.py b/nxtomomill/app/h5_3dxrd2nx.py index 03ac598..e027a77 100644 --- a/nxtomomill/app/h5_3dxrd2nx.py +++ b/nxtomomill/app/h5_3dxrd2nx.py @@ -103,8 +103,8 @@ import logging from nxtomomill import utils from nxtomomill.utils import Progress from nxtomomill.converter import from_h5_to_nx -from nxtomomill.io.confighandler import XRD3DHDF5ConfigHandler -from nxtomomill.io.confighandler import SETTABLE_PARAMETERS_UNITS +from nxtomomill.io.config.confighandler import XRD3DHDF5ConfigHandler +from nxtomomill.io.config.confighandler import SETTABLE_PARAMETERS_UNITS from collections.abc import Iterable from nxtomomill.utils import Format from nxtomomill.io.config import XRD3DHDF5Config diff --git a/nxtomomill/io/config/confighandler.py b/nxtomomill/io/config/confighandler.py new file mode 100644 index 0000000..2785d89 --- /dev/null +++ b/nxtomomill/io/config/confighandler.py @@ -0,0 +1,358 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2015-2020 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ + +""" +contains the HDF5ConfigHandler +""" + +__authors__ = ["H. Payno"] +__license__ = "MIT" +__date__ = "08/03/2021" + + +from .hdf5config import TomoHDF5Config +from .hdf5config import XRD3DHDF5Config +from nxtomomill import utils +from nxtomomill.utils import Format +from nxtomomill.io.utils import filter_str_def +from typing import Union +import logging +import os + +_logger = logging.getLogger(__name__) + +SETTABLE_PARAMETERS_UNITS = { + "energy": "kev", + "x_pixel_size": "m", + "y_pixel_size": "m", + "detector_sample_distance": "m", +} + +SETTABLE_PARAMETERS_TYPE = { + "energy": float, + "x_pixel_size": float, + "y_pixel_size": float, + "detector_sample_distance": float, +} + +SETTABLE_PARAMETERS = SETTABLE_PARAMETERS_UNITS.keys() + + +def _extract_param_value(key_values): + """extract all the key / values elements from the str_list. Expected + format is `param_1_name param_1_value param_2_name param_2_value ...` + + :param str str_list: raw input string as `param_1_name param_1_value + param_2_name param_2_value ...` + :return: dict of tuple (param_name, param_value) + :rtype: dict + """ + if len(key_values) % 2 != 0: + raise ValueError( + "Expect a pair `param_name, param_value` for each " "parameters" + ) + + def pairwise(it): + it = iter(it) + while True: + try: + yield next(it), next(it) + except StopIteration: + # no more elements in the iterator + return + + res = {} + for name, value in pairwise(key_values): + if name not in SETTABLE_PARAMETERS: + raise ValueError("parameters {} is not managed".format(name)) + if name in SETTABLE_PARAMETERS_TYPE: + type_ = SETTABLE_PARAMETERS_TYPE[name] + if type_ is not None: + res[name] = type_(value) + continue + res[name] = value + return res + + +class BaseHDF5ConfigHandler: + """Class to handle inputs to HDF5Config. + And insure there is no opposite Information + """ + + @staticmethod + def get_tuple_of_keys_from_cmd(cmd_value): + return utils.get_tuple_of_keys_from_cmd(cmd_value) + + @staticmethod + def conv_str_to_bool(bstr): + return bstr in ("True", True) + + @staticmethod + def conv_log_level(bool_debug): + if bool_debug is True: + return "debug" + else: + return "warning" + + @staticmethod + def conv_xrd_ct_to_format(str_bool): + if str_bool in ("True", True): + return Format.XRD_CT + elif str_bool in ("False", False): + return Format.STANDARD + else: + return None + + def __init__(self, argparse_options, raise_error=True): + self._argparse_options = argparse_options + self._config = None + self.build_configuration(raise_error=raise_error) + + @property + def configuration(self) -> Union[None, TomoHDF5Config]: + return self._config + + @property + def argparse_options(self): + return self._argparse_options + + def _check_argparse_options(self, raise_error): + raise NotImplementedError("BaseClass") + + def _create_HDF5_config(self): + raise NotImplementedError("BAse class") + + def build_configuration(self, raise_error) -> bool: + """ + :param bool raise_error: raise error if encounter some errors. Else + display a log message + :return: True if the settings are valid + """ + self._check_argparse_options(raise_error=raise_error) + options = self.argparse_options + config = self._create_HDF5_config() + + # check input and output file + if config.input_file is None: + config.input_file = options.input_file + elif options.input_file is not None and config.input_file != options.input_file: + raise ValueError( + "Two different input files provided from " + "command line and from the configuration file" + ) + if config.input_file is None: + err = "No input file provided" + if raise_error: + raise ValueError(err) + else: + _logger.error(err) + + if config.output_file is None: + config.output_file = options.output_file + elif ( + options.output_file is not None + and config.output_file != options.output_file + ): + raise ValueError( + "Two different output files provided from " + "command line and from the configuration file" + ) + if config.output_file is None: + input_file, input_file_ext = os.path.splitext(config.input_file) + if config.file_extension is None: + err = "If no outputfile provided you should provide the " "extension" + if raise_error: + raise ValueError(err) + else: + _logger.error(err) + config.output_file = input_file + config.file_extension.value + # set parameter from the arg parse options + # key is the name of the argparse option. + # value is a tuple: (name of the setter in the HDF5Config, + # function to format the input) + self._config = self._map_option_to_config_param(config, options) + + def _map_option_to_config_param(self, config, options): + raise NotImplementedError("Base class") + + def __str__(self): + raise NotImplementedError("") + + +class TomoHDF5ConfigHandler(BaseHDF5ConfigHandler): + def _create_HDF5_config(self): + if self.argparse_options.config: + return TomoHDF5Config.from_cfg_file(self.argparse_options.config) + else: + return TomoHDF5Config() + + def _map_option_to_config_param(self, config, options): + mapping = { + "valid_camera_names": ( + "valid_camera_names", + self.get_tuple_of_keys_from_cmd, + ), + "overwrite": ("overwrite", self.conv_str_to_bool), + "file_extension": ("file_extension", filter_str_def), + "single_file": ("single_file", self.conv_str_to_bool), + "debug": ("log_level", self.conv_log_level), + "entries": ("entries", self.get_tuple_of_keys_from_cmd), + "ignore_sub_entries": ( + "sub_entries_to_ignore", + self.get_tuple_of_keys_from_cmd, + ), + "duplicate_data": ("default_copy_behavior", self.conv_str_to_bool), + "raises_error": ("raises_error", self.conv_str_to_bool), + "field_of_view": ("field_of_view", filter_str_def), + "request_input": ("request_input", self.conv_str_to_bool), + "x_trans_keys": ("x_trans_keys", self.get_tuple_of_keys_from_cmd), + "y_trans_keys": ("y_trans_keys", self.get_tuple_of_keys_from_cmd), + "z_trans_keys": ("z_trans_keys", self.get_tuple_of_keys_from_cmd), + "rot_angle_keys": ("rotation_angle_keys", self.get_tuple_of_keys_from_cmd), + "sample_detector_distance": ( + "sample_detector_distance", + self.get_tuple_of_keys_from_cmd, + ), + "acq_expo_time_keys": ( + "exposition_time_keys", + self.get_tuple_of_keys_from_cmd, + ), + "x_pixel_size_key": ("x_pixel_size_paths", self.get_tuple_of_keys_from_cmd), + "y_pixel_size_key": ("y_pixel_size_paths", self.get_tuple_of_keys_from_cmd), + "init_titles": ("init_titles", self.get_tuple_of_keys_from_cmd), + "init_zserie_titles": ( + "zserie_init_titles", + self.get_tuple_of_keys_from_cmd, + ), + "init_pcotomo_titles": ( + "pcotomo_init_titles", + self.get_tuple_of_keys_from_cmd, + ), + "dark_titles": ("dark_titles", self.get_tuple_of_keys_from_cmd), + "flat_titles": ("flat_titles", self.get_tuple_of_keys_from_cmd), + "proj_titles": ("projections_titles", self.get_tuple_of_keys_from_cmd), + "align_titles": ("alignment_titles", self.get_tuple_of_keys_from_cmd), + "set_params": ("param_already_defined", _extract_param_value), + } + for argparse_name, (config_name, format_fct) in mapping.items(): + argparse_value = getattr(options, argparse_name) + if argparse_value is not None: + value = format_fct(argparse_value) + setattr(config, config_name, value) + return config + + def _check_argparse_options(self, raise_error): + if self.argparse_options is None: + err = "No argparse options provided" + if raise_error: + raise ValueError(err) + else: + _logger.error(err) + return False + + options = self.argparse_options + if options.config is not None: + # check no other option are provided + duplicated_inputs = [] + for opt in ( + "set_params", + "align_titles", + "proj_titles", + "flat_titles", + "dark_titles", + "init_zserie_titles", + "init_titles", + "init_pcotomo_titles", + "x_pixel_size_key", + "y_pixel_size_key", + "acq_expo_time_keys", + "rot_angle_keys", + "valid_camera_names", + "z_trans_keys", + "y_trans_keys", + "x_trans_keys", + "request_input", + "raises_error", + "ignore_sub_entries", + "entries", + "debug", + "overwrite", + "single_file", + "file_extension", + "field_of_view", + "duplicate_data", + ): + if getattr(options, opt): + duplicated_inputs.append(opt) + if len(duplicated_inputs) > 0: + err = "You provided a configuration file and inputs " "for {}".format( + duplicated_inputs + ) + if raise_error: + raise ValueError(err) + else: + _logger.error(err) + return False + + +class XRD3DHDF5ConfigHandler(TomoHDF5ConfigHandler): + def _create_HDF5_config(self): + if self.argparse_options.config: + return XRD3DHDF5Config.from_cfg_file(self.argparse_options.config) + else: + return XRD3DHDF5Config() + + def _map_option_to_config_param(self, config, options): + config = super()._map_option_to_config_param(config, options) + + mapping = { + "rocking_keys": ("rocking_keys", self.get_tuple_of_keys_from_cmd), + } + for argparse_name, (config_name, format_fct) in mapping.items(): + argparse_value = getattr(options, argparse_name) + if argparse_value is not None: + value = format_fct(argparse_value) + setattr(config, config_name, value) + return config + + def _check_argparse_options(self, raise_error): + + super()._check_argparse_options(raise_error) + options = self.argparse_options + if options.config is not None: + # check no other option are provided + duplicated_inputs = [] + for opt in ("rocking_keys",): + if getattr(options, opt): + duplicated_inputs.append(opt) + if len(duplicated_inputs) > 0: + err = "You provided a configuration file and inputs " "for {}".format( + duplicated_inputs + ) + if raise_error: + raise ValueError(err) + else: + _logger.error(err) + return False diff --git a/nxtomomill/io/test/test_config.py b/nxtomomill/io/config/test/test_config.py similarity index 100% rename from nxtomomill/io/test/test_config.py rename to nxtomomill/io/config/test/test_config.py diff --git a/nxtomomill/io/test/test_confighandler.py b/nxtomomill/io/config/test/test_confighandler.py similarity index 98% rename from nxtomomill/io/test/test_confighandler.py rename to nxtomomill/io/config/test/test_confighandler.py index e8ad444..4810915 100644 --- a/nxtomomill/io/test/test_confighandler.py +++ b/nxtomomill/io/config/test/test_confighandler.py @@ -28,8 +28,8 @@ __license__ = "MIT" __date__ = "10/03/2021" -from nxtomomill.io.confighandler import TomoHDF5ConfigHandler -from nxtomomill.io.confighandler import TomoHDF5Config +from nxtomomill.io.config.confighandler import TomoHDF5ConfigHandler +from nxtomomill.io.config.hdf5config import TomoHDF5Config from nxtomomill.io.framegroup import FrameGroup import unittest import tempfile diff --git a/nxtomomill/io/confighandler.py b/nxtomomill/io/confighandler.py index f17c069..7bf5832 100644 --- a/nxtomomill/io/confighandler.py +++ b/nxtomomill/io/confighandler.py @@ -32,8 +32,8 @@ __license__ = "MIT" __date__ = "08/03/2021" -from .config import TomoHDF5Config -from .config import XRD3DHDF5Config +from .config.hdf5config import TomoHDF5Config +from .config.hdf5config import XRD3DHDF5Config from nxtomomill import utils from nxtomomill.utils import Format from nxtomomill.io.utils import filter_str_def -- GitLab From f3003f58267f48e5cec70fc3075881869100d2a7 Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Fri, 22 Apr 2022 14:05:24 +0200 Subject: [PATCH 08/30] io: improve TomoEDFConfig --- nxtomomill/io/config/configbase.py | 48 ++++++++++++++++++-- nxtomomill/io/config/edfconfig.py | 73 ++++++++++++++++++++++++++---- nxtomomill/io/config/hdf5config.py | 25 +++------- 3 files changed, 116 insertions(+), 30 deletions(-) diff --git a/nxtomomill/io/config/configbase.py b/nxtomomill/io/config/configbase.py index cc1f65b..37b3882 100644 --- a/nxtomomill/io/config/configbase.py +++ b/nxtomomill/io/config/configbase.py @@ -32,6 +32,7 @@ __license__ = "MIT" __date__ = "21/04/2022" +import configparser from typing import Union, Iterable from nxtomomill.utils import FileExtension from nxtomomill.nexus.nxdetector import FieldOfView @@ -157,12 +158,53 @@ class ConfigBase: def to_dict(self) -> dict: """convert the configuration to a dictionary""" - raise NotADirectoryError("Base class") + raise NotImplementedError("Base class") def load_from_dict(self, dict_: dict) -> None: """Load the configuration from a dictionary""" - raise NotADirectoryError("Base class") + raise NotImplementedError("Base class") @staticmethod def from_dict(dict_: dict): - raise NotADirectoryError("Base class") + raise NotImplementedError("Base class") + + def to_cfg_file(self, file_path: str): + # TODO: add some generic information like:provided order of the tuple + # will be the effective one. You can provide a key from it names if + # it is contained in the positioners group + # maybe split in sub section ? + self.dict_to_cfg(file_path=file_path, dict_=self.to_dict()) + + @staticmethod + def dict_to_cfg(file_path, dict_): + """ """ + raise NotImplementedError("Base class") + + @staticmethod + def _dict_to_cfg(file_path, dict_, comments_fct, logger): + """ """ + if not file_path.lower().endswith((".cfg", ".config")): + logger.warning("add a valid extension to the output file") + file_path += ".cfg" + config = configparser.ConfigParser(allow_no_value=True) + for section_name, values in dict_.items(): + config.add_section(section_name) + config.set(section_name, "# " + comments_fct(section_name), None) + for key, value in values.items(): + # adopt nabu design: comments are set prior to the key + config.set(section_name, "# " + comments_fct(key), None) + config.set(section_name, key, str(value)) + + with open(file_path, "w") as config_file: + config.write(config_file) + + @staticmethod + def from_cfg_file(file_path: str, encoding=None): + raise NotImplementedError("Base class") + + @staticmethod + def get_comments(key): + raise NotImplementedError("Base class") + + def __str__(self): + return str(self.to_dict()) diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index 71de25f..80d5053 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -31,6 +31,7 @@ __authors__ = [ __license__ = "MIT" __date__ = "21/04/2022" +import configparser from tomoscan.unitsystem.metricsystem import MetricSystem from tomoscan.unitsystem.energysystem import EnergySI from nxtomomill.io.config.configbase import ConfigBase @@ -525,24 +526,80 @@ class TomoEDFConfig(ConfigBase): self.ignore_file_patterns = dict_.get(TomoEDFConfig.IGNORE_FILE_PATTERN_DK, ()) def load_keys_section(self, dict_: dict) -> None: - self.motor_position_keys = dict_.get(TomoEDFConfig.MOTOR_POSITION_KEY_DK) - self.motor_mne_keys = dict_.get(TomoEDFConfig.MOTOR_MNE_KEY_DK) - self.x_trans_unit = dict_.get(TomoEDFConfig.X_TRANS_KEY_DK) + if TomoEDFConfig.MOTOR_POSITION_KEY_DK in dict_: + self.motor_position_keys = dict_.get(TomoEDFConfig.MOTOR_POSITION_KEY_DK) + if TomoEDFConfig.MOTOR_MNE_KEY_DK in dict_: + self.motor_mne_keys = dict_.get(TomoEDFConfig.MOTOR_MNE_KEY_DK) + if TomoEDFConfig.X_TRANS_KEY_DK in dict_: + self.x_trans_keys = dict_.get(TomoEDFConfig.X_TRANS_KEY_DK) + if TomoEDFConfig.Y_TRANS_KEY_DK in dict_: + self.y_trans_keys = dict_.get(TomoEDFConfig.Y_TRANS_KEY_DK) + if TomoEDFConfig.Z_TRANS_KEY_DK in dict_: + self.z_trans_keys = dict_.get(TomoEDFConfig.Z_TRANS_KEY_DK) def load_flat_dark_section(self, dict_: dict) -> None: - pass + if TomoEDFConfig.DARK_NAMES_DK in dict_: + self.dark_names = dict_.get(TomoEDFConfig.DARK_NAMES_DK) + if TomoEDFConfig.FLAT_NAMES_DK in dict_: + self.flat_names = dict_.get(TomoEDFConfig.FLAT_NAMES_DK) def load_unit_section(self, dict_: dict) -> None: - pass + if TomoEDFConfig.PIXEL_SIZE_EXPECTED_UNIT in dict_: + self.pixel_size_unit = dict_.get(TomoEDFConfig.PIXEL_SIZE_EXPECTED_UNIT) + if TomoEDFConfig.DISTANCE_EXPECTED_UNIT in dict_: + self.distance_unit = dict_.get(TomoEDFConfig.DISTANCE_EXPECTED_UNIT) + if TomoEDFConfig.ENERGY_EXPECTED_UNIT in dict_: + self.energy_unit = dict_.get(TomoEDFConfig.ENERGY_EXPECTED_UNIT) + if TomoEDFConfig.X_TRANS_EXPECTED_UNIT in dict_: + self.x_trans_unit = dict_.get(TomoEDFConfig.X_TRANS_EXPECTED_UNIT) + if TomoEDFConfig.Y_TRANS_EXPECTED_UNIT in dict_: + self.y_trans_unit = dict_.get(TomoEDFConfig.Y_TRANS_EXPECTED_UNIT) + if TomoEDFConfig.Z_TRANS_EXPECTED_UNIT in dict_: + self.z_trans_unit = dict_.get(TomoEDFConfig.Z_TRANS_EXPECTED_UNIT) def load_sample_section(self, dict_: dict) -> None: - pass + if TomoEDFConfig.SAMPLE_NAME_DK in dict_: + self.sample_name = dict_.get(TomoEDFConfig.SAMPLE_NAME_DK) def load_source_section(self, dict_: dict) -> None: - pass + if TomoEDFConfig.INSTRUMENT_NAME_DK in dict_: + self.instrument_name = dict_.get(TomoEDFConfig.INSTRUMENT_NAME_DK) + if TomoEDFConfig.SOURCE_NAME_DK in dict_: + self.source_name = dict_.get(TomoEDFConfig.SOURCE_NAME_DK) + if TomoEDFConfig.SOURCE_TYPE_DK in dict_: + self.source_type = dict_[TomoEDFConfig.SOURCE_TYPE_DK] def load_detector_section(self, dict_: dict) -> None: - pass + if TomoEDFConfig.FIELD_OF_VIEW_DK in dict_: + self.field_of_view = dict_[TomoEDFConfig.FIELD_OF_VIEW_DK] + + def to_cfg_file(self, file_path: str): + # TODO: add some generic information like:provided order of the tuple + # will be the effective one. You can provide a key from it names if + # it is contained in the positioners group + # maybe split in sub section ? + self.dict_to_cfg(file_path=file_path, dict_=self.to_dict()) + + @staticmethod + def dict_to_cfg(file_path, dict_): + """ """ + return ConfigBase._dict_to_cfg( + file_path=file_path, + dict_=dict_, + comments_fct=TomoEDFConfig.get_comments, + logger=_logger, + ) + + @staticmethod + def from_cfg_file(file_path: str, encoding=None): + assert file_path is not None, "file_path should not be None" + config_parser = configparser.ConfigParser(allow_no_value=True) + config_parser.read(file_path, encoding=encoding) + return TomoEDFConfig.from_dict(config_parser) + + @staticmethod + def get_comments(key): + return TomoEDFConfig.COMMENTS[key] def generate_default_edf_config() -> dict: diff --git a/nxtomomill/io/config/hdf5config.py b/nxtomomill/io/config/hdf5config.py index 749f646..58b967f 100644 --- a/nxtomomill/io/config/hdf5config.py +++ b/nxtomomill/io/config/hdf5config.py @@ -1103,22 +1103,12 @@ class TomoHDF5Config(ConfigBase): @staticmethod def dict_to_cfg(file_path, dict_): """ """ - if not file_path.lower().endswith((".cfg", ".config")): - _logger.warning("add a valid extension to the output file") - file_path += ".cfg" - config = configparser.ConfigParser(allow_no_value=True) - for section_name, values in dict_.items(): - config.add_section(section_name) - config.set( - section_name, "# " + TomoHDF5Config.get_comments(section_name), None - ) - for key, value in values.items(): - # adopt nabu design: comments are set prior to the key - config.set(section_name, "# " + TomoHDF5Config.get_comments(key), None) - config.set(section_name, key, str(value)) - - with open(file_path, "w") as config_file: - config.write(config_file) + return ConfigBase._dict_to_cfg( + file_path=file_path, + dict_=dict_, + comments_fct=TomoHDF5Config.get_comments, + logger=_logger, + ) @staticmethod def from_cfg_file(file_path: str, encoding=None): @@ -1131,9 +1121,6 @@ class TomoHDF5Config(ConfigBase): def get_comments(key): return TomoHDF5Config.COMMENTS[key] - def __str__(self): - return str(self.to_dict()) - class XRD3DHDF5Config(TomoHDF5Config): -- GitLab From 3d3e04de6ad20cdd1fe7f896e3c8e97fba7ece98 Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Fri, 22 Apr 2022 14:13:12 +0200 Subject: [PATCH 09/30] move tomoscan dependency to 0.9.0 Now can use dataset_basename and scan_info --- nxtomomill/converter/edf/edfconverter.py | 6 ++++++ setup.cfg | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/nxtomomill/converter/edf/edfconverter.py b/nxtomomill/converter/edf/edfconverter.py index bbdc31f..ab9ffe2 100644 --- a/nxtomomill/converter/edf/edfconverter.py +++ b/nxtomomill/converter/edf/edfconverter.py @@ -271,10 +271,12 @@ def edf_to_nx( distance = scan.retrieve_information( scan=os.path.abspath(scan.path), + dataset_basename=scan.dataset_basename, ref_file=None, key="Distance", type_=float, key_aliases=["distance"], + scan_info=scan.scan_info, ) if distance is not None: h5d["/entry/instrument/detector/distance"] = distance @@ -282,10 +284,12 @@ def edf_to_nx( pixel_size = scan.retrieve_information( scan=os.path.abspath(scan.path), + dataset_basename=scan.dataset_basename, ref_file=None, key="PixelSize", type_=float, key_aliases=["pixelSize"], + scan_info=scan.scan_info, ) h5d["/entry/instrument/detector/x_pixel_size"] = ( pixel_size * metricsystem.micrometer.value @@ -298,10 +302,12 @@ def edf_to_nx( energy = scan.retrieve_information( scan=os.path.abspath(scan.path), + dataset_basename=scan.dataset_basename, ref_file=None, key="Energy", type_=float, key_aliases=["energy"], + scan_info=scan.scan_info, ) if energy is not None: h5d["/entry/instrument/beam/incident_energy"] = energy diff --git a/setup.cfg b/setup.cfg index 37ad67f..a2aab5f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ install_requires = setuptools h5py>=3.0 silx>=0.14a - tomoscan>=0.8.0a + tomoscan>=0.9.0a [options.entry_points] console_scripts = -- GitLab From d6f1ca5171c2d724b57b0028654fca55f2ec50ed Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Fri, 22 Apr 2022 14:22:52 +0200 Subject: [PATCH 10/30] io: rename config/test/test_config.py config/test/test_hdf5_config.py --- nxtomomill/io/config/test/{test_config.py => test_hdf5_config.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nxtomomill/io/config/test/{test_config.py => test_hdf5_config.py} (100%) diff --git a/nxtomomill/io/config/test/test_config.py b/nxtomomill/io/config/test/test_hdf5_config.py similarity index 100% rename from nxtomomill/io/config/test/test_config.py rename to nxtomomill/io/config/test/test_hdf5_config.py -- GitLab From 51c6d8e21b1b8d7b014786c6c025d5a37286c253 Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Fri, 22 Apr 2022 16:18:09 +0200 Subject: [PATCH 11/30] io: config: first iteration on the TomoEDFConfig class --- nxtomomill/io/config/configbase.py | 27 +- nxtomomill/io/config/edfconfig.py | 346 ++++++++++++++----- nxtomomill/io/config/hdf5config.py | 7 +- nxtomomill/io/config/test/test_edf_config.py | 277 +++++++++++++++ nxtomomill/settings.py | 4 +- 5 files changed, 560 insertions(+), 101 deletions(-) create mode 100644 nxtomomill/io/config/test/test_edf_config.py diff --git a/nxtomomill/io/config/configbase.py b/nxtomomill/io/config/configbase.py index 37b3882..2da41fa 100644 --- a/nxtomomill/io/config/configbase.py +++ b/nxtomomill/io/config/configbase.py @@ -45,6 +45,13 @@ class ConfigBase: # to ease API and avoid setting wrong attributes we 'freeze' the attributes # see https://stackoverflow.com/questions/3603502/prevent-creating-new-attributes-outside-init + def __init__(self) -> None: + self._output_file = False + self._overwrite = False + self._file_extension = FileExtension.NX + self._log_level = logging.WARNING + self._field_of_view = None + def __setattr__(self, __name, __value): if self.__isfrozen and not hasattr(self, __name): raise AttributeError("can't set attribute", __name) @@ -118,9 +125,12 @@ class ConfigBase: @rotation_angle_keys.setter def rotation_angle_keys(self, keys: Iterable): - if not isinstance(keys, Iterable): + if not isinstance(keys, Iterable) or isinstance(keys, str): raise TypeError("'keys' should be an Iterable") else: + for elmt in keys: + if not isinstance(elmt, str): + raise TypeError("keys elmts are expected to be str") self._rot_angle_keys = keys @property @@ -129,9 +139,12 @@ class ConfigBase: @x_trans_keys.setter def x_trans_keys(self, keys) -> None: - if not isinstance(keys, Iterable): + if not isinstance(keys, Iterable) or isinstance(keys, str): raise TypeError("'keys' should be an Iterable") else: + for elmt in keys: + if not isinstance(elmt, str): + raise TypeError("keys elmts are expected to be str") self._x_trans_keys = keys @property @@ -140,9 +153,12 @@ class ConfigBase: @y_trans_keys.setter def y_trans_keys(self, keys) -> None: - if not isinstance(keys, Iterable): + if not isinstance(keys, Iterable) or isinstance(keys, str): raise TypeError("'keys' should be an Iterable") else: + for elmt in keys: + if not isinstance(elmt, str): + raise TypeError("keys elmts are expected to be str") self._y_trans_keys = keys @property @@ -151,9 +167,12 @@ class ConfigBase: @z_trans_keys.setter def z_trans_keys(self, keys) -> None: - if not isinstance(keys, Iterable): + if not isinstance(keys, Iterable) or isinstance(keys, str): raise TypeError("'keys' should be an Iterable") else: + for elmt in keys: + if not isinstance(elmt, str): + raise TypeError("keys elmts are expected to be str") self._z_trans_keys = keys def to_dict(self) -> dict: diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index 80d5053..99dc000 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -35,7 +35,7 @@ import configparser from tomoscan.unitsystem.metricsystem import MetricSystem from tomoscan.unitsystem.energysystem import EnergySI from nxtomomill.io.config.configbase import ConfigBase -from nxtomomill.io.utils import filter_str_def +from nxtomomill.io.utils import convert_str_to_tuple, filter_str_def from nxtomomill.nexus.nxsource import SourceType from nxtomomill.utils import FileExtension from nxtomomill.settings import Tomo @@ -66,7 +66,7 @@ class TomoEDFConfig(ConfigBase): OVERWRITE_DK = "overwrite" - DATASET_BASENAME_DK = "file_prefix" + DATASET_BASENAME_DK = "dataset_basename" LOG_LEVEL_DK = "log_level" @@ -143,7 +143,7 @@ class TomoEDFConfig(ConfigBase): Z_TRANS_EXPECTED_UNIT = "expected_unit_for_z_translation" COMMENTS_UNIT_SECTION_DK = { - GENERAL_SECTION_DK: "Details units system used on SPEC side to save data. All will ne converted to NXtomo default (SI at the exception of energy-keV) \n", + UNIT_SECTION_DK: "Details units system used on SPEC side to save data. All will ne converted to NXtomo default (SI at the exception of energy-keV) \n", PIXEL_SIZE_EXPECTED_UNIT: f"Size used to save pixel size. Must be in of {_valid_metric_values}", DISTANCE_EXPECTED_UNIT: f"Unit used by SPEC to save sample to detector distance. Must be in of {_valid_metric_values}", ENERGY_EXPECTED_UNIT: f"Unit used by SPEC to save energy. Must be in of {_valid_energy_values}", @@ -202,6 +202,8 @@ class TomoEDFConfig(ConfigBase): COMMENTS.update(COMMENTS_DETECTOR_SECTION_DK) def __init__(self): + super().__init__() + self._set_freeze(False) # general information self._input_folder = None self._output_file = None @@ -249,10 +251,26 @@ class TomoEDFConfig(ConfigBase): def input_folder(self) -> Optional[str]: return self._input_folder + @input_folder.setter + def input_folder(self, folder: Optional[str]) -> None: + if not isinstance(folder, (type(None), str)): + raise TypeError( + f"folder is expected to be None or an instance of str. Not {type(folder)}" + ) + self._input_folder = folder + @property def dataset_basename(self) -> Optional[str]: return self._dataset_basename + @dataset_basename.setter + def dataset_basename(self, dataset_basename: Optional[str]) -> None: + if not isinstance(dataset_basename, (type(None), str)): + raise TypeError( + f"dataset_basename is expected to be None or an instance of str. Not {type(dataset_basename)}" + ) + self._dataset_basename = dataset_basename + @property def title(self) -> Optional[str]: return self._title @@ -266,63 +284,99 @@ class TomoEDFConfig(ConfigBase): self._title = title @property - def ignore_file_patterns(self) -> Optional[Iterable]: + def ignore_file_patterns(self) -> tuple: return self._ignore_file_patterns @ignore_file_patterns.setter - def ignore_file_patterns(self, patterns): - if not isinstance(patterns, Iterable): - raise TypeError("patterns is expected to be an Iterable") - self._patterns = patterns + def ignore_file_patterns(self, patterns: Optional[Union[tuple, list]]): + if not isinstance(patterns, (type(None), tuple, list)): + raise TypeError("patterns is expected to be a tuple or a list") + if patterns is None: + self._ignore_file_patterns = tuple() + else: + for elmt in patterns: + if not isinstance(elmt, str): + raise TypeError("patterns elmts are expected to be str") + self._ignore_file_patterns = tuple(patterns) @property - def motor_position_keys(self) -> Iterable: + def motor_position_keys(self) -> tuple: return self._motor_position_keys @motor_position_keys.setter - def motor_position_keys(self, keys: Optional[Iterable]) -> None: - if not isinstance(keys, (type(None), Iterable)): - raise TypeError("keys is expected to be None or an Iterable") - self._motor_position_keys = keys + def motor_position_keys(self, keys: Optional[Union[tuple, list]]) -> None: + if not isinstance(keys, (type(None), tuple, list)): + raise TypeError( + "keys is expected to be None or an instance of list or tuple" + ) + if keys is None: + self._motor_position_keys = tuple() + else: + for elmt in keys: + if not isinstance(elmt, str): + raise TypeError("keys elmts are expected to be str") + self._motor_position_keys = tuple(keys) @property - def motor_mne_keys(self) -> Iterable: + def motor_mne_keys(self) -> tuple: return self._motor_mne_keys @motor_mne_keys.setter - def motor_mne_keys(self, keys) -> None: - if not isinstance(keys, (type(None), Iterable)): - raise TypeError("keys is expected to be None or an Iterable") - self._motor_mne_keys = keys + def motor_mne_keys(self, keys: Optional[Union[tuple, list]]) -> None: + if not isinstance(keys, (type(None), tuple, list)): + raise TypeError( + "keys is expected to be None or an instance of list or tuple" + ) + if keys is None: + self._motor_mne_keys = tuple() + else: + for elmt in keys: + if not isinstance(elmt, str): + raise TypeError("keys elmts are expected to be str") + self._motor_mne_keys = tuple(keys) @property - def dark_names(self) -> Iterable: + def dark_names(self) -> tuple: return self._dark_names - @dark_names - def dark_names(self, names: Iterable) -> None: - if not isinstance(names, Iterable): - raise TypeError("names is expected to be an Iterable") - self._dark_names = names + @dark_names.setter + def dark_names(self, names: Optional[Union[tuple, list]]) -> None: + if not isinstance(names, (type(None), tuple, list)): + raise TypeError("names is expected to be a tuple or a list") + if names is None: + self._dark_names = tuple() + else: + for elmt in names: + if not isinstance(elmt, str): + raise TypeError("names elmts are expected to be str") + self._dark_names = tuple(names) @property - def flat_names(self) -> Iterable: + def flat_names(self) -> tuple: return self._flat_names @flat_names.setter - def flat_names(self, names: Iterable) -> None: - if not isinstance(names, Iterable): - raise TypeError("names is expected to be an Iterable") - self._flat_names = names + def flat_names(self, names: Optional[Union[tuple, list]]) -> None: + if not isinstance(names, (type(None), tuple, list)): + raise TypeError("names is expected to be a tuple or a list") + if names is None: + self._flat_names = tuple() + else: + for elmt in names: + if not isinstance(elmt, str): + raise TypeError("names elmts are expected to be str") + self._flat_names = tuple(names) @property def pixel_size_unit(self) -> MetricSystem: return self._pixel_size_unit @pixel_size_unit.setter - def pixel_size_unit(self, unit: MetricSystem) -> None: - if not isinstance(unit, MetricSystem): + def pixel_size_unit(self, unit: Union[MetricSystem, str]) -> None: + if not isinstance(unit, (MetricSystem, str)): raise TypeError("unit is expected to be an instance of MetricSystem") + if isinstance(unit, str): + unit = MetricSystem.from_str(unit) self._pixel_size_unit = unit @property @@ -330,9 +384,11 @@ class TomoEDFConfig(ConfigBase): return self._distance_unit @distance_unit.setter - def distance_unit(self, unit: MetricSystem) -> None: - if not isinstance(unit, MetricSystem): + def distance_unit(self, unit: Union[MetricSystem, str]) -> None: + if not isinstance(unit, (MetricSystem, str)): raise TypeError("unit is expected to be an instance of MetricSystem") + if isinstance(unit, str): + unit = MetricSystem.from_str(unit) self._distance_unit = unit @property @@ -340,9 +396,11 @@ class TomoEDFConfig(ConfigBase): return self._energy_unit @energy_unit.setter - def energy_unit(self, unit: EnergySI) -> None: - if not isinstance(unit, EnergySI): + def energy_unit(self, unit: Union[EnergySI, str]) -> None: + if not isinstance(unit, (EnergySI, str)): raise TypeError("unit is expected to be an instance of EnergySI") + if isinstance(unit, str): + unit = EnergySI.from_str(unit) self._energy_unit = unit @property @@ -350,9 +408,11 @@ class TomoEDFConfig(ConfigBase): return self._x_trans_unit @x_trans_unit.setter - def x_trans_unit(self, unit: MetricSystem) -> None: - if not isinstance(unit, MetricSystem): + def x_trans_unit(self, unit: Union[MetricSystem, str]) -> None: + if not isinstance(unit, (MetricSystem, str)): raise TypeError("unit is expected to be an instance of MetricSystem") + if isinstance(unit, str): + unit = MetricSystem.from_str(unit) self._x_trans_unit = unit @property @@ -360,9 +420,11 @@ class TomoEDFConfig(ConfigBase): return self._y_trans_unit @y_trans_unit.setter - def y_trans_unit(self, unit: MetricSystem) -> None: - if not isinstance(unit, MetricSystem): + def y_trans_unit(self, unit: Union[MetricSystem, str]) -> None: + if not isinstance(unit, (MetricSystem, str)): raise TypeError("unit is expected to be an instance of MetricSystem") + if isinstance(unit, str): + unit = MetricSystem.from_str(unit) self._y_trans_unit = unit @property @@ -370,9 +432,11 @@ class TomoEDFConfig(ConfigBase): return self._z_trans_unit @z_trans_unit.setter - def z_trans_unit(self, unit: MetricSystem) -> None: - if not isinstance(unit, MetricSystem): + def z_trans_unit(self, unit: Union[MetricSystem, str]) -> None: + if not isinstance(unit, (MetricSystem, str)): raise TypeError("unit is expected to be an instance of MetricSystem") + if isinstance(unit, str): + unit = MetricSystem.from_str(unit) self._z_trans_unit = unit @property @@ -418,49 +482,75 @@ class TomoEDFConfig(ConfigBase): if source_type is None: self._source_type = None else: - self._source_name = SourceType.from_value(source_type) + self._source_type = SourceType.from_value(source_type) def to_dict(self) -> dict: """convert the configuration to a dictionary""" return { self.GENERAL_SECTION_DK: { - self.INPUT_FOLDER_DK: self.input_folder, - self.OUTPUT_FILE_DK: self.output_file, + self.INPUT_FOLDER_DK: self.input_folder + if self.input_folder is not None + else "", + self.OUTPUT_FILE_DK: self.output_file + if self.output_file is not None + else "", self.OVERWRITE_DK: self.overwrite, - self.FILE_EXTENSION_DK: self.file_extension, - self.DATASET_BASENAME_DK: self.dataset_basename, - self.LOG_LEVEL_DK: self.log_level, - self.TITLE_DK: self.title, - self.IGNORE_FILE_PATTERN_DK: self.ignore_file_patterns, + self.FILE_EXTENSION_DK: self.file_extension.value, + self.DATASET_BASENAME_DK: self.dataset_basename + if self.dataset_basename is not None + else "", + self.LOG_LEVEL_DK: logging.getLevelName(self.log_level).lower(), + self.TITLE_DK: self.title if self.title is not None else "", + self.IGNORE_FILE_PATTERN_DK: self.ignore_file_patterns + if self.ignore_file_patterns != tuple() + else "", }, self.EDF_KEYS_SECTION_DK: { - self.MOTOR_POSITION_KEY_DK: self.motor_position_keys, - self.MOTOR_MNE_KEY_DK: self.motor_mne_keys, - self.rotation_angle_keys: self.rotation_angle_keys, - self.X_TRANS_KEY_DK: self.x_trans_keys, - self.Y_TRANS_KEY_DK: self.y_trans_keys, - self.Z_TRANS_KEY_DK: self.z_trans_keys, + self.MOTOR_POSITION_KEY_DK: self.motor_position_keys + if self.motor_position_keys != tuple() + else "", + self.MOTOR_MNE_KEY_DK: self.motor_mne_keys + if self.motor_mne_keys != tuple() + else "", + self.ROT_ANGLE_KEY_DK: self.rotation_angle_keys + if self.rotation_angle_keys != tuple() + else "", + self.X_TRANS_KEY_DK: self.x_trans_keys + if self.x_trans_keys != tuple() + else "", + self.Y_TRANS_KEY_DK: self.y_trans_keys + if self.y_trans_keys != tuple() + else "", + self.Z_TRANS_KEY_DK: self.z_trans_keys + if self.z_trans_keys != tuple() + else "", }, self.FLAT_DARK_SECTION_DK: { - self.DARK_NAMES_DK: self.dark_names, - self.FLAT_NAMES_DK: self.flat_names, + self.DARK_NAMES_DK: self.dark_names + if self.dark_names != tuple() + else "", + self.FLAT_NAMES_DK: self.flat_names + if self.dark_names != tuple() + else "", }, self.UNIT_SECTION_DK: { - self.PIXEL_SIZE_EXPECTED_UNIT: self.pixel_size_unit, - self.DISTANCE_EXPECTED_UNIT: self.distance_unit, - self.ENERGY_EXPECTED_UNIT: self.energy_unit, - self.X_TRANS_EXPECTED_UNIT: self.x_trans_unit, - self.Y_TRANS_EXPECTED_UNIT: self.y_trans_unit, - self.Z_TRANS_EXPECTED_UNIT: self.z_trans_unit, + self.PIXEL_SIZE_EXPECTED_UNIT: str(self.pixel_size_unit), + self.DISTANCE_EXPECTED_UNIT: str(self.distance_unit), + self.ENERGY_EXPECTED_UNIT: str(self.energy_unit), + self.X_TRANS_EXPECTED_UNIT: str(self.x_trans_unit), + self.Y_TRANS_EXPECTED_UNIT: str(self.y_trans_unit), + self.Z_TRANS_EXPECTED_UNIT: str(self.z_trans_unit), }, self.SAMPLE_SECTION_DK: { - self.SAMPLE_NAME_DK: self.sample_name, + self.SAMPLE_NAME_DK: self.sample_name + if self.sample_name is not None + else "", }, self.SOURCE_SECTION_DK: { self.INSTRUMENT_NAME_DK: self.instrument_name or "", self.SOURCE_NAME_DK: self.source_name or "", self.SOURCE_TYPE_DK: self.source_type.value - if self.source is not None + if self.source_type is not None else "", }, self.DETECTOR_SECTION_DK: { @@ -515,47 +605,120 @@ class TomoEDFConfig(ConfigBase): overwrite = dict_.get(TomoEDFConfig.OVERWRITE_DK, None) if overwrite is not None: self.overwrite = cast_bool(overwrite) + file_extension = dict_.get(TomoEDFConfig.FILE_EXTENSION_DK, None) - if file_extension is not None: + if file_extension not in (None, ""): self.file_extension = filter_str_def(file_extension) - self.dataset_basename = dict_.get(TomoEDFConfig.DATASET_BASENAME_DK, None) + + dataset_basename = dict_.get(TomoEDFConfig.DATASET_BASENAME_DK, None) + if dataset_basename is not None: + if dataset_basename == "": + dataset_basename = None + self.dataset_basename = dataset_basename + log_level = dict_.get(TomoEDFConfig.LOG_LEVEL_DK, None) if log_level is not None: self.log_level = log_level self.title = dict_.get(TomoEDFConfig.TITLE_DK) - self.ignore_file_patterns = dict_.get(TomoEDFConfig.IGNORE_FILE_PATTERN_DK, ()) + ignore_file_patterns = dict_.get(TomoEDFConfig.IGNORE_FILE_PATTERN_DK, None) + if ignore_file_patterns is not None: + if ignore_file_patterns == "": + ignore_file_patterns = tuple() + else: + ignore_file_patterns = convert_str_to_tuple(ignore_file_patterns) + self.ignore_file_patterns = ignore_file_patterns def load_keys_section(self, dict_: dict) -> None: - if TomoEDFConfig.MOTOR_POSITION_KEY_DK in dict_: - self.motor_position_keys = dict_.get(TomoEDFConfig.MOTOR_POSITION_KEY_DK) - if TomoEDFConfig.MOTOR_MNE_KEY_DK in dict_: - self.motor_mne_keys = dict_.get(TomoEDFConfig.MOTOR_MNE_KEY_DK) - if TomoEDFConfig.X_TRANS_KEY_DK in dict_: - self.x_trans_keys = dict_.get(TomoEDFConfig.X_TRANS_KEY_DK) - if TomoEDFConfig.Y_TRANS_KEY_DK in dict_: - self.y_trans_keys = dict_.get(TomoEDFConfig.Y_TRANS_KEY_DK) - if TomoEDFConfig.Z_TRANS_KEY_DK in dict_: - self.z_trans_keys = dict_.get(TomoEDFConfig.Z_TRANS_KEY_DK) + motor_position_keys = dict_.get(TomoEDFConfig.MOTOR_POSITION_KEY_DK, None) + if motor_position_keys is not None: + if motor_position_keys == "": + motor_position_keys = tuple() + else: + motor_position_keys = convert_str_to_tuple(motor_position_keys) + self.motor_position_keys = motor_position_keys + + motor_mne_keys = dict_.get(TomoEDFConfig.MOTOR_MNE_KEY_DK, None) + if motor_mne_keys is not None: + if motor_mne_keys == "": + motor_mne_keys = tuple() + else: + motor_mne_keys = convert_str_to_tuple(motor_mne_keys) + self.motor_mne_keys = motor_mne_keys + + rotation_angle_keys = dict_.get(TomoEDFConfig.ROT_ANGLE_KEY_DK, None) + if rotation_angle_keys is not None: + if rotation_angle_keys == "": + rotation_angle_keys = tuple() + else: + rotation_angle_keys = convert_str_to_tuple(rotation_angle_keys) + self.rotation_angle_keys = rotation_angle_keys + + x_trans_keys = dict_.get(TomoEDFConfig.X_TRANS_KEY_DK, None) + if x_trans_keys is not None: + if x_trans_keys == "": + x_trans_keys = tuple() + else: + x_trans_keys = convert_str_to_tuple(x_trans_keys) + self.x_trans_keys = x_trans_keys + + y_trans_keys = dict_.get(TomoEDFConfig.Y_TRANS_KEY_DK, None) + if y_trans_keys is not None: + if y_trans_keys == "": + y_trans_keys = tuple() + else: + y_trans_keys = convert_str_to_tuple(y_trans_keys) + self.y_trans_keys = y_trans_keys + + z_trans_keys = dict_.get(TomoEDFConfig.Z_TRANS_KEY_DK, None) + if z_trans_keys is not None: + if z_trans_keys == "": + z_trans_keys = tuple() + else: + z_trans_keys = convert_str_to_tuple(z_trans_keys) + self.z_trans_keys = z_trans_keys def load_flat_dark_section(self, dict_: dict) -> None: - if TomoEDFConfig.DARK_NAMES_DK in dict_: - self.dark_names = dict_.get(TomoEDFConfig.DARK_NAMES_DK) - if TomoEDFConfig.FLAT_NAMES_DK in dict_: - self.flat_names = dict_.get(TomoEDFConfig.FLAT_NAMES_DK) + dark_names = dict_.get(TomoEDFConfig.DARK_NAMES_DK, None) + if dark_names is not None: + if dark_names == "": + dark_names = tuple() + else: + dark_names = convert_str_to_tuple(dark_names) + self.dark_names = dark_names + + flat_names = dict_.get(TomoEDFConfig.FLAT_NAMES_DK, None) + if flat_names is not None: + if flat_names == "": + flat_names = tuple() + else: + flat_names = convert_str_to_tuple(flat_names) + self.flat_names = flat_names def load_unit_section(self, dict_: dict) -> None: if TomoEDFConfig.PIXEL_SIZE_EXPECTED_UNIT in dict_: - self.pixel_size_unit = dict_.get(TomoEDFConfig.PIXEL_SIZE_EXPECTED_UNIT) + self.pixel_size_unit = MetricSystem.from_str( + dict_.get(TomoEDFConfig.PIXEL_SIZE_EXPECTED_UNIT) + ) if TomoEDFConfig.DISTANCE_EXPECTED_UNIT in dict_: - self.distance_unit = dict_.get(TomoEDFConfig.DISTANCE_EXPECTED_UNIT) + self.distance_unit = MetricSystem.from_str( + dict_.get(TomoEDFConfig.DISTANCE_EXPECTED_UNIT) + ) if TomoEDFConfig.ENERGY_EXPECTED_UNIT in dict_: - self.energy_unit = dict_.get(TomoEDFConfig.ENERGY_EXPECTED_UNIT) + self.energy_unit = EnergySI.from_str( + dict_.get(TomoEDFConfig.ENERGY_EXPECTED_UNIT) + ) if TomoEDFConfig.X_TRANS_EXPECTED_UNIT in dict_: - self.x_trans_unit = dict_.get(TomoEDFConfig.X_TRANS_EXPECTED_UNIT) + self.x_trans_unit = MetricSystem.from_str( + dict_.get(TomoEDFConfig.X_TRANS_EXPECTED_UNIT) + ) if TomoEDFConfig.Y_TRANS_EXPECTED_UNIT in dict_: - self.y_trans_unit = dict_.get(TomoEDFConfig.Y_TRANS_EXPECTED_UNIT) + self.y_trans_unit = MetricSystem.from_str( + dict_.get(TomoEDFConfig.Y_TRANS_EXPECTED_UNIT) + ) if TomoEDFConfig.Z_TRANS_EXPECTED_UNIT in dict_: - self.z_trans_unit = dict_.get(TomoEDFConfig.Z_TRANS_EXPECTED_UNIT) + self.z_trans_unit = MetricSystem.from_str( + dict_.get(TomoEDFConfig.Z_TRANS_EXPECTED_UNIT) + ) def load_sample_section(self, dict_: dict) -> None: if TomoEDFConfig.SAMPLE_NAME_DK in dict_: @@ -570,8 +733,11 @@ class TomoEDFConfig(ConfigBase): self.source_type = dict_[TomoEDFConfig.SOURCE_TYPE_DK] def load_detector_section(self, dict_: dict) -> None: - if TomoEDFConfig.FIELD_OF_VIEW_DK in dict_: - self.field_of_view = dict_[TomoEDFConfig.FIELD_OF_VIEW_DK] + field_of_view = dict_.get(TomoEDFConfig.FIELD_OF_VIEW_DK, None) + if field_of_view is not None: + if field_of_view == "": + field_of_view = None + self.field_of_view = field_of_view def to_cfg_file(self, file_path: str): # TODO: add some generic information like:provided order of the tuple diff --git a/nxtomomill/io/config/hdf5config.py b/nxtomomill/io/config/hdf5config.py index 58b967f..80519be 100644 --- a/nxtomomill/io/config/hdf5config.py +++ b/nxtomomill/io/config/hdf5config.py @@ -317,12 +317,10 @@ class TomoHDF5Config(ConfigBase): raise ValueError(f"No default unit for {key}") def __init__(self): + super().__init__(self) + self._set_freeze(False) # general information - self._output_file = None self._input_file = None - self._overwrite = False - self._file_extension = FileExtension.NX - self._log_level = logging.WARNING self._raises_error = False self._no_input = False self._format = Format.STANDARD @@ -330,7 +328,6 @@ class TomoHDF5Config(ConfigBase): self._bam_single_file = False # a single file is create by default if there is only one entry per file. # but we can enfore multi-file writing - self._field_of_view = None # information regarding keys and paths self._valid_camera_names = settings.Tomo.H5.VALID_CAMERA_NAMES diff --git a/nxtomomill/io/config/test/test_edf_config.py b/nxtomomill/io/config/test/test_edf_config.py new file mode 100644 index 0000000..2dd6091 --- /dev/null +++ b/nxtomomill/io/config/test/test_edf_config.py @@ -0,0 +1,277 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ + +__authors__ = ["H. Payno", "J.Garriga"] +__license__ = "MIT" +__date__ = "22/04/2022" + + +from tempfile import TemporaryDirectory + +import pytest +from nxtomomill import settings +from nxtomomill.io.config.edfconfig import TomoEDFConfig, generate_default_edf_config +import os +from tomoscan.unitsystem.metricsystem import MetricSystem +from tomoscan.unitsystem.energysystem import EnergySI + +from nxtomomill.nexus.nxsource import SourceType +from nxtomomill.nexus.nxdetector import FieldOfView + + +def test_TomoEDFConfig_to_dict(): + """Test processing of the to_dict""" + config = TomoEDFConfig() + + config_dict = config.to_dict() + for key in ( + TomoEDFConfig.GENERAL_SECTION_DK, + TomoEDFConfig.EDF_KEYS_SECTION_DK, + TomoEDFConfig.FLAT_DARK_SECTION_DK, + TomoEDFConfig.UNIT_SECTION_DK, + TomoEDFConfig.SAMPLE_SECTION_DK, + TomoEDFConfig.SOURCE_SECTION_DK, + TomoEDFConfig.DETECTOR_SECTION_DK, + ): + assert key in config_dict + + TomoEDFConfig.from_dict(config_dict) + + with TemporaryDirectory() as folder: + file_path = os.path.join(folder, "config.cfg") + assert not os.path.exists(file_path) + config.to_cfg_file(file_path) + assert os.path.exists(file_path) + config_loaded = TomoEDFConfig.from_cfg_file(file_path=file_path) + + assert config_loaded.to_dict() == config.to_dict() + + +def test_TomoEDFConfig_default_config(): + """insure default configuration generation works""" + with TemporaryDirectory() as folder: + file_path = os.path.join(folder, "config.cfg") + assert not os.path.exists(file_path) + config = generate_default_edf_config() + assert isinstance(config, dict) + TomoEDFConfig.from_dict(config).to_cfg_file(file_path=file_path) + assert os.path.exists(file_path) + + +def test_TomoEDFConfig_setters(): + """test the different setters and getter of the EDFTomoConfig""" + config = TomoEDFConfig() + + # try to test a new attribut (insure class is frozeen) + with pytest.raises(AttributeError): + config.new_attrs = "toto" + + # test general section setters + with pytest.raises(TypeError): + config.input_folder = 12.0 + config.input_folder = "my_folder" + with pytest.raises(TypeError): + config.output_file = 12.0 + config.output_file = "my_nx.nx" + + config.file_extension = ".nx" + + with pytest.raises(TypeError): + config.dataset_basename = 12.0 + config.dataset_basename = None + config.dataset_basename = "test_" + + config.overwrite = True + + with pytest.raises(TypeError): + config.title = 12.0 + config.title = None + config.title = "my title" + + with pytest.raises(TypeError): + config.ignore_file_patterns = 12.0 + with pytest.raises(TypeError): + config.ignore_file_patterns = "toto" + with pytest.raises(TypeError): + config.ignore_file_patterns = (1.0,) + config.ignore_file_patterns = ("toto",) + config.ignore_file_patterns = None + + # test header keys + dark and flat setters + attributes_value = { + "motor_position_keys": settings.Tomo.EDF.MOTOR_POS, + "motor_mne_keys": settings.Tomo.EDF.MOTOR_MNE, + "x_trans_keys": settings.Tomo.EDF.X_TRANS, + "y_trans_keys": settings.Tomo.EDF.Y_TRANS, + "z_trans_keys": settings.Tomo.EDF.Z_TRANS, + "rotation_angle_keys": settings.Tomo.EDF.ROT_ANGLE, + "dark_names": settings.Tomo.EDF.DARK_NAMES, + "flat_names": settings.Tomo.EDF.REFS_NAMES, + } + for attr, key in attributes_value.items(): + setattr(config, attr, key) + with pytest.raises(TypeError): + setattr(config, attr, "toto") + with pytest.raises(TypeError): + setattr(config, attr, 12.20) + with pytest.raises(TypeError): + setattr(config, attr, (12.20,)) + + # test units setters + attributes_value = { + "pixel_size_unit": MetricSystem.MICROMETER, + "distance_unit": MetricSystem.METER, + "energy_unit": EnergySI.KILOELECTRONVOLT, + "x_trans_unit": MetricSystem.METER, + "y_trans_unit": MetricSystem.METER, + "z_trans_unit": MetricSystem.METER, + } + for attr, key in attributes_value.items(): + # test providing an instance of a unit + setattr(config, attr, key) + # test providing a sting if of a unit + setattr(config, attr, str(key)) + with pytest.raises(TypeError): + setattr(config, attr, None) + with pytest.raises(TypeError): + setattr(config, attr, 12.0) + + # test sample setters + config.sample_name = None + with pytest.raises(TypeError): + config.sample_name = (12,) + with pytest.raises(TypeError): + config.sample_name = 12.0 + config.sample_name = "my sample" + + # test source setters + config.instrument_name = None + config.instrument_name = "BMXX" + with pytest.raises(TypeError): + config.instrument_name = 12.3 + + config.source_name = None + config.source_name = "ESRF" + with pytest.raises(TypeError): + config.source_name = 12.2 + config.source_name = "XFEL" + + config.source_type = None + config.source_type = SourceType.FIXED_TUBE_X_RAY.value + config.source_type = SourceType.FIXED_TUBE_X_RAY + with pytest.raises(ValueError): + config.source_type = "trsts" + config.source_type = 123 + + # test detector setters + config.field_of_view = None + with pytest.raises(TypeError): + config.field_of_view = 12 + with pytest.raises(ValueError): + config.field_of_view = "toto" + config.field_of_view = FieldOfView.FULL.value + config.field_of_view = FieldOfView.HALF + + config_dict = config.to_dict() + general_section_dict = config_dict[TomoEDFConfig.GENERAL_SECTION_DK] + assert general_section_dict[TomoEDFConfig.INPUT_FOLDER_DK] == "my_folder" + assert general_section_dict[TomoEDFConfig.OUTPUT_FILE_DK] == "my_nx.nx" + assert general_section_dict[TomoEDFConfig.FILE_EXTENSION_DK] == ".nx" + assert general_section_dict[TomoEDFConfig.OVERWRITE_DK] == True + assert general_section_dict[TomoEDFConfig.DATASET_BASENAME_DK] == "test_" + assert general_section_dict[TomoEDFConfig.TITLE_DK] == "my title" + assert general_section_dict[TomoEDFConfig.IGNORE_FILE_PATTERN_DK] == "" + + edf_headers_section_dict = config_dict[TomoEDFConfig.EDF_KEYS_SECTION_DK] + assert ( + edf_headers_section_dict[TomoEDFConfig.MOTOR_POSITION_KEY_DK] + == settings.Tomo.EDF.MOTOR_POS + ) + assert ( + edf_headers_section_dict[TomoEDFConfig.MOTOR_MNE_KEY_DK] + == settings.Tomo.EDF.MOTOR_MNE + ) + assert ( + edf_headers_section_dict[TomoEDFConfig.X_TRANS_KEY_DK] + == settings.Tomo.EDF.X_TRANS + ) + assert ( + edf_headers_section_dict[TomoEDFConfig.Y_TRANS_KEY_DK] + == settings.Tomo.EDF.Y_TRANS + ) + assert ( + edf_headers_section_dict[TomoEDFConfig.Z_TRANS_KEY_DK] + == settings.Tomo.EDF.Z_TRANS + ) + assert ( + edf_headers_section_dict[TomoEDFConfig.ROT_ANGLE_KEY_DK] + == settings.Tomo.EDF.ROT_ANGLE + ) + + dark_flat_section_dict = config_dict[TomoEDFConfig.FLAT_DARK_SECTION_DK] + assert ( + dark_flat_section_dict[TomoEDFConfig.DARK_NAMES_DK] + == settings.Tomo.EDF.DARK_NAMES + ) + assert ( + dark_flat_section_dict[TomoEDFConfig.FLAT_NAMES_DK] + == settings.Tomo.EDF.REFS_NAMES + ) + + units_section_dict = config_dict[TomoEDFConfig.UNIT_SECTION_DK] + assert units_section_dict[TomoEDFConfig.PIXEL_SIZE_EXPECTED_UNIT] == str( + MetricSystem.MICROMETER + ) + assert units_section_dict[TomoEDFConfig.DISTANCE_EXPECTED_UNIT] == str( + MetricSystem.METER + ) + assert units_section_dict[TomoEDFConfig.ENERGY_EXPECTED_UNIT] == str( + EnergySI.KILOELECTRONVOLT + ) + assert units_section_dict[TomoEDFConfig.X_TRANS_EXPECTED_UNIT] == str( + MetricSystem.METER + ) + assert units_section_dict[TomoEDFConfig.Y_TRANS_EXPECTED_UNIT] == str( + MetricSystem.METER + ) + assert units_section_dict[TomoEDFConfig.Z_TRANS_EXPECTED_UNIT] == str( + MetricSystem.METER + ) + + sample_section_dict = config_dict[TomoEDFConfig.SAMPLE_SECTION_DK] + assert sample_section_dict[TomoEDFConfig.SAMPLE_NAME_DK] == "my sample" + + source_section_dict = config_dict[TomoEDFConfig.SOURCE_SECTION_DK] + assert source_section_dict[TomoEDFConfig.INSTRUMENT_NAME_DK] == "BMXX" + assert source_section_dict[TomoEDFConfig.SOURCE_NAME_DK] == "XFEL" + assert ( + source_section_dict[TomoEDFConfig.SOURCE_TYPE_DK] + == SourceType.FIXED_TUBE_X_RAY.value + ) + + detector_section_dict = config_dict[TomoEDFConfig.DETECTOR_SECTION_DK] + assert ( + detector_section_dict[TomoEDFConfig.FIELD_OF_VIEW_DK] == FieldOfView.HALF.value + ) diff --git a/nxtomomill/settings.py b/nxtomomill/settings.py index ea34f5e..f8e1a5d 100644 --- a/nxtomomill/settings.py +++ b/nxtomomill/settings.py @@ -125,9 +125,9 @@ class Tomo: class EDF: """EDF settings for tomography""" - MOTOR_POS = "motor_pos" + MOTOR_POS = ("motor_pos",) - MOTOR_MNE = "motor_mne" + MOTOR_MNE = ("motor_mne",) ROT_ANGLE = ("srot",) -- GitLab From e7e00a9d11715f9ad8123d88e5cca7252dd4663b Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Fri, 22 Apr 2022 17:04:53 +0200 Subject: [PATCH 12/30] edf2nx: add handlin of the TomoEDFConfig --- nxtomomill/app/edf2nx.py | 59 ++---- nxtomomill/converter/edf/edfconverter.py | 193 +++++++++++++------ nxtomomill/io/config/edfconfig.py | 25 +++ nxtomomill/io/config/test/test_edf_config.py | 8 + nxtomomill/nexus/nxtomo.py | 2 +- 5 files changed, 178 insertions(+), 109 deletions(-) diff --git a/nxtomomill/app/edf2nx.py b/nxtomomill/app/edf2nx.py index 1b2e3c2..e2e2964 100644 --- a/nxtomomill/app/edf2nx.py +++ b/nxtomomill/app/edf2nx.py @@ -104,57 +104,22 @@ def main(argv): parser.add_argument("scan_path", help="folder containing the edf files") parser.add_argument("output_file", help="foutput .h5 file") parser.add_argument( - "--file_extension", - action="store_true", - default=".h5", - help="extension of the output file. Valid values are " - "" + "/".join(utils.FileExtension.values()), + "--dataset-basename", + "--file-prefix", + default=None, + help="file prefix to be used to deduce projections", ) parser.add_argument( - "--motor_pos_key", - default=EDF_MOTOR_POS, - help="motor position key in EDF HEADER", + "--info-file", + default=None, + help=".info file containing acquisition information (ScanRange, Energy, TOMO_N...)", ) parser.add_argument( - "--motor_mne_key", default=EDF_MOTOR_MNE, help="motor mne key in EDF HEADER" - ) - parser.add_argument( - "--refs_name_keys", - default=",".join(EDF_REFS_NAMES), - help="prefix of flat field file", - ) - parser.add_argument( - "--ignore_file_containing", - default=",".join(EDF_TO_IGNORE), - help="substring that lead to ignoring the file if " "contained in the name", - ) - parser.add_argument( - "--rot_angle_key", - default=EDF_ROT_ANGLE, - help="rotation angle key in EDF HEADER", - ) - parser.add_argument( - "--dark_names", - default=",".join(EDF_DARK_NAMES), - help="prefix of the dark field file", - ) - parser.add_argument( - "--x_trans_key", default=EDF_X_TRANS, help="x translation key in EDF HEADER" - ) - parser.add_argument( - "--y_trans_key", default=EDF_Y_TRANS, help="y translation key in EDF HEADER" - ) - parser.add_argument( - "--z_trans_key", default=EDF_Z_TRANS, help="z translation key in EDF HEADER" - ) - parser.add_argument("--sample_name", default=None, help="name of the sample") - parser.add_argument("--title", default=None, help="title") - parser.add_argument("--instrument_name", default=None, help="instrument name used") - parser.add_argument("--source_name", default="ESRF", help="name of the source used") - parser.add_argument( - "--source_type", - default=SourceType.SYNCHROTRON_X_RAY_SOURCE, - help="type of the source used", + "--config", + "--configuration-file", + "--configuration", + default=None, + help="file containing the full configuration to convert from SPEC-EDF to bliss to nexus", ) options = parser.parse_args(argv[1:]) diff --git a/nxtomomill/converter/edf/edfconverter.py b/nxtomomill/converter/edf/edfconverter.py index ab9ffe2..b18963c 100644 --- a/nxtomomill/converter/edf/edfconverter.py +++ b/nxtomomill/converter/edf/edfconverter.py @@ -36,6 +36,7 @@ __date__ = "27/11/2020" from collections import namedtuple from typing import Optional +from nxtomomill.io.config.edfconfig import TomoEDFConfig from nxtomomill.nexus.nxsource import SourceType from nxtomomill import utils from nxtomomill.utils import ImageKey @@ -43,6 +44,9 @@ from nxtomomill.converter.version import version as converter_version from nxtomomill.nexus.utils import create_nx_data_group from nxtomomill.nexus.utils import link_nxbeam_to_root from nxtomomill.settings import Tomo +from silx.utils.deprecation import deprecated +from tomoscan.esrf.utils import get_parameters_frm_par_or_info +from tomoscan.unitsystem.energysystem import EnergySI try: from tomoscan.esrf.scan.edfscan import EDFTomoScan @@ -71,12 +75,12 @@ _logger = logging.getLogger(__name__) EDFFileKeys = namedtuple( "EDFFileKeys", [ - "motor_pos_key", - "motor_mne_key", - "rot_angle_key", - "x_trans_key", - "y_trans_key", - "z_trans_key", + "motor_pos_keys", + "motor_mne_keys", + "rot_angle_keys", + "x_trans_keys", + "y_trans_keys", + "z_trans_keys", "to_ignore", "dark_names", "ref_names", @@ -96,6 +100,7 @@ DEFAULT_EDF_KEYS = EDFFileKeys( ) +@deprecated(replacement="from_edf_to_nx", since_version="0.9.0") def edf_to_nx( scan: EDFTomoScan, output_file: str, @@ -126,14 +131,74 @@ def edf_to_nx( :rtype:tuple """ if not isinstance(scan, EDFTomoScan): - raise TypeError("scan should be an instance of {}".format(EDFTomoScan)) + raise TypeError("scan is expected to be an instance of EDFTomoScan") + + config = TomoEDFConfig() + config.input_folder = scan.path + config.dataset_basename = scan.dataset_basename + config.output_file = output_file + config.file_extension = file_extension + config.sample_name = sample_name + config.title = title + config.instrument_name = instrument_name + config.source_name = source_name + config.source_type = source_type + # handle file_keys + config.motor_position_keys = file_keys.motor_pos_keys + config.motor_mne_keys = file_keys.motor_mne_keys + config.rotation_angle_keys = file_keys.rot_angle_keys + config.x_trans_keys = file_keys.x_trans_keys + config.y_trans_keys = file_keys.y_trans_keys + config.z_trans_keys = file_keys.z_trans_keys + config.ignore_file_patterns = file_keys.to_ignore + config.dark_names = file_keys.dark_names + config.flat_names = file_keys.ref_names + + return from_edf_to_nx(config, progress=progress) + + +def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: + """ + Convert an edf file to a nexus file. + For now duplicate data. + + :param TomoEDFConfig config: configuration to use to process the data + :return: (nexus_file, entry) + :rtype:tuple + """ # in old data, rot ange is unknown. Compute it as a function of the proj number compute_rotangle = True + if config.input_folder is None: + raise ValueError("input_folder should be provided") + + if config.output_file is None: + raise ValueError("output_file should be provided") + + # TODO: handle file info + if config.dataset_info_file is not None: + if not os.path.isfile(config.dataset_info_file): + raise ValueError(f"{config.dataset_info_file} is not a file") + else: + scan_info = get_parameters_frm_par_or_info(config.dataset_info_file) + else: + scan_info = None + + scan = EDFTomoScan( + scan=config.input_folder, + dataset_basename=config.dataset_basename, + scan_info=scan_info, + # TODO: add n frames ? + ) + fileout_h5 = utils.get_file_name( - file_name=output_file, extension=file_extension, check=True + file_name=config.output_file, + extension=config.file_extension, + check=True, ) - _logger.info("Output file will be " + fileout_h5) + _logger.info(f"Output file will be {fileout_h5}") + + output_data_path = "entry" if source_type is not None: source_type = SourceType.from_value(source_type) @@ -143,14 +208,18 @@ def edf_to_nx( metadata = [] proj_urls = scan.get_proj_urls(scan=scan.path) - for dark_to_find in file_keys.dark_names: + for dark_to_find in config.dark_names: dk_urls = scan.get_darks_url(scan_path=scan.path, prefix=dark_to_find) if len(dk_urls) > 0: if dark_to_find == "dark": DARK_ACCUM_FACT = False break - _edf_to_ignore = list(file_keys.to_ignore) - for refs_to_find in file_keys.ref_names: + if config.ignore_file_patterns is None: + _edf_to_ignore = tuple() + else: + _edf_to_ignore = config.ignore_file_patterns + + for refs_to_find in config.ref_names: if refs_to_find == "ref": _edf_to_ignore.append("HST") else: @@ -167,49 +236,27 @@ def edf_to_nx( n_frames = len(proj_urls) + len(refs_urls) + len(dk_urls) - # TODO: should be managed by tomoscan as well def getExtraInfo(scan): projections_urls = scan.projections indexes = sorted(projections_urls.keys()) first_proj_file = projections_urls[indexes[0]] fid = fabio.open(first_proj_file.file_path()) + try: if hasattr(fid, "header"): hd = fid.header else: hd = fid.getHeader() - try: - rotangle_index = ( - hd[file_keys.motor_mne_key] - .split(" ") - .index(file_keys.rot_angle_key) - ) - except (KeyError, ValueError): - rotangle_index = -1 - try: - xtrans_index = ( - hd[file_keys.motor_mne_key] - .split(" ") - .index(file_keys.x_trans_key) - ) - except (KeyError, ValueError): - xtrans_index = -1 - try: - ytrans_index = ( - hd[file_keys.motor_mne_key] - .split(" ") - .index(file_keys.y_trans_key) - ) - except (KeyError, ValueError): - ytrans_index = -1 - try: - ztrans_index = ( - hd[file_keys.motor_mne_key] - .split(" ") - .index(file_keys.z_trans_key) - ) - except (KeyError, ValueError): - ztrans_index = -1 + + motor_mne_key = _get_valid_key(header, config.motor_mne_keys) + motors = hd.get(motor_mne_key, "").split(" ") + + rotangle_index = _get_valid_key_index( + motors, config.rotation_angle_keys + ) + xtrans_index = _get_valid_key_index(motors, config.x_trans_keys) + ytrans_index = _get_valid_key_index(motors, config.y_trans_keys) + ztrans_index = _get_valid_key_index(motors, config.z_trans_keys) if hasattr(fid, "bytecode"): frame_type = fid.bytecode @@ -245,7 +292,7 @@ def edf_to_nx( ) h5d["/entry/title"] = ( - title if title is not None else os.path.basename(scan.path) + config.title if config.title is not None else os.path.basename(scan.path) ) if sample_name is None: @@ -256,13 +303,13 @@ def edf_to_nx( except Exception: sample_name = "unknow" h5d["/entry/sample/name"] = sample_name - if instrument_name is not None: + if config.instrument_name is not None: instrument_grp = h5d["/entry"].require_group("instrument") - instrument_grp["name"] = instrument_name + instrument_grp["name"] = config.instrument_name - if source_name is not None: + if config.source_name is not None: source_grp = h5d["/entry/instrument"].require_group("source") - source_grp["name"] = source_name + source_grp["name"] = config.source_name if source_type is not None: source_grp = h5d["/entry/instrument"].require_group("source") source_grp["type"] = source_type.value @@ -292,11 +339,11 @@ def edf_to_nx( scan_info=scan.scan_info, ) h5d["/entry/instrument/detector/x_pixel_size"] = ( - pixel_size * metricsystem.micrometer.value + pixel_size * config.pixel_size_unit.value ) h5d["/entry/instrument/detector/x_pixel_size"].attrs["unit"] = "m" h5d["/entry/instrument/detector/y_pixel_size"] = ( - pixel_size * metricsystem.micrometer.value + pixel_size * config.pixel_size_unit.value ) h5d["/entry/instrument/detector/y_pixel_size"].attrs["unit"] = "m" @@ -310,6 +357,8 @@ def edf_to_nx( scan_info=scan.scan_info, ) if energy is not None: + if config.energy_unit != EnergySI.KILOELECTRONVOLT: + energy = energy * config.energy_unit / EnergySI.KILOELECTRONVOLT h5d["/entry/instrument/beam/incident_energy"] = energy h5d["/entry/instrument/beam/incident_energy"].attrs["unit"] = "keV" @@ -401,6 +450,7 @@ def edf_to_nx( dk_indexes = sorted(dk_urls.keys()) if progress is not None: progress.reset(len(dk_urls)) + for dk_index in dk_indexes: dk_url = dk_urls[dk_index] if ignore(os.path.basename(dk_url.file_path())): @@ -413,8 +463,9 @@ def edf_to_nx( keys_dataset[nf] = ImageKey.DARK_FIELD.value keys_control_dataset[nf] = ImageKey.DARK_FIELD.value - if file_keys.motor_pos_key in header: - str_mot_val = header[file_keys.motor_pos_key].split(" ") + motor_pos_key = _get_valid_key(header, config.motor_position_keys) + if motor_pos_key: + str_mot_val = header[motor_pos_key].split(" ") if rot_angle_index == -1: rotation_dataset[nf] = 0.0 else: @@ -487,8 +538,10 @@ def edf_to_nx( dataDataset[nfr, :, :] = data + test_val keysDataset[nfr] = ImageKey.FLAT_FIELD.value keysCDataset[nfr] = ImageKey.FLAT_FIELD.value - if file_keys.motor_pos_key in header: - str_mot_val = header[file_keys.motor_pos_key].split(" ") + motor_pos_key = _get_valid_key(header, config.motor_position_keys) + + if motor_pos_key in header: + str_mot_val = header[motor_pos_key].split(" ") if raix == -1: rotationDataset[nfr] = 0.0 else: @@ -553,8 +606,8 @@ def edf_to_nx( if nproj >= scan.tomo_n: keys_control_dataset[nf] = ImageKey.ALIGNMENT.value - if file_keys.motor_pos_key in header: - str_mot_val = header[file_keys.motor_pos_key].split(" ") + if motor_pos_key in header: + str_mot_val = header[motor_pos_key].split(" ") # continuous scan - rot angle is unknown. Compute it if compute_rotangle is True and nproj < scan.tomo_n: @@ -634,15 +687,33 @@ def edf_to_nx( try: create_nx_data_group( - file_path=output_file, entry_path="entry", axis_scale=["linear", "linear"] + file_path=fileout_h5, + entry_path=output_data_path, + axis_scale=["linear", "linear"], ) except Exception as e: _logger.error("Fail to create NXdata group. Reason is {}".format(str(e))) # create beam group at root for compatibility try: - link_nxbeam_to_root(file_path=output_file, entry_path="entry") + link_nxbeam_to_root(file_path=fileout_h5, entry_path=output_data_path) except Exception: pass - return fileout_h5, "entry" + return fileout_h5, output_data_path + + +def _get_valid_key(header: dict, keys: tuple) -> Optional[str]: + """Return the first existing key in header""" + for key in keys: + if key in header: + return key + else: + return None + + +def _get_valid_key_index(motors: list, keys: tuple) -> Optional[int]: + try: + return motors.index(keys) + except ValueError: + return -1 diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index 99dc000..7d3a3fc 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -68,6 +68,8 @@ class TomoEDFConfig(ConfigBase): DATASET_BASENAME_DK = "dataset_basename" + DATASET_FILE_INFO_DK = "dataset_info_file" + LOG_LEVEL_DK = "log_level" TITLE_DK = "title" @@ -81,6 +83,7 @@ class TomoEDFConfig(ConfigBase): OVERWRITE_DK: "overwrite output files if exists without asking", FILE_EXTENSION_DK: "file extension. Ignored if the output file is provided and contains an extension", DATASET_BASENAME_DK: "dataset file prefix. If not provided will take the folder basename", + DATASET_FILE_INFO_DK: f"path to .info file containing dataset information (Energy, ScanRange, TOMO_N...). If not will deduce it from {INPUT_FOLDER_DK} and {DATASET_BASENAME_DK}", LOG_LEVEL_DK: 'Log level. Valid levels are "debug", "info", "warning" and "error"', TITLE_DK: "NXtomo title", IGNORE_FILE_PATTERN_DK: "some file pattern leading to ignoring the file", @@ -213,6 +216,7 @@ class TomoEDFConfig(ConfigBase): self._log_level = logging.WARNING self._title = None self._ignore_file_patterns = Tomo.EDF.TO_IGNORE + self._dataset_info_file = None # edf header keys self._motor_position_keys = Tomo.EDF.MOTOR_POS @@ -271,6 +275,18 @@ class TomoEDFConfig(ConfigBase): ) self._dataset_basename = dataset_basename + @property + def dataset_info_file(self) -> Optional[str]: + return self._dataset_info_file + + @dataset_info_file.setter + def dataset_info_file(self, file_path: Optional[str]) -> None: + if not isinstance(file_path, (type(None), str)): + raise TypeError( + f"file_path is expected to be None or an instance of str. Not {type(file_path)}" + ) + self._dataset_info_file = file_path + @property def title(self) -> Optional[str]: return self._title @@ -499,6 +515,9 @@ class TomoEDFConfig(ConfigBase): self.DATASET_BASENAME_DK: self.dataset_basename if self.dataset_basename is not None else "", + self.DATASET_FILE_INFO_DK: self.dataset_info_file + if self.dataset_info_file is not None + else "", self.LOG_LEVEL_DK: logging.getLevelName(self.log_level).lower(), self.TITLE_DK: self.title if self.title is not None else "", self.IGNORE_FILE_PATTERN_DK: self.ignore_file_patterns @@ -616,6 +635,12 @@ class TomoEDFConfig(ConfigBase): dataset_basename = None self.dataset_basename = dataset_basename + dataset_info_file = dict_.get(TomoEDFConfig.DATASET_FILE_INFO_DK, None) + if dataset_info_file is not None: + if dataset_info_file == "": + dataset_info_file = None + self.dataset_info_file = dataset_info_file + log_level = dict_.get(TomoEDFConfig.LOG_LEVEL_DK, None) if log_level is not None: self.log_level = log_level diff --git a/nxtomomill/io/config/test/test_edf_config.py b/nxtomomill/io/config/test/test_edf_config.py index 2dd6091..fae2267 100644 --- a/nxtomomill/io/config/test/test_edf_config.py +++ b/nxtomomill/io/config/test/test_edf_config.py @@ -103,6 +103,11 @@ def test_TomoEDFConfig_setters(): config.dataset_basename = None config.dataset_basename = "test_" + with pytest.raises(TypeError): + config.dataset_info_file = 12.3 + config.dataset_info_file = None + config.dataset_info_file = "my_info_file.info" + config.overwrite = True with pytest.raises(TypeError): @@ -201,6 +206,9 @@ def test_TomoEDFConfig_setters(): assert general_section_dict[TomoEDFConfig.FILE_EXTENSION_DK] == ".nx" assert general_section_dict[TomoEDFConfig.OVERWRITE_DK] == True assert general_section_dict[TomoEDFConfig.DATASET_BASENAME_DK] == "test_" + assert ( + general_section_dict[TomoEDFConfig.DATASET_FILE_INFO_DK] == "my_info_file.info" + ) assert general_section_dict[TomoEDFConfig.TITLE_DK] == "my title" assert general_section_dict[TomoEDFConfig.IGNORE_FILE_PATTERN_DK] == "" diff --git a/nxtomomill/nexus/nxtomo.py b/nxtomomill/nexus/nxtomo.py index a17b4f3..f7ca9d7 100644 --- a/nxtomomill/nexus/nxtomo.py +++ b/nxtomomill/nexus/nxtomo.py @@ -38,7 +38,7 @@ from .nxsample import NXsample from .utils import get_data_and_unit, get_data from datetime import datetime from silx.utils.proxy import docstring -from tomoscan.unitsystem.metricsystem import EnergySI +from tomoscan.unitsystem.energysystem import EnergySI from tomoscan.nexus.paths.nxtomo import get_paths as get_nexus_paths from silx.io.url import DataUrl import logging -- GitLab From b5cef6114da4a35ea358bcca77c89ba5056bd1b0 Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Mon, 25 Apr 2022 09:23:58 +0200 Subject: [PATCH 13/30] edf2nx: add a first vesion handling the angle_calculation options --- nxtomomill/app/edf2nx.py | 1 - nxtomomill/converter/__init__.py | 1 + nxtomomill/converter/edf/edfconverter.py | 52 ++++++++------ nxtomomill/converter/edf/test/test_edf2nx.py | 71 ++++++++++++++++-- nxtomomill/io/config/edfconfig.py | 76 +++++++++++++++++++- nxtomomill/io/config/hdf5config.py | 1 - nxtomomill/io/config/test/test_edf_config.py | 20 +++++- nxtomomill/utils/utils.py | 4 +- 8 files changed, 197 insertions(+), 29 deletions(-) diff --git a/nxtomomill/app/edf2nx.py b/nxtomomill/app/edf2nx.py index e2e2964..09a6649 100644 --- a/nxtomomill/app/edf2nx.py +++ b/nxtomomill/app/edf2nx.py @@ -69,7 +69,6 @@ __date__ = "16/01/2020" import argparse import logging -from nxtomomill.nexus.nxsource import SourceType from nxtomomill import utils from nxtomomill import converter from nxtomomill.utils import Progress diff --git a/nxtomomill/converter/__init__.py b/nxtomomill/converter/__init__.py index 8f16a93..7982fae 100644 --- a/nxtomomill/converter/__init__.py +++ b/nxtomomill/converter/__init__.py @@ -1,4 +1,5 @@ from .edf.edfconverter import edf_to_nx # noqa F401 +from .edf.edfconverter import from_edf_to_nx # noqa F401 from .hdf5.hdf5converter import from_h5_to_nx # noqa F401 from .hdf5.hdf5converter import h5_to_nx # noqa F401 from .hdf5.hdf5converter import get_bliss_tomo_entries # noqa F401 diff --git a/nxtomomill/converter/edf/edfconverter.py b/nxtomomill/converter/edf/edfconverter.py index b18963c..c4345ef 100644 --- a/nxtomomill/converter/edf/edfconverter.py +++ b/nxtomomill/converter/edf/edfconverter.py @@ -166,9 +166,6 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: :return: (nexus_file, entry) :rtype:tuple """ - # in old data, rot ange is unknown. Compute it as a function of the proj number - compute_rotangle = True - if config.input_folder is None: raise ValueError("input_folder should be provided") @@ -200,9 +197,6 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: output_data_path = "entry" - if source_type is not None: - source_type = SourceType.from_value(source_type) - DARK_ACCUM_FACT = True with HDF5File(fileout_h5, "w") as h5d: metadata = [] @@ -215,11 +209,11 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: DARK_ACCUM_FACT = False break if config.ignore_file_patterns is None: - _edf_to_ignore = tuple() + _edf_to_ignore = list() else: - _edf_to_ignore = config.ignore_file_patterns + _edf_to_ignore = list(config.ignore_file_patterns) - for refs_to_find in config.ref_names: + for refs_to_find in config.flat_names: if refs_to_find == "ref": _edf_to_ignore.append("HST") else: @@ -242,13 +236,18 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: first_proj_file = projections_urls[indexes[0]] fid = fabio.open(first_proj_file.file_path()) + rotangle_index = -1 + xtrans_index = -1 + ytrans_index = -1 + ztrans_index = -1 + frame_type = None + try: if hasattr(fid, "header"): hd = fid.header else: hd = fid.getHeader() - - motor_mne_key = _get_valid_key(header, config.motor_mne_keys) + motor_mne_key = _get_valid_key(hd, config.motor_mne_keys) motors = hd.get(motor_mne_key, "").split(" ") rotangle_index = _get_valid_key_index( @@ -265,6 +264,7 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: finally: fid.close() fid = None + return frame_type, rotangle_index, xtrans_index, ytrans_index, ztrans_index ( @@ -295,7 +295,8 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: config.title if config.title is not None else os.path.basename(scan.path) ) - if sample_name is None: + if config.sample_name is None: + sample_name = config.sample_name # try to deduce sample name from scan path. try: sample_name = os.path.abspath(scan.path).split(os.sep)[-3:] @@ -310,11 +311,14 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: if config.source_name is not None: source_grp = h5d["/entry/instrument"].require_group("source") source_grp["name"] = config.source_name - if source_type is not None: + if config.source_type is not None: source_grp = h5d["/entry/instrument"].require_group("source") - source_grp["type"] = source_type.value + source_grp["type"] = config.source_type.value - proj_angle = scan.scan_range / scan.tomo_n + if config.force_angle_calculation_endpoint: + proj_angle = scan.scan_range / (scan.tomo_n - 1) + else: + proj_angle = scan.scan_range / scan.tomo_n distance = scan.retrieve_information( scan=os.path.abspath(scan.path), @@ -610,8 +614,15 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: str_mot_val = header[motor_pos_key].split(" ") # continuous scan - rot angle is unknown. Compute it - if compute_rotangle is True and nproj < scan.tomo_n: - rotation_dataset[nf] = nproj * proj_angle + if config.force_angle_calculation is True and nproj < scan.tomo_n: + angle = nproj * proj_angle + if ( + scan.scan_range < 0 + and config.angle_calculation_rev_neg_scan_range is False + ): + angle = scan.scan_range - angle + + rotation_dataset[nf] = angle else: if rot_angle_index == -1: rotation_dataset[nf] = 0.0 @@ -713,7 +724,8 @@ def _get_valid_key(header: dict, keys: tuple) -> Optional[str]: def _get_valid_key_index(motors: list, keys: tuple) -> Optional[int]: - try: - return motors.index(keys) - except ValueError: + for key in keys: + if key in motors: + return motors.index(key) + else: return -1 diff --git a/nxtomomill/converter/edf/test/test_edf2nx.py b/nxtomomill/converter/edf/test/test_edf2nx.py index 3a10864..cd10bb0 100644 --- a/nxtomomill/converter/edf/test/test_edf2nx.py +++ b/nxtomomill/converter/edf/test/test_edf2nx.py @@ -31,7 +31,9 @@ __date__ = "09/10/2020" import tempfile import os from nxtomomill import converter +from nxtomomill.converter.edf.edfconverter import TomoEDFConfig from tomoscan.esrf.mock import MockEDF +import numpy try: from tomoscan.esrf.scan.hdf5scan import HDF5TomoScan @@ -69,10 +71,11 @@ def test_edf_to_nx_converter(progress): assert scan.dark_n == 1 output_file = os.path.join(folder, "nexus_file.nx") - nx_file, nx_entry = converter.edf_to_nx( - scan=scan, - output_file=output_file, - file_extension=".nx", + config = TomoEDFConfig() + config.input_folder = scan_path + config.output_file = output_file + nx_file, nx_entry = converter.from_edf_to_nx( + config=config, progress=progress, ) hdf5_scan = HDF5TomoScan(scan=nx_file, entry=nx_entry) @@ -81,3 +84,63 @@ def test_edf_to_nx_converter(progress): assert hdf5_scan.dim_1 == dim assert hdf5_scan.dim_2 == dim assert is_valid_for_reconstruction(hdf5_scan) + + +@pytest.mark.parametrize("scan_range", (-180, 180, 360)) +@pytest.mark.parametrize("endpoint", (True, False)) +@pytest.mark.parametrize("revert", (True, False)) +@pytest.mark.parametrize("force_angles", (True, False)) +def test_rotation_angle_infos(scan_range, endpoint, revert, force_angles): + """test conversion fits TomoEDFConfig parameters regarding the rotation angle calculation options""" + with tempfile.TemporaryDirectory() as folder_scan_range_180: + scan_path = os.path.join(folder_scan_range_180, "myscan") + n_proj = 12 + # TODO: check alignement + n_alignment_proj = 5 + dim = 4 + + MockEDF( + scan_path=scan_path, + n_radio=n_proj, + n_ini_radio=n_proj, + n_extra_radio=n_alignment_proj, + dim=dim, + dark_n=1, + ref_n=1, + scan_range=scan_range, + rotation_angle_endpoint=endpoint, + ) + + output_file = os.path.join(folder_scan_range_180, "nexus_file.nx") + + config = TomoEDFConfig() + config.input_folder = scan_path + config.output_file = output_file + config.force_angle_calculation = force_angles + config.force_angle_calculation_endpoint = endpoint + config.angle_calculation_rev_neg_scan_range = revert + + nx_file, nx_entry = converter.from_edf_to_nx( + config=config, + ) + hdf5_scan = HDF5TomoScan(scan=nx_file, entry=nx_entry) + + # compute expected rotation angles + converted_rotation_angles = numpy.asarray(hdf5_scan.rotation_angle) + converted_rotation_angles = converted_rotation_angles[ + hdf5_scan.image_key_control == 0 + ] + + if force_angles and revert and scan_range < 0 and not endpoint: + expected_angles = numpy.linspace(0, scan_range, n_proj, endpoint=endpoint) + else: + expected_angles = numpy.linspace( + min(0, scan_range), max(0, scan_range), n_proj, endpoint=endpoint + ) + revert_angles_in_nx = force_angles and revert and (scan_range < 0) + if revert_angles_in_nx: + expected_angles = expected_angles[::-1] + + numpy.testing.assert_almost_equal( + converted_rotation_angles, expected_angles, decimal=3 + ) diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index 7d3a3fc..26f5ac5 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -39,7 +39,7 @@ from nxtomomill.io.utils import convert_str_to_tuple, filter_str_def from nxtomomill.nexus.nxsource import SourceType from nxtomomill.utils import FileExtension from nxtomomill.settings import Tomo -from typing import Iterable, Optional, Union +from typing import Optional, Union import logging @@ -161,9 +161,20 @@ class TomoEDFConfig(ConfigBase): SAMPLE_NAME_DK = "sample_name" + FORCE_ANGLE_CALCULATION = "force_angle_calculation" + + FORCE_ANGLE_CALCULATION_ENDPOINT = "angle_calculation_endpoint" + + FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE = ( + "angle_calculation_rev_neg_scan_range" + ) + COMMENTS_SAMPLE_SECTION_DK = { SAMPLE_SECTION_DK: "section dedicated to sample definition.\n", SAMPLE_NAME_DK: "name of the sample", + FORCE_ANGLE_CALCULATION: "if set to False try first to deduce rotation angle from edf metadata. Else compute them from numpy.linspace", + FORCE_ANGLE_CALCULATION_ENDPOINT: "if rotation angles have to be calculated set numpy.linspace endpoint parameter to this value. If True then the rotation angle value of the last projection will be equal to the `ScanRange` value", + FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE: "Invert rotation angle values in the case of negative `ScanRange` value", } # SOURCE SECTION @@ -240,6 +251,9 @@ class TomoEDFConfig(ConfigBase): # sample self._sample_name = None + self._force_angle_calculation = False + self._force_angle_calculation_endpoint = False + self._force_angle_calculation_revert_neg_scan_range = True # source self._instrument_name = None @@ -465,6 +479,45 @@ class TomoEDFConfig(ConfigBase): raise TypeError("name is expected to be None or an instance of str") self._sample_name = name + @property + def force_angle_calculation(self) -> bool: + return self._force_angle_calculation + + @force_angle_calculation.setter + def force_angle_calculation(self, force: bool): + if not isinstance(force, bool): + raise TypeError( + f"force is expected to be an instance of bool. Not {type(force)}" + ) + else: + self._force_angle_calculation = force + + @property + def force_angle_calculation_endpoint(self) -> bool: + return self._force_angle_calculation_endpoint + + @force_angle_calculation_endpoint.setter + def force_angle_calculation_endpoint(self, endpoint: bool) -> None: + if not isinstance(endpoint, bool): + raise TypeError( + f"endpoint is expected to be an instance of bool. Not {type(endpoint)}" + ) + else: + self._force_angle_calculation_endpoint = endpoint + + @property + def angle_calculation_rev_neg_scan_range(self) -> bool: + return self._force_angle_calculation_revert_neg_scan_range + + @angle_calculation_rev_neg_scan_range.setter + def angle_calculation_rev_neg_scan_range(self, revert: bool): + if not isinstance(revert, bool): + raise TypeError( + f"revert is expected to be an instance of bool. Not {type(revert)}" + ) + else: + self._force_angle_calculation_revert_neg_scan_range = revert + @property def instrument_name(self) -> Optional[str]: return self._instrument_name @@ -564,6 +617,9 @@ class TomoEDFConfig(ConfigBase): self.SAMPLE_NAME_DK: self.sample_name if self.sample_name is not None else "", + self.FORCE_ANGLE_CALCULATION: self.force_angle_calculation, + self.FORCE_ANGLE_CALCULATION_ENDPOINT: self.force_angle_calculation_endpoint, + self.FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE: self.angle_calculation_rev_neg_scan_range, }, self.SOURCE_SECTION_DK: { self.INSTRUMENT_NAME_DK: self.instrument_name or "", @@ -749,6 +805,24 @@ class TomoEDFConfig(ConfigBase): if TomoEDFConfig.SAMPLE_NAME_DK in dict_: self.sample_name = dict_.get(TomoEDFConfig.SAMPLE_NAME_DK) + force_angle_calculation = dict_.get(TomoEDFConfig.FORCE_ANGLE_CALCULATION, None) + if force_angle_calculation is not None: + self.force_angle_calculation = force_angle_calculation + + force_angle_calculation_endpoint = dict_.get( + TomoEDFConfig.FORCE_ANGLE_CALCULATION_ENDPOINT, None + ) + if force_angle_calculation_endpoint is not None: + self.force_angle_calculation_endpoint = force_angle_calculation_endpoint + + angle_calculation_rev_neg_scan_range = dict_.get( + TomoEDFConfig.FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE, None + ) + if angle_calculation_rev_neg_scan_range is not None: + self.angle_calculation_rev_neg_scan_range = ( + angle_calculation_rev_neg_scan_range + ) + def load_source_section(self, dict_: dict) -> None: if TomoEDFConfig.INSTRUMENT_NAME_DK in dict_: self.instrument_name = dict_.get(TomoEDFConfig.INSTRUMENT_NAME_DK) diff --git a/nxtomomill/io/config/hdf5config.py b/nxtomomill/io/config/hdf5config.py index 80519be..916cb77 100644 --- a/nxtomomill/io/config/hdf5config.py +++ b/nxtomomill/io/config/hdf5config.py @@ -35,7 +35,6 @@ __date__ = "08/07/2021" from nxtomomill import settings from nxtomomill.io.config.configbase import ConfigBase -from nxtomomill.utils import FileExtension from nxtomomill.utils import Format from nxtomomill.nexus.nxdetector import FieldOfView from nxtomomill.io.utils import is_url_path diff --git a/nxtomomill/io/config/test/test_edf_config.py b/nxtomomill/io/config/test/test_edf_config.py index fae2267..afdfc10 100644 --- a/nxtomomill/io/config/test/test_edf_config.py +++ b/nxtomomill/io/config/test/test_edf_config.py @@ -171,6 +171,18 @@ def test_TomoEDFConfig_setters(): config.sample_name = 12.0 config.sample_name = "my sample" + with pytest.raises(TypeError): + config.force_angle_calculation = "toto" + config.force_angle_calculation = True + + with pytest.raises(TypeError): + config.force_angle_calculation_endpoint = "toto" + config.force_angle_calculation_endpoint = True + + with pytest.raises(TypeError): + config.angle_calculation_revert_neg_scan_range = "toto" + config.angle_calculation_revert_neg_scan_range = False + # test source setters config.instrument_name = None config.instrument_name = "BMXX" @@ -204,7 +216,7 @@ def test_TomoEDFConfig_setters(): assert general_section_dict[TomoEDFConfig.INPUT_FOLDER_DK] == "my_folder" assert general_section_dict[TomoEDFConfig.OUTPUT_FILE_DK] == "my_nx.nx" assert general_section_dict[TomoEDFConfig.FILE_EXTENSION_DK] == ".nx" - assert general_section_dict[TomoEDFConfig.OVERWRITE_DK] == True + assert general_section_dict[TomoEDFConfig.OVERWRITE_DK] is True assert general_section_dict[TomoEDFConfig.DATASET_BASENAME_DK] == "test_" assert ( general_section_dict[TomoEDFConfig.DATASET_FILE_INFO_DK] == "my_info_file.info" @@ -270,6 +282,12 @@ def test_TomoEDFConfig_setters(): sample_section_dict = config_dict[TomoEDFConfig.SAMPLE_SECTION_DK] assert sample_section_dict[TomoEDFConfig.SAMPLE_NAME_DK] == "my sample" + assert sample_section_dict[TomoEDFConfig.FORCE_ANGLE_CALCULATION] is True + assert sample_section_dict[TomoEDFConfig.FORCE_ANGLE_CALCULATION_ENDPOINT] is True + assert ( + sample_section_dict[TomoEDFConfig.FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE] + is False + ) source_section_dict = config_dict[TomoEDFConfig.SOURCE_SECTION_DK] assert source_section_dict[TomoEDFConfig.INSTRUMENT_NAME_DK] == "BMXX" diff --git a/nxtomomill/utils/utils.py b/nxtomomill/utils/utils.py index 35ca4ef..3024376 100644 --- a/nxtomomill/utils/utils.py +++ b/nxtomomill/utils/utils.py @@ -105,7 +105,9 @@ def get_file_name(file_name, extension, check=True): :param bool check: if check, already check if the file as one of the '_FileExtension' """ - extension = FileExtension.from_value(extension.lower()) + if isinstance(extension, str): + extension = FileExtension.from_value(extension.lower()) + assert isinstance(extension, FileExtension) if check: for value in FileExtension.values(): if file_name.lower().endswith(value): -- GitLab From dae253a483111e1065b57337aad1fcd5b009210d Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Mon, 25 Apr 2022 17:03:25 +0200 Subject: [PATCH 14/30] edf2nx: fix some typo and use `from_h5_to_nx` instead of `edf_to_nx` from the edf2nx application --- nxtomomill/app/edf2nx.py | 66 ++++++++++++++---------------- nxtomomill/io/config/edfconfig.py | 2 +- nxtomomill/io/config/hdf5config.py | 2 +- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/nxtomomill/app/edf2nx.py b/nxtomomill/app/edf2nx.py index 09a6649..124fdb4 100644 --- a/nxtomomill/app/edf2nx.py +++ b/nxtomomill/app/edf2nx.py @@ -69,28 +69,16 @@ __date__ = "16/01/2020" import argparse import logging -from nxtomomill import utils from nxtomomill import converter +from nxtomomill.io.config.edfconfig import TomoEDFConfig from nxtomomill.utils import Progress -from nxtomomill.settings import Tomo try: from tomoscan.esrf.scan.edfscan import EDFTomoScan except ImportError: from tomoscan.esrf.edfscan import EDFTomoScan -EDF_MOTOR_POS = Tomo.EDF.MOTOR_POS -EDF_MOTOR_MNE = Tomo.EDF.MOTOR_MNE -EDF_REFS_NAMES = Tomo.EDF.REFS_NAMES -EDF_TO_IGNORE = Tomo.EDF.TO_IGNORE -EDF_ROT_ANGLE = Tomo.EDF.ROT_ANGLE -EDF_DARK_NAMES = Tomo.EDF.DARK_NAMES -EDF_X_TRANS = Tomo.EDF.X_TRANS -EDF_Y_TRANS = Tomo.EDF.Y_TRANS -EDF_Z_TRANS = Tomo.EDF.Z_TRANS - logging.basicConfig(level=logging.INFO) -_logger = logging.getLogger(__name__) def main(argv): @@ -122,33 +110,41 @@ def main(argv): ) options = parser.parse_args(argv[1:]) - - conv = utils.get_tuple_of_keys_from_cmd - file_keys = converter.EDFFileKeys( - motor_mne_key=options.motor_mne_key, - motor_pos_key=options.motor_pos_key, - ref_names=conv(options.refs_name_keys), - to_ignore=conv(options.ignore_file_containing), - rot_angle_key=options.rot_angle_key, - dark_names=conv(options.dark_names), - x_trans_key=options.x_trans_key, - y_trans_key=options.y_trans_key, - z_trans_key=options.z_trans_key, - ) + config = TomoEDFConfig() + if options.config is not None: + config.from_cfg_file(options.config) + + check_input = { + "dataset basename": (options.dataset_basename, config.dataset_basename), + "scan path": (options.scan_path, config.input_folder), + "output file": (options.output_file, config.output_file), + "info file": (options.info_file, config.dataset_info_file), + } + for input_name, (opt_value, config_value) in check_input.items(): + if ( + opt_value is not None + and config_value is not None + and opt_value != config_value + ): + raise ValueError( + f"two values for {input_name} are provided from arguments and configuration file ({opt_value, config_value})" + ) + + if options.dataset_basename is not None: + config.dataset_basename = options.dataset_basename + if options.info_file is not None: + config.dataset_info_file = options.info_file + if options.scan_path is not None: + config.input_folder = options.scan_path + if options.output_file is not None: + config.output_file = options.output_file input_dir = options.scan_path scan = EDFTomoScan(input_dir) - converter.edf_to_nx( + converter.from_h5_to_nx( scan=scan, - output_file=options.output_file, - file_extension=options.file_extension, - file_keys=file_keys, + configuration=config, progress=Progress(""), - sample_name=options.sample_name, - title=options.title, - instrument_name=options.instrument_name, - source_name=options.source_name, - source_type=options.source_type, ) diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index 26f5ac5..8c79e4d 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -487,7 +487,7 @@ class TomoEDFConfig(ConfigBase): def force_angle_calculation(self, force: bool): if not isinstance(force, bool): raise TypeError( - f"force is expected to be an instance of bool. Not {type(force)}" + f"force is expected to be an instance of bool. Not {type(force)} - {force}" ) else: self._force_angle_calculation = force diff --git a/nxtomomill/io/config/hdf5config.py b/nxtomomill/io/config/hdf5config.py index 916cb77..fe88087 100644 --- a/nxtomomill/io/config/hdf5config.py +++ b/nxtomomill/io/config/hdf5config.py @@ -316,7 +316,7 @@ class TomoHDF5Config(ConfigBase): raise ValueError(f"No default unit for {key}") def __init__(self): - super().__init__(self) + super().__init__() self._set_freeze(False) # general information self._input_file = None -- GitLab From 1eec819e00597e0e1b73cdd037dd2e0c05639cae Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 07:11:33 +0200 Subject: [PATCH 15/30] edf2nx: add somega to the valid rotation angle close #101 --- nxtomomill/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nxtomomill/settings.py b/nxtomomill/settings.py index f8e1a5d..0b72558 100644 --- a/nxtomomill/settings.py +++ b/nxtomomill/settings.py @@ -129,7 +129,10 @@ class Tomo: MOTOR_MNE = ("motor_mne",) - ROT_ANGLE = ("srot",) + ROT_ANGLE = ( + "srot", + "somega", + ) X_TRANS = ("sx",) -- GitLab From 1cdf4271c02eb20838ad0277a9ab2e274d828f37 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 07:58:45 +0200 Subject: [PATCH 16/30] edf2nx: fix typo: call from_edf_to_nx instead of from_h5_to_nx --- nxtomomill/app/edf2nx.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nxtomomill/app/edf2nx.py b/nxtomomill/app/edf2nx.py index 124fdb4..427df5b 100644 --- a/nxtomomill/app/edf2nx.py +++ b/nxtomomill/app/edf2nx.py @@ -139,10 +139,7 @@ def main(argv): if options.output_file is not None: config.output_file = options.output_file - input_dir = options.scan_path - scan = EDFTomoScan(input_dir) - converter.from_h5_to_nx( - scan=scan, + converter.from_edf_to_nx( configuration=config, progress=Progress(""), ) -- GitLab From 5ff02134929a82e9b7641060e74dae996316a69c Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 07:59:30 +0200 Subject: [PATCH 17/30] add convert_str_to_bool to nxtomomill.io.utils --- nxtomomill/io/config/hdf5config.py | 69 +++++++++++++----------------- nxtomomill/io/utils.py | 11 +++++ 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/nxtomomill/io/config/hdf5config.py b/nxtomomill/io/config/hdf5config.py index fe88087..63b9143 100644 --- a/nxtomomill/io/config/hdf5config.py +++ b/nxtomomill/io/config/hdf5config.py @@ -37,10 +37,7 @@ from nxtomomill import settings from nxtomomill.io.config.configbase import ConfigBase from nxtomomill.utils import Format from nxtomomill.nexus.nxdetector import FieldOfView -from nxtomomill.io.utils import is_url_path -from nxtomomill.io.utils import convert_str_to_tuple -from nxtomomill.io.utils import convert_str_to_frame_grp -from nxtomomill.io.utils import filter_str_def +from nxtomomill.io import utils from silx.io.url import DataUrl from typing import Union from typing import Iterable @@ -690,7 +687,7 @@ class TomoHDF5Config(ConfigBase): if isinstance(url, str): if url == "": continue - elif is_url_path(url): + elif utils.is_url_path(url): url = DataUrl(path=url) else: url = DataUrl(data_path=url, scheme="silx") @@ -896,41 +893,31 @@ class TomoHDF5Config(ConfigBase): ) def load_general_section(self, dict_): - def cast_bool(value): - if isinstance(value, bool): - return value - elif isinstance(value, str): - if value not in ("False", "True"): - raise ValueError("value should be 'True' or 'False'") - return value == "True" - else: - raise TypeError("value should be a string") - self.input_file = dict_.get(TomoHDF5Config.INPUT_FILE_DK, None) self.output_file = dict_.get(TomoHDF5Config.OUTPUT_FILE_DK, None) overwrite = dict_.get(TomoHDF5Config.OVERWRITE_DK, None) if overwrite is not None: - self.overwrite = cast_bool(overwrite) + self.overwrite = utils.convert_str_to_bool(overwrite) file_extension = dict_.get(TomoHDF5Config.FILE_EXTENSION_DK, None) if file_extension is not None: - self.file_extension = filter_str_def(file_extension) + self.file_extension = utils.filter_str_def(file_extension) log_level = dict_.get(TomoHDF5Config.LOG_LEVEL_DK, None) if log_level is not None: self.log_level = log_level raises_error = dict_.get(TomoHDF5Config.RAISES_ERROR_DK, None) if raises_error is not None: - self.raises_error = cast_bool(raises_error) + self.raises_error = utils.convert_str_to_bool(raises_error) no_input = dict_.get(TomoHDF5Config.NO_INPUT_DK, None) if no_input is not None: - self.no_input = cast_bool(no_input) + self.no_input = utils.convert_str_to_bool(no_input) single_file = dict_.get(TomoHDF5Config.SINGLE_FILE_DK, None) if single_file is not None: - self.single_file = cast_bool(single_file) + self.single_file = utils.convert_str_to_bool(single_file) input_format = dict_.get(TomoHDF5Config.INPUT_FORMAT_DK, None) if input_format is not None: if input_format == "": input_format = None - self.format = filter_str_def(input_format) + self.format = utils.filter_str_def(input_format) field_of_view = dict_.get(TomoHDF5Config.FIELD_OF_VIEW_DK, None) if field_of_view is not None: if field_of_view == "": @@ -943,31 +930,31 @@ class TomoHDF5Config(ConfigBase): if valid_camera_names in ("", "none", "None", None): valid_camera_names = None else: - valid_camera_names = convert_str_to_tuple( + valid_camera_names = utils.convert_str_to_tuple( valid_camera_names, none_if_empty=True ) self.valid_camera_names = valid_camera_names # handle rotation angles. rotation_angle_keys = dict_.get(TomoHDF5Config.ROT_ANGLE_DK, None) if rotation_angle_keys is not None: - rotation_angle_keys = convert_str_to_tuple( + rotation_angle_keys = utils.convert_str_to_tuple( rotation_angle_keys, none_if_empty=True ) self.rotation_angle_keys = rotation_angle_keys # handle x translation x_trans_keys = dict_.get(TomoHDF5Config.X_TRANS_KEYS_DK, None) if x_trans_keys is not None: - x_trans_keys = convert_str_to_tuple(x_trans_keys, none_if_empty=True) + x_trans_keys = utils.convert_str_to_tuple(x_trans_keys, none_if_empty=True) self.x_trans_keys = x_trans_keys # handle y translation y_trans_keys = dict_.get(TomoHDF5Config.Y_TRANS_KEYS_DK, None) if y_trans_keys is not None: - y_trans_keys = convert_str_to_tuple(y_trans_keys, none_if_empty=True) + y_trans_keys = utils.convert_str_to_tuple(y_trans_keys, none_if_empty=True) self.y_trans_keys = y_trans_keys # handle z translation z_trans_keys = dict_.get(TomoHDF5Config.Z_TRANS_KEYS_DK, None) if z_trans_keys is not None: - z_trans_keys = convert_str_to_tuple(z_trans_keys, none_if_empty=True) + z_trans_keys = utils.convert_str_to_tuple(z_trans_keys, none_if_empty=True) self.z_trans_keys = z_trans_keys # handle y rotation keys y_rot_key = dict_.get(TomoHDF5Config.Y_ROT_KEYS_DK, None) @@ -976,28 +963,28 @@ class TomoHDF5Config(ConfigBase): # handle diode keys diode_keys = dict_.get(TomoHDF5Config.DIODE_KEYS_DK, None) if diode_keys is not None: - diode_keys = convert_str_to_tuple(diode_keys, none_if_empty=True) + diode_keys = utils.convert_str_to_tuple(diode_keys, none_if_empty=True) self.diode_keys = diode_keys # handle exposure time exposition_time_keys = dict_.get( TomoHDF5Config.ACQUISITION_EXPO_TIME_KEYS_DK, None ) if exposition_time_keys is not None: - exposition_time_keys = convert_str_to_tuple( + exposition_time_keys = utils.convert_str_to_tuple( exposition_time_keys, none_if_empty=True ) self.exposition_time_keys = exposition_time_keys # handle x pixel paths x_pixel_size_paths = dict_.get(TomoHDF5Config.X_PIXEL_SIZE_KEYS_DK, None) if x_pixel_size_paths is not None: - x_pixel_size_paths = convert_str_to_tuple( + x_pixel_size_paths = utils.convert_str_to_tuple( x_pixel_size_paths, none_if_empty=True ) self.x_pixel_size_paths = x_pixel_size_paths # handle y pixel paths y_pixel_size_paths = dict_.get(TomoHDF5Config.Y_PIXEL_SIZE_KEYS_DK, None) if y_pixel_size_paths is not None: - y_pixel_size_paths = convert_str_to_tuple( + y_pixel_size_paths = utils.convert_str_to_tuple( y_pixel_size_paths, none_if_empty=True ) self.y_pixel_size_paths = y_pixel_size_paths @@ -1006,7 +993,7 @@ class TomoHDF5Config(ConfigBase): TomoHDF5Config.SAMPLE_DETECTOR_DISTANCE_DK, None ) if sample_detector_distance is not None: - sample_detector_distance = convert_str_to_tuple( + sample_detector_distance = utils.convert_str_to_tuple( sample_detector_distance, none_if_empty=True ) self.sample_detector_distance_paths = sample_detector_distance @@ -1015,24 +1002,24 @@ class TomoHDF5Config(ConfigBase): # handle entries to convert entries = dict_.get(TomoHDF5Config.ENTRIES_DK) if entries is not None: - entries = convert_str_to_tuple(entries, none_if_empty=True) + entries = utils.convert_str_to_tuple(entries, none_if_empty=True) self.entries = entries # handle init titles. empty string is consider as a valid value init_titles = dict_.get(TomoHDF5Config.INIT_TITLES_DK, None) if init_titles is not None: - init_titles = convert_str_to_tuple(init_titles, none_if_empty=True) + init_titles = utils.convert_str_to_tuple(init_titles, none_if_empty=True) self.init_titles = init_titles # handle zserie init titles. empty string is consider as a valid value zserie_init_titles = dict_.get(TomoHDF5Config.ZSERIE_INIT_TITLES_DK, None) if zserie_init_titles is not None: - zserie_init_titles = convert_str_to_tuple( + zserie_init_titles = utils.convert_str_to_tuple( zserie_init_titles, none_if_empty=True ) self.zserie_init_titles = zserie_init_titles # handle dark titles. empty string is consider as a valid value dark_titles = dict_.get(TomoHDF5Config.DARK_TITLES_DK, None) if dark_titles is not None: - dark_titles = convert_str_to_tuple(dark_titles, none_if_empty=True) + dark_titles = utils.convert_str_to_tuple(dark_titles, none_if_empty=True) self.dark_titles = dark_titles # handle ref titles. empty string is consider as a valid value flat_titles_dks = [ @@ -1049,7 +1036,9 @@ class TomoHDF5Config(ConfigBase): f"flat titles are provided twice under {flat_title_key_picked} and {alias}. Please clean your configuration file. {flat_title_key_picked} will be used" ) else: - flat_titles = convert_str_to_tuple(flat_titles, none_if_empty=True) + flat_titles = utils.convert_str_to_tuple( + flat_titles, none_if_empty=True + ) self.flat_titles = flat_titles flat_title_key_picked = alias if ( @@ -1062,12 +1051,12 @@ class TomoHDF5Config(ConfigBase): # handle projection titles. empty string is consider as a valid value proj_titles = dict_.get(TomoHDF5Config.PROJ_TITLES_DK, None) if proj_titles is not None: - proj_titles = convert_str_to_tuple(proj_titles, none_if_empty=True) + proj_titles = utils.convert_str_to_tuple(proj_titles, none_if_empty=True) self.projections_titles = proj_titles # handle alignment titles. empty string is consider as a valid value alignment_titles = dict_.get(TomoHDF5Config.ALIGNMENT_TITLES_DK, None) if alignment_titles is not None: - alignment_titles = convert_str_to_tuple( + alignment_titles = utils.convert_str_to_tuple( alignment_titles, none_if_empty=True ) self.alignment_titles = alignment_titles @@ -1076,7 +1065,7 @@ class TomoHDF5Config(ConfigBase): # urls data_urls = dict_.get(TomoHDF5Config.DATA_DK, None) if data_urls is not None: - data_urls = convert_str_to_frame_grp(data_urls) + data_urls = utils.convert_str_to_frame_grp(data_urls) self.data_frame_grps = data_urls default_copy_behavior = dict_.get(TomoHDF5Config.DEFAULT_DATA_COPY_DK, None) if default_copy_behavior is not None: @@ -1157,7 +1146,7 @@ class XRD3DHDF5Config(TomoHDF5Config): rocking_keys = dict_.get(XRD3DHDF5Config.ROCKING_KEYS_DK, None) # handle rocking if rocking_keys is not None: - rocking_keys = convert_str_to_tuple(rocking_keys, none_if_empty=True) + rocking_keys = utils.convert_str_to_tuple(rocking_keys, none_if_empty=True) self.rocking_keys = rocking_keys def load_from_dict(self, dict_: dict) -> None: diff --git a/nxtomomill/io/utils.py b/nxtomomill/io/utils.py index 96aadd3..48bac68 100644 --- a/nxtomomill/io/utils.py +++ b/nxtomomill/io/utils.py @@ -93,6 +93,17 @@ def convert_str_to_tuple( return tuple(elmts) +def convert_str_to_bool(value: Union[str, bool]): + if isinstance(value, bool): + return value + elif isinstance(value, str): + if value not in ("False", "True"): + raise ValueError("value should be 'True' or 'False'") + return value == "True" + else: + raise TypeError("value should be a string") + + def is_url_path(url_str: str) -> bool: """ :return: True if the provided string fit DataUrl pattern -- GitLab From f2592a5b552c5dda3ee51e4772f454d60acc194f Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 07:59:44 +0200 Subject: [PATCH 18/30] test_edf_converter: fix typo --- nxtomomill/io/config/test/test_edf_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nxtomomill/io/config/test/test_edf_config.py b/nxtomomill/io/config/test/test_edf_config.py index afdfc10..f0fc86d 100644 --- a/nxtomomill/io/config/test/test_edf_config.py +++ b/nxtomomill/io/config/test/test_edf_config.py @@ -180,8 +180,8 @@ def test_TomoEDFConfig_setters(): config.force_angle_calculation_endpoint = True with pytest.raises(TypeError): - config.angle_calculation_revert_neg_scan_range = "toto" - config.angle_calculation_revert_neg_scan_range = False + config.angle_calculation_rev_neg_scan_range = "toto" + config.angle_calculation_rev_neg_scan_range = False # test source setters config.instrument_name = None -- GitLab From 084e070b777ea39963c69b405a1625ef508cbe5a Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 08:00:04 +0200 Subject: [PATCH 19/30] test_config_handler: fix typo --- nxtomomill/io/config/test/test_confighandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nxtomomill/io/config/test/test_confighandler.py b/nxtomomill/io/config/test/test_confighandler.py index 4810915..ce195aa 100644 --- a/nxtomomill/io/config/test/test_confighandler.py +++ b/nxtomomill/io/config/test/test_confighandler.py @@ -126,7 +126,7 @@ class TestH5Config(unittest.TestCase): # try another random para,eter input_config.output_file = None - input_config.x_trans_keys = "xtrans2" + input_config.x_trans_keys = ("xtrans2",) options.x_trans_keys = "xtrans3" input_config.to_cfg_file(input_file_path) with self.assertRaises(ValueError): -- GitLab From b8f4a2d8f2071aa581945cdd52c6b0f81976fa50 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 08:01:36 +0200 Subject: [PATCH 20/30] edfconfig: cast loaded bool option --- nxtomomill/io/config/edfconfig.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index 8c79e4d..fb6daca 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -35,7 +35,11 @@ import configparser from tomoscan.unitsystem.metricsystem import MetricSystem from tomoscan.unitsystem.energysystem import EnergySI from nxtomomill.io.config.configbase import ConfigBase -from nxtomomill.io.utils import convert_str_to_tuple, filter_str_def +from nxtomomill.io.utils import ( + convert_str_to_tuple, + filter_str_def, + convert_str_to_bool, +) from nxtomomill.nexus.nxsource import SourceType from nxtomomill.utils import FileExtension from nxtomomill.settings import Tomo @@ -665,21 +669,11 @@ class TomoEDFConfig(ConfigBase): _logger.error("No {} section found".format(section_key)) def load_general_section(self, dict_: dict) -> None: - def cast_bool(value): - if isinstance(value, bool): - return value - elif isinstance(value, str): - if value not in ("False", "True"): - raise ValueError("value should be 'True' or 'False'") - return value == "True" - else: - raise TypeError("value should be a string") - self.input_folder = dict_.get(TomoEDFConfig.INPUT_FOLDER_DK, None) self.output_file = dict_.get(TomoEDFConfig.OUTPUT_FILE_DK, None) overwrite = dict_.get(TomoEDFConfig.OVERWRITE_DK, None) if overwrite is not None: - self.overwrite = cast_bool(overwrite) + self.overwrite = convert_str_to_bool(overwrite) file_extension = dict_.get(TomoEDFConfig.FILE_EXTENSION_DK, None) if file_extension not in (None, ""): @@ -807,19 +801,21 @@ class TomoEDFConfig(ConfigBase): force_angle_calculation = dict_.get(TomoEDFConfig.FORCE_ANGLE_CALCULATION, None) if force_angle_calculation is not None: - self.force_angle_calculation = force_angle_calculation + self.force_angle_calculation = convert_str_to_bool(force_angle_calculation) force_angle_calculation_endpoint = dict_.get( TomoEDFConfig.FORCE_ANGLE_CALCULATION_ENDPOINT, None ) if force_angle_calculation_endpoint is not None: - self.force_angle_calculation_endpoint = force_angle_calculation_endpoint + self.force_angle_calculation_endpoint = convert_str_to_bool( + force_angle_calculation_endpoint + ) angle_calculation_rev_neg_scan_range = dict_.get( TomoEDFConfig.FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE, None ) if angle_calculation_rev_neg_scan_range is not None: - self.angle_calculation_rev_neg_scan_range = ( + self.angle_calculation_rev_neg_scan_range = convert_str_to_bool( angle_calculation_rev_neg_scan_range ) -- GitLab From d644c4edfa4268c84c3475c2845c26ff8afe4878 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 08:01:56 +0200 Subject: [PATCH 21/30] configbase: fix typo. Output file default value is None --- nxtomomill/io/config/configbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nxtomomill/io/config/configbase.py b/nxtomomill/io/config/configbase.py index 2da41fa..07a680f 100644 --- a/nxtomomill/io/config/configbase.py +++ b/nxtomomill/io/config/configbase.py @@ -46,7 +46,7 @@ class ConfigBase: # see https://stackoverflow.com/questions/3603502/prevent-creating-new-attributes-outside-init def __init__(self) -> None: - self._output_file = False + self._output_file = None self._overwrite = False self._file_extension = FileExtension.NX self._log_level = logging.WARNING -- GitLab From 59801c285b719f03578ac0b3ea21534039a059f9 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 08:02:57 +0200 Subject: [PATCH 22/30] from_edf_to_nx: rename `config` parameter to `configuration` to be homogeneous with from_h5_to_nx --- nxtomomill/converter/edf/edfconverter.py | 87 +++++++++++--------- nxtomomill/converter/edf/test/test_edf2nx.py | 4 +- 2 files changed, 50 insertions(+), 41 deletions(-) diff --git a/nxtomomill/converter/edf/edfconverter.py b/nxtomomill/converter/edf/edfconverter.py index c4345ef..f05e9c3 100644 --- a/nxtomomill/converter/edf/edfconverter.py +++ b/nxtomomill/converter/edf/edfconverter.py @@ -157,7 +157,7 @@ def edf_to_nx( return from_edf_to_nx(config, progress=progress) -def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: +def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: """ Convert an edf file to a nexus file. For now duplicate data. @@ -166,31 +166,32 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: :return: (nexus_file, entry) :rtype:tuple """ - if config.input_folder is None: + if configuration.input_folder is None: raise ValueError("input_folder should be provided") + if not os.path.isdir(configuration.input_folder): + raise OSError(f"{configuration.input_folder} is not a valid folder path") - if config.output_file is None: + if configuration.output_file is None: raise ValueError("output_file should be provided") - # TODO: handle file info - if config.dataset_info_file is not None: - if not os.path.isfile(config.dataset_info_file): - raise ValueError(f"{config.dataset_info_file} is not a file") + if configuration.dataset_info_file is not None: + if not os.path.isfile(configuration.dataset_info_file): + raise ValueError(f"{configuration.dataset_info_file} is not a file") else: - scan_info = get_parameters_frm_par_or_info(config.dataset_info_file) + scan_info = get_parameters_frm_par_or_info(configuration.dataset_info_file) else: scan_info = None scan = EDFTomoScan( - scan=config.input_folder, - dataset_basename=config.dataset_basename, + scan=configuration.input_folder, + dataset_basename=configuration.dataset_basename, scan_info=scan_info, # TODO: add n frames ? ) fileout_h5 = utils.get_file_name( - file_name=config.output_file, - extension=config.file_extension, + file_name=configuration.output_file, + extension=configuration.file_extension, check=True, ) _logger.info(f"Output file will be {fileout_h5}") @@ -202,18 +203,18 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: metadata = [] proj_urls = scan.get_proj_urls(scan=scan.path) - for dark_to_find in config.dark_names: + for dark_to_find in configuration.dark_names: dk_urls = scan.get_darks_url(scan_path=scan.path, prefix=dark_to_find) if len(dk_urls) > 0: if dark_to_find == "dark": DARK_ACCUM_FACT = False break - if config.ignore_file_patterns is None: + if configuration.ignore_file_patterns is None: _edf_to_ignore = list() else: - _edf_to_ignore = list(config.ignore_file_patterns) + _edf_to_ignore = list(configuration.ignore_file_patterns) - for refs_to_find in config.flat_names: + for refs_to_find in configuration.flat_names: if refs_to_find == "ref": _edf_to_ignore.append("HST") else: @@ -247,15 +248,15 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: hd = fid.header else: hd = fid.getHeader() - motor_mne_key = _get_valid_key(hd, config.motor_mne_keys) + motor_mne_key = _get_valid_key(hd, configuration.motor_mne_keys) motors = hd.get(motor_mne_key, "").split(" ") rotangle_index = _get_valid_key_index( - motors, config.rotation_angle_keys + motors, configuration.rotation_angle_keys ) - xtrans_index = _get_valid_key_index(motors, config.x_trans_keys) - ytrans_index = _get_valid_key_index(motors, config.y_trans_keys) - ztrans_index = _get_valid_key_index(motors, config.z_trans_keys) + xtrans_index = _get_valid_key_index(motors, configuration.x_trans_keys) + ytrans_index = _get_valid_key_index(motors, configuration.y_trans_keys) + ztrans_index = _get_valid_key_index(motors, configuration.z_trans_keys) if hasattr(fid, "bytecode"): frame_type = fid.bytecode @@ -292,11 +293,13 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: ) h5d["/entry/title"] = ( - config.title if config.title is not None else os.path.basename(scan.path) + configuration.title + if configuration.title is not None + else os.path.basename(scan.path) ) - if config.sample_name is None: - sample_name = config.sample_name + if configuration.sample_name is None: + sample_name = configuration.sample_name # try to deduce sample name from scan path. try: sample_name = os.path.abspath(scan.path).split(os.sep)[-3:] @@ -304,18 +307,18 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: except Exception: sample_name = "unknow" h5d["/entry/sample/name"] = sample_name - if config.instrument_name is not None: + if configuration.instrument_name is not None: instrument_grp = h5d["/entry"].require_group("instrument") - instrument_grp["name"] = config.instrument_name + instrument_grp["name"] = configuration.instrument_name - if config.source_name is not None: + if configuration.source_name is not None: source_grp = h5d["/entry/instrument"].require_group("source") - source_grp["name"] = config.source_name - if config.source_type is not None: + source_grp["name"] = configuration.source_name + if configuration.source_type is not None: source_grp = h5d["/entry/instrument"].require_group("source") - source_grp["type"] = config.source_type.value + source_grp["type"] = configuration.source_type.value - if config.force_angle_calculation_endpoint: + if configuration.force_angle_calculation_endpoint: proj_angle = scan.scan_range / (scan.tomo_n - 1) else: proj_angle = scan.scan_range / scan.tomo_n @@ -343,11 +346,11 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: scan_info=scan.scan_info, ) h5d["/entry/instrument/detector/x_pixel_size"] = ( - pixel_size * config.pixel_size_unit.value + pixel_size * configuration.pixel_size_unit.value ) h5d["/entry/instrument/detector/x_pixel_size"].attrs["unit"] = "m" h5d["/entry/instrument/detector/y_pixel_size"] = ( - pixel_size * config.pixel_size_unit.value + pixel_size * configuration.pixel_size_unit.value ) h5d["/entry/instrument/detector/y_pixel_size"].attrs["unit"] = "m" @@ -361,8 +364,8 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: scan_info=scan.scan_info, ) if energy is not None: - if config.energy_unit != EnergySI.KILOELECTRONVOLT: - energy = energy * config.energy_unit / EnergySI.KILOELECTRONVOLT + if configuration.energy_unit != EnergySI.KILOELECTRONVOLT: + energy = energy * configuration.energy_unit / EnergySI.KILOELECTRONVOLT h5d["/entry/instrument/beam/incident_energy"] = energy h5d["/entry/instrument/beam/incident_energy"].attrs["unit"] = "keV" @@ -467,7 +470,7 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: keys_dataset[nf] = ImageKey.DARK_FIELD.value keys_control_dataset[nf] = ImageKey.DARK_FIELD.value - motor_pos_key = _get_valid_key(header, config.motor_position_keys) + motor_pos_key = _get_valid_key(header, configuration.motor_position_keys) if motor_pos_key: str_mot_val = header[motor_pos_key].split(" ") if rot_angle_index == -1: @@ -542,7 +545,9 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: dataDataset[nfr, :, :] = data + test_val keysDataset[nfr] = ImageKey.FLAT_FIELD.value keysCDataset[nfr] = ImageKey.FLAT_FIELD.value - motor_pos_key = _get_valid_key(header, config.motor_position_keys) + motor_pos_key = _get_valid_key( + header, configuration.motor_position_keys + ) if motor_pos_key in header: str_mot_val = header[motor_pos_key].split(" ") @@ -610,15 +615,19 @@ def from_edf_to_nx(config: TomoEDFConfig, progress=None) -> tuple: if nproj >= scan.tomo_n: keys_control_dataset[nf] = ImageKey.ALIGNMENT.value + motor_pos_key = _get_valid_key(header, configuration.motor_position_keys) if motor_pos_key in header: str_mot_val = header[motor_pos_key].split(" ") # continuous scan - rot angle is unknown. Compute it - if config.force_angle_calculation is True and nproj < scan.tomo_n: + if ( + configuration.force_angle_calculation is True + and nproj < scan.tomo_n + ): angle = nproj * proj_angle if ( scan.scan_range < 0 - and config.angle_calculation_rev_neg_scan_range is False + and configuration.angle_calculation_rev_neg_scan_range is False ): angle = scan.scan_range - angle diff --git a/nxtomomill/converter/edf/test/test_edf2nx.py b/nxtomomill/converter/edf/test/test_edf2nx.py index cd10bb0..206397b 100644 --- a/nxtomomill/converter/edf/test/test_edf2nx.py +++ b/nxtomomill/converter/edf/test/test_edf2nx.py @@ -75,7 +75,7 @@ def test_edf_to_nx_converter(progress): config.input_folder = scan_path config.output_file = output_file nx_file, nx_entry = converter.from_edf_to_nx( - config=config, + configuration=config, progress=progress, ) hdf5_scan = HDF5TomoScan(scan=nx_file, entry=nx_entry) @@ -121,7 +121,7 @@ def test_rotation_angle_infos(scan_range, endpoint, revert, force_angles): config.angle_calculation_rev_neg_scan_range = revert nx_file, nx_entry = converter.from_edf_to_nx( - config=config, + configuration=config, ) hdf5_scan = HDF5TomoScan(scan=nx_file, entry=nx_entry) -- GitLab From 388549fdb9708423fa14bcdc8490c478eee50a06 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 08:07:55 +0200 Subject: [PATCH 23/30] PEP8 --- nxtomomill/app/edf2nx.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nxtomomill/app/edf2nx.py b/nxtomomill/app/edf2nx.py index 427df5b..bdb497c 100644 --- a/nxtomomill/app/edf2nx.py +++ b/nxtomomill/app/edf2nx.py @@ -73,11 +73,6 @@ from nxtomomill import converter from nxtomomill.io.config.edfconfig import TomoEDFConfig from nxtomomill.utils import Progress -try: - from tomoscan.esrf.scan.edfscan import EDFTomoScan -except ImportError: - from tomoscan.esrf.edfscan import EDFTomoScan - logging.basicConfig(level=logging.INFO) -- GitLab From 6107da0524efc0d54fda9482b97b719084f09fc4 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 11:31:46 +0200 Subject: [PATCH 24/30] edf2nx: fix processing when dataset_basename and info_file are provided. Improve alignement handling. --- nxtomomill/converter/edf/edfconverter.py | 31 ++-- nxtomomill/converter/edf/test/test_edf2nx.py | 172 ++++++++++++++++++- 2 files changed, 186 insertions(+), 17 deletions(-) diff --git a/nxtomomill/converter/edf/edfconverter.py b/nxtomomill/converter/edf/edfconverter.py index f05e9c3..7ee2a57 100644 --- a/nxtomomill/converter/edf/edfconverter.py +++ b/nxtomomill/converter/edf/edfconverter.py @@ -201,7 +201,9 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: DARK_ACCUM_FACT = True with HDF5File(fileout_h5, "w") as h5d: metadata = [] - proj_urls = scan.get_proj_urls(scan=scan.path) + proj_urls = scan.get_proj_urls( + scan=scan.path, dataset_basename=scan.dataset_basename + ) for dark_to_find in configuration.dark_names: dk_urls = scan.get_darks_url(scan_path=scan.path, prefix=dark_to_find) @@ -219,12 +221,12 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: _edf_to_ignore.append("HST") else: _edf_to_ignore.remove("HST") - try: - get_flats_url = scan.get_flats_url - except ImportError: - get_flats_url = scan.get_refs_url - refs_urls = get_flats_url( - scan_path=scan.path, prefix=refs_to_find, ignore=_edf_to_ignore + + refs_urls = scan.get_flats_url( + scan_path=scan.path, + prefix=refs_to_find, + ignore=_edf_to_ignore, + dataset_basename=scan.dataset_basename, ) if len(refs_urls) > 0: break @@ -232,7 +234,12 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: n_frames = len(proj_urls) + len(refs_urls) + len(dk_urls) def getExtraInfo(scan): + assert isinstance(scan, EDFTomoScan) projections_urls = scan.projections + if len(projections_urls) == 0: + raise ValueError( + f"No projections found in {scan.path} with dataset basename: {configuration.dataset_basename if configuration.dataset_basename is not None else 'Default'} and dataset info file: {configuration.dataset_info_file if configuration.dataset_info_file is not None else 'Default'}. " + ) indexes = sorted(projections_urls.keys()) first_proj_file = projections_urls[indexes[0]] fid = fabio.open(first_proj_file.file_path()) @@ -473,7 +480,7 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: motor_pos_key = _get_valid_key(header, configuration.motor_position_keys) if motor_pos_key: str_mot_val = header[motor_pos_key].split(" ") - if rot_angle_index == -1: + if rot_angle_index == -1 or configuration.force_angle_calculation: rotation_dataset[nf] = 0.0 else: rotation_dataset[nf] = float(str_mot_val[rot_angle_index]) @@ -551,7 +558,7 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: if motor_pos_key in header: str_mot_val = header[motor_pos_key].split(" ") - if raix == -1: + if raix == -1 or configuration.force_angle_calculation: rotationDataset[nfr] = 0.0 else: rotationDataset[nfr] = float(str_mot_val[raix]) @@ -633,7 +640,11 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: rotation_dataset[nf] = angle else: - if rot_angle_index == -1: + if rot_angle_index == -1 or configuration.force_angle_calculation: + # FIXMe: for alignment projection when calculation is force the + # angle will always be set to 0.0 when it should be computed. + # it would be wiser to compute all the rotation angle at the + # end when each frame is tag. rotation_dataset[nf] = 0.0 else: rotation_dataset[nf] = float(str_mot_val[rot_angle_index]) diff --git a/nxtomomill/converter/edf/test/test_edf2nx.py b/nxtomomill/converter/edf/test/test_edf2nx.py index 206397b..1b2b079 100644 --- a/nxtomomill/converter/edf/test/test_edf2nx.py +++ b/nxtomomill/converter/edf/test/test_edf2nx.py @@ -25,9 +25,10 @@ __authors__ = ["H. Payno"] __license__ = "MIT" -__date__ = "09/10/2020" +__date__ = "26/04/2022" +import shutil import tempfile import os from nxtomomill import converter @@ -35,6 +36,8 @@ from nxtomomill.converter.edf.edfconverter import TomoEDFConfig from tomoscan.esrf.mock import MockEDF import numpy +from tomoscan.esrf.utils import dump_info_file + try: from tomoscan.esrf.scan.hdf5scan import HDF5TomoScan from tomoscan.esrf.scan.edfscan import EDFTomoScan @@ -92,10 +95,9 @@ def test_edf_to_nx_converter(progress): @pytest.mark.parametrize("force_angles", (True, False)) def test_rotation_angle_infos(scan_range, endpoint, revert, force_angles): """test conversion fits TomoEDFConfig parameters regarding the rotation angle calculation options""" - with tempfile.TemporaryDirectory() as folder_scan_range_180: - scan_path = os.path.join(folder_scan_range_180, "myscan") + with tempfile.TemporaryDirectory() as data_folder: + scan_path = os.path.join(data_folder, "myscan") n_proj = 12 - # TODO: check alignement n_alignment_proj = 5 dim = 4 @@ -111,7 +113,7 @@ def test_rotation_angle_infos(scan_range, endpoint, revert, force_angles): rotation_angle_endpoint=endpoint, ) - output_file = os.path.join(folder_scan_range_180, "nexus_file.nx") + output_file = os.path.join(data_folder, "nexus_file.nx") config = TomoEDFConfig() config.input_folder = scan_path @@ -126,8 +128,8 @@ def test_rotation_angle_infos(scan_range, endpoint, revert, force_angles): hdf5_scan = HDF5TomoScan(scan=nx_file, entry=nx_entry) # compute expected rotation angles - converted_rotation_angles = numpy.asarray(hdf5_scan.rotation_angle) - converted_rotation_angles = converted_rotation_angles[ + raw_rotation_angles = numpy.asarray(hdf5_scan.rotation_angle) + converted_rotation_angles = raw_rotation_angles[ hdf5_scan.image_key_control == 0 ] @@ -144,3 +146,159 @@ def test_rotation_angle_infos(scan_range, endpoint, revert, force_angles): numpy.testing.assert_almost_equal( converted_rotation_angles, expected_angles, decimal=3 ) + # test alignment projection and flat are contained in the projections range + dark_angles = raw_rotation_angles[hdf5_scan.image_key_control == 2] + for angle in dark_angles: + assert angle == 0 or min(converted_rotation_angles) <= angle <= max( + converted_rotation_angles + ) + + flat_angles = raw_rotation_angles[hdf5_scan.image_key_control == 1] + + for angle in flat_angles: + assert angle == 0 or min(converted_rotation_angles) <= angle <= max( + converted_rotation_angles + ) + + alignment_angles = raw_rotation_angles[hdf5_scan.image_key_control == -1] + for angle in alignment_angles: + assert angle == 0 or min(converted_rotation_angles) <= angle <= max( + converted_rotation_angles + ) + + +def test_different_info_file(): + """insure providing a different spec info file will be taken into account""" + with tempfile.TemporaryDirectory() as data_folder: + scan_path = os.path.join(data_folder, "myscan") + n_proj = 12 + n_alignment_proj = 0 + dim = 4 + flat_n = 1 + dark_n = 1 + + original_scan_range = 180 + original_energy = 2.3 + original_pixel_size = 0.03 + original_distance = 0.36 + + new_scan_range = -180 + new_energy = 6.6 + new_pixel_size = 0.002 + new_distance = 0.458 + new_n_proj = n_proj - 2 + + other_info_file = os.path.join(data_folder, "new_info_file.info") + dump_info_file( + file_path=other_info_file, + tomo_n=new_n_proj, + scan_range=new_scan_range, + flat_n=flat_n, + flat_on=new_n_proj, + dark_n=dark_n, + dim_1=dim, + dim_2=dim, + col_beg=0, + col_end=dim, + row_beg=0, + row_end=dim, + pixel_size=new_pixel_size, + distance=new_distance, + energy=new_energy, + ) + assert os.path.exists(other_info_file) + + MockEDF( + scan_path=scan_path, + n_radio=n_proj, + n_ini_radio=n_proj, + n_extra_radio=n_alignment_proj, + dim=dim, + dark_n=dark_n, + flat_n=flat_n, + scan_range=original_scan_range, + energy=original_energy, + pixel_size=original_pixel_size, + distance=original_distance, + ) + + output_file = os.path.join(data_folder, "nexus_file.nx") + config = TomoEDFConfig() + config.input_folder = scan_path + config.output_file = output_file + config.force_angle_calculation = True + config.pixel_size_unit = "m" + config.distance_unit = "m" + config.energy_unit = "kev" + + # test step 1: check scan info is correctly read from the original parameters + nx_file, nx_entry = converter.from_edf_to_nx( + configuration=config, + ) + hdf5_scan_original_info_file = HDF5TomoScan(scan=nx_file, entry=nx_entry) + assert hdf5_scan_original_info_file.energy == original_energy + assert hdf5_scan_original_info_file.scan_range == original_scan_range + assert hdf5_scan_original_info_file.pixel_size == original_pixel_size + assert hdf5_scan_original_info_file.distance == original_distance + assert len(hdf5_scan_original_info_file.projections) == n_proj + + # test step 2: check scan info is correctly read from new parameters + config.dataset_info_file = other_info_file + config.overwrite = True + nx_file, nx_entry = converter.from_edf_to_nx( + configuration=config, + ) + hdf5_scan_new_info_file = HDF5TomoScan(scan=nx_file, entry=nx_entry) + assert hdf5_scan_new_info_file.energy == new_energy + # for now HDF5TomoScan expect range to be 180 or 360. There is no such -180 as in EDF + assert abs(hdf5_scan_new_info_file.scan_range) == abs(new_scan_range) + assert hdf5_scan_new_info_file.pixel_size == new_pixel_size + assert hdf5_scan_new_info_file.distance == new_distance + assert len(hdf5_scan_new_info_file.projections) == new_n_proj + + +def test_different_dataset_basename(): + """test conversion succeed if we provide a dataset with a different basename""" + with tempfile.TemporaryDirectory() as data_folder: + + original_scan_path = os.path.join(data_folder, "myscan") + new_scan_path = os.path.join(data_folder, "myscan_435") + + n_proj = 12 + n_alignment_proj = 5 + dim = 4 + energy = 12.35 + n_darks = 2 + n_flats = 1 + + MockEDF( + scan_path=original_scan_path, + n_radio=n_proj, + n_ini_radio=n_proj, + n_extra_radio=n_alignment_proj, + dim=dim, + dark_n=n_darks, + flat_n=n_flats, + energy=energy, + ) + + shutil.move( + original_scan_path, + new_scan_path, + ) + + output_file = os.path.join(data_folder, "nexus_file.nx") + + config = TomoEDFConfig() + config.input_folder = new_scan_path + config.output_file = output_file + config.dataset_basename = "myscan" + nx_file, nx_entry = converter.from_edf_to_nx( + configuration=config, + ) + hdf5_scan = HDF5TomoScan(scan=nx_file, entry=nx_entry) + assert len(hdf5_scan.projections) == n_proj + assert len(hdf5_scan.alignment_projections) == n_alignment_proj + assert len(hdf5_scan.darks) == n_darks + assert len(hdf5_scan.flats) == n_flats + assert hdf5_scan.energy == energy -- GitLab From cce5379281517d7fb6ad24e7ac52ac3598de7068 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 11:45:20 +0200 Subject: [PATCH 25/30] edf2nx: improve value set for alignment projection --- nxtomomill/converter/edf/edfconverter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nxtomomill/converter/edf/edfconverter.py b/nxtomomill/converter/edf/edfconverter.py index 7ee2a57..e6ffe56 100644 --- a/nxtomomill/converter/edf/edfconverter.py +++ b/nxtomomill/converter/edf/edfconverter.py @@ -586,6 +586,7 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: nproj = 0 iref_pj = 0 + alignment_indices = [] for proj_index in proj_indexes: proj_url = proj_urls[proj_index] if ignore(os.path.basename(proj_url.file_path())): @@ -646,6 +647,8 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: # it would be wiser to compute all the rotation angle at the # end when each frame is tag. rotation_dataset[nf] = 0.0 + if configuration.force_angle_calculation: + alignment_indices.append(nf) else: rotation_dataset[nf] = float(str_mot_val[rot_angle_index]) @@ -668,6 +671,17 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: if progress is not None: progress.increase_advancement(i=1) + # we need to update alignement angles values. I wanted to avoid to redo all the previous existing processing. + n_alignment_angles = len(alignment_indices) + if n_alignment_angles == 3: + alignments_angles = numpy.linspace( + scan.scan_range, + 0, + n_alignment_angles, + endpoint=(n_alignment_angles % 2) == 0, + ) + for index, angle in zip(alignments_angles, alignment_indices): + rotation_dataset[index] = angle # store last flat if any remaining in the list if iref_pj < len(ref_projs): nf = store_refs( -- GitLab From 0256478cfae1ba408aa1016dd93690ed25189eac Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 13:41:48 +0200 Subject: [PATCH 26/30] edf2nx: make scan_path and output_file optional inputs (can be provided by the configuration file now) --- nxtomomill/app/edf2nx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nxtomomill/app/edf2nx.py b/nxtomomill/app/edf2nx.py index bdb497c..d8ebe92 100644 --- a/nxtomomill/app/edf2nx.py +++ b/nxtomomill/app/edf2nx.py @@ -83,8 +83,8 @@ def main(argv): "edf to hdf5 - nexus " "compliant file format." ) - parser.add_argument("scan_path", help="folder containing the edf files") - parser.add_argument("output_file", help="foutput .h5 file") + parser.add_argument("scan_path", help="folder containing the edf files", nargs="?") + parser.add_argument("output_file", help="foutput .h5 file", nargs="?") parser.add_argument( "--dataset-basename", "--file-prefix", -- GitLab From 461f5a075405e8c22760f18b8e26692773a8bf4c Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 13:42:39 +0200 Subject: [PATCH 27/30] add `edf-quick-start`-application that can create a configuration file to convert from spec-edf to NXtomo --- nxtomomill/__main__.py | 5 + nxtomomill/app/edfquickstart.py | 70 ++++++++ nxtomomill/io/config/__init__.py | 6 + nxtomomill/io/config/edfconfig.py | 285 +++++++++++++++++++++--------- 4 files changed, 285 insertions(+), 81 deletions(-) create mode 100644 nxtomomill/app/edfquickstart.py diff --git a/nxtomomill/__main__.py b/nxtomomill/__main__.py index 0797bbb..e4c9495 100644 --- a/nxtomomill/__main__.py +++ b/nxtomomill/__main__.py @@ -219,6 +219,11 @@ def main(): description="convert spec-edf acquisition to nexus - hdf5" "format to nx compliant file format", ) + launcher.add_command( + "edf-quick-start", + module_name="nxtomomill.app.edfquickstart", + description="create a configuration file to convert from spec-edf to NXtomo", + ) launcher.add_command( "tomoh52nx", module_name="nxtomomill.app.h52nx", diff --git a/nxtomomill/app/edfquickstart.py b/nxtomomill/app/edfquickstart.py new file mode 100644 index 0000000..b47ef8f --- /dev/null +++ b/nxtomomill/app/edfquickstart.py @@ -0,0 +1,70 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2015-2020 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ + +""" +Application to create a default configuration file to be used by edf2nx application. + +.. code-block:: bash + + usage: nxtomomill edf-quick-start [-h] output_file + + Create a default configuration file + + positional arguments: + output_file output .cfg file + +""" + +__authors__ = [ + "H. Payno", +] +__license__ = "MIT" +__date__ = "23/02/2021" + + +import logging +import argparse +from nxtomomill.io import generate_default_edf_config +from nxtomomill.io import TomoEDFConfig + +logging.basicConfig(level=logging.INFO) +_logger = logging.getLogger(__name__) + + +def main(argv): + """ """ + parser = argparse.ArgumentParser(description="Create a default configuration file") + parser.add_argument("output_file", help="output .cfg file") + parser.add_argument( + "--level", + "--option-level", + help="Level of options to embed in the configuration file. Can be 'required' or 'advanced'.", + default="required", + ) + + options = parser.parse_args(argv[1:]) + + configuration = generate_default_edf_config(level=options.level) + TomoEDFConfig.dict_to_cfg(file_path=options.output_file, dict_=configuration) diff --git a/nxtomomill/io/config/__init__.py b/nxtomomill/io/config/__init__.py index 7a262b8..8388358 100644 --- a/nxtomomill/io/config/__init__.py +++ b/nxtomomill/io/config/__init__.py @@ -37,3 +37,9 @@ from .hdf5config import ( # noqa F401,F403 DXFileConfiguration, generate_default_h5_config, ) + + +from .edfconfig import ( # noqa F401,F403 + TomoEDFConfig, + generate_default_edf_config, +) diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index fb6daca..b7d8994 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -44,12 +44,18 @@ from nxtomomill.nexus.nxsource import SourceType from nxtomomill.utils import FileExtension from nxtomomill.settings import Tomo from typing import Optional, Union +from silx.utils.enum import Enum as _Enum import logging _logger = logging.getLogger(__name__) +class OptionLevel(_Enum): + REQUIRED = "required" + ADVANCED = "advanced" + + class TomoEDFConfig(ConfigBase): """ Configuration class to provide to the convert from h5 to nx @@ -93,6 +99,18 @@ class TomoEDFConfig(ConfigBase): IGNORE_FILE_PATTERN_DK: "some file pattern leading to ignoring the file", } + LEVEL_GENERAL_SECTION = { + INPUT_FOLDER_DK: OptionLevel.REQUIRED, + OUTPUT_FILE_DK: OptionLevel.REQUIRED, + OVERWRITE_DK: OptionLevel.REQUIRED, + FILE_EXTENSION_DK: OptionLevel.ADVANCED, + DATASET_BASENAME_DK: OptionLevel.REQUIRED, + DATASET_FILE_INFO_DK: OptionLevel.REQUIRED, + LOG_LEVEL_DK: OptionLevel.REQUIRED, + TITLE_DK: OptionLevel.ADVANCED, + IGNORE_FILE_PATTERN_DK: OptionLevel.REQUIRED, + } + # EDF KEYS SECTION EDF_KEYS_SECTION_DK = "EDF_KEYS_SECTION" @@ -119,6 +137,15 @@ class TomoEDFConfig(ConfigBase): ROT_ANGLE_KEY_DK: "key to be used for rotation angle", } + LEVEL_KEYS_SECTION = { + MOTOR_POSITION_KEY_DK: OptionLevel.REQUIRED, + MOTOR_MNE_KEY_DK: OptionLevel.REQUIRED, + X_TRANS_KEY_DK: OptionLevel.REQUIRED, + Y_TRANS_KEY_DK: OptionLevel.REQUIRED, + Z_TRANS_KEY_DK: OptionLevel.REQUIRED, + ROT_ANGLE_KEY_DK: OptionLevel.REQUIRED, + } + # DARK AND FLAT SECTION FLAT_DARK_SECTION_DK = "DARK_AND_FLAT_SECTION" @@ -133,6 +160,11 @@ class TomoEDFConfig(ConfigBase): FLAT_NAMES_DK: "prefix of the flat field file(s)", } + LEVEL_DARK_FLAT_SECTION = { + DARK_NAMES_DK: OptionLevel.REQUIRED, + FLAT_NAMES_DK: OptionLevel.REQUIRED, + } + # UNITS SECTION UNIT_SECTION_DK = "UNIT_SECTION" @@ -159,6 +191,15 @@ class TomoEDFConfig(ConfigBase): Z_TRANS_EXPECTED_UNIT: f"Unit used by bliss to save z translation. Must be in of {_valid_metric_values}", } + LEVEL_UNIT_SECTION = { + PIXEL_SIZE_EXPECTED_UNIT: OptionLevel.ADVANCED, + DISTANCE_EXPECTED_UNIT: OptionLevel.ADVANCED, + ENERGY_EXPECTED_UNIT: OptionLevel.ADVANCED, + X_TRANS_EXPECTED_UNIT: OptionLevel.ADVANCED, + Y_TRANS_EXPECTED_UNIT: OptionLevel.ADVANCED, + Z_TRANS_EXPECTED_UNIT: OptionLevel.ADVANCED, + } + # SAMPLE SECTION SAMPLE_SECTION_DK = "SAMPLE_SECTION" @@ -181,6 +222,13 @@ class TomoEDFConfig(ConfigBase): FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE: "Invert rotation angle values in the case of negative `ScanRange` value", } + LEVEL_SAMPLE_SECTION = { + SAMPLE_NAME_DK: OptionLevel.ADVANCED, + FORCE_ANGLE_CALCULATION: OptionLevel.REQUIRED, + FORCE_ANGLE_CALCULATION_ENDPOINT: OptionLevel.REQUIRED, + FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE: OptionLevel.ADVANCED, + } + # SOURCE SECTION SOURCE_SECTION_DK = "SOURCE_SECTION" @@ -198,6 +246,12 @@ class TomoEDFConfig(ConfigBase): SOURCE_TYPE_DK: f"type of the source. Must be one of {SourceType.values()}", } + LEVEL_SOURCE_SECTION = { + INSTRUMENT_NAME_DK: OptionLevel.ADVANCED, + SOURCE_NAME_DK: OptionLevel.ADVANCED, + SOURCE_TYPE_DK: OptionLevel.ADVANCED, + } + # DETECTOR SECTION DETECTOR_SECTION_DK = "DETECTOR_SECTION" @@ -209,6 +263,10 @@ class TomoEDFConfig(ConfigBase): FIELD_OF_VIEW_DK: "Detector field of view. Must be in `Half` or `Full`", } + LEVEL_DETECTOR_SECTION = { + FIELD_OF_VIEW_DK: OptionLevel.ADVANCED, + } + # create comments COMMENTS = COMMENTS_GENERAL_SECTION @@ -219,6 +277,16 @@ class TomoEDFConfig(ConfigBase): COMMENTS.update(COMMENTS_SOURCE_SECTION_DK) COMMENTS.update(COMMENTS_DETECTOR_SECTION_DK) + SECTIONS_LEVEL = { + GENERAL_SECTION_DK: OptionLevel.REQUIRED, + EDF_KEYS_SECTION_DK: OptionLevel.REQUIRED, + FLAT_DARK_SECTION_DK: OptionLevel.REQUIRED, + UNIT_SECTION_DK: OptionLevel.ADVANCED, + SAMPLE_SECTION_DK: OptionLevel.REQUIRED, + SOURCE_SECTION_DK: OptionLevel.ADVANCED, + DETECTOR_SECTION_DK: OptionLevel.ADVANCED, + } + def __init__(self): super().__init__() self._set_freeze(False) @@ -557,87 +625,142 @@ class TomoEDFConfig(ConfigBase): else: self._source_type = SourceType.from_value(source_type) - def to_dict(self) -> dict: + def to_dict(self, level="advanced") -> dict: """convert the configuration to a dictionary""" - return { - self.GENERAL_SECTION_DK: { - self.INPUT_FOLDER_DK: self.input_folder - if self.input_folder is not None - else "", - self.OUTPUT_FILE_DK: self.output_file - if self.output_file is not None - else "", - self.OVERWRITE_DK: self.overwrite, - self.FILE_EXTENSION_DK: self.file_extension.value, - self.DATASET_BASENAME_DK: self.dataset_basename - if self.dataset_basename is not None - else "", - self.DATASET_FILE_INFO_DK: self.dataset_info_file - if self.dataset_info_file is not None - else "", - self.LOG_LEVEL_DK: logging.getLevelName(self.log_level).lower(), - self.TITLE_DK: self.title if self.title is not None else "", - self.IGNORE_FILE_PATTERN_DK: self.ignore_file_patterns - if self.ignore_file_patterns != tuple() - else "", - }, - self.EDF_KEYS_SECTION_DK: { - self.MOTOR_POSITION_KEY_DK: self.motor_position_keys - if self.motor_position_keys != tuple() - else "", - self.MOTOR_MNE_KEY_DK: self.motor_mne_keys - if self.motor_mne_keys != tuple() - else "", - self.ROT_ANGLE_KEY_DK: self.rotation_angle_keys - if self.rotation_angle_keys != tuple() - else "", - self.X_TRANS_KEY_DK: self.x_trans_keys - if self.x_trans_keys != tuple() - else "", - self.Y_TRANS_KEY_DK: self.y_trans_keys - if self.y_trans_keys != tuple() - else "", - self.Z_TRANS_KEY_DK: self.z_trans_keys - if self.z_trans_keys != tuple() - else "", - }, - self.FLAT_DARK_SECTION_DK: { - self.DARK_NAMES_DK: self.dark_names - if self.dark_names != tuple() - else "", - self.FLAT_NAMES_DK: self.flat_names - if self.dark_names != tuple() - else "", - }, - self.UNIT_SECTION_DK: { - self.PIXEL_SIZE_EXPECTED_UNIT: str(self.pixel_size_unit), - self.DISTANCE_EXPECTED_UNIT: str(self.distance_unit), - self.ENERGY_EXPECTED_UNIT: str(self.energy_unit), - self.X_TRANS_EXPECTED_UNIT: str(self.x_trans_unit), - self.Y_TRANS_EXPECTED_UNIT: str(self.y_trans_unit), - self.Z_TRANS_EXPECTED_UNIT: str(self.z_trans_unit), - }, - self.SAMPLE_SECTION_DK: { - self.SAMPLE_NAME_DK: self.sample_name - if self.sample_name is not None - else "", - self.FORCE_ANGLE_CALCULATION: self.force_angle_calculation, - self.FORCE_ANGLE_CALCULATION_ENDPOINT: self.force_angle_calculation_endpoint, - self.FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE: self.angle_calculation_rev_neg_scan_range, - }, - self.SOURCE_SECTION_DK: { - self.INSTRUMENT_NAME_DK: self.instrument_name or "", - self.SOURCE_NAME_DK: self.source_name or "", - self.SOURCE_TYPE_DK: self.source_type.value - if self.source_type is not None - else "", - }, - self.DETECTOR_SECTION_DK: { - self.FIELD_OF_VIEW_DK: self.field_of_view.value - if self.field_of_view is not None - else "", - }, + level = OptionLevel.from_value(level) + sections_callback = { + self.GENERAL_SECTION_DK: self._general_section_to_dict, + self.EDF_KEYS_SECTION_DK: self._edf_keys_section_to_dict, + self.FLAT_DARK_SECTION_DK: self._flat_keys_section_to_dict, + self.UNIT_SECTION_DK: self._unit_section_to_dict, + self.SAMPLE_SECTION_DK: self._sample_section_to_dict, + self.SOURCE_SECTION_DK: self._source_section_to_dict, + self.DETECTOR_SECTION_DK: self._detector_section_to_dict, + } + res = {} + for section, callback in sections_callback.items(): + if ( + level == OptionLevel.ADVANCED + or TomoEDFConfig.SECTIONS_LEVEL[section] == OptionLevel.REQUIRED + ): + res[section] = callback(level=level) + return res + + @staticmethod + def _filter_dict_keys(dict_, level, level_ref) -> dict: + keys = tuple(dict_.keys()) + for key in keys: + if level == OptionLevel.REQUIRED and level_ref[key] == OptionLevel.ADVANCED: + del dict_[key] + return dict_ + + def _general_section_to_dict(self, level) -> dict: + res = { + self.INPUT_FOLDER_DK: self.input_folder + if self.input_folder is not None + else "", + self.OUTPUT_FILE_DK: self.output_file + if self.output_file is not None + else "", + self.OVERWRITE_DK: self.overwrite, + self.FILE_EXTENSION_DK: self.file_extension.value, + self.DATASET_BASENAME_DK: self.dataset_basename + if self.dataset_basename is not None + else "", + self.DATASET_FILE_INFO_DK: self.dataset_info_file + if self.dataset_info_file is not None + else "", + self.LOG_LEVEL_DK: logging.getLevelName(self.log_level).lower(), + self.TITLE_DK: self.title if self.title is not None else "", + self.IGNORE_FILE_PATTERN_DK: self.ignore_file_patterns + if self.ignore_file_patterns != tuple() + else "", + } + return self._filter_dict_keys( + dict_=res, level=level, level_ref=TomoEDFConfig.LEVEL_GENERAL_SECTION + ) + + def _edf_keys_section_to_dict(self, level) -> dict: + res = { + self.MOTOR_POSITION_KEY_DK: self.motor_position_keys + if self.motor_position_keys != tuple() + else "", + self.MOTOR_MNE_KEY_DK: self.motor_mne_keys + if self.motor_mne_keys != tuple() + else "", + self.ROT_ANGLE_KEY_DK: self.rotation_angle_keys + if self.rotation_angle_keys != tuple() + else "", + self.X_TRANS_KEY_DK: self.x_trans_keys + if self.x_trans_keys != tuple() + else "", + self.Y_TRANS_KEY_DK: self.y_trans_keys + if self.y_trans_keys != tuple() + else "", + self.Z_TRANS_KEY_DK: self.z_trans_keys + if self.z_trans_keys != tuple() + else "", + } + return self._filter_dict_keys( + dict_=res, level=level, level_ref=TomoEDFConfig.LEVEL_KEYS_SECTION + ) + + def _flat_keys_section_to_dict(self, level) -> dict: + res = { + self.DARK_NAMES_DK: self.dark_names if self.dark_names != tuple() else "", + self.FLAT_NAMES_DK: self.flat_names if self.dark_names != tuple() else "", + } + return self._filter_dict_keys( + dict_=res, level=level, level_ref=TomoEDFConfig.LEVEL_DARK_FLAT_SECTION + ) + + def _unit_section_to_dict(self, level) -> dict: + res = { + self.PIXEL_SIZE_EXPECTED_UNIT: str(self.pixel_size_unit), + self.DISTANCE_EXPECTED_UNIT: str(self.distance_unit), + self.ENERGY_EXPECTED_UNIT: str(self.energy_unit), + self.X_TRANS_EXPECTED_UNIT: str(self.x_trans_unit), + self.Y_TRANS_EXPECTED_UNIT: str(self.y_trans_unit), + self.Z_TRANS_EXPECTED_UNIT: str(self.z_trans_unit), } + return self._filter_dict_keys( + dict_=res, level=level, level_ref=TomoEDFConfig.LEVEL_UNIT_SECTION + ) + + def _sample_section_to_dict(self, level) -> dict: + res = { + self.SAMPLE_NAME_DK: self.sample_name + if self.sample_name is not None + else "", + self.FORCE_ANGLE_CALCULATION: self.force_angle_calculation, + self.FORCE_ANGLE_CALCULATION_ENDPOINT: self.force_angle_calculation_endpoint, + self.FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE: self.angle_calculation_rev_neg_scan_range, + } + return self._filter_dict_keys( + dict_=res, level=level, level_ref=TomoEDFConfig.LEVEL_SAMPLE_SECTION + ) + + def _source_section_to_dict(self, level) -> dict: + res = { + self.INSTRUMENT_NAME_DK: self.instrument_name or "", + self.SOURCE_NAME_DK: self.source_name or "", + self.SOURCE_TYPE_DK: self.source_type.value + if self.source_type is not None + else "", + } + return self._filter_dict_keys( + dict_=res, level=level, level_ref=TomoEDFConfig.LEVEL_SOURCE_SECTION + ) + + def _detector_section_to_dict(self, level) -> dict: + res = { + self.FIELD_OF_VIEW_DK: self.field_of_view.value + if self.field_of_view is not None + else "", + } + return self._filter_dict_keys( + dict_=res, level=level, level_ref=TomoEDFConfig.LEVEL_DETECTOR_SECTION + ) @staticmethod def from_dict(dict_: dict): @@ -863,6 +986,6 @@ class TomoEDFConfig(ConfigBase): return TomoEDFConfig.COMMENTS[key] -def generate_default_edf_config() -> dict: +def generate_default_edf_config(level: str = "required") -> dict: """generate a default configuration for converting spec-edf to NXtomo""" - return TomoEDFConfig().to_dict() + return TomoEDFConfig().to_dict(level=level) -- GitLab From ccf4f12a9786cc6e28ec2a313b42d218ed9debf9 Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 16:48:32 +0200 Subject: [PATCH 28/30] edf2nx: improvements: * fallback on rotation angle creation if unable to find a valid key in headers and add a warning * use distance_unit * fix title handling * improve field of view creation --- nxtomomill/converter/edf/edfconverter.py | 38 ++++++++++++---- nxtomomill/converter/edf/test/test_edf2nx.py | 46 ++++++++++++++++++++ 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/nxtomomill/converter/edf/edfconverter.py b/nxtomomill/converter/edf/edfconverter.py index e6ffe56..4c811dc 100644 --- a/nxtomomill/converter/edf/edfconverter.py +++ b/nxtomomill/converter/edf/edfconverter.py @@ -43,6 +43,7 @@ from nxtomomill.utils import ImageKey from nxtomomill.converter.version import version as converter_version from nxtomomill.nexus.utils import create_nx_data_group from nxtomomill.nexus.utils import link_nxbeam_to_root +from nxtomomill.nexus.nxdetector import FieldOfView from nxtomomill.settings import Tomo from silx.utils.deprecation import deprecated from tomoscan.esrf.utils import get_parameters_frm_par_or_info @@ -283,6 +284,12 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: z_trans_index, ) = getExtraInfo(scan=scan) + if rot_angle_index == -1 and configuration.force_angle_calculation is False: + _logger.warning( + f"Unable to find one of the defined key for rotation in header ({configuration.rotation_angle_keys}). Will force angle calculation" + ) + configuration.force_angle_calculation = True + data_dataset = h5d.create_dataset( "/entry/instrument/detector/data", shape=(n_frames, scan.dim_2, scan.dim_1), @@ -299,14 +306,13 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: dtype=numpy.int32, ) - h5d["/entry/title"] = ( - configuration.title - if configuration.title is not None - else os.path.basename(scan.path) - ) + title = configuration.title + if title is None: + title = os.path.basename(scan.path) + h5d["/entry/title"] = title + sample_name = configuration.sample_name if configuration.sample_name is None: - sample_name = configuration.sample_name # try to deduce sample name from scan path. try: sample_name = os.path.abspath(scan.path).split(os.sep)[-3:] @@ -340,7 +346,9 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: scan_info=scan.scan_info, ) if distance is not None: - h5d["/entry/instrument/detector/distance"] = distance + h5d["/entry/instrument/detector/distance"] = ( + distance * configuration.pixel_size_unit.value + ) h5d["/entry/instrument/detector/distance"].attrs["unit"] = "m" pixel_size = scan.retrieve_information( @@ -641,7 +649,7 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: rotation_dataset[nf] = angle else: - if rot_angle_index == -1 or configuration.force_angle_calculation: + if configuration.force_angle_calculation: # FIXMe: for alignment projection when calculation is force the # angle will always be set to 0.0 when it should be computed. # it would be wiser to compute all the rotation angle at the @@ -680,8 +688,9 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: n_alignment_angles, endpoint=(n_alignment_angles % 2) == 0, ) - for index, angle in zip(alignments_angles, alignment_indices): + for index, angle in zip(alignment_indices, alignments_angles): rotation_dataset[index] = angle + # store last flat if any remaining in the list if iref_pj < len(ref_projs): nf = store_refs( @@ -710,6 +719,17 @@ def from_edf_to_nx(configuration: TomoEDFConfig, progress=None) -> tuple: h5d["/entry/instrument"].attrs["NX_class"] = "NXinstrument" h5d["/entry/instrument/detector"].attrs["NX_class"] = "NXdetector" h5d["/entry/instrument/detector/data"].attrs["interpretation"] = "image" + if configuration.field_of_view is not None: + field_of_view = configuration.field_of_view + elif abs(scan.scan_range) == 180: + field_of_view = "Half" + elif abs(scan.scan_range) == 360: + field_of_view = "Full" + + if field_of_view is not None: + field_of_view = FieldOfView.from_value(field_of_view) + h5d["/entry/instrument/detector/field_of_view"] = field_of_view.value + h5d["/entry/sample"].attrs["NX_class"] = "NXsample" h5d["/entry/definition"] = "NXtomo" source_grp = h5d["/entry/instrument"].get("source", None) diff --git a/nxtomomill/converter/edf/test/test_edf2nx.py b/nxtomomill/converter/edf/test/test_edf2nx.py index 1b2b079..8447c26 100644 --- a/nxtomomill/converter/edf/test/test_edf2nx.py +++ b/nxtomomill/converter/edf/test/test_edf2nx.py @@ -167,6 +167,52 @@ def test_rotation_angle_infos(scan_range, endpoint, revert, force_angles): ) +def test_rot_angle_key_does_not_exists(): + """test conversion fits TomoEDFConfig parameters regarding the rotation angle calculation options""" + with tempfile.TemporaryDirectory() as data_folder: + scan_path = os.path.join(data_folder, "myscan") + n_proj = 12 + n_alignment_proj = 5 + dim = 4 + + MockEDF( + scan_path=scan_path, + n_radio=n_proj, + n_ini_radio=n_proj, + n_extra_radio=n_alignment_proj, + dim=dim, + dark_n=1, + ref_n=1, + scan_range=180, + ) + + output_file = os.path.join(data_folder, "nexus_file.nx") + + config = TomoEDFConfig() + config.input_folder = scan_path + config.output_file = output_file + config.force_angle_calculation = False + config.rotation_angle_keys = tuple() + + nx_file, nx_entry = converter.from_edf_to_nx( + configuration=config, + ) + hdf5_scan = HDF5TomoScan(scan=nx_file, entry=nx_entry) + + # compute expected rotation angles + raw_rotation_angles = numpy.asarray(hdf5_scan.rotation_angle) + converted_rotation_angles = raw_rotation_angles[ + hdf5_scan.image_key_control == 0 + ] + + expected_angles = numpy.linspace( + 0, 180, n_proj, endpoint=config.force_angle_calculation_endpoint + ) + numpy.testing.assert_almost_equal( + converted_rotation_angles, expected_angles, decimal=3 + ) + + def test_different_info_file(): """insure providing a different spec info file will be taken into account""" with tempfile.TemporaryDirectory() as data_folder: -- GitLab From da9685107bc122e3c78c101b1d5fdeda7ac6f16c Mon Sep 17 00:00:00 2001 From: payno Date: Tue, 26 Apr 2022 17:03:03 +0200 Subject: [PATCH 29/30] edf2nx: set default metric unit to cm for distance, x, y and z translations --- nxtomomill/io/config/edfconfig.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index b7d8994..c0c26dd 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -315,11 +315,11 @@ class TomoEDFConfig(ConfigBase): # units self._pixel_size_unit = MetricSystem.MICROMETER - self._distance_unit = MetricSystem.METER + self._distance_unit = MetricSystem.CENTIMETER self._energy_unit = EnergySI.KILOELECTRONVOLT - self._x_trans_unit = MetricSystem.METER - self._y_trans_unit = MetricSystem.METER - self._z_trans_unit = MetricSystem.METER + self._x_trans_unit = MetricSystem.CENTIMETER + self._y_trans_unit = MetricSystem.CENTIMETER + self._z_trans_unit = MetricSystem.CENTIMETER # sample self._sample_name = None -- GitLab From fac97c78e319b8b86e5fcbd9e70247cf2a9831aa Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Wed, 27 Apr 2022 11:35:31 +0200 Subject: [PATCH 30/30] doc: improve documentation regarding edf 2 nx tutorial + applications --- doc/tutorials/dxfile2nx.rst | 3 +- doc/tutorials/edf2nx.rst | 76 +++++++++++++++++++++++++------ nxtomomill/app/dxfile2nx.py | 3 ++ nxtomomill/app/edf2nx.py | 40 ++++++---------- nxtomomill/app/edfquickstart.py | 1 + nxtomomill/app/h52nx.py | 2 + nxtomomill/app/h5quickstart.py | 1 + nxtomomill/io/config/edfconfig.py | 22 ++++----- 8 files changed, 95 insertions(+), 53 deletions(-) diff --git a/doc/tutorials/dxfile2nx.rst b/doc/tutorials/dxfile2nx.rst index c8aa3b3..5c322e7 100644 --- a/doc/tutorials/dxfile2nx.rst +++ b/doc/tutorials/dxfile2nx.rst @@ -1,7 +1,8 @@ +.. _dxfile2nxtutorial: + dxfile2nx tutorial ================== - the `dxfile2nx` application is used to convert an acquisition stored with the `dxfile format `_ to the `NXTomo `_ format. To call this application you can call directly diff --git a/doc/tutorials/edf2nx.rst b/doc/tutorials/edf2nx.rst index e764a40..b929b62 100644 --- a/doc/tutorials/edf2nx.rst +++ b/doc/tutorials/edf2nx.rst @@ -1,3 +1,5 @@ +.. _edf2nxtutorial: + edf2nx tutorial =============== @@ -6,11 +8,11 @@ the `edf2nx` application is used to convert acquisition from edf standard tomogr This format will also be stored in .h5 / .hdf5 / .nx file. For comprehension we will use the nexus format (.nx) in this tutorial. -To call this application you can call directly +To access this application you can call directly .. code-block:: bash - nxtomomill edf2nx [options] + nxtomomill edf2nx [-h] [--dataset-basename DATASET_BASENAME] [--info-file INFO_FILE] [--config CONFIG] [scan_path] [output_file] if nxtomomill has been installed in the global scope. Otherwise you can call @@ -18,20 +20,25 @@ Otherwise you can call .. code-block:: bash - python -m nxtomomill edf2nx [options] + nxtomomill edf2nx [options] + +simple convertion (no configuration file) +----------------------------------------- -The first two parameters should be: +To execute a conversion from EDF-Spec to NXtomo using the default parameters the first two parameters should be: -* input-file-directory: the root directory containing all the .edf frames and the .info file of the acquisition -* output-file: filename where the tomography scan will be stored +* input folder directory aka scan_path: root directory containing all the .edf frames and the .info file of the acquisition. + By default it expects this folder name to be the .edf file prefix (as `folder_name_0000.edf...`) and .info file to be named `folder_name.info`. + To have advanced option on this please have a look at :ref:`edf2nxtutorialConfigFile` and on `dataset_basename` and `dataset_info_file` fields. +* output_file: output filename which will contain the NXtomo created Sor for example to convert an edf-like tomography dataset '/data/idxx/inhouse/myname/sample1_' to 'sample1_.nx' you should call: .. code-block:: bash - python -m nxtomomill edf2nx /data/idxx/inhouse/myname/sample1_ /data/idxx/inhouse/myname/sample1_.nx + nxtomomill edf2nx /data/idxx/inhouse/myname/sample1_ /data/idxx/inhouse/myname/sample1_.nx Normally the resulting file should have more or less the same size than the initial directory. @@ -42,7 +49,7 @@ You can also access the help of edf2nx by calling: .. code-block:: bash - python -m nxtomomill edf2nx --help + nxtomomill edf2nx --help The result can be displayed using any hdf5 display or using silx: @@ -55,14 +62,53 @@ All the .edf files in the origin directory are considered except those having '_ The algorithm selects raw dark fields - darkendxxxx.edf - and raw flat fields refxxxx.edf. However, if processed darks (dark.edf) and refs (refHST) exist, they are stored in the destination file instead of the raw files. -The names of the motors are hard-coded: 'srot' for the rotation, 'sx', 'sy' and 'sz' for the positioning motors. +The names of the motors are defined to some defauls ('srot', 'somega') for the rotation, 'sx', 'sy' and 'sz' for the positioning motors. +You can defined different keys from the configuration file. If you have 'redundant' keys to be used you can also let us know so we can add those as default keys. + + +.. edf2nxtutorialConfigFile: -.. note:: The conversion is based on a set of key value that are contained in the EDF file headers. - Those values are set to a default value. Those values can be tune either by updating the nxtomomill.settings file - or defined on the fly when calling edf2nx. See edf2nx --help to access the different keys. - Example: +advanced convertion (using configuration file) +---------------------------------------------- + +The conversion is based on settings (defined in settings.py module and EDFTomoConfig class). In order to define: + +1. which key of the EDF headers should be used to get rotation angle, translations +2. prefix to be used to deduce dark and flat files +3. rules to compute rotation angle if we cannot deduce them from edf headers +4. pattern to recognize some file to be ignored (like pyhst reconstruction files) + +All of the settings used can be defined on a configuration file. A configuration file can be created from: + +.. code-block:: bash + + nxtomomill edf-quick-start [edf_2nx_config.cfg] [--level] + +For now user can get configuration file with two level of details: `required` and `advanced`. + +Sections and fields comments should be clear enought for you to understand the meaning of each elements (otherwise let us know). +We can notice that for the previous explained case: + +1. define EDF header keys to be used: this will be defined in the *EDF_KEYS_SECTION* section +2. define prefix to be used for dark and flat: this will be defined in the *DARK_AND_FLAT_SECTION* section +3. rules to compute rotation angle if we cannot deduce them from edf headers: this will be defined in the *SAMPLE_SECTION* section +4. pattern to recognize some file to be ignored (like pyhst reconstruction files): this will be defined in the *GENERAL_SECTION* section + +Regarding the *GENERAL_SECTION* we can also provide more hint on: + +* `dataset_basename`: the `dataset_basename` will be used to deduce all the information required to build a NXtomo: get projections files (like dataset_basename_XXXX.edf) and retrieve infomration like energy, sample / detector distance from the >info file. + If not provided then the `dataset_basename` will be the name of the provided folder. +* `dataset_info_file`: In order to retrieve scan range, energy, distance, pixel size... we use a .info file with predefined keys. + If not provided the converted will look for a `dataset_basename.info` file. But you can provide provide path to another .info file to be used. + + +Once your configuration is edited you can use it from the `nxtomomill edf2nx` using the `--config` option like: + +.. code-block:: bash - .. code-block:: bash + nxtomomill edf2nx --config edf_2nx_config.cfg - python -m nxtomomill tomoedf2nx /data/idxx/inhouse/myname/sample1_ /data/idxx/inhouse/myname/sample1_.nx --rot_angle_key=srot --ignore_file_containing=_slice_,test +.. note:: to ease usage the "dataset basename" and the "info file" can also be provided from command line (`--dataset-basename` and `--info-file` options). + If you provide one of those parameters from the command line option then it should not be provided from the configuration. + This is the same for `scan_path` aka folder path (containing raw data / .edf) and `output_file` diff --git a/nxtomomill/app/dxfile2nx.py b/nxtomomill/app/dxfile2nx.py index e0deecc..1ae5a45 100644 --- a/nxtomomill/app/dxfile2nx.py +++ b/nxtomomill/app/dxfile2nx.py @@ -58,6 +58,9 @@ Application to convert from dx file (HDF5) to NXTomo (HDF5) file incident beam energy in keV --overwrite Do not ask for user permission to overwrite output files --data-copy Force data duplication of frames. This will permit to have an 'all-embed' file. Otherwise we will have link between the dxfile and the NXTomo. + +For a complete tutorial you can have a look at :ref:`dxfile2nxtutorial` + """ __authors__ = [ diff --git a/nxtomomill/app/edf2nx.py b/nxtomomill/app/edf2nx.py index d8ebe92..3dc21be 100644 --- a/nxtomomill/app/edf2nx.py +++ b/nxtomomill/app/edf2nx.py @@ -28,37 +28,24 @@ Application to convert a tomo dataset written in edf into and hdf5/nexus file. .. code-block:: bash - usage: nxtomomill edf2nx [-h] [--file_extension] [--motor_pos_key MOTOR_POS_KEY] [--motor_mne_key MOTOR_MNE_KEY] [--refs_name_keys REFS_NAME_KEYS] [--ignore_file_containing IGNORE_FILE_CONTAINING] - [--rot_angle_key ROT_ANGLE_KEY] [--dark_names DARK_NAMES] [--x_trans_key X_TRANS_KEY] [--y_trans_key Y_TRANS_KEY] [--z_trans_key Z_TRANS_KEY] - scan_path output_file + usage: nxtomomill edf2nx [-h] [--dataset-basename DATASET_BASENAME] [--info-file INFO_FILE] [--config CONFIG] [scan_path] [output_file] convert data acquired as edf to hdf5 - nexus compliant file format. positional arguments: - scan_path folder containing the edf files - output_file foutput .h5 file + scan_path folder containing the edf files + output_file foutput .h5 file optional arguments: - -h, --help show this help message and exit - --file_extension extension of the output file. Valid values are .h5/.hdf5/.nx - --motor_pos_key MOTOR_POS_KEY - motor position key in EDF HEADER - --motor_mne_key MOTOR_MNE_KEY - motor mne key in EDF HEADER - --refs_name_keys REFS_NAME_KEYS - prefix of flat field file - --ignore_file_containing IGNORE_FILE_CONTAINING - substring that lead to ignoring the file if contained in the name - --rot_angle_key ROT_ANGLE_KEY - rotation angle key in EDF HEADER - --dark_names DARK_NAMES - prefix of the dark field file - --x_trans_key X_TRANS_KEY - x translation key in EDF HEADER - --y_trans_key Y_TRANS_KEY - y translation key in EDF HEADER - --z_trans_key Z_TRANS_KEY - z translation key in EDF HEADER + -h, --help show this help message and exit + --dataset-basename DATASET_BASENAME, --file-prefix DATASET_BASENAME + file prefix to be used to deduce projections + --info-file INFO_FILE + .info file containing acquisition information (ScanRange, Energy, TOMO_N...) + --config CONFIG, --configuration-file CONFIG, --configuration CONFIG + file containing the full configuration to convert from SPEC-EDF to bliss to nexus. Default configuration file can be created from `nxtomomill edf-quick-start` command + +For a complete tutorial you can have a look at :ref:`edf2nxtutorial` """ __authors__ = ["C. Nemoz", "H. Payno", "A.Sole"] @@ -101,7 +88,8 @@ def main(argv): "--configuration-file", "--configuration", default=None, - help="file containing the full configuration to convert from SPEC-EDF to bliss to nexus", + help="file containing the full configuration to convert from SPEC-EDF to bliss to nexus. " + "Default configuration file can be created from `nxtomomill edf-quick-start` command", ) options = parser.parse_args(argv[1:]) diff --git a/nxtomomill/app/edfquickstart.py b/nxtomomill/app/edfquickstart.py index b47ef8f..ce4f2e9 100644 --- a/nxtomomill/app/edfquickstart.py +++ b/nxtomomill/app/edfquickstart.py @@ -35,6 +35,7 @@ Application to create a default configuration file to be used by edf2nx applicat positional arguments: output_file output .cfg file +For a complete tutorial you can have a look at :ref:`edf2nxtutorial` """ __authors__ = [ diff --git a/nxtomomill/app/h52nx.py b/nxtomomill/app/h52nx.py index 92374a3..8de587e 100644 --- a/nxtomomill/app/h52nx.py +++ b/nxtomomill/app/h52nx.py @@ -95,6 +95,8 @@ Application to convert a bliss-hdf5 tomography dataset to Nexus - NXtomo (hdf5) --config CONFIG, --config-file CONFIG, --configuration CONFIG, --configuration-file CONFIG file containing the full configuration to convert from h5 bliss to nexus +For a complete tutorial you can have a look at: :ref:`Tomoh52nx` + """ __authors__ = ["C. Nemoz", "H. Payno", "A.Sole"] diff --git a/nxtomomill/app/h5quickstart.py b/nxtomomill/app/h5quickstart.py index b2c239f..d3ebcb3 100644 --- a/nxtomomill/app/h5quickstart.py +++ b/nxtomomill/app/h5quickstart.py @@ -40,6 +40,7 @@ Application to create a default configuration file to be used by h52nx applicati --from-title-names Provide minimalistic configuration to make a conversion from titles names. (FRAME TYPE section is ignored). Exclusive with `from-scan-urls` option --from-scan-urls Provide minimalistic configuration to make a conversion from scan urls. (ENTRIES and TITLES section is ignored). Exclusive with `from-title-names` option +For a complete tutorial you can have a look at: :ref:`Tomoh52nx` """ __authors__ = [ diff --git a/nxtomomill/io/config/edfconfig.py b/nxtomomill/io/config/edfconfig.py index c0c26dd..d8b8959 100644 --- a/nxtomomill/io/config/edfconfig.py +++ b/nxtomomill/io/config/edfconfig.py @@ -88,15 +88,15 @@ class TomoEDFConfig(ConfigBase): COMMENTS_GENERAL_SECTION = { GENERAL_SECTION_DK: "general information. \n", - INPUT_FOLDER_DK: "input file if not provided must be provided from the command line", - OUTPUT_FILE_DK: "output file name. If not provided will use the input file basename and the file extension", + INPUT_FOLDER_DK: "Folder containing .edf files. if not provided from the configuration file must be provided from the command line", + OUTPUT_FILE_DK: "output file name. If not provided from the configuration file must be provided from the command line", OVERWRITE_DK: "overwrite output files if exists without asking", FILE_EXTENSION_DK: "file extension. Ignored if the output file is provided and contains an extension", - DATASET_BASENAME_DK: "dataset file prefix. If not provided will take the folder basename", - DATASET_FILE_INFO_DK: f"path to .info file containing dataset information (Energy, ScanRange, TOMO_N...). If not will deduce it from {INPUT_FOLDER_DK} and {DATASET_BASENAME_DK}", + DATASET_BASENAME_DK: f"dataset file prefix. Usde to determine projections file and info file. If not provided will take the name of {INPUT_FOLDER_DK}", + DATASET_FILE_INFO_DK: f"path to .info file containing dataset information (Energy, ScanRange, TOMO_N...). If not will deduce it from {DATASET_BASENAME_DK}", LOG_LEVEL_DK: 'Log level. Valid levels are "debug", "info", "warning" and "error"', TITLE_DK: "NXtomo title", - IGNORE_FILE_PATTERN_DK: "some file pattern leading to ignoring the file", + IGNORE_FILE_PATTERN_DK: "some file pattern leading to ignoring the file. Like reconstructed slice files.", } LEVEL_GENERAL_SECTION = { @@ -128,9 +128,9 @@ class TomoEDFConfig(ConfigBase): ROT_ANGLE_KEY_DK = "rot_angle_key" COMMENTS_KEYS_SECTION = { - EDF_KEYS_SECTION_DK: "section to define EDF keys to pick in headers to deduce several information.\n", + EDF_KEYS_SECTION_DK: "section to define EDF keys to pick from headers to deduce information like rotation angle.\n", MOTOR_POSITION_KEY_DK: "motor position key", - MOTOR_MNE_KEY_DK: "motor mne key", + MOTOR_MNE_KEY_DK: "key to retrieve indices of each motor in metadata", X_TRANS_KEY_DK: "key to be used for x translation", Y_TRANS_KEY_DK: "key to be used for y translation", Z_TRANS_KEY_DK: "key to be used for z translation", @@ -156,8 +156,8 @@ class TomoEDFConfig(ConfigBase): COMMENTS_DARK_FLAT_SECTION = { FLAT_DARK_SECTION_DK: "section to define dark and flat detection. \n", - DARK_NAMES_DK: "prefix of the dark field file(s)", - FLAT_NAMES_DK: "prefix of the flat field file(s)", + DARK_NAMES_DK: "prefix of dark field file(s)", + FLAT_NAMES_DK: "prefix of flat field file(s)", } LEVEL_DARK_FLAT_SECTION = { @@ -217,8 +217,8 @@ class TomoEDFConfig(ConfigBase): COMMENTS_SAMPLE_SECTION_DK = { SAMPLE_SECTION_DK: "section dedicated to sample definition.\n", SAMPLE_NAME_DK: "name of the sample", - FORCE_ANGLE_CALCULATION: "if set to False try first to deduce rotation angle from edf metadata. Else compute them from numpy.linspace", - FORCE_ANGLE_CALCULATION_ENDPOINT: "if rotation angles have to be calculated set numpy.linspace endpoint parameter to this value. If True then the rotation angle value of the last projection will be equal to the `ScanRange` value", + FORCE_ANGLE_CALCULATION: "Should the rotation angle be computed from scan range and numpy.linspace or should we try to load it from .edf header.", + FORCE_ANGLE_CALCULATION_ENDPOINT: "If rotation angles have to be calculated set numpy.linspace endpoint parameter to this value. If True then the rotation angle value of the last projection will be equal to the `ScanRange` value", FORCE_ANGLE_CALCULATION_REVERT_NEG_SCAN_RANGE: "Invert rotation angle values in the case of negative `ScanRange` value", } -- GitLab